Skip to content

Commit 1228299

Browse files
hmottestadhmottestad
committed
GH-5691 CLI for running and storing query explanations
1 parent 64ce9ac commit 1228299

3 files changed

Lines changed: 165 additions & 10 deletions

File tree

‎testsuites/benchmark/src/main/java/org/eclipse/rdf4j/benchmark/plan/QueryPlanSnapshotCli.java‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -171,9 +171,9 @@ private void runCaptureMode(QueryPlanSnapshotCliOptions options) throws Exceptio
171171

172172
private void runSingleQueryCapture(QueryPlanSnapshotCliOptions options,
173173
QueryPlanSnapshotStoreSupport.StoreRuntime storeRuntime) throws Exception {
174-
for (Theme theme : Theme.values()) {
175-
QueryPlanSnapshotStoreSupport.loadThemeData(storeRuntime.repository, theme);
176-
}
174+
QueryPlanSnapshotStoreSupport.ThemeDataLoadStatus themeDataLoadStatus = QueryPlanSnapshotStoreSupport
175+
.ensureThemeDataLoaded(storeRuntime);
176+
printThemeDataLoadStatus(themeDataLoadStatus);
177177
BenchmarkQuery benchmarkQuery = resolveBenchmarkQuery(options);
178178
String queryText = resolveQueryText(options, benchmarkQuery);
179179
String querySource = benchmarkQuery == null ? "direct" : "theme-index";
@@ -182,7 +182,8 @@ private void runSingleQueryCapture(QueryPlanSnapshotCliOptions options,
182182
? options.outputDirectory
183183
: defaultOutputDirectory(options.store);
184184

185-
FeatureFlagCollector featureFlags = createFeatureFlagCollector(options, storeRuntime, querySource);
185+
FeatureFlagCollector featureFlags = createFeatureFlagCollector(options, storeRuntime, querySource,
186+
themeDataLoadStatus);
186187
QueryPlanCaptureContext context = createContext(options, benchmarkQuery, queryText, querySource, queryId,
187188
outputDirectory, featureFlags);
188189
QueryPlanCapture capture = new QueryPlanCapture();
@@ -218,15 +219,14 @@ private void runAllThemeQueriesCapture(QueryPlanSnapshotCliOptions options,
218219
Path outputDirectory = options.outputDirectory != null
219220
? options.outputDirectory
220221
: defaultOutputDirectory(options.store);
222+
QueryPlanSnapshotStoreSupport.ThemeDataLoadStatus themeDataLoadStatus = QueryPlanSnapshotStoreSupport
223+
.ensureThemeDataLoaded(storeRuntime);
224+
printThemeDataLoadStatus(themeDataLoadStatus);
221225
QueryPlanCapture capture = new QueryPlanCapture();
222226
Theme[] allThemes = Theme.values();
223227
int total = allThemes.length * ThemeQueryCatalog.QUERY_COUNT;
224228
int current = 0;
225229

226-
for (Theme theme : allThemes) {
227-
QueryPlanSnapshotStoreSupport.loadThemeData(storeRuntime.repository, theme);
228-
}
229-
230230
for (Theme theme : allThemes) {
231231
for (int queryIndex = 0; queryIndex < ThemeQueryCatalog.QUERY_COUNT; queryIndex++) {
232232
current++;
@@ -239,7 +239,7 @@ private void runAllThemeQueriesCapture(QueryPlanSnapshotCliOptions options,
239239
String queryId = defaultQueryId(perQueryOptions, benchmarkQuery);
240240

241241
FeatureFlagCollector featureFlags = createFeatureFlagCollector(perQueryOptions, storeRuntime,
242-
querySource);
242+
querySource, themeDataLoadStatus);
243243
QueryPlanCaptureContext context = createContext(perQueryOptions, benchmarkQuery, queryText, querySource,
244244
queryId, outputDirectory, featureFlags);
245245

@@ -883,6 +883,19 @@ private void printThemeQueries(Theme theme) {
883883
}
884884
}
885885

886+
private void printThemeDataLoadStatus(QueryPlanSnapshotStoreSupport.ThemeDataLoadStatus themeDataLoadStatus) {
887+
if (themeDataLoadStatus.lmdbFullyLoadedSizeBytes == null) {
888+
return;
889+
}
890+
if (themeDataLoadStatus.reusedLmdbData) {
891+
output.println("LMDB data already fully loaded (" + themeDataLoadStatus.lmdbFullyLoadedSizeBytes
892+
+ " bytes). Skipping reload.");
893+
return;
894+
}
895+
output.println("LMDB data loaded. Recorded fully-loaded size=" + themeDataLoadStatus.lmdbFullyLoadedSizeBytes
896+
+ " bytes.");
897+
}
898+
886899
private static QueryPlanCaptureContext createContext(QueryPlanSnapshotCliOptions options,
887900
BenchmarkQuery benchmarkQuery,
888901
String queryText, String querySource, String queryId, Path outputDirectory,
@@ -916,7 +929,8 @@ private static QueryPlanCaptureContext createContext(QueryPlanSnapshotCliOptions
916929
}
917930

918931
private static FeatureFlagCollector createFeatureFlagCollector(QueryPlanSnapshotCliOptions options,
919-
QueryPlanSnapshotStoreSupport.StoreRuntime storeRuntime, String querySource) {
932+
QueryPlanSnapshotStoreSupport.StoreRuntime storeRuntime, String querySource,
933+
QueryPlanSnapshotStoreSupport.ThemeDataLoadStatus themeDataLoadStatus) {
920934
FeatureFlagCollector featureFlags = new FeatureFlagCollector()
921935
.addValue("cli.store", options.store.id)
922936
.addValue("cli.theme", options.theme.name())
@@ -944,6 +958,11 @@ private static FeatureFlagCollector createFeatureFlagCollector(QueryPlanSnapshot
944958
.addReflectiveField("lmdbConfig.autoGrow", storeRuntime.lmdbStoreConfig, "autoGrow")
945959
.addReflectiveGetter("lmdbConfig.valueDbSize", storeRuntime.lmdbStoreConfig, "getValueDBSize")
946960
.addReflectiveGetter("lmdbConfig.tripleDbSize", storeRuntime.lmdbStoreConfig, "getTripleDBSize");
961+
if (themeDataLoadStatus.lmdbFullyLoadedSizeBytes != null) {
962+
featureFlags.addValue("lmdbData.fullyLoadedSizeBytes",
963+
themeDataLoadStatus.lmdbFullyLoadedSizeBytes.toString());
964+
}
965+
featureFlags.addValue("lmdbData.reusedWithoutReload", Boolean.toString(themeDataLoadStatus.reusedLmdbData));
947966
}
948967

949968
QueryPlanCapture.registerConfiguredFeatureFlags(featureFlags);

‎testsuites/benchmark/src/main/java/org/eclipse/rdf4j/benchmark/plan/QueryPlanSnapshotStoreSupport.java‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@
1313

1414
import java.io.IOException;
1515
import java.io.UncheckedIOException;
16+
import java.nio.charset.StandardCharsets;
1617
import java.nio.file.Files;
1718
import java.nio.file.Path;
19+
import java.nio.file.StandardOpenOption;
1820
import java.util.Comparator;
1921
import java.util.stream.Stream;
2022

@@ -30,6 +32,9 @@
3032

3133
final class QueryPlanSnapshotStoreSupport {
3234

35+
private static final String LMDB_FULLY_LOADED_SIZE_FILE = ".rdf4j-query-plan-cli-fully-loaded-size-bytes";
36+
private static final long LMDB_SIZE_MATCH_TOLERANCE_BYTES = 1_048_576L;
37+
3338
private QueryPlanSnapshotStoreSupport() {
3439
}
3540

@@ -41,6 +46,26 @@ static void loadThemeData(SailRepository repository, Theme theme) throws IOExcep
4146
}
4247
}
4348

49+
static ThemeDataLoadStatus ensureThemeDataLoaded(StoreRuntime storeRuntime) throws IOException {
50+
if (storeRuntime.lmdbStore == null) {
51+
loadAllThemes(storeRuntime.repository);
52+
return ThemeDataLoadStatus.memoryStore();
53+
}
54+
55+
Path dataDirectory = storeRuntime.dataDirectory;
56+
Long recordedSize = readRecordedFullyLoadedSize(dataDirectory);
57+
long currentSize = computeLmdbDataSizeBytes(dataDirectory);
58+
if (recordedSize != null && recordedSize.longValue() > 0
59+
&& currentSize + LMDB_SIZE_MATCH_TOLERANCE_BYTES >= recordedSize.longValue()) {
60+
return ThemeDataLoadStatus.lmdbReused(recordedSize.longValue());
61+
}
62+
63+
loadAllThemes(storeRuntime.repository);
64+
long fullyLoadedSize = computeLmdbDataSizeBytes(dataDirectory);
65+
recordFullyLoadedSize(dataDirectory, fullyLoadedSize);
66+
return ThemeDataLoadStatus.lmdbLoaded(fullyLoadedSize);
67+
}
68+
4469
static StoreRuntime createStoreRuntime(QueryPlanSnapshotCliOptions options) throws IOException {
4570
if (options.store == QueryPlanSnapshotCliOptions.StoreType.MEMORY) {
4671
MemoryStore memoryStore = new MemoryStore();
@@ -60,6 +85,68 @@ static StoreRuntime createStoreRuntime(QueryPlanSnapshotCliOptions options) thro
6085
return new StoreRuntime(repository, null, lmdbStore, config, dataDirectory, deleteDataDirectory);
6186
}
6287

88+
private static void loadAllThemes(SailRepository repository) throws IOException {
89+
for (Theme theme : Theme.values()) {
90+
loadThemeData(repository, theme);
91+
}
92+
}
93+
94+
private static long computeLmdbDataSizeBytes(Path dataDirectory) throws IOException {
95+
if (dataDirectory == null || !Files.exists(dataDirectory)) {
96+
return 0L;
97+
}
98+
99+
Path marker = fullyLoadedSizeMarker(dataDirectory);
100+
try (Stream<Path> walk = Files.walk(dataDirectory)) {
101+
return walk
102+
.filter(Files::isRegularFile)
103+
.filter(path -> !path.equals(marker))
104+
.mapToLong(path -> {
105+
try {
106+
return Files.size(path);
107+
} catch (IOException e) {
108+
throw new UncheckedIOException(e);
109+
}
110+
})
111+
.sum();
112+
} catch (UncheckedIOException e) {
113+
throw e.getCause();
114+
}
115+
}
116+
117+
private static Long readRecordedFullyLoadedSize(Path dataDirectory) throws IOException {
118+
if (dataDirectory == null) {
119+
return null;
120+
}
121+
Path marker = fullyLoadedSizeMarker(dataDirectory);
122+
if (!Files.isRegularFile(marker)) {
123+
return null;
124+
}
125+
String raw = Files.readString(marker, StandardCharsets.UTF_8).trim();
126+
if (raw.isEmpty()) {
127+
return null;
128+
}
129+
try {
130+
long parsed = Long.parseLong(raw);
131+
return parsed >= 0 ? parsed : null;
132+
} catch (NumberFormatException ignored) {
133+
return null;
134+
}
135+
}
136+
137+
private static void recordFullyLoadedSize(Path dataDirectory, long fullyLoadedSizeBytes) throws IOException {
138+
if (dataDirectory == null) {
139+
return;
140+
}
141+
Path marker = fullyLoadedSizeMarker(dataDirectory);
142+
Files.writeString(marker, Long.toString(fullyLoadedSizeBytes), StandardCharsets.UTF_8,
143+
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
144+
}
145+
146+
private static Path fullyLoadedSizeMarker(Path dataDirectory) {
147+
return dataDirectory.resolve(LMDB_FULLY_LOADED_SIZE_FILE);
148+
}
149+
63150
private static void deleteDirectory(Path directory) throws IOException {
64151
if (directory == null || !Files.exists(directory)) {
65152
return;
@@ -107,4 +194,26 @@ public void close() throws IOException {
107194
}
108195
}
109196
}
197+
198+
static final class ThemeDataLoadStatus {
199+
final boolean reusedLmdbData;
200+
final Long lmdbFullyLoadedSizeBytes;
201+
202+
private ThemeDataLoadStatus(boolean reusedLmdbData, Long lmdbFullyLoadedSizeBytes) {
203+
this.reusedLmdbData = reusedLmdbData;
204+
this.lmdbFullyLoadedSizeBytes = lmdbFullyLoadedSizeBytes;
205+
}
206+
207+
static ThemeDataLoadStatus memoryStore() {
208+
return new ThemeDataLoadStatus(false, null);
209+
}
210+
211+
static ThemeDataLoadStatus lmdbLoaded(long lmdbFullyLoadedSizeBytes) {
212+
return new ThemeDataLoadStatus(false, lmdbFullyLoadedSizeBytes);
213+
}
214+
215+
static ThemeDataLoadStatus lmdbReused(long lmdbFullyLoadedSizeBytes) {
216+
return new ThemeDataLoadStatus(true, lmdbFullyLoadedSizeBytes);
217+
}
218+
}
110219
}

‎testsuites/benchmark/src/test/java/org/eclipse/rdf4j/benchmark/plan/QueryPlanSnapshotCliTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,33 @@ void runModePrintsExecutionVerificationSummary() throws Exception {
260260
assertTrue(printed.contains("runs="), printed);
261261
}
262262

263+
@Test
264+
void lmdbRunRecordsLoadedSizeAndSkipsReloadWhenSizeMatches() throws Exception {
265+
Path lmdbDataDirectory = Files.createTempDirectory("rdf4j-cli-lmdb-reuse-");
266+
QueryPlanSnapshotCliOptions options = QueryPlanSnapshotCli.parseArgs(new String[] {
267+
"--no-interactive",
268+
"--store", "lmdb",
269+
"--lmdb-data-dir", lmdbDataDirectory.toString(),
270+
"--theme", "MEDICAL_RECORDS",
271+
"--query-index", "0",
272+
"--persist", "false"
273+
});
274+
275+
ByteArrayOutputStream firstRunOutput = new ByteArrayOutputStream();
276+
QueryPlanSnapshotCli firstRunCli = new QueryPlanSnapshotCli(new BufferedReader(new StringReader("")),
277+
new PrintStream(firstRunOutput, true, StandardCharsets.UTF_8.name()));
278+
firstRunCli.run(options);
279+
280+
ByteArrayOutputStream secondRunOutput = new ByteArrayOutputStream();
281+
QueryPlanSnapshotCli secondRunCli = new QueryPlanSnapshotCli(new BufferedReader(new StringReader("")),
282+
new PrintStream(secondRunOutput, true, StandardCharsets.UTF_8.name()));
283+
secondRunCli.run(options);
284+
285+
String secondRunPrinted = secondRunOutput.toString(StandardCharsets.UTF_8);
286+
assertTrue(secondRunPrinted.contains("LMDB data already fully loaded"),
287+
"Expected second run to skip reloading LMDB data when byte size matches: " + secondRunPrinted);
288+
}
289+
263290
@Test
264291
void runModePrintsConfiguredQueryTimeoutInResultsSection() throws Exception {
265292
ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream();

0 commit comments

Comments
 (0)