GH-5691 CLI for running and storing query explanations

Author: hmottestad

testsuites/benchmark/README.md

@@ -10,6 +10,9 @@ Scope:
 - Metadata + feature-flag state for reproducibility.
 - Smart semantic diff modes.
 
+Status:
+- Experimental CLI and capture APIs (`@Experimental`).
+
 ## What You Get
 
 Each snapshot stores:
@@ -24,6 +27,9 @@ Each snapshot stores:
   - IR-rendered query (optimized/executed by default)
 - Metadata (git commit, benchmark/store/theme context, custom keys).
 - Feature flags (system properties, direct values, reflection probes).
+- Execution verification summary printed by CLI:
+  - repeated query executions
+  - dynamic run count with a soft 60-second cap per query
 
 ## Where Things Live
 
@@ -59,7 +65,7 @@ Interactive menu behavior:
   - `list themes`
   - `list queries`
   - `help`
-- Run mode prompt order starts with: `store` -> `query source` (`themed`, `manual`, or `file`) -> remaining fields.
+- Run mode prompt order starts with: `store` -> `query source` (`themed`, `manual`, `file`, or `all-themed`) -> remaining fields.
 - No-argument run wizard also prompts optional CLI flags: persist, compare-latest, diff-mode, query-id, output-dir, lmdb-data-dir (for LMDB).
 
 Show help:
@@ -115,7 +121,7 @@ Interactive selection (browse/view/compare):
 ```
 
 Interactive run browser actions:
-- `view run`: pick a run and print full details (captured time, query id/fingerprint, metadata, feature flags, per-level explanation availability).
+- `view run`: pick a run and print full details, original query, and full per-level explanation blocks (including IR-rendered query text when present).
 - `compare runs`: pick left/right runs and print semantic diff.
 - `quit`: exit browser.
 
@@ -156,6 +162,44 @@ Useful for ad-hoc compare-only checks:
 ... -Dexec.args="--store memory --theme MEDICAL_RECORDS --query-index 0 --no-persist --compare-latest"
 ```
 
+Every CLI run also prints:
+
+- `=== Execution Verification ===`
+- `runs`, `totalMillis`, `averageMillis`, `resultCount`
+- `softLimitMillis` (currently `60000`)
+- whether stopping hit the soft-limit projection or max repeat-run cap
+
+### 7) Run all themed queries across all themes for one store
+
+Memory store:
+
+```bash
+... -Dexec.args="--store memory --all-theme-queries"
+```
+
+LMDB store:
+
+```bash
+... -Dexec.args="--store lmdb --all-theme-queries"
+```
+
+With in-memory-only capture:
+
+```bash
+... -Dexec.args="--store memory --all-theme-queries --no-persist"
+```
+
+With compare-latest per query run:
+
+```bash
+... -Dexec.args="--store lmdb --all-theme-queries --compare-latest --diff-mode structure+estimates"
+```
+
+Notes:
+- `--all-theme-queries` is run mode only (not compare mode).
+- Do not combine `--all-theme-queries` with single-query selectors (`--theme`, `--theme-query`, `--query-index`, `--query`, `--query-file`).
+- In interactive mode this is available via query source `all-themed`.
+
 ## Smart Diff Modes
 
 Set with `--diff-mode`:
@@ -347,6 +391,7 @@ This path stores themed benchmark artifacts without CLI wrapper.
 - `--no-persist`
 - `--compare-latest`
 - `--compare-existing`
+- `--all-theme-queries`
 - `--query-id <id>`
 - `--fingerprint <hash>`
 - `--compare-indices <i,j>`

testsuites/benchmark/src/main/java/org/eclipse/rdf4j/benchmark/plan/QueryPlanSnapshotCli.java

@@ -24,6 +24,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.TimeUnit;
 
 import org.eclipse.rdf4j.benchmark.common.BenchmarkQuery;
 import org.eclipse.rdf4j.benchmark.common.ThemeQueryCatalog;
@@ -56,6 +57,9 @@ public final class QueryPlanSnapshotCli {
 			"optimized",
 			"executed");
 	private static final String MANUAL_QUERY_ID_ENTRY = "<manual entry>";
+	private static final long EXECUTION_REPEAT_SOFT_LIMIT_NANOS = TimeUnit.SECONDS.toNanos(60);
+	private static final int EXECUTION_REPEAT_MIN_RUNS = 2;
+	private static final int EXECUTION_REPEAT_MAX_RUNS = 128;
 
 	public static void main(String[] args) throws Exception {
 		QueryPlanSnapshotCliOptions options = parseArgs(args);
@@ -164,6 +168,7 @@ private void runSingleQueryCapture(QueryPlanSnapshotCliOptions options,
 
 		QueryPlanSnapshot currentSnapshot;
 		Path snapshotPath = null;
+		QueryExecutionVerification executionVerification;
 		try (SailRepositoryConnection connection = storeRuntime.repository.getConnection()) {
 			if (options.persist) {
 				snapshotPath = capture.captureAndWrite(context, () -> connection.prepareTupleQuery(queryText));
@@ -173,10 +178,12 @@ private void runSingleQueryCapture(QueryPlanSnapshotCliOptions options,
 				currentSnapshot = capture.capture(context, () -> connection.prepareTupleQuery(queryText));
 				output.println("Snapshot captured in-memory only (--persist=false).");
 			}
+			executionVerification = verifyRepeatedExecution(connection, queryText);
 		}
 
 		printResultsSection(options, queryId, queryText);
 		printPrettyExplanations(currentSnapshot);
+		printExecutionVerification(executionVerification);
 
 		if (options.compareLatest) {
 			compareWithLatest(outputDirectory, queryId, currentSnapshot, snapshotPath, capture, options.diffMode);
@@ -212,6 +219,7 @@ private void runAllThemeQueriesCapture(QueryPlanSnapshotCliOptions options,
 
 				QueryPlanSnapshot currentSnapshot;
 				Path snapshotPath = null;
+				QueryExecutionVerification executionVerification;
 				try (SailRepositoryConnection connection = storeRuntime.repository.getConnection()) {
 					if (options.persist) {
 						snapshotPath = capture.captureAndWrite(context, () -> connection.prepareTupleQuery(queryText));
@@ -221,6 +229,7 @@ private void runAllThemeQueriesCapture(QueryPlanSnapshotCliOptions options,
 						currentSnapshot = capture.capture(context, () -> connection.prepareTupleQuery(queryText));
 						output.println("Snapshot captured in-memory only (--persist=false).");
 					}
+					executionVerification = verifyRepeatedExecution(connection, queryText);
 				}
 
 				output.println();
@@ -229,6 +238,7 @@ private void runAllThemeQueriesCapture(QueryPlanSnapshotCliOptions options,
 						"Theme=" + theme + ", QueryIndex=" + queryIndex + ", QueryName=" + benchmarkQuery.getName());
 				printResultsSection(perQueryOptions, queryId, queryText);
 				printPrettyExplanations(currentSnapshot);
+				printExecutionVerification(executionVerification);
 
 				if (options.compareLatest) {
 					compareWithLatest(outputDirectory, queryId, currentSnapshot, snapshotPath, capture,
@@ -942,6 +952,84 @@ private void printExplanation(String levelKey, QueryPlanExplanation explanation)
 		}
 	}
 
+	private QueryExecutionVerification verifyRepeatedExecution(SailRepositoryConnection connection, String queryText) {
+		long elapsedNanos = 0;
+		long stableResultCount = Long.MIN_VALUE;
+		int runs = 0;
+		boolean softLimitReached = false;
+
+		while (runs < EXECUTION_REPEAT_MAX_RUNS) {
+			if (runs >= EXECUTION_REPEAT_MIN_RUNS) {
+				long averageNanos = Math.max(1L, elapsedNanos / runs);
+				if (elapsedNanos + averageNanos > EXECUTION_REPEAT_SOFT_LIMIT_NANOS) {
+					softLimitReached = true;
+					break;
+				}
+			} else if (elapsedNanos >= EXECUTION_REPEAT_SOFT_LIMIT_NANOS) {
+				softLimitReached = true;
+				break;
+			}
+
+			long startedAt = System.nanoTime();
+			long currentResultCount = connection.prepareTupleQuery(queryText).evaluate().stream().count();
+			long runNanos = Math.max(1L, System.nanoTime() - startedAt);
+			elapsedNanos += runNanos;
+			runs++;
+
+			if (stableResultCount == Long.MIN_VALUE) {
+				stableResultCount = currentResultCount;
+			} else if (stableResultCount != currentResultCount) {
+				throw new IllegalStateException("Result count changed between repeated runs: expected "
+						+ stableResultCount + " but got " + currentResultCount + " on run " + runs);
+			}
+		}
+
+		boolean maxRunsReached = runs >= EXECUTION_REPEAT_MAX_RUNS;
+		if (runs == 0) {
+			return new QueryExecutionVerification(0, 0, 0, softLimitReached, maxRunsReached);
+		}
+
+		return new QueryExecutionVerification(runs, elapsedNanos, stableResultCount, softLimitReached,
+				maxRunsReached);
+	}
+
+	private void printExecutionVerification(QueryExecutionVerification executionVerification) {
+		output.println();
+		output.println("=== Execution Verification ===");
+		if (executionVerification.runs == 0) {
+			output.println("No repeated runs executed.");
+			return;
+		}
+
+		long totalMillis = TimeUnit.NANOSECONDS.toMillis(executionVerification.elapsedNanos);
+		long averageMillis = TimeUnit.NANOSECONDS.toMillis(
+				executionVerification.elapsedNanos / executionVerification.runs);
+		output.println("runs=" + executionVerification.runs
+				+ ", totalMillis=" + totalMillis
+				+ ", averageMillis=" + averageMillis
+				+ ", resultCount=" + executionVerification.resultCount
+				+ ", softLimitMillis=" + TimeUnit.NANOSECONDS.toMillis(EXECUTION_REPEAT_SOFT_LIMIT_NANOS)
+				+ ", softLimitReached=" + executionVerification.softLimitReached
+				+ ", maxRunsReached=" + executionVerification.maxRunsReached);
+	}
+
+	private static final class QueryExecutionVerification {
+		private final int runs;
+		private final long elapsedNanos;
+		private final long resultCount;
+		private final boolean softLimitReached;
+		private final boolean maxRunsReached;
+
+		private QueryExecutionVerification(int runs, long elapsedNanos, long resultCount, boolean softLimitReached,
+				boolean maxRunsReached) {
+			this.runs = runs;
+			this.elapsedNanos = elapsedNanos;
+			this.resultCount = resultCount;
+			this.softLimitReached = softLimitReached;
+			this.maxRunsReached = maxRunsReached;
+		}
+	}
+
 	static QueryPlanSnapshotCliOptions parseArgs(String[] args) {
 		return QueryPlanSnapshotCliOptions.parseArgs(args);
 	}

testsuites/benchmark/src/main/java/org/eclipse/rdf4j/benchmark/plan/QueryPlanSnapshotComparator.java

@@ -18,7 +18,9 @@
 import java.nio.file.Path;
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.TreeSet;
@@ -98,35 +100,61 @@ static void printRunDetails(PrintStream out, SnapshotRun run) {
 		out.println("  capturedAt: " + normalize(snapshot.getCapturedAt()));
 		out.println("  queryId: " + normalize(snapshot.getQueryId()));
 		out.println("  unoptimizedFingerprint: " + normalize(snapshot.getUnoptimizedFingerprint()));
-		out.println("  queryString:");
-		printTextBlock(out, snapshot.getQueryString(), "    ");
 		if (run.path != null) {
 			out.println("  path: " + run.path.toAbsolutePath());
 		}
-
 		printStringMap(out, "metadata", snapshot.getMetadata());
 		printStringMap(out, "featureFlags", snapshot.getFeatureFlags());
 
-		Map<String, QueryPlanExplanation> explanations = snapshot.getExplanations();
-		out.println("  explanations:");
+		out.println();
+		out.println("=== Results ===");
+		out.println("Original query:");
+		printTextBlock(out, snapshot.getQueryString(), "");
+		printFullExplanations(out, snapshot.getExplanations());
+	}
+
+	private static void printFullExplanations(PrintStream out, Map<String, QueryPlanExplanation> explanations) {
 		if (explanations == null || explanations.isEmpty()) {
-			out.println("    <none>");
+			out.println();
+			out.println("No query explanations captured.");
 			return;
 		}
 
-		for (Map.Entry<String, QueryPlanExplanation> entry : explanations.entrySet()) {
-			QueryPlanExplanation explanation = entry.getValue();
-			out.println("    " + entry.getKey() + ":");
+		LinkedHashSet<String> orderedLevels = new LinkedHashSet<>(List.of("unoptimized", "optimized", "executed"));
+		orderedLevels.addAll(explanations.keySet());
+		for (String levelKey : orderedLevels) {
+			QueryPlanExplanation explanation = explanations.get(levelKey);
 			if (explanation == null) {
-				out.println("      <null>");
 				continue;
 			}
-			out.println("      tupleExprJson: " + (isPresent(explanation.getTupleExprJson()) ? "present" : "missing"));
-			out.println(
-					"      irRenderedQuery: " + (isPresent(explanation.getIrRenderedQuery()) ? "present" : "missing"));
-			out.println("      explanationText:");
-			printTextBlock(out, explanation.getExplanationText(), "        ");
+
+			out.println();
+			out.println("=== " + displayLevelName(levelKey, explanation.getLevel()) + " Explanation ===");
+			String explanationText = explanation.getExplanationText();
+			if (isPresent(explanationText)) {
+				printTextBlock(out, explanationText, "");
+			} else {
+				out.println("(no explanation text)");
+			}
+
+			if (isPresent(explanation.getIrRenderedQuery())) {
+				out.println("--- IR Rendered Query ---");
+				printTextBlock(out, explanation.getIrRenderedQuery(), "");
+			}
+			if (isPresent(explanation.getIrRenderingError())) {
+				out.println("--- IR Rendering Error ---");
+				printTextBlock(out, explanation.getIrRenderingError(), "");
+			}
+		}
+	}
+
+	private static String displayLevelName(String levelKey, String levelValue) {
+		String base = isPresent(levelValue) ? levelValue : levelKey;
+		if (!isPresent(base)) {
+			return "Unknown";
 		}
+		String lower = base.toLowerCase(Locale.ROOT);
+		return lower.substring(0, 1).toUpperCase(Locale.ROOT) + lower.substring(1);
 	}
 
 	static void printComparison(PrintStream out, SnapshotRun left, SnapshotRun right) {

testsuites/benchmark/src/test/java/org/eclipse/rdf4j/benchmark/plan/QueryPlanSnapshotCliTest.java

@@ -194,6 +194,27 @@ void runModePrintsPrettyExplanationForAllLevels() throws Exception {
 		assertTrue(executedIndex > optimizedIndex);
 	}
 
+	@Test
+	void runModePrintsExecutionVerificationSummary() throws Exception {
+		ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream();
+		QueryPlanSnapshotCli cli = new QueryPlanSnapshotCli(new BufferedReader(new StringReader("")),
+				new PrintStream(outputBuffer, true, StandardCharsets.UTF_8.name()));
+
+		QueryPlanSnapshotCliOptions options = QueryPlanSnapshotCli.parseArgs(new String[] {
+				"--no-interactive",
+				"--store", "memory",
+				"--theme", "MEDICAL_RECORDS",
+				"--query-index", "0",
+				"--persist", "false"
+		});
+
+		cli.run(options);
+
+		String printed = outputBuffer.toString(StandardCharsets.UTF_8);
+		assertTrue(printed.contains("=== Execution Verification ==="), printed);
+		assertTrue(printed.contains("runs="), printed);
+	}
+
 	@Test
 	void runModePrintsOriginalQueryAtStartOfResultsSection() throws Exception {
 		String query = "SELECT * WHERE { ?s ?p ?o } LIMIT 5";
@@ -242,7 +263,7 @@ void interactiveRunAsksForQuerySourceBeforeTheme() throws Exception {
 
 		String printed = outputBuffer.toString(StandardCharsets.UTF_8);
 		int storeIndex = printed.indexOf("Store [memory|lmdb]");
-		int querySourceIndex = printed.indexOf("Query source [themed|manual|file]");
+		int querySourceIndex = printed.indexOf("Query source [themed|manual|file|all-themed]");
 		int themeIndex = printed.indexOf("Theme");
 		assertTrue(storeIndex >= 0);
 		assertTrue(querySourceIndex > storeIndex);