Skip to content

Commit adbd443

Browse files
hmottestadhmottestad
committed
slower and faster
1 parent fb7c471 commit adbd443

26 files changed

Lines changed: 5408 additions & 2048 deletions

File tree

‎core/query/src/main/java/org/eclipse/rdf4j/query/explanation/GenericPlanNode.java‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,9 @@ public class GenericPlanNode {
144144
private Map<String, Long> longMetricsActual = new LinkedHashMap<>();
145145
private Map<String, Double> doubleMetricsActual = new LinkedHashMap<>();
146146
private Map<String, String> stringMetricsActual = new LinkedHashMap<>();
147+
private Map<String, Long> longMetricsPlanned = new LinkedHashMap<>();
148+
private Map<String, Double> doubleMetricsPlanned = new LinkedHashMap<>();
149+
private Map<String, String> stringMetricsPlanned = new LinkedHashMap<>();
147150

148151
// true if this node introduces a new scope
149152
private Boolean newScope;
@@ -403,6 +406,11 @@ public void setRuntimeTelemetryEnabled(boolean runtimeTelemetryEnabled) {
403406
this.runtimeTelemetryEnabled = runtimeTelemetryEnabled;
404407
}
405408

409+
@JsonIgnore
410+
public boolean isRuntimeTelemetryEnabled() {
411+
return runtimeTelemetryEnabled;
412+
}
413+
406414
public void setEstimateStabilityMetricsEnabled(boolean estimateStabilityMetricsEnabled) {
407415
this.estimateStabilityMetricsEnabled = estimateStabilityMetricsEnabled;
408416
}
@@ -502,6 +510,66 @@ public void setStringMetricActual(String metricName, String metricValue) {
502510
stringMetricsActual.put(metricName, metricValue);
503511
}
504512

513+
public Map<String, Long> getLongMetricsPlanned() {
514+
return longMetricsPlanned.isEmpty() ? null : longMetricsPlanned;
515+
}
516+
517+
public void setLongMetricsPlanned(Map<String, Long> longMetricsPlanned) {
518+
this.longMetricsPlanned = longMetricsPlanned == null ? new LinkedHashMap<>()
519+
: new LinkedHashMap<>(longMetricsPlanned);
520+
}
521+
522+
public Long getLongMetricPlanned(String metricName) {
523+
return longMetricsPlanned.get(metricName);
524+
}
525+
526+
public void setLongMetricPlanned(String metricName, Long metricValue) {
527+
if (metricName == null || metricValue == null || metricValue < 0) {
528+
return;
529+
}
530+
longMetricsPlanned.put(metricName, metricValue);
531+
}
532+
533+
public Map<String, Double> getDoubleMetricsPlanned() {
534+
return doubleMetricsPlanned.isEmpty() ? null : doubleMetricsPlanned;
535+
}
536+
537+
public void setDoubleMetricsPlanned(Map<String, Double> doubleMetricsPlanned) {
538+
this.doubleMetricsPlanned = doubleMetricsPlanned == null ? new LinkedHashMap<>()
539+
: new LinkedHashMap<>(doubleMetricsPlanned);
540+
}
541+
542+
public Double getDoubleMetricPlanned(String metricName) {
543+
return doubleMetricsPlanned.get(metricName);
544+
}
545+
546+
public void setDoubleMetricPlanned(String metricName, Double metricValue) {
547+
if (metricName == null || metricValue == null || metricValue < 0) {
548+
return;
549+
}
550+
doubleMetricsPlanned.put(metricName, metricValue);
551+
}
552+
553+
public Map<String, String> getStringMetricsPlanned() {
554+
return stringMetricsPlanned.isEmpty() ? null : stringMetricsPlanned;
555+
}
556+
557+
public void setStringMetricsPlanned(Map<String, String> stringMetricsPlanned) {
558+
this.stringMetricsPlanned = stringMetricsPlanned == null ? new LinkedHashMap<>()
559+
: new LinkedHashMap<>(stringMetricsPlanned);
560+
}
561+
562+
public String getStringMetricPlanned(String metricName) {
563+
return stringMetricsPlanned.get(metricName);
564+
}
565+
566+
public void setStringMetricPlanned(String metricName, String metricValue) {
567+
if (metricName == null || metricValue == null || metricValue.isEmpty()) {
568+
return;
569+
}
570+
stringMetricsPlanned.put(metricName, metricValue);
571+
}
572+
505573
public void setTimedOut(Boolean timedOut) {
506574
this.timedOut = timedOut;
507575
}
@@ -919,6 +987,8 @@ private Long sourceRowsFilteredForDisplay() {
919987
}
920988

921989
private void appendMapTelemetry(Map<String, String> metrics) {
990+
appendPlannedMapTelemetry(metrics);
991+
922992
Map<String, Long> visibleLongMetrics = getLongMetricsActual();
923993
if (visibleLongMetrics != null) {
924994
for (Map.Entry<String, Long> entry : visibleLongMetrics.entrySet()) {
@@ -962,13 +1032,66 @@ private void appendMapTelemetry(Map<String, String> metrics) {
9621032
}
9631033
}
9641034

1035+
private void appendPlannedMapTelemetry(Map<String, String> metrics) {
1036+
for (Map.Entry<String, Long> entry : longMetricsPlanned.entrySet()) {
1037+
Long metricValue = entry.getValue();
1038+
if (metricValue == null || metricValue < 0 || metrics.containsKey(entry.getKey())) {
1039+
continue;
1040+
}
1041+
putIfKnown(metrics, entry.getKey(), toHumanReadableNumber(metricValue));
1042+
}
1043+
for (Map.Entry<String, Double> entry : doubleMetricsPlanned.entrySet()) {
1044+
Double metricValue = entry.getValue();
1045+
if (metricValue == null || metricValue < 0 || metrics.containsKey(entry.getKey())) {
1046+
continue;
1047+
}
1048+
putIfKnown(metrics, entry.getKey(), toHumanReadableNumber(metricValue));
1049+
}
1050+
for (Map.Entry<String, String> entry : orderedStringMetricsPlanned()) {
1051+
String metricName = entry.getKey();
1052+
String metricValue = entry.getValue();
1053+
if (metricValue == null || metricValue.isEmpty() || metrics.containsKey(metricName)) {
1054+
continue;
1055+
}
1056+
metrics.put(metricName, metricValue);
1057+
}
1058+
}
1059+
1060+
private List<Map.Entry<String, String>> orderedStringMetricsPlanned() {
1061+
List<Map.Entry<String, String>> orderedEntries = new ArrayList<>();
1062+
appendPreferredStringMetric(orderedEntries, stringMetricsPlanned, TelemetryMetricNames.PLANNED_INDEX_NAME);
1063+
appendPreferredStringMetric(orderedEntries, stringMetricsPlanned,
1064+
TelemetryMetricNames.PLANNED_INDEX_ACCESS_MODE);
1065+
appendPreferredStringMetric(orderedEntries, stringMetricsPlanned,
1066+
TelemetryMetricNames.PLANNED_LOOKUP_COMPONENTS);
1067+
appendPreferredStringMetric(orderedEntries, stringMetricsPlanned,
1068+
TelemetryMetricNames.PLANNED_MISSING_LOOKUP_COMPONENTS);
1069+
appendPreferredStringMetric(orderedEntries, stringMetricsPlanned, TelemetryMetricNames.PLANNED_BOUND_VARS);
1070+
appendPreferredStringMetric(orderedEntries, stringMetricsPlanned, TelemetryMetricNames.PLANNER_ID);
1071+
appendPreferredStringMetric(orderedEntries, stringMetricsPlanned, TelemetryMetricNames.PLANNER_ALGORITHM);
1072+
appendPreferredStringMetric(orderedEntries, stringMetricsPlanned, TelemetryMetricNames.PLANNER_PATH);
1073+
for (Map.Entry<String, String> entry : stringMetricsPlanned.entrySet()) {
1074+
if (orderedEntries.stream().anyMatch(ordered -> ordered.getKey().equals(entry.getKey()))) {
1075+
continue;
1076+
}
1077+
orderedEntries.add(entry);
1078+
}
1079+
return orderedEntries;
1080+
}
1081+
9651082
private List<Map.Entry<String, String>> orderedStringMetricsActual() {
9661083
Map<String, String> visibleStringMetrics = visibleStringMetricsActual();
9671084
List<Map.Entry<String, String>> orderedEntries = new ArrayList<>();
9681085
appendPreferredStringMetric(orderedEntries, visibleStringMetrics, TelemetryMetricNames.BINDING_STATE);
9691086
appendPreferredStringMetric(orderedEntries, visibleStringMetrics, TelemetryMetricNames.JOIN_TYPE);
9701087
appendPreferredStringMetric(orderedEntries, visibleStringMetrics, TelemetryMetricNames.INDEX_NAME);
9711088
appendPreferredStringMetric(orderedEntries, visibleStringMetrics, TelemetryMetricNames.INDEX_NAMES);
1089+
appendPreferredStringMetric(orderedEntries, visibleStringMetrics, TelemetryMetricNames.OPTIMIZER_PLANNER_ID);
1090+
appendPreferredStringMetric(orderedEntries, visibleStringMetrics, TelemetryMetricNames.OPTIMIZER_PLANNER_PATH);
1091+
appendPreferredStringMetric(orderedEntries, visibleStringMetrics,
1092+
TelemetryMetricNames.OPTIMIZER_PLANNER_REJECTED_FACTOR);
1093+
appendPreferredStringMetric(orderedEntries, visibleStringMetrics,
1094+
TelemetryMetricNames.OPTIMIZER_PLANNER_DIAGNOSTICS);
9721095
for (Map.Entry<String, String> entry : visibleStringMetrics.entrySet()) {
9731096
if (isPreferredExplainAnnotationMetric(entry.getKey())) {
9741097
continue;
@@ -1470,13 +1593,24 @@ private String toDotInternal(double maxResultSizeActual, double maxTotalTime, do
14701593
private void appendExplainAnnotationDotRows(List<String> rows) {
14711594
appendExplainAnnotationDotRow(rows, "Binding state", TelemetryMetricNames.BINDING_STATE);
14721595
appendExplainAnnotationDotRow(rows, "Join type", TelemetryMetricNames.JOIN_TYPE);
1596+
appendPlannedExplainAnnotationDotRow(rows, "Planned index", TelemetryMetricNames.PLANNED_INDEX_NAME);
1597+
appendPlannedExplainAnnotationDotRow(rows, "Planned bound vars", TelemetryMetricNames.PLANNED_BOUND_VARS);
14731598
if (getStringMetricActual(TelemetryMetricNames.INDEX_NAME) != null) {
14741599
appendExplainAnnotationDotRow(rows, "Index", TelemetryMetricNames.INDEX_NAME);
14751600
} else {
14761601
appendExplainAnnotationDotRow(rows, "Indexes", TelemetryMetricNames.INDEX_NAMES);
14771602
}
14781603
}
14791604

1605+
private void appendPlannedExplainAnnotationDotRow(List<String> rows, String label, String metricName) {
1606+
String metricValue = getStringMetricPlanned(metricName);
1607+
if (metricValue == null || metricValue.isEmpty()) {
1608+
return;
1609+
}
1610+
rows.add("<tr><td>" + StringEscapeUtils.escapeHtml4(label) + "</td><td>"
1611+
+ StringEscapeUtils.escapeHtml4(metricValue) + "</td></tr>");
1612+
}
1613+
14801614
private void appendExplainAnnotationDotRow(List<String> rows, String label, String metricName) {
14811615
String metricValue = getStringMetricActual(metricName);
14821616
if (metricValue == null || metricValue.isEmpty()) {

‎core/query/src/main/java/org/eclipse/rdf4j/query/explanation/TelemetryMetricNames.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,25 @@ private TelemetryMetricNames() {
107107
public static final String BINDING_STATE = "bindingState";
108108
public static final String JOIN_TYPE = "joinType";
109109

110+
public static final String PLANNER_ID = "plannerId";
111+
public static final String PLANNER_ALGORITHM = "plannerAlgorithm";
112+
public static final String PLANNER_PATH = "plannerPath";
113+
public static final String PLANNED_INDEX_NAME = "plannedIndexName";
114+
public static final String PLANNED_INDEX_PREFIX_LENGTH = "plannedIndexPrefixLength";
115+
public static final String PLANNED_LOOKUP_COMPONENTS = "plannedLookupComponents";
116+
public static final String PLANNED_BOUND_VARS = "plannedBoundVars";
117+
public static final String PLANNED_WORK_ROWS = "plannedWorkRows";
118+
public static final String PLANNED_FILTER_PASS_RATIO = "plannedFilterPassRatio";
119+
public static final String PLANNED_FILTER_EVIDENCE_COUNT = "plannedFilterEvidenceCount";
120+
public static final String FILTER_SELECTIVITY_SOURCE = "filterSelectivitySource";
121+
public static final String PLANNED_INDEX_ACCESS_MODE = "plannedIndexAccessMode";
122+
public static final String PLANNED_ACCESS_ROWS = "plannedAccessRows";
123+
public static final String PLANNED_ACCESS_ROWS_AFTER_FILTER = "plannedAccessRowsAfterFilter";
124+
public static final String PLANNED_MISSING_LOOKUP_COMPONENTS = "plannedMissingLookupComponents";
125+
public static final String DEFERRED_FILTER_SCOPE = "deferredFilterScope";
126+
public static final String SHARED_JOIN_VARS = "sharedJoinVars";
127+
public static final String UNLOCKED_FILTERS = "unlockedFilters";
128+
110129
public static final String METRIC_ORIGIN = "metricOrigin";
111130
public static final String SAMPLE_COUNT_ACTUAL = "sampleCountActual";
112131
public static final String CONFIDENCE_SCORE_ACTUAL = "confidenceScoreActual";
@@ -128,6 +147,10 @@ private TelemetryMetricNames() {
128147
public static final String OPTIMIZER_CANDIDATE_COUNT = OPTIMIZER_PREFIX + "candidateCount";
129148
public static final String OPTIMIZER_REJECTION_REASON = OPTIMIZER_PREFIX + "rejectionReason";
130149
public static final String OPTIMIZER_ESTIMATE_SOURCE = OPTIMIZER_PREFIX + "estimateSource";
150+
public static final String OPTIMIZER_PLANNER_ID = OPTIMIZER_PREFIX + "plannerId";
151+
public static final String OPTIMIZER_PLANNER_PATH = OPTIMIZER_PREFIX + "plannerPath";
152+
public static final String OPTIMIZER_PLANNER_REJECTED_FACTOR = OPTIMIZER_PREFIX + "plannerRejectedFactor";
153+
public static final String OPTIMIZER_PLANNER_DIAGNOSTICS = OPTIMIZER_PREFIX + "plannerDiagnostics";
131154

132155
public static boolean isOptimizerMetric(String metricName) {
133156
return metricName != null && metricName.startsWith(OPTIMIZER_PREFIX);

‎core/query/src/test/java/org/eclipse/rdf4j/query/explanation/GenericPlanNodeTest.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,27 @@ void toDotIncludesExplainAnnotationRows() {
248248
assertTrue(actual.contains("<tr><td>Index</td><td>spoc</td></tr>"), actual);
249249
}
250250

251+
@Test
252+
void plannedIndexMetricsDoNotOverwriteRuntimeIndexMetrics() {
253+
GenericPlanNode node = new GenericPlanNode("StatementPattern");
254+
node.setRuntimeTelemetryEnabled(false);
255+
node.setStringMetricPlanned(TelemetryMetricNames.PLANNED_INDEX_NAME, "posc");
256+
257+
String optimized = node.toString();
258+
259+
assertTrue(optimized.contains("plannedIndexName=posc"), optimized);
260+
261+
node.setRuntimeTelemetryEnabled(true);
262+
node.setStringMetricActual(TelemetryMetricNames.INDEX_NAME, "spoc");
263+
String telemetry = node.toString();
264+
String dot = node.toDot();
265+
266+
assertTrue(telemetry.contains("plannedIndexName=posc"), telemetry);
267+
assertTrue(telemetry.contains("indexName=spoc"), telemetry);
268+
assertTrue(dot.contains("<tr><td>Planned index</td><td>posc</td></tr>"), dot);
269+
assertTrue(dot.contains("<tr><td>Index</td><td>spoc</td></tr>"), dot);
270+
}
271+
251272
@Test
252273
void regularJoinTypeIsHidden() {
253274
GenericPlanNode join = new GenericPlanNode("Join");

‎core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/impl/EvaluationStatistics.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
*
99
* SPDX-License-Identifier: BSD-3-Clause
1010
*******************************************************************************/
11+
// Some portions generated by Codex
1112
package org.eclipse.rdf4j.query.algebra.evaluation.impl;
1213

1314
import java.util.Collection;
@@ -91,10 +92,54 @@ public double estimateFilterPassRatio(Filter filter) {
9192
return -1.0d;
9293
}
9394

95+
public FilterPassEstimate estimateFilterPass(Filter filter) {
96+
double passRatio = estimateFilterPassRatio(filter);
97+
FilterPassEstimate.Source source = Double.isFinite(passRatio) && passRatio >= 0.0d
98+
? FilterPassEstimate.Source.HEURISTIC
99+
: FilterPassEstimate.Source.UNKNOWN;
100+
return new FilterPassEstimate(passRatio, source, -1L);
101+
}
102+
94103
public void recordFilterOutcome(Filter filter, long passedCount, long filteredCount) {
95104
// no-op by default
96105
}
97106

107+
public static final class FilterPassEstimate {
108+
public enum Source {
109+
LEARNED_FILTER,
110+
LEARNED_PATTERN,
111+
SAMPLED,
112+
HEURISTIC,
113+
UNKNOWN
114+
}
115+
116+
private final double passRatio;
117+
private final Source source;
118+
private final long evidenceCount;
119+
120+
public FilterPassEstimate(double passRatio, Source source) {
121+
this(passRatio, source, -1L);
122+
}
123+
124+
public FilterPassEstimate(double passRatio, Source source, long evidenceCount) {
125+
this.passRatio = passRatio;
126+
this.source = source == null ? Source.UNKNOWN : source;
127+
this.evidenceCount = evidenceCount;
128+
}
129+
130+
public double getPassRatio() {
131+
return passRatio;
132+
}
133+
134+
public Source getSource() {
135+
return source;
136+
}
137+
138+
public long getEvidenceCount() {
139+
return evidenceCount;
140+
}
141+
}
142+
98143
/*-----------------------------------*
99144
* Inner class CardinalityCalculator *
100145
*-----------------------------------*/

0 commit comments

Comments
 (0)