Skip to content

Commit fe7864d

Browse files
hmottestadhmottestad
committed
more potential optimisations
1 parent cc6f415 commit fe7864d

3 files changed

Lines changed: 270 additions & 5 deletions

File tree

‎core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/optimizer/OptionalFilterJoinOptimizer.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,11 @@ private static boolean requiresRightVars(ValueExpr expr, Set<String> rightOnly)
9696
return requiresRightVars(and.getLeftArg(), rightOnly) || requiresRightVars(and.getRightArg(), rightOnly);
9797
}
9898
if (expr instanceof Not) {
99-
return requiresRightVars(((Not) expr).getArg(), rightOnly);
99+
ValueExpr arg = ((Not) expr).getArg();
100+
if (arg instanceof Bound && rightOnly.contains(((Bound) arg).getArg().getName())) {
101+
return false;
102+
}
103+
return requiresRightVars(arg, rightOnly);
100104
}
101105
if (expr instanceof UnaryValueOperator) {
102106
return requiresRightVars(((UnaryValueOperator) expr).getArg(), rightOnly);

‎core/queryrender/src/test/java/org/eclipse/rdf4j/queryrender/SparqlUoOptimizerIrDiffTest.java‎

Lines changed: 73 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,27 @@
1818
import java.nio.file.Files;
1919
import java.nio.file.Path;
2020
import java.nio.file.Paths;
21+
import java.util.Comparator;
2122

23+
import org.eclipse.rdf4j.common.iteration.CloseableIteration;
24+
import org.eclipse.rdf4j.common.order.StatementOrder;
25+
import org.eclipse.rdf4j.model.IRI;
26+
import org.eclipse.rdf4j.model.Resource;
27+
import org.eclipse.rdf4j.model.Statement;
28+
import org.eclipse.rdf4j.model.Value;
29+
import org.eclipse.rdf4j.model.ValueFactory;
30+
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
2231
import org.eclipse.rdf4j.query.BindingSet;
2332
import org.eclipse.rdf4j.query.Dataset;
2433
import org.eclipse.rdf4j.query.MalformedQueryException;
34+
import org.eclipse.rdf4j.query.QueryEvaluationException;
2535
import org.eclipse.rdf4j.query.QueryLanguage;
2636
import org.eclipse.rdf4j.query.algebra.TupleExpr;
37+
import org.eclipse.rdf4j.query.algebra.evaluation.TripleSource;
38+
import org.eclipse.rdf4j.query.algebra.evaluation.impl.DefaultEvaluationStrategy;
2739
import org.eclipse.rdf4j.query.algebra.evaluation.impl.EvaluationStatistics;
28-
import org.eclipse.rdf4j.query.algebra.evaluation.optimizer.SparqlUoOptimizer;
29-
import org.eclipse.rdf4j.query.algebra.evaluation.optimizer.UnionScopeChangeOptimizer;
40+
import org.eclipse.rdf4j.query.algebra.evaluation.optimizer.SparqlUoQueryOptimizerPipeline;
41+
import org.eclipse.rdf4j.query.algebra.evaluation.optimizer.sparqluo.SparqlUoConfig;
3042
import org.eclipse.rdf4j.query.impl.EmptyBindingSet;
3143
import org.eclipse.rdf4j.query.impl.SimpleDataset;
3244
import org.eclipse.rdf4j.query.parser.ParsedQuery;
@@ -77,8 +89,7 @@ private static void dump(String baseName, String sparql, TupleExprIRRenderer.Con
7789
assertNotNull(before);
7890
TupleExpr after = before.clone();
7991

80-
new UnionScopeChangeOptimizer().optimize(after, DATASET, BINDINGS);
81-
new SparqlUoOptimizer(new EvaluationStatistics(), true).optimize(after, DATASET, BINDINGS);
92+
optimizeWithPipeline(after);
8293

8394
TupleExprIRRenderer renderer = new TupleExprIRRenderer(style);
8495
writeReportFile(baseName, "SPARQL_input", sparql);
@@ -92,6 +103,17 @@ private static void dump(String baseName, String sparql, TupleExprIRRenderer.Con
92103
writeReportFile(baseName, "TupleExpr_after", VarNameNormalizer.normalizeVars(after.toString()));
93104
}
94105

106+
private static void optimizeWithPipeline(TupleExpr expr) {
107+
EmptyTripleSource tripleSource = new EmptyTripleSource();
108+
EvaluationStatistics evaluationStatistics = new EvaluationStatistics();
109+
DefaultEvaluationStrategy strategy = new DefaultEvaluationStrategy(tripleSource, null, null, 0L,
110+
evaluationStatistics);
111+
SparqlUoConfig config = SparqlUoConfig.builder().allowNonImprovingTransforms(true).build();
112+
strategy.setOptimizerPipeline(
113+
new SparqlUoQueryOptimizerPipeline(strategy, tripleSource, evaluationStatistics, config));
114+
strategy.optimize(expr, evaluationStatistics, BINDINGS);
115+
}
116+
95117
@Test
96118
@DisplayName("IR diff: UNION common-prefix pull-up")
97119
void irDiff_unionCommonPrefixPullUp() {
@@ -103,6 +125,17 @@ void irDiff_unionCommonPrefixPullUp() {
103125
dump("SparqlUo_ir_union_common_prefix", q, cfg());
104126
}
105127

128+
@Test
129+
@DisplayName("IR diff: UNION common-filter hoist")
130+
void irDiff_unionCommonFilterHoist() {
131+
String q = "SELECT * WHERE {\n" +
132+
" { ?s <urn:p1> ?name . FILTER(?name != \"\") FILTER(?name != \"x\") }\n" +
133+
" UNION\n" +
134+
" { ?s <urn:p2> ?name . FILTER(?name != \"x\") FILTER(?name != \"\") }\n" +
135+
"}";
136+
dump("SparqlUo_ir_union_filter_hoist", q, cfg());
137+
}
138+
106139
@Test
107140
@DisplayName("IR diff: OPTIONAL-safe lifting")
108141
void irDiff_optionalSafeLift() {
@@ -124,4 +157,40 @@ void irDiff_optionalLiftNegative() {
124157
"}";
125158
dump("SparqlUo_ir_optional_lift_negative", q, cfg());
126159
}
160+
161+
@Test
162+
@DisplayName("IR diff: MINUS UNION split")
163+
void irDiff_minusUnionSplit() {
164+
String q = "SELECT * WHERE {\n" +
165+
" ?s <urn:p1> ?o\n" +
166+
" MINUS { { ?s <urn:p2> ?o2 } UNION { ?s <urn:p3> ?o3 } }\n" +
167+
"}";
168+
dump("SparqlUo_ir_minus_union_split", q, cfg());
169+
}
170+
171+
private static final class EmptyTripleSource implements TripleSource {
172+
private final ValueFactory vf = SimpleValueFactory.getInstance();
173+
174+
@Override
175+
public ValueFactory getValueFactory() {
176+
return vf;
177+
}
178+
179+
@Override
180+
public CloseableIteration<? extends Statement> getStatements(Resource subj, IRI pred, Value obj,
181+
Resource... contexts) throws QueryEvaluationException {
182+
return TripleSource.EMPTY_ITERATION;
183+
}
184+
185+
@Override
186+
public CloseableIteration<? extends Statement> getStatements(StatementOrder order, Resource subj, IRI pred,
187+
Value obj, Resource... contexts) throws QueryEvaluationException {
188+
return TripleSource.EMPTY_ITERATION;
189+
}
190+
191+
@Override
192+
public Comparator<Value> getComparator() {
193+
return null;
194+
}
195+
}
127196
}

‎core/sail/memory/src/test/java/org/eclipse/rdf4j/sail/memory/benchmark/ThemeQueryPlanDump.java‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
/*******************************************************************************
2+
* Copyright (c) 2025 Eclipse RDF4J contributors.
3+
*
4+
* All rights reserved. This program and the accompanying materials
5+
* are made available under the terms of the Eclipse Distribution License v1.0
6+
* which accompanies this distribution, and is available at
7+
* http://www.eclipse.org/org/documents/edl-v10.php.
8+
*
9+
* SPDX-License-Identifier: BSD-3-Clause
10+
*******************************************************************************/
11+
// Some portions generated by Codex
12+
package org.eclipse.rdf4j.sail.memory.benchmark;
13+
14+
import java.io.IOException;
15+
import java.nio.charset.StandardCharsets;
16+
import java.nio.file.Files;
17+
import java.nio.file.Path;
18+
import java.nio.file.Paths;
19+
import java.util.ArrayList;
20+
import java.util.Arrays;
21+
import java.util.List;
22+
import java.util.Locale;
23+
24+
import org.eclipse.rdf4j.benchmark.common.ThemeQueryCatalog;
25+
import org.eclipse.rdf4j.benchmark.rio.util.ThemeDataSetGenerator;
26+
import org.eclipse.rdf4j.benchmark.rio.util.ThemeDataSetGenerator.Theme;
27+
import org.eclipse.rdf4j.common.transaction.IsolationLevels;
28+
import org.eclipse.rdf4j.query.Dataset;
29+
import org.eclipse.rdf4j.query.algebra.evaluation.EvaluationStrategy;
30+
import org.eclipse.rdf4j.query.algebra.evaluation.EvaluationStrategyFactory;
31+
import org.eclipse.rdf4j.query.algebra.evaluation.TripleSource;
32+
import org.eclipse.rdf4j.query.algebra.evaluation.impl.DefaultEvaluationStrategyFactory;
33+
import org.eclipse.rdf4j.query.algebra.evaluation.impl.EvaluationStatistics;
34+
import org.eclipse.rdf4j.query.algebra.evaluation.optimizer.StandardQueryOptimizerPipeline;
35+
import org.eclipse.rdf4j.query.explanation.Explanation;
36+
import org.eclipse.rdf4j.repository.sail.SailRepository;
37+
import org.eclipse.rdf4j.repository.sail.SailRepositoryConnection;
38+
import org.eclipse.rdf4j.repository.util.RDFInserter;
39+
import org.eclipse.rdf4j.sail.memory.MemoryStore;
40+
41+
public final class ThemeQueryPlanDump {
42+
43+
private static final String[] THEMES = {
44+
"MEDICAL_RECORDS",
45+
"SOCIAL_MEDIA",
46+
"LIBRARY",
47+
"ENGINEERING",
48+
"HIGHLY_CONNECTED",
49+
"TRAIN",
50+
"ELECTRICAL_GRID",
51+
"PHARMA"
52+
};
53+
54+
private static final int[] QUERY_INDEXES = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
55+
56+
public static void main(String[] args) throws Exception {
57+
Config config = Config.parse(args);
58+
List<String> themeNames = config.themeNames;
59+
List<Integer> queryIndexes = config.queryIndexes;
60+
boolean[] useSparqlUoValues = config.useSparqlUoValues;
61+
Path outputRoot = config.outputRoot;
62+
63+
for (String themeName : themeNames) {
64+
Theme theme = Theme.valueOf(themeName);
65+
for (boolean useSparqlUo : useSparqlUoValues) {
66+
SailRepository repository = createRepository(useSparqlUo);
67+
try {
68+
loadData(repository, theme);
69+
for (int queryIndex : queryIndexes) {
70+
writeExplanation(repository, theme, queryIndex, useSparqlUo, outputRoot);
71+
}
72+
} finally {
73+
repository.shutDown();
74+
}
75+
}
76+
}
77+
}
78+
79+
private static void writeExplanation(SailRepository repository, Theme theme, int queryIndex, boolean useSparqlUo,
80+
Path outputRoot) throws IOException {
81+
String query = ThemeQueryCatalog.queryFor(theme, queryIndex);
82+
long expected = ThemeQueryCatalog.expectedCountFor(theme, queryIndex);
83+
String explanation = explain(repository, query);
84+
Path outputPath = outputRoot
85+
.resolve(useSparqlUo ? "sparqluo" : "standard")
86+
.resolve(theme.name())
87+
.resolve(String.format(Locale.ROOT, "q%02d.txt", queryIndex));
88+
Files.createDirectories(outputPath.getParent());
89+
StringBuilder builder = new StringBuilder();
90+
builder.append("Theme: ").append(theme.name()).append('\n');
91+
builder.append("QueryIndex: ").append(queryIndex).append('\n');
92+
builder.append("UseSparqlUo: ").append(useSparqlUo).append('\n');
93+
builder.append("ExpectedCount: ").append(expected).append('\n');
94+
builder.append("Query:\n").append(query).append('\n');
95+
builder.append("Explanation:\n").append(explanation).append('\n');
96+
Files.writeString(outputPath, builder.toString(), StandardCharsets.UTF_8);
97+
}
98+
99+
private static String explain(SailRepository repository, String query) {
100+
try (SailRepositoryConnection connection = repository.getConnection()) {
101+
return connection.prepareTupleQuery(query).explain(Explanation.Level.Executed).toString();
102+
}
103+
}
104+
105+
private static void loadData(SailRepository repository, Theme theme) throws IOException {
106+
try (SailRepositoryConnection connection = repository.getConnection()) {
107+
connection.begin(IsolationLevels.NONE);
108+
RDFInserter inserter = new RDFInserter(connection);
109+
ThemeDataSetGenerator.generate(theme, inserter);
110+
connection.commit();
111+
}
112+
}
113+
114+
private static SailRepository createRepository(boolean useSparqlUo) {
115+
MemoryStore store = new MemoryStore();
116+
if (!useSparqlUo) {
117+
store.setEvaluationStrategyFactory(createStandardPipelineFactory(store));
118+
}
119+
return new SailRepository(store);
120+
}
121+
122+
private static EvaluationStrategyFactory createStandardPipelineFactory(MemoryStore store) {
123+
DefaultEvaluationStrategyFactory factory = new DefaultEvaluationStrategyFactory(
124+
store.getFederatedServiceResolver()) {
125+
@Override
126+
public EvaluationStrategy createEvaluationStrategy(Dataset dataset, TripleSource tripleSource,
127+
EvaluationStatistics evaluationStatistics) {
128+
EvaluationStrategy strategy = super.createEvaluationStrategy(dataset, tripleSource,
129+
evaluationStatistics);
130+
strategy.setOptimizerPipeline(
131+
new StandardQueryOptimizerPipeline(strategy, tripleSource, evaluationStatistics));
132+
return strategy;
133+
}
134+
};
135+
factory.setQuerySolutionCacheThreshold(store.getIterationCacheSyncThreshold());
136+
factory.setTrackResultSize(store.isTrackResultSize());
137+
return factory;
138+
}
139+
140+
private static final class Config {
141+
private final List<String> themeNames;
142+
private final List<Integer> queryIndexes;
143+
private final boolean[] useSparqlUoValues;
144+
private final Path outputRoot;
145+
146+
private Config(List<String> themeNames, List<Integer> queryIndexes, boolean[] useSparqlUoValues,
147+
Path outputRoot) {
148+
this.themeNames = themeNames;
149+
this.queryIndexes = queryIndexes;
150+
this.useSparqlUoValues = useSparqlUoValues;
151+
this.outputRoot = outputRoot;
152+
}
153+
154+
private static Config parse(String[] args) {
155+
List<String> themeNames = new ArrayList<>(Arrays.asList(THEMES));
156+
List<Integer> queryIndexes = new ArrayList<>();
157+
for (int queryIndex : QUERY_INDEXES) {
158+
queryIndexes.add(queryIndex);
159+
}
160+
String useSparqlUo = "both";
161+
Path outputRoot = Paths.get("target", "theme-query-plans");
162+
163+
if (args != null && args.length >= 1) {
164+
themeNames = List.of(args[0].toUpperCase(Locale.ROOT));
165+
}
166+
if (args != null && args.length >= 2) {
167+
queryIndexes = List.of(Integer.parseInt(args[1]));
168+
}
169+
if (args != null && args.length >= 3) {
170+
useSparqlUo = args[2];
171+
}
172+
if (args != null && args.length >= 4) {
173+
outputRoot = Paths.get(args[3]);
174+
}
175+
176+
return new Config(themeNames, queryIndexes, parseUseSparqlUo(useSparqlUo), outputRoot);
177+
}
178+
}
179+
180+
private static boolean[] parseUseSparqlUo(String value) {
181+
if (value == null || value.isBlank() || "both".equalsIgnoreCase(value)) {
182+
return new boolean[] { true, false };
183+
}
184+
if ("true".equalsIgnoreCase(value)) {
185+
return new boolean[] { true };
186+
}
187+
if ("false".equalsIgnoreCase(value)) {
188+
return new boolean[] { false };
189+
}
190+
throw new IllegalArgumentException("Unexpected z_useSparqlUo value: " + value);
191+
}
192+
}

0 commit comments

Comments
 (0)