Skip to content

Commit b7f58bd

Browse files
hmottestadhmottestad
committed
wip
1 parent 7e53d02 commit b7f58bd

9 files changed

Lines changed: 945 additions & 4 deletions

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbEvaluationStatistics.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,12 @@
88
*
99
* SPDX-License-Identifier: BSD-3-Clause
1010
*******************************************************************************/
11+
// Some portions generated by Codex
1112
package org.eclipse.rdf4j.sail.lmdb;
1213

1314
import java.io.IOException;
15+
import java.util.HashMap;
16+
import java.util.Map;
1417

1518
import org.eclipse.rdf4j.model.IRI;
1619
import org.eclipse.rdf4j.model.Resource;
@@ -32,6 +35,7 @@ class LmdbEvaluationStatistics extends EvaluationStatistics {
3235
private final ValueStore valueStore;
3336

3437
private final TripleStore tripleStore;
38+
private final Map<CardinalityKey, Double> cardinalityCache = new HashMap<>();
3539

3640
public LmdbEvaluationStatistics(ValueStore valueStore, TripleStore tripleStore) {
3741
this.valueStore = valueStore;
@@ -113,5 +117,54 @@ private double cardinality(Resource subj, IRI pred, Value obj, Resource context)
113117
}
114118

115119
return tripleStore.cardinality(subjID, predID, objID, contextID);
120+
121+
122+
// CardinalityKey key = new CardinalityKey(subjID, predID, objID, contextID);
123+
// Double cached = cardinalityCache.get(key);
124+
// if (cached != null) {
125+
// return cached;
126+
// }
127+
//
128+
// double cardinality = tripleStore.cardinality(subjID, predID, objID, contextID);
129+
// cardinalityCache.put(key, cardinality);
130+
// return cardinality;
131+
}
132+
133+
private static final class CardinalityKey {
134+
private final long subjID;
135+
private final long predID;
136+
private final long objID;
137+
private final long contextID;
138+
139+
private CardinalityKey(long subjID, long predID, long objID, long contextID) {
140+
this.subjID = subjID;
141+
this.predID = predID;
142+
this.objID = objID;
143+
this.contextID = contextID;
144+
}
145+
146+
@Override
147+
public boolean equals(Object obj) {
148+
if (this == obj) {
149+
return true;
150+
}
151+
if (!(obj instanceof CardinalityKey)) {
152+
return false;
153+
}
154+
CardinalityKey other = (CardinalityKey) obj;
155+
return subjID == other.subjID
156+
&& predID == other.predID
157+
&& objID == other.objID
158+
&& contextID == other.contextID;
159+
}
160+
161+
@Override
162+
public int hashCode() {
163+
int result = Long.hashCode(subjID);
164+
result = 31 * result + Long.hashCode(predID);
165+
result = 31 * result + Long.hashCode(objID);
166+
result = 31 * result + Long.hashCode(contextID);
167+
return result;
168+
}
116169
}
117170
}

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/estimate/LmdbBtreeRangeCounter.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,14 +216,18 @@ private SearchResult findFirstGreaterOrEqual(LmdbPage page, byte[] key, int keyL
216216

217217
private boolean matches(LmdbNode node, ByteBuffer pageBuffer, GroupMatcher matcher) {
218218
ByteBuffer keySlice = pageBuffer.duplicate();
219+
keySlice.order(pageBuffer.order());
219220
keySlice.position(node.keyOffset);
220221
keySlice.limit(node.keyOffset + node.keySize);
221-
return matcher.matches(keySlice.slice());
222+
ByteBuffer keyView = keySlice.slice();
223+
keyView.order(pageBuffer.order());
224+
return matcher.matches(keyView);
222225
}
223226

224227
private long countNodeEntries(LmdbNode node, LmdbPage page, RangeCountResult stats) throws IOException {
225228
if ((node.nodeFlags & LmdbFormat.F_SUBDATA) != 0 && node.valueSize >= LmdbFormat.META_DB_SIZE) {
226229
ByteBuffer dup = page.buffer.duplicate();
230+
dup.order(page.buffer.order());
227231
dup.position(node.valueOffset);
228232
LmdbDb subDb = LmdbDb.parse(dup, node.valueOffset);
229233
return subDb.entries;

‎core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/LmdbEvaluationStatisticsMemoizationTest.java‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*******************************************************************************
2+
* Copyright (c) 2026 Eclipse RDF4J contributors.
3+
*
4+
* All rights reserved. This program and the accompanying materials
5+
* are made available under the terms of the Eclipse Distribution License v1.0
6+
* which accompanies this distribution, and is available at
7+
* http://www.eclipse.org/org/documents/edl-v10.php.
8+
*
9+
* SPDX-License-Identifier: BSD-3-Clause
10+
*******************************************************************************/
11+
// Some portions generated by Codex
12+
package org.eclipse.rdf4j.sail.lmdb;
13+
14+
import static org.junit.jupiter.api.Assertions.assertEquals;
15+
import static org.junit.jupiter.api.Assertions.assertTrue;
16+
17+
import java.io.File;
18+
import java.lang.reflect.Field;
19+
import java.lang.reflect.Method;
20+
import java.nio.file.Files;
21+
import java.util.List;
22+
import java.util.Map;
23+
24+
import org.apache.commons.io.FileUtils;
25+
import org.eclipse.rdf4j.common.transaction.IsolationLevels;
26+
import org.eclipse.rdf4j.model.IRI;
27+
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
28+
import org.eclipse.rdf4j.query.QueryLanguage;
29+
import org.eclipse.rdf4j.query.algebra.StatementPattern;
30+
import org.eclipse.rdf4j.query.algebra.evaluation.impl.EvaluationStatistics;
31+
import org.eclipse.rdf4j.query.algebra.helpers.collectors.StatementPatternCollector;
32+
import org.eclipse.rdf4j.query.parser.ParsedTupleQuery;
33+
import org.eclipse.rdf4j.query.parser.QueryParserUtil;
34+
import org.eclipse.rdf4j.repository.sail.SailRepository;
35+
import org.eclipse.rdf4j.repository.sail.SailRepositoryConnection;
36+
import org.eclipse.rdf4j.sail.base.SailStore;
37+
import org.eclipse.rdf4j.sail.lmdb.config.LmdbStoreConfig;
38+
import org.junit.jupiter.api.Test;
39+
40+
class LmdbEvaluationStatisticsMemoizationTest {
41+
42+
@Test
43+
void cachesEquivalentStatementPatternCardinalitiesByResolvedIds() throws Exception {
44+
File dataDir = Files.createTempDirectory("lmdb-eval-stats-memoization").toFile();
45+
SailRepository repository = new SailRepository(new LmdbStore(dataDir, new LmdbStoreConfig()));
46+
try {
47+
loadData(repository);
48+
49+
EvaluationStatistics statistics = extractEvaluationStatistics(repository);
50+
StatementPattern followsA = statementPattern("SELECT * WHERE { ?s <urn:test:follows> ?o . }");
51+
StatementPattern followsB = statementPattern("SELECT * WHERE { ?x <urn:test:follows> ?y . }");
52+
StatementPattern name = statementPattern("SELECT * WHERE { ?u <urn:test:name> ?label . }");
53+
54+
statistics.getCardinality(followsA);
55+
statistics.getCardinality(followsB);
56+
statistics.getCardinality(name);
57+
58+
Map<?, ?> cache = cardinalityCache(statistics);
59+
assertEquals(2, cache.size(), "Equivalent follows patterns should share one memoized entry");
60+
} finally {
61+
repository.shutDown();
62+
FileUtils.deleteDirectory(dataDir);
63+
}
64+
}
65+
66+
private static void loadData(SailRepository repository) {
67+
SimpleValueFactory vf = SimpleValueFactory.getInstance();
68+
IRI follows = vf.createIRI("urn:test:follows");
69+
IRI name = vf.createIRI("urn:test:name");
70+
try (SailRepositoryConnection connection = repository.getConnection()) {
71+
connection.begin(IsolationLevels.NONE);
72+
for (int i = 0; i < 8; i++) {
73+
IRI subject = vf.createIRI("urn:test:user:" + i);
74+
IRI object = vf.createIRI("urn:test:user:" + ((i + 1) % 8));
75+
connection.add(subject, follows, object);
76+
connection.add(subject, name, vf.createLiteral("u" + i));
77+
}
78+
connection.commit();
79+
}
80+
}
81+
82+
private static EvaluationStatistics extractEvaluationStatistics(SailRepository repository) throws Exception {
83+
Object sail = repository.getSail();
84+
Method getSailStoreMethod = sail.getClass().getDeclaredMethod("getSailStore");
85+
getSailStoreMethod.setAccessible(true);
86+
SailStore sailStore = (SailStore) getSailStoreMethod.invoke(sail);
87+
return sailStore.getEvaluationStatistics();
88+
}
89+
90+
private static StatementPattern statementPattern(String query) {
91+
ParsedTupleQuery parsed = QueryParserUtil.parseTupleQuery(QueryLanguage.SPARQL, query, null);
92+
List<StatementPattern> patterns = StatementPatternCollector.process(parsed.getTupleExpr());
93+
return patterns.get(0);
94+
}
95+
96+
private static Map<?, ?> cardinalityCache(EvaluationStatistics statistics) throws Exception {
97+
assertTrue(statistics.getClass().getName().endsWith("LmdbEvaluationStatistics"));
98+
Field field = statistics.getClass().getDeclaredField("cardinalityCache");
99+
field.setAccessible(true);
100+
return (Map<?, ?>) field.get(statistics);
101+
}
102+
}

‎core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/benchmark/ThemeQueryBenchmark.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,13 @@
2323
import org.eclipse.rdf4j.benchmark.rio.util.ThemeDataSetGenerator;
2424
import org.eclipse.rdf4j.benchmark.rio.util.ThemeDataSetGenerator.Theme;
2525
import org.eclipse.rdf4j.common.transaction.IsolationLevels;
26+
import org.eclipse.rdf4j.query.TupleQuery;
2627
import org.eclipse.rdf4j.query.explanation.Explanation;
2728
import org.eclipse.rdf4j.repository.sail.SailRepository;
2829
import org.eclipse.rdf4j.repository.sail.SailRepositoryConnection;
2930
import org.eclipse.rdf4j.repository.util.RDFInserter;
3031
import org.eclipse.rdf4j.sail.lmdb.LmdbStore;
32+
import org.eclipse.rdf4j.sail.lmdb.config.LmdbStoreConfig;
3133
import org.junit.jupiter.api.Disabled;
3234
import org.junit.jupiter.api.Test;
3335
import org.openjdk.jmh.annotations.Benchmark;
@@ -103,7 +105,9 @@ public void setup() throws IOException {
103105
query = ThemeQueryCatalog.queryFor(theme, z_queryIndex);
104106
expected = ThemeQueryCatalog.expectedCountFor(theme, z_queryIndex);
105107
dataDir = Files.newTemporaryFolder();
106-
repository = new SailRepository(new LmdbStore(dataDir, ConfigUtil.createConfig()));
108+
LmdbStoreConfig config = ConfigUtil.createConfig();
109+
config.setPageCardinalityEstimator(false );
110+
repository = new SailRepository(new LmdbStore(dataDir, config));
107111
loadData();
108112
}
109113

@@ -125,8 +129,10 @@ public void tearDown() throws IOException {
125129
@Benchmark
126130
public long executeQuery() {
127131
try (SailRepositoryConnection connection = repository.getConnection()) {
128-
long count = connection
129-
.prepareTupleQuery(query)
132+
TupleQuery tupleQuery = connection
133+
.prepareTupleQuery(query);
134+
tupleQuery.setMaxExecutionTime(180);
135+
long count = tupleQuery
130136
.evaluate()
131137
.stream()
132138
.count();

0 commit comments

Comments
 (0)