Skip to content

Commit 3047534

Browse files
hmottestadhmottestad
committed
GH-5553 Defer LMDB index usage recording to explanation
1 parent 83b6bbe commit 3047534

2 files changed

Lines changed: 108 additions & 21 deletions

File tree

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbRecordIterator.java‎

Lines changed: 75 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.util.Collection;
2929
import java.util.Collections;
3030
import java.util.List;
31+
import java.util.Set;
3132
import java.util.concurrent.ConcurrentHashMap;
3233
import java.util.concurrent.ConcurrentMap;
3334
import java.util.concurrent.atomic.LongAdder;
@@ -50,12 +51,14 @@ class LmdbRecordIterator implements RecordIterator {
5051
private static final Logger log = LoggerFactory.getLogger(LmdbRecordIterator.class);
5152
private static final List<Character> COMPONENT_ORDER = List.of('s', 'p', 'o', 'c');
5253
private static final ConcurrentMap<String, LongAdder> RECOMMENDED_INDEX_TRACKER = new ConcurrentHashMap<>();
54+
private static final ConcurrentMap<LmdbRecordIterator, String> INDEX_NAME_CACHE = new ConcurrentHashMap<>();
55+
private static final Set<LmdbRecordIterator> PENDING_RECOMMENDATIONS = ConcurrentHashMap.newKeySet();
56+
private static final Set<LmdbRecordIterator> RECORDED_RECOMMENDATIONS = ConcurrentHashMap.newKeySet();
57+
5358
private final Pool pool;
5459

5560
private final TripleIndex index;
5661

57-
private volatile String indexName;
58-
5962
private final long subj;
6063
private final long pred;
6164
private final long obj;
@@ -156,9 +159,13 @@ static ConcurrentMap<String, LongAdder> getRecommendedIndexTracker() {
156159

157160
static void resetIndexRecommendationTracker() {
158161
RECOMMENDED_INDEX_TRACKER.clear();
162+
INDEX_NAME_CACHE.clear();
163+
PENDING_RECOMMENDATIONS.clear();
164+
RECORDED_RECOMMENDATIONS.clear();
159165
}
160166

161-
private static String computeIndexName(TripleIndex index, long subj, long pred, long obj, long context) {
167+
private static String computeIndexName(TripleIndex index, long subj, long pred, long obj, long context,
168+
boolean recordUsage) {
162169
String actual = new String(index.getFieldSeq());
163170
int boundCount = countBound(subj, pred, obj, context);
164171
if (boundCount <= 0) {
@@ -170,7 +177,7 @@ private static String computeIndexName(TripleIndex index, long subj, long pred,
170177
return actual;
171178
}
172179

173-
CandidateIndex recommendation = selectRecommendedIndex(actual, subj, pred, obj, context);
180+
CandidateIndex recommendation = selectRecommendedIndex(actual, subj, pred, obj, context, recordUsage);
174181
if (recommendation == null) {
175182
return actual;
176183
}
@@ -196,8 +203,8 @@ private static int countBound(long subj, long pred, long obj, long context) {
196203
}
197204

198205
private static CandidateIndex selectRecommendedIndex(String actual, long subj, long pred, long obj,
199-
long context) {
200-
List<CandidateIndex> candidates = buildCandidateIndexes(subj, pred, obj, context);
206+
long context, boolean recordUsage) {
207+
List<CandidateIndex> candidates = buildCandidateIndexes(subj, pred, obj, context, recordUsage);
201208
if (candidates.isEmpty()) {
202209
return null;
203210
}
@@ -223,14 +230,19 @@ private static CandidateIndex selectRecommendedIndex(String actual, long subj, l
223230
best = candidates.get(0);
224231
}
225232

226-
for (CandidateIndex candidate : candidates) {
227-
candidate.counter.increment();
233+
if (recordUsage) {
234+
for (CandidateIndex candidate : candidates) {
235+
if (candidate.counter != null) {
236+
candidate.counter.increment();
237+
}
238+
}
228239
}
229240

230241
return best;
231242
}
232243

233-
private static List<CandidateIndex> buildCandidateIndexes(long subj, long pred, long obj, long context) {
244+
private static List<CandidateIndex> buildCandidateIndexes(long subj, long pred, long obj, long context,
245+
boolean recordUsage) {
234246
List<Character> boundComponents = gatherBoundComponents(subj, pred, obj, context);
235247
if (boundComponents.isEmpty()) {
236248
return Collections.emptyList();
@@ -250,22 +262,44 @@ private static List<CandidateIndex> buildCandidateIndexes(long subj, long pred,
250262
for (String bound : boundPermutations) {
251263
for (String suffix : unboundPermutations) {
252264
String candidate = bound + suffix;
253-
addCandidate(result, candidate, preferredOrder, subj, pred, obj, context);
265+
addCandidate(result, candidate, preferredOrder, subj, pred, obj, context, recordUsage);
254266
}
255267
}
256268

257269
return result;
258270
}
259271

260272
private static void addCandidate(Collection<CandidateIndex> candidates, String candidate,
261-
List<Character> preferredOrder, long subj, long pred, long obj, long context) {
262-
LongAdder counter = RECOMMENDED_INDEX_TRACKER.computeIfAbsent(candidate, key -> new LongAdder());
263-
long count = counter.sum();
273+
List<Character> preferredOrder, long subj, long pred, long obj, long context, boolean recordUsage) {
274+
LongAdder counter = RECOMMENDED_INDEX_TRACKER.get(candidate);
275+
if (counter == null && recordUsage) {
276+
counter = RECOMMENDED_INDEX_TRACKER.computeIfAbsent(candidate, key -> new LongAdder());
277+
}
278+
long count = counter != null ? counter.sum() : 0L;
264279
int score = computePatternScore(candidate, subj, pred, obj, context);
265280
int deviation = computeDeviation(candidate, preferredOrder);
266281
candidates.add(new CandidateIndex(candidate, count, score, deviation, counter));
267282
}
268283

284+
private static boolean shouldRecordUsage() {
285+
StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
286+
for (StackTraceElement element : stackTrace) {
287+
String className = element.getClassName();
288+
String methodName = element.getMethodName();
289+
if (("org.eclipse.rdf4j.query.algebra.StatementPattern".equals(className)
290+
&& "getIndexName".equals(methodName))
291+
|| ("org.eclipse.rdf4j.repository.sail.SailQuery".equals(className) && "explain".equals(methodName))
292+
|| ("org.eclipse.rdf4j.sail.base.SailSourceConnection".equals(className)
293+
&& "explain".equals(methodName))
294+
|| "org.eclipse.rdf4j.query.algebra.helpers.QueryModelTreeToGenericPlanNode".equals(className)
295+
|| ("org.eclipse.rdf4j.query.explanation.Explanation".equals(className)
296+
&& "toString".equals(methodName))) {
297+
return true;
298+
}
299+
}
300+
return false;
301+
}
302+
269303
private static List<Character> gatherBoundComponents(long subj, long pred, long obj, long context) {
270304
List<Character> bound = new ArrayList<>(4);
271305
if (subj >= 0) {
@@ -536,17 +570,34 @@ public long[] next() {
536570

537571
@Override
538572
public String getIndexName() {
539-
String current = indexName;
540-
if (current == null) {
541-
synchronized (this) {
542-
current = indexName;
543-
if (current == null) {
544-
current = computeIndexName(index, subj, pred, obj, context);
545-
indexName = current;
573+
while (true) {
574+
boolean explanationContext = shouldRecordUsage();
575+
String cached = INDEX_NAME_CACHE.get(this);
576+
if (cached != null) {
577+
if (explanationContext && RECORDED_RECOMMENDATIONS.add(this)) {
578+
String computed = computeIndexName(index, subj, pred, obj, context, true);
579+
INDEX_NAME_CACHE.put(this, computed);
580+
return computed;
581+
}
582+
return cached;
583+
}
584+
585+
if (PENDING_RECOMMENDATIONS.add(this)) {
586+
try {
587+
boolean recordUsage = explanationContext;
588+
String computed = computeIndexName(index, subj, pred, obj, context, recordUsage);
589+
INDEX_NAME_CACHE.put(this, computed);
590+
if (recordUsage) {
591+
RECORDED_RECOMMENDATIONS.add(this);
592+
}
593+
return computed;
594+
} finally {
595+
PENDING_RECOMMENDATIONS.remove(this);
546596
}
547597
}
598+
599+
Thread.onSpinWait();
548600
}
549-
return current;
550601
}
551602

552603
private boolean matches() {
@@ -563,6 +614,9 @@ private boolean matches() {
563614

564615
private void closeInternal(boolean maybeCalledAsync) {
565616
if (!closed) {
617+
INDEX_NAME_CACHE.remove(this);
618+
PENDING_RECOMMENDATIONS.remove(this);
619+
RECORDED_RECOMMENDATIONS.remove(this);
566620
long writeStamp = 0L;
567621
boolean writeLocked = false;
568622
if (maybeCalledAsync && ownerThread != Thread.currentThread()) {

‎core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/LmdbExplainIndexRecommendationTest.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@
1515
import java.io.File;
1616
import java.util.ArrayList;
1717
import java.util.List;
18+
import java.util.Map;
1819
import java.util.Set;
1920
import java.util.concurrent.ConcurrentMap;
2021
import java.util.concurrent.atomic.LongAdder;
22+
import java.util.stream.Collectors;
2123
import java.util.stream.Stream;
2224

2325
import org.eclipse.rdf4j.model.IRI;
@@ -115,6 +117,36 @@ void trackerRemainsEmptyUntilIndexNameRequested() {
115117
assertThat(tracked).isNotEmpty();
116118
}
117119

120+
@Test
121+
void explanationStringOnlyCountsCandidatesOnce() {
122+
File storeDir = new File(dataDir, "psoc-repeat-" + System.nanoTime());
123+
storeDir.mkdirs();
124+
125+
SailRepository repository = new SailRepository(new LmdbStore(storeDir, new LmdbStoreConfig("psoc")));
126+
repository.init();
127+
128+
try (SailRepositoryConnection connection = repository.getConnection()) {
129+
connection.add(connection.getValueFactory().createIRI("http://example.com/alice"), RDF.TYPE, FOAF.PERSON);
130+
Explanation explanation = connection.prepareTupleQuery(PERSON_QUERY).explain(Explanation.Level.Optimized);
131+
132+
explanation.toString();
133+
134+
ConcurrentMap<String, LongAdder> tracked = LmdbRecordIterator.getRecommendedIndexTracker();
135+
assertThat(tracked).isNotEmpty();
136+
Map<String, Long> before = tracked.entrySet()
137+
.stream()
138+
.collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().sum()));
139+
140+
explanation.toString();
141+
142+
tracked.forEach((name, counter) -> assertThat(counter.sum()).isEqualTo(before.get(name)));
143+
} catch (RepositoryException e) {
144+
throw new RuntimeException(e);
145+
} finally {
146+
repository.shutDown();
147+
}
148+
}
149+
118150
private static Stream<Arguments> allTripleIndexPermutations() {
119151
List<String> permutations = new ArrayList<>();
120152
permute("spoc".toCharArray(), new boolean[4], new StringBuilder(), permutations);
@@ -178,4 +210,5 @@ private void runSelectQuery(String index, String query, RepositoryConnectionCons
178210
private interface RepositoryConnectionConsumer {
179211
void accept(SailRepositoryConnection connection) throws RepositoryException;
180212
}
213+
181214
}

0 commit comments

Comments
 (0)