Skip to content

Commit ab00f92

Browse files
hmottestadhmottestad
committed
working on new ID based join iterator
1 parent 980cddd commit ab00f92

10 files changed

Lines changed: 161 additions & 90 deletions

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbDelegatingSailDataset.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,8 @@ public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predI
113113
}
114114
// Fallback via TripleSource with Value conversion
115115
return new LmdbSailDatasetTripleSource(valueStore, delegate)
116-
.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, reuse, quadReuse);
116+
.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null, reuse,
117+
quadReuse, null);
117118
}
118119

119120
@Override

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbEvaluationDataset.java‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,6 @@ default RecordIterator getRecordIterator(StatementPattern pattern, BindingSet bi
6868
@InternalUseOnly
6969
default RecordIterator getRecordIterator(StatementPattern pattern, BindingSet bindings, KeyRangeBuffers keyBuffers,
7070
RecordIterator iteratorReuse) throws QueryEvaluationException {
71-
if (iteratorReuse != null) {
72-
iteratorReuse.close();
73-
}
7471
return getRecordIterator(pattern, bindings, keyBuffers);
7572
}
7673

@@ -105,6 +102,8 @@ RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, i
105102
@InternalUseOnly
106103
default RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
107104
long[] patternIds, KeyRangeBuffers keyBuffers) throws QueryEvaluationException {
105+
assert keyBuffers == null : "We are not passing keyBuffers into the next method";
106+
108107
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds);
109108
}
110109

@@ -144,9 +143,6 @@ default RecordIterator getRecordIterator(long[] binding, int subjIndex, int pred
144143
long[] patternIds, KeyRangeBuffers keyBuffers, long[] bindingReuse, long[] quadReuse,
145144
RecordIterator iteratorReuse)
146145
throws QueryEvaluationException {
147-
if (iteratorReuse != null) {
148-
iteratorReuse.close();
149-
}
150146
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, keyBuffers,
151147
bindingReuse, quadReuse);
152148
}
@@ -196,9 +192,6 @@ default RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, i
196192
default RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
197193
int ctxIndex, long[] patternIds, StatementOrder order, KeyRangeBuffers keyBuffers, long[] bindingReuse,
198194
long[] quadReuse, RecordIterator iteratorReuse) throws QueryEvaluationException {
199-
if (iteratorReuse != null) {
200-
iteratorReuse.close();
201-
}
202195
return getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, order,
203196
keyBuffers,
204197
bindingReuse, quadReuse);

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbIdTripleSource.java‎

Lines changed: 9 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -31,40 +31,17 @@ public interface LmdbIdTripleSource {
3131
* @param patternIds constants for S/P/O/C positions (UNKNOWN_ID for wildcard)
3232
*/
3333
RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
34-
long[] patternIds) throws QueryEvaluationException;
35-
36-
default RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
37-
long[] patternIds, LmdbEvaluationDataset.KeyRangeBuffers keyBuffers) throws QueryEvaluationException {
38-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds);
39-
}
40-
41-
/**
42-
* Variant that accepts reusable scratch buffers for bindings and quads.
43-
*/
44-
default RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
45-
long[] patternIds, long[] bindingReuse) throws QueryEvaluationException {
46-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, bindingReuse, null);
47-
}
48-
49-
default RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
50-
long[] patternIds, long[] bindingReuse, long[] quadReuse) throws QueryEvaluationException {
51-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds);
52-
}
53-
54-
default RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
55-
long[] patternIds, LmdbEvaluationDataset.KeyRangeBuffers keyBuffers, long[] bindingReuse, long[] quadReuse)
56-
throws QueryEvaluationException {
57-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, bindingReuse,
58-
quadReuse);
59-
}
34+
long[] patternIds, LmdbEvaluationDataset.KeyRangeBuffers keyBuffers, long[] bindingReuse, long[] quadReuse,
35+
RecordIterator reuse) throws QueryEvaluationException;
6036

6137
/**
6238
* Create an ordered iterator over ID-level bindings; may fall back to the unordered iterator if unsupported.
6339
*/
6440
default RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
6541
int ctxIndex, long[] patternIds, StatementOrder order) throws QueryEvaluationException {
6642
if (order == null) {
67-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds);
43+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null, null, null,
44+
null);
6845
}
6946
return null;
7047
}
@@ -75,7 +52,8 @@ default RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, i
7552
default RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
7653
int ctxIndex, long[] patternIds, StatementOrder order, long[] reuse) throws QueryEvaluationException {
7754
if (order == null) {
78-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, reuse, null);
55+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null, reuse, null,
56+
null);
7957
}
8058
return null;
8159
}
@@ -84,8 +62,8 @@ default RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, i
8462
int ctxIndex, long[] patternIds, StatementOrder order, long[] bindingReuse, long[] quadReuse)
8563
throws QueryEvaluationException {
8664
if (order == null) {
87-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, bindingReuse,
88-
quadReuse);
65+
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null, bindingReuse,
66+
quadReuse, null);
8967
}
9068
return null;
9169
}
@@ -95,7 +73,7 @@ default RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, i
9573
long[] bindingReuse, long[] quadReuse) throws QueryEvaluationException {
9674
if (order == null) {
9775
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, keyBuffers,
98-
bindingReuse, quadReuse);
76+
bindingReuse, quadReuse, null);
9977
}
10078
return null;
10179
}

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbIdTripleSourceAdapter.java‎

Lines changed: 5 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -79,33 +79,15 @@ public ValueFactory getValueFactory() {
7979
return delegate.getValueFactory();
8080
}
8181

82-
// ID-level implementation
8382
@Override
8483
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
85-
long[] patternIds) throws QueryEvaluationException {
86-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null, null, null);
87-
}
88-
89-
@Override
90-
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
91-
long[] patternIds, long[] reuse) throws QueryEvaluationException {
92-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null, reuse, null);
93-
}
94-
95-
@Override
96-
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
97-
long[] patternIds, long[] reuse, long[] quadReuse) throws QueryEvaluationException {
98-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null, reuse, quadReuse);
99-
}
100-
101-
@Override
102-
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
103-
long[] patternIds, LmdbEvaluationDataset.KeyRangeBuffers keyBuffers, long[] reuse, long[] quadReuse)
84+
long[] patternIds, LmdbEvaluationDataset.KeyRangeBuffers keyBuffers, long[] reuse, long[] quadReuse,
85+
RecordIterator iteratorReuse)
10486
throws QueryEvaluationException {
10587
// Prefer direct ID-level access if the delegate already supports it
10688
if (delegate instanceof LmdbIdTripleSource) {
10789
return ((LmdbIdTripleSource) delegate).getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex,
108-
patternIds, keyBuffers, reuse, quadReuse);
90+
patternIds, keyBuffers, reuse, quadReuse, iteratorReuse);
10991
}
11092

11193
// If no active connection changes, delegate to the current LMDB dataset to avoid materialization
@@ -114,7 +96,7 @@ public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predI
11496
if (dsOpt.isPresent()) {
11597
return dsOpt.get()
11698
.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, keyBuffers,
117-
reuse, quadReuse);
99+
reuse, quadReuse, iteratorReuse);
118100
}
119101
}
120102

@@ -196,7 +178,7 @@ public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, in
196178
long[] bindingReuse, long[] quadReuse) throws QueryEvaluationException {
197179
if (order == null) {
198180
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, keyBuffers,
199-
bindingReuse, quadReuse);
181+
bindingReuse, quadReuse, null);
200182
}
201183
if (delegate instanceof LmdbIdTripleSource) {
202184
return ((LmdbIdTripleSource) delegate).getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex,

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbOverlayEvaluationDataset.java‎

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,16 @@
2525
import org.eclipse.rdf4j.query.algebra.evaluation.TripleSource;
2626
import org.eclipse.rdf4j.sail.lmdb.join.LmdbIdJoinIterator;
2727
import org.eclipse.rdf4j.sail.lmdb.model.LmdbValue;
28+
import org.slf4j.Logger;
29+
import org.slf4j.LoggerFactory;
2830

2931
/**
3032
* Delegates reads via the Sail {@link TripleSource} to honor transaction overlays and isolation while exposing
3133
* LMDB-style {@link RecordIterator}s for ID-based joins.
3234
*/
3335
final class LmdbOverlayEvaluationDataset implements LmdbEvaluationDataset {
3436

37+
private static final Logger log = LoggerFactory.getLogger(LmdbOverlayEvaluationDataset.class);
3538
private final TripleSource tripleSource;
3639
private final ValueStore valueStore;
3740

@@ -52,6 +55,15 @@ public RecordIterator getRecordIterator(StatementPattern pattern, BindingSet bin
5255
return getRecordIteratorInternal(pattern, bindings, keyBuffers);
5356
}
5457

58+
@Override
59+
public RecordIterator getRecordIterator(StatementPattern pattern, BindingSet bindings, KeyRangeBuffers keyBuffers,
60+
RecordIterator iteratorReuse) throws QueryEvaluationException {
61+
if (iteratorReuse != null) {
62+
iteratorReuse.close();
63+
}
64+
return getRecordIteratorInternal(pattern, bindings, keyBuffers);
65+
}
66+
5567
private RecordIterator getRecordIteratorInternal(StatementPattern pattern, BindingSet bindings,
5668
KeyRangeBuffers keyBuffers)
5769
throws QueryEvaluationException {
@@ -132,40 +144,49 @@ public void close() {
132144
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
133145
long[] patternIds) throws QueryEvaluationException {
134146
return getRecordIteratorInternal(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null, null,
135-
null);
147+
null, null);
136148
}
137149

138150
@Override
139151
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
140152
long[] patternIds, KeyRangeBuffers keyBuffers) throws QueryEvaluationException {
141153
return getRecordIteratorInternal(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, keyBuffers,
142-
null, null);
154+
null, null, null);
143155
}
144156

145157
@Override
146158
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
147159
long[] patternIds, long[] reuse) throws QueryEvaluationException {
148160
return getRecordIteratorInternal(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null, reuse,
149-
null);
161+
null, null);
150162
}
151163

152164
@Override
153165
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
154166
long[] patternIds, long[] reuse, long[] quadReuse) throws QueryEvaluationException {
155167
return getRecordIteratorInternal(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null, reuse,
156-
quadReuse);
168+
quadReuse, null);
157169
}
158170

159171
@Override
160172
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
161173
long[] patternIds, KeyRangeBuffers keyBuffers, long[] reuse, long[] quadReuse)
162174
throws QueryEvaluationException {
163175
return getRecordIteratorInternal(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, keyBuffers,
164-
reuse, quadReuse);
176+
reuse, quadReuse, null);
177+
}
178+
179+
@Override
180+
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
181+
long[] patternIds, KeyRangeBuffers keyBuffers, long[] reuse, long[] quadReuse, RecordIterator iteratorReuse)
182+
throws QueryEvaluationException {
183+
return getRecordIteratorInternal(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, keyBuffers,
184+
reuse, quadReuse, iteratorReuse);
165185
}
166186

167187
private RecordIterator getRecordIteratorInternal(long[] binding, int subjIndex, int predIndex, int objIndex,
168-
int ctxIndex, long[] patternIds, KeyRangeBuffers keyBuffers, long[] reuse, long[] quadReuse)
188+
int ctxIndex, long[] patternIds, KeyRangeBuffers keyBuffers, long[] reuse, long[] quadReuse,
189+
RecordIterator iteratorReuse)
169190
throws QueryEvaluationException {
170191
// Prefer an ID-level path if the TripleSource supports it and we can trust overlay correctness.
171192
if (tripleSource instanceof LmdbIdTripleSource) {
@@ -174,7 +195,7 @@ private RecordIterator getRecordIteratorInternal(long[] binding, int subjIndex,
174195
// correct when available.
175196
RecordIterator viaIds = ((LmdbIdTripleSource) tripleSource)
176197
.getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, keyBuffers, reuse,
177-
quadReuse);
198+
quadReuse, iteratorReuse);
178199
if (viaIds != null) {
179200
return viaIds;
180201
}
@@ -290,12 +311,24 @@ public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, in
290311
long[] quadReuse) throws QueryEvaluationException {
291312
if (order == null) {
292313
return getRecordIteratorInternal(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, keyBuffers,
293-
bindingReuse, quadReuse);
314+
bindingReuse, quadReuse, null);
294315
}
295316
return LmdbEvaluationDataset.super.getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex,
296317
patternIds, order, keyBuffers, bindingReuse, quadReuse);
297318
}
298319

320+
@Override
321+
public RecordIterator getOrderedRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex,
322+
int ctxIndex, long[] patternIds, StatementOrder order, KeyRangeBuffers keyBuffers, long[] bindingReuse,
323+
long[] quadReuse, RecordIterator iteratorReuse) throws QueryEvaluationException {
324+
if (iteratorReuse != null) {
325+
iteratorReuse.close();
326+
log.warn("getOrderedRecordIterator does not support reusing RecordIterator");
327+
}
328+
return getOrderedRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, order,
329+
keyBuffers, bindingReuse, quadReuse);
330+
}
331+
299332
@Override
300333
public RecordIterator getOrderedRecordIterator(StatementPattern pattern, BindingSet bindings, StatementOrder order,
301334
KeyRangeBuffers keyBuffers) throws QueryEvaluationException {

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailDatasetTripleSource.java‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -42,24 +42,13 @@ public LmdbSailDatasetTripleSource(ValueFactory vf, SailDataset dataset) {
4242

4343
@Override
4444
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
45-
long[] patternIds) throws QueryEvaluationException {
46-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, null, null);
47-
}
48-
49-
@Override
50-
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
51-
long[] patternIds, long[] reuse) throws QueryEvaluationException {
52-
return getRecordIterator(binding, subjIndex, predIndex, objIndex, ctxIndex, patternIds, reuse, null);
53-
}
54-
55-
@Override
56-
public RecordIterator getRecordIterator(long[] binding, int subjIndex, int predIndex, int objIndex, int ctxIndex,
57-
long[] patternIds, long[] reuse, long[] quadReuse) throws QueryEvaluationException {
45+
long[] patternIds, LmdbEvaluationDataset.KeyRangeBuffers keyBuffers, long[] bindingReuse, long[] quadReuse,
46+
RecordIterator iteratorReuse) throws QueryEvaluationException {
5847

5948
// Fast path: backing dataset supports ID-level access
6049
if (dataset instanceof LmdbEvaluationDataset) {
6150
return ((LmdbEvaluationDataset) dataset).getRecordIterator(binding, subjIndex, predIndex, objIndex,
62-
ctxIndex, patternIds, reuse, quadReuse);
51+
ctxIndex, patternIds, keyBuffers, bindingReuse, quadReuse, iteratorReuse);
6352
}
6453

6554
// Fallback path: value-level iteration converted to IDs using ValueStore

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1585,8 +1585,6 @@ public long[] next() throws QueryEvaluationException {
15851585
public void close() {
15861586
if (base != null) {
15871587
base.close();
1588-
base = null;
1589-
reusableBase = null;
15901588
}
15911589
}
15921590
}

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/join/LmdbIdBGPQueryEvaluationStep.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -572,6 +572,9 @@ public long[] next() throws QueryEvaluationException {
572572
return next;
573573
}
574574
currentRight.close();
575+
if (currentRight != LmdbIdJoinIterator.EMPTY_RECORD_ITERATOR) {
576+
reusableRight = currentRight;
577+
}
575578
currentRight = null;
576579
}
577580

@@ -587,7 +590,7 @@ public long[] next() throws QueryEvaluationException {
587590
}
588591
currentRight = dataset.getRecordIterator(leftBinding, plan.subjIndex, plan.predIndex, plan.objIndex,
589592
plan.ctxIndex, plan.patternIds, keyBuffers, rightScratch, quadScratch, reusableRight);
590-
if (currentRight != null && currentRight != LmdbIdJoinIterator.emptyRecordIterator()) {
593+
if (currentRight != null && currentRight != LmdbIdJoinIterator.EMPTY_RECORD_ITERATOR) {
591594
reusableRight = currentRight;
592595
} else {
593596
reusableRight = null;

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/join/LmdbIdJoinIterator.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ interface RecordIteratorFactory {
4141
RecordIterator apply(long[] leftRecord, RecordIterator reuse) throws QueryEvaluationException;
4242
}
4343

44-
private static final RecordIterator EMPTY_RECORD_ITERATOR = new RecordIterator() {
44+
public static final RecordIterator EMPTY_RECORD_ITERATOR = new RecordIterator() {
4545
@Override
4646
public long[] next() {
4747
return null;

0 commit comments

Comments
 (0)