GH-5447 general LMDB optimisation

Author: hmottestad

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbRecordIterator.java

@@ -29,7 +29,7 @@
 import org.eclipse.rdf4j.sail.SailException;
 import org.eclipse.rdf4j.sail.lmdb.TripleStore.TripleIndex;
 import org.eclipse.rdf4j.sail.lmdb.TxnManager.Txn;
-import org.eclipse.rdf4j.sail.lmdb.Varint.GroupMatcher;
+import org.eclipse.rdf4j.sail.lmdb.util.GroupMatcher;
 import org.lwjgl.PointerBuffer;
 import org.lwjgl.system.MemoryStack;
 import org.lwjgl.util.lmdb.MDBVal;
@@ -45,11 +45,17 @@ class LmdbRecordIterator implements RecordIterator {
 
 	private final TripleIndex index;
 
+	private final long subj;
+	private final long pred;
+	private final long obj;
+	private final long context;
+
 	private final long cursor;
 
 	private final MDBVal maxKey;
 
-	private final GroupMatcher groupMatcher;
+	private final boolean matchValues;
+	private GroupMatcher groupMatcher;
 
 	private final Txn txnRef;
 
@@ -71,7 +77,8 @@ class LmdbRecordIterator implements RecordIterator {
 
 	private int lastResult;
 
-	private final long[] quad = new long[4];
+	private final long[] quad;
+	private final long[] originalQuad;
 
 	private boolean fetchNext = false;
 
@@ -81,6 +88,12 @@ class LmdbRecordIterator implements RecordIterator {
 
 	LmdbRecordIterator(TripleIndex index, boolean rangeSearch, long subj, long pred, long obj,
 			long context, boolean explicit, Txn txnRef) throws IOException {
+		this.subj = subj;
+		this.pred = pred;
+		this.obj = obj;
+		this.context = context;
+		this.originalQuad = new long[] { subj, pred, obj, context };
+		this.quad = new long[] { subj, pred, obj, context };
 		this.pool = Pool.get();
 		this.keyData = pool.getVal();
 		this.valueData = pool.getVal();
@@ -100,12 +113,8 @@ class LmdbRecordIterator implements RecordIterator {
 			this.maxKey = null;
 		}
 
-		boolean matchValues = subj > 0 || pred > 0 || obj > 0 || context >= 0;
-		if (matchValues) {
-			this.groupMatcher = index.createMatcher(subj, pred, obj, context);
-		} else {
-			this.groupMatcher = null;
-		}
+		this.matchValues = subj > 0 || pred > 0 || obj > 0 || context >= 0;
+
 		this.dbi = index.getDB(explicit);
 		this.txnRef = txnRef;
 		this.txnLockManager = txnRef.lockManager();
@@ -145,6 +154,7 @@ public long[] next() {
 			}
 
 			if (txnRefVersion != txnRef.version()) {
+				// TODO: None of the tests in the LMDB Store cover this case!
 				// cursor must be renewed
 				mdb_cursor_renew(txn, cursor);
 				if (fetchNext) {
@@ -188,12 +198,12 @@ public long[] next() {
 				// if (maxKey != null && TripleStore.COMPARATOR.compare(keyData.mv_data(), maxKey.mv_data()) > 0) {
 				if (maxKey != null && mdb_cmp(txn, dbi, keyData, maxKey) > 0) {
 					lastResult = MDB_NOTFOUND;
-				} else if (groupMatcher != null && !groupMatcher.matches(keyData.mv_data())) {
+				} else if (matches()) {
 					// value doesn't match search key/mask, fetch next value
 					lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT);
 				} else {
 					// Matching value found
-					index.keyToQuad(keyData.mv_data(), quad);
+					index.keyToQuad(keyData.mv_data(), originalQuad, quad);
 					// fetch next value
 					fetchNext = true;
 					return quad;
@@ -206,6 +216,18 @@ public long[] next() {
 		}
 	}
 
+	private boolean matches() {
+
+		if (groupMatcher != null) {
+			return !this.groupMatcher.matches(keyData.mv_data());
+		} else if (matchValues) {
+			this.groupMatcher = index.createMatcher(subj, pred, obj, context);
+			return !this.groupMatcher.matches(keyData.mv_data());
+		} else {
+			return false;
+		}
+	}
+
 	private void closeInternal(boolean maybeCalledAsync) {
 		if (!closed) {
 			long writeStamp = 0L;

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java

@@ -29,7 +29,6 @@
 import org.eclipse.rdf4j.common.iteration.CloseableIteration;
 import org.eclipse.rdf4j.common.iteration.CloseableIteratorIteration;
 import org.eclipse.rdf4j.common.iteration.ConvertingIteration;
-import org.eclipse.rdf4j.common.iteration.EmptyIteration;
 import org.eclipse.rdf4j.common.iteration.FilterIteration;
 import org.eclipse.rdf4j.common.iteration.UnionIteration;
 import org.eclipse.rdf4j.common.order.StatementOrder;
@@ -72,6 +71,7 @@ class LmdbSailStore implements SailStore {
 	private boolean multiThreadingActive;
 	private volatile boolean asyncTransactionFinished;
 	private volatile boolean nextTransactionAsync;
+	private volatile boolean mayHaveInferred;
 
 	boolean enableMultiThreading = true;
 
@@ -143,6 +143,9 @@ class AddQuadOperation implements Operation {
 
 		@Override
 		public void execute() throws IOException {
+			if (!explicit) {
+				mayHaveInferred = true;
+			}
 			if (!unusedIds.isEmpty()) {
 				// these ids are used again
 				unusedIds.remove(s);
@@ -191,8 +194,10 @@ public LmdbSailStore(File dataDir, LmdbStoreConfig config) throws IOException, S
 		boolean initialized = false;
 		try {
 			namespaceStore = new NamespaceStore(dataDir);
-			valueStore = new ValueStore(new File(dataDir, "values"), config);
-			tripleStore = new TripleStore(new File(dataDir, "triples"), config);
+			var valueStore = new ValueStore(new File(dataDir, "values"), config);
+			this.valueStore = valueStore;
+			tripleStore = new TripleStore(new File(dataDir, "triples"), config, valueStore);
+			mayHaveInferred = tripleStore.hasTriples(false);
 			initialized = true;
 		} finally {
 			if (!initialized) {
@@ -348,19 +353,23 @@ protected void handleClose() throws SailException {
 	 */
 	CloseableIteration<? extends Statement> createStatementIterator(
 			Txn txn, Resource subj, IRI pred, Value obj, boolean explicit, Resource... contexts) throws IOException {
+		if (!explicit && !mayHaveInferred) {
+			// there are no inferred statements and the iterator should only return inferred statements
+			return CloseableIteration.EMPTY_STATEMENT_ITERATION;
+		}
 		long subjID = LmdbValue.UNKNOWN_ID;
 		if (subj != null) {
 			subjID = valueStore.getId(subj);
 			if (subjID == LmdbValue.UNKNOWN_ID) {
-				return new EmptyIteration<>();
+				return CloseableIteration.EMPTY_STATEMENT_ITERATION;
 			}
 		}
 
 		long predID = LmdbValue.UNKNOWN_ID;
 		if (pred != null) {
 			predID = valueStore.getId(pred);
 			if (predID == LmdbValue.UNKNOWN_ID) {
-				return new EmptyIteration<>();
+				return CloseableIteration.EMPTY_STATEMENT_ITERATION;
 			}
 		}
 
@@ -369,7 +378,7 @@ CloseableIteration<? extends Statement> createStatementIterator(
 			objID = valueStore.getId(obj);
 
 			if (objID == LmdbValue.UNKNOWN_ID) {
-				return new EmptyIteration<>();
+				return CloseableIteration.EMPTY_STATEMENT_ITERATION;
 			}
 		}
 

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbStatementIterator.java

@@ -11,8 +11,9 @@
 package org.eclipse.rdf4j.sail.lmdb;
 
 import java.io.IOException;
+import java.util.NoSuchElementException;
 
-import org.eclipse.rdf4j.common.iteration.LookAheadIteration;
+import org.eclipse.rdf4j.common.iteration.AbstractCloseableIteration;
 import org.eclipse.rdf4j.model.IRI;
 import org.eclipse.rdf4j.model.Resource;
 import org.eclipse.rdf4j.model.Statement;
@@ -23,7 +24,7 @@
  * A statement iterator that wraps a RecordIterator containing statement records and translates these records to
  * {@link Statement} objects.
  */
-class LmdbStatementIterator extends LookAheadIteration<Statement> {
+class LmdbStatementIterator extends AbstractCloseableIteration<Statement> {
 
 	/*-----------*
 	 * Variables *
@@ -32,6 +33,7 @@ class LmdbStatementIterator extends LookAheadIteration<Statement> {
 	private final RecordIterator recordIt;
 
 	private final ValueStore valueStore;
+	private Statement nextElement;
 
 	/*--------------*
 	 * Constructors *
@@ -49,7 +51,6 @@ public LmdbStatementIterator(RecordIterator recordIt, ValueStore valueStore) {
 	 * Methods *
 	 *---------*/
 
-	@Override
 	public Statement getNextElement() throws SailException {
 		try {
 			long[] quad = recordIt.next();
@@ -86,4 +87,52 @@ protected void handleClose() throws SailException {
 	private SailException causeIOException(IOException e) {
 		return new SailException(e);
 	}
+
+	@Override
+	public final boolean hasNext() {
+		if (isClosed()) {
+			return false;
+		}
+
+		return lookAhead() != null;
+	}
+
+	@Override
+	public final Statement next() {
+		if (isClosed()) {
+			throw new NoSuchElementException("The iteration has been closed.");
+		}
+		Statement result = lookAhead();
+
+		if (result != null) {
+			nextElement = null;
+			return result;
+		} else {
+			throw new NoSuchElementException();
+		}
+	}
+
+	/**
+	 * Fetches the next element if it hasn't been fetched yet and stores it in {@link #nextElement}.
+	 *
+	 * @return The next element, or null if there are no more results.
+	 */
+	private Statement lookAhead() {
+		if (nextElement == null) {
+			nextElement = getNextElement();
+
+			if (nextElement == null) {
+				close();
+			}
+		}
+		return nextElement;
+	}
+
+	/**
+	 * Throws an {@link UnsupportedOperationException}.
+	 */
+	@Override
+	public void remove() {
+		throw new UnsupportedOperationException();
+	}
 }