GH-4950 LMDB: extensible ID scheme

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java

@@ -240,7 +240,8 @@ abstract static class StatefulOperation implements Operation {
 	/**
 	 * Creates a new {@link LmdbSailStore}.
 	 */
-	public LmdbSailStore(File dataDir, LmdbStoreConfig config) throws IOException, SailException {
+	public LmdbSailStore(File dataDir, StoreProperties properties, LmdbStoreConfig config)
+			throws IOException, SailException {
 		this.setFactory = new PersistentSetFactory<>(dataDir);
 		this.bulkOperationSize = config.getBulkOperationSize();
 		Function<Long, byte[]> encode = element -> {
@@ -254,9 +255,9 @@ public LmdbSailStore(File dataDir, LmdbStoreConfig config) throws IOException, S
 		boolean initialized = false;
 		try {
 			namespaceStore = new NamespaceStore(dataDir);
-			var valueStore = new ValueStore(new File(dataDir, "values"), config);
+			var valueStore = new ValueStore(new File(dataDir, "values"), properties, config);
 			this.valueStore = valueStore;
-			tripleStore = new TripleStore(new File(dataDir, "triples"), config, valueStore);
+			tripleStore = new TripleStore(new File(dataDir, "triples"), properties, config, valueStore);
 			mayHaveInferred = tripleStore.hasTriples(false);
 			initialized = true;
 		} finally {
@@ -977,7 +978,8 @@ private long removeStatements(long subj, long pred, long obj, boolean explicit,
 				tripleStore.removeTriplesByContext(subj, pred, obj, contextId, explicit, quad -> {
 					removeCount[0]++;
 					for (long id : quad) {
-						if (id != 0L) {
+						if (id != 0L && !ValueIds.isInlined(id)) {
+							// only add references, exclude inlined values
 							unusedIds.add(id);
 						}
 					}

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbStore.java

@@ -12,29 +12,26 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Comparator;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
 
-import org.apache.commons.io.FileUtils;
 import org.eclipse.rdf4j.collection.factory.api.CollectionFactory;
 import org.eclipse.rdf4j.collection.factory.mapdb.MapDb3CollectionFactory;
 import org.eclipse.rdf4j.common.annotation.Experimental;
 import org.eclipse.rdf4j.common.concurrent.locks.Lock;
 import org.eclipse.rdf4j.common.concurrent.locks.LockManager;
-import org.eclipse.rdf4j.common.io.MavenUtil;
 import org.eclipse.rdf4j.common.transaction.IsolationLevel;
 import org.eclipse.rdf4j.common.transaction.IsolationLevels;
 import org.eclipse.rdf4j.model.ValueFactory;
 import org.eclipse.rdf4j.query.algebra.evaluation.EvaluationStrategy;
 import org.eclipse.rdf4j.query.algebra.evaluation.EvaluationStrategyFactory;
 import org.eclipse.rdf4j.query.algebra.evaluation.federation.FederatedServiceResolver;
 import org.eclipse.rdf4j.query.algebra.evaluation.federation.FederatedServiceResolverClient;
-import org.eclipse.rdf4j.query.algebra.evaluation.impl.StrictEvaluationStrategyFactory;
+import org.eclipse.rdf4j.query.algebra.evaluation.impl.DefaultEvaluationStrategyFactory;
 import org.eclipse.rdf4j.repository.sparql.federation.SPARQLServiceResolver;
 import org.eclipse.rdf4j.sail.InterruptedSailException;
 import org.eclipse.rdf4j.sail.NotifyingSailConnection;
@@ -62,8 +59,10 @@ public class LmdbStore extends AbstractNotifyingSail implements FederatedService
 	/*-----------*
 	 * Variables *
 	 *-----------*/
-
-	private static final String VERSION = MavenUtil.loadVersion("org.eclipse.rdf4j", "rdf4j-sail-lmdb", "devel");
+	/**
+	 * The current version of the LMDB store.
+	 */
+	static final int VERSION = 2;
 
 	/**
 	 * Specifies which triple indexes this lmdb store must use.
@@ -169,7 +168,7 @@ public void setDataDir(File dataDir) {
 	 */
 	public synchronized EvaluationStrategyFactory getEvaluationStrategyFactory() {
 		if (evalStratFactory == null) {
-			evalStratFactory = new StrictEvaluationStrategyFactory(getFederatedServiceResolver());
+			evalStratFactory = new DefaultEvaluationStrategyFactory(getFederatedServiceResolver());
 		}
 		evalStratFactory.setQuerySolutionCacheThreshold(getIterationCacheSyncThreshold());
 		evalStratFactory.setTrackResultSize(isTrackResultSize());
@@ -252,18 +251,33 @@ protected void initializeInternal() throws SailException {
 		logger.debug("Data dir is " + dataDir);
 
 		try {
-			File versionFile = new File(dataDir, "lmdbrdf.ver");
-			String version = versionFile.exists() ? FileUtils.readFileToString(versionFile, StandardCharsets.UTF_8)
-					: null;
-			if (!VERSION.equals(version) && upgradeStore(dataDir, version)) {
-				FileUtils.writeStringToFile(versionFile, VERSION, StandardCharsets.UTF_8);
+			StoreProperties properties = new StoreProperties(dataDir);
+			// ensure that it is an error if an unsupported version of LmdbStore already exists
+			if (new File(dataDir, "lmdbrdf.ver").exists()) {
+				throw new SailException("Directory contains data from an older unsupported version of LmdbStore");
+			}
+			boolean updateVersion = false;
+			if (properties.load()) {
+				if (!String.valueOf(VERSION).equals(properties.getVersion())) {
+					updateVersion = upgradeStore(dataDir, properties.getVersion());
+				}
+			} else {
+				properties.setVersion(String.valueOf(VERSION));
 			}
-			backingStore = new LmdbSailStore(dataDir, config);
+
+			backingStore = new LmdbSailStore(dataDir, properties, config);
+
+			// update version afer loading and potential internal migration within value and triple store
+			if (updateVersion) {
+				properties.setVersion(String.valueOf(VERSION));
+			}
+			properties.save();
+
 			this.store = new SnapshotSailStore(backingStore, () -> new MemoryOverflowModel() {
 				@Override
 				protected LmdbSailStore createSailStore(File dataDir) throws IOException, SailException {
 					// Model can't fit into memory, use another LmdbSailStore to store delta
-					LmdbSailStore lmdbSailStore = new LmdbSailStore(dataDir, config);
+					LmdbSailStore lmdbSailStore = new LmdbSailStore(dataDir, new StoreProperties(), config);
 					lmdbSailStore.enableMultiThreading = false;
 					return lmdbSailStore;
 				}

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbUtil.java

@@ -21,23 +21,18 @@
 import static org.lwjgl.util.lmdb.LMDB.MDB_RDONLY;
 import static org.lwjgl.util.lmdb.LMDB.MDB_SUCCESS;
 import static org.lwjgl.util.lmdb.LMDB.mdb_dbi_open;
-import static org.lwjgl.util.lmdb.LMDB.mdb_set_compare;
 import static org.lwjgl.util.lmdb.LMDB.mdb_strerror;
 import static org.lwjgl.util.lmdb.LMDB.mdb_txn_abort;
 import static org.lwjgl.util.lmdb.LMDB.mdb_txn_begin;
 import static org.lwjgl.util.lmdb.LMDB.mdb_txn_commit;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.nio.IntBuffer;
-import java.util.Comparator;
 
 import org.lwjgl.PointerBuffer;
 import org.lwjgl.system.MemoryStack;
 import org.lwjgl.system.MemoryUtil;
 import org.lwjgl.system.Pointer;
-import org.lwjgl.util.lmdb.MDBCmpFuncI;
-import org.lwjgl.util.lmdb.MDBVal;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -121,22 +116,16 @@ static <T> T transaction(long env, Transaction<T> transaction) throws IOExceptio
 		return ret;
 	}
 
-	static int openDatabase(long env, String name, int flags, Comparator<ByteBuffer> comparator) throws IOException {
-		return transaction(env, (stack, txn) -> {
-			IntBuffer ip = stack.mallocInt(1);
+	static int openDatabase(long env, String name, int flags) throws IOException {
+		return transaction(env, (stack, txn) -> openDatabaseWithTxn(txn, name, flags));
+	}
 
+	static int openDatabaseWithTxn(long txn, String name, int flags) throws IOException {
+		try (MemoryStack stack = stackPush()) {
+			IntBuffer ip = stack.mallocInt(1);
 			E(mdb_dbi_open(txn, name, flags, ip));
-			int dbi = ip.get(0);
-			if (comparator != null) {
-				MDBCmpFuncI cmp = (a, b) -> {
-					MDBVal aVal = MDBVal.create(a);
-					MDBVal bVal = MDBVal.create(b);
-					return comparator.compare(aVal.mv_data(), bVal.mv_data());
-				};
-				mdb_set_compare(txn, dbi, cmp);
-			}
-			return dbi;
-		});
+			return ip.get(0);
+		}
 	}
 
 	/**
@@ -173,7 +162,6 @@ static boolean requiresResize(long mapSize, long pageSize, long txn, long requir
 		if (percentageUsed > PERCENTAGE_FULL_TRIGGERS_RESIZE) {
 			return true;
 		}
-
 		return mapSize - nextPageNo * pageSize < Math.max(requiredSize, MIN_FREE_SPACE);
 	}
 

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/PersistentSetFactory.java

@@ -74,7 +74,7 @@ class PersistentSetFactory<T extends Serializable> {
 
 			dbDir = Files.createTempDirectory(cacheDir.toPath(), "set");
 			E(mdb_env_open(env, dbDir.toAbsolutePath().toString(), flags, 0664));
-			this.defaultDbi = openDatabase(env, null, MDB_CREATE, null);
+			this.defaultDbi = openDatabase(env, null, MDB_CREATE);
 
 			MDBStat stat = MDBStat.malloc(stack);
 			readTransaction(env, (stack2, txn) -> {
@@ -132,7 +132,7 @@ void ensureResize() throws IOException, InterruptedException {
 
 	PersistentSet<T> createSet(String name, Function<T, byte[]> writeFunc, Function<ByteBuffer, T> readFunc)
 			throws IOException {
-		int dbi = openDatabase(env, name, MDB_CREATE, null);
+		int dbi = openDatabase(env, name, MDB_CREATE);
 		return new PersistentSet<>(this, dbi) {
 			@Override
 			protected byte[] write(T element) throws IOException {

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/StoreProperties.java

@@ -0,0 +1,126 @@
+/*******************************************************************************
+ * Copyright (c) 2026 Eclipse RDF4J contributors.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Distribution License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *******************************************************************************/
+package org.eclipse.rdf4j.sail.lmdb;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Optional;
+import java.util.Properties;
+
+class StoreProperties {
+	/**
+	 * The file name for the properties file.
+	 */
+	static final String FILE_NAME = "store.properties";
+
+	/**
+	 * The key used to store the triple store version in the properties file.
+	 */
+	static final String VERSION_KEY = "version";
+	/**
+	 * The key used to store the triple indexes specification that specifies which triple indexes exist.
+	 */
+	static final String INDEXES_KEY = "triple-indexes";
+
+	protected final File propertiesFile;
+
+	protected String version;
+
+	protected String tripleIndexes;
+
+	protected boolean loaded;
+
+	protected boolean dirty;
+
+	StoreProperties() {
+		this.propertiesFile = null;
+	}
+
+	StoreProperties(File dir) {
+		this.propertiesFile = new File(dir, FILE_NAME);
+	}
+
+	/**
+	 * Load from properties file.
+	 *
+	 * @return <code>true</code> if loaded from file, else <code>false</code>
+	 */
+	boolean load() {
+		Optional.ofNullable(propertiesFile).filter(File::isFile).ifPresent(file -> {
+			Properties properties = new Properties();
+			try (InputStream in = new FileInputStream(file)) {
+				properties.load(in);
+			} catch (IOException e) {
+				throw new IllegalStateException("Unable to load store properties from " + file, e);
+			}
+			version = properties.getProperty(VERSION_KEY);
+			tripleIndexes = properties.getProperty(INDEXES_KEY);
+			loaded = true;
+		});
+		return loaded;
+	}
+
+	/**
+	 * Save to properties file.
+	 */
+	void save() {
+		if (!dirty) {
+			return;
+		}
+		Optional.ofNullable(propertiesFile).ifPresent(file -> {
+			Properties properties = new Properties();
+			if (version != null) {
+				properties.setProperty(VERSION_KEY, version);
+			}
+			if (tripleIndexes != null) {
+				properties.setProperty(INDEXES_KEY, tripleIndexes);
+			}
+			File parent = file.getParentFile();
+			if (parent != null) {
+				parent.mkdirs();
+			}
+			try (OutputStream out = new FileOutputStream(file)) {
+				properties.store(out, "LmdbStore meta-data");
+				dirty = false;
+			} catch (IOException e) {
+				throw new IllegalStateException("Unable to store properties to " + file, e);
+			}
+		});
+	}
+
+	boolean isLoaded() {
+		return loaded;
+	}
+
+	String getVersion() {
+		return version;
+	}
+
+	StoreProperties setVersion(String version) {
+		this.version = version;
+		this.dirty = true;
+		return this;
+	}
+
+	String getTripleIndexes() {
+		return tripleIndexes;
+	}
+
+	StoreProperties setTripleIndexes(String tripleIndexes) {
+		this.tripleIndexes = tripleIndexes;
+		this.dirty = true;
+		return this;
+	}
+}