Skip to content

Commit 7806479

Browse files
hmottestadhmottestad
authored
GH-4950 LMDB: extensible ID scheme (#4956)
2 parents 6304f3a + 3ca8e03 commit 7806479

34 files changed

Lines changed: 2285 additions & 641 deletions

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,8 @@ abstract static class StatefulOperation implements Operation {
240240
/**
241241
* Creates a new {@link LmdbSailStore}.
242242
*/
243-
public LmdbSailStore(File dataDir, LmdbStoreConfig config) throws IOException, SailException {
243+
public LmdbSailStore(File dataDir, StoreProperties properties, LmdbStoreConfig config)
244+
throws IOException, SailException {
244245
this.setFactory = new PersistentSetFactory<>(dataDir);
245246
this.bulkOperationSize = config.getBulkOperationSize();
246247
Function<Long, byte[]> encode = element -> {
@@ -254,9 +255,9 @@ public LmdbSailStore(File dataDir, LmdbStoreConfig config) throws IOException, S
254255
boolean initialized = false;
255256
try {
256257
namespaceStore = new NamespaceStore(dataDir);
257-
var valueStore = new ValueStore(new File(dataDir, "values"), config);
258+
var valueStore = new ValueStore(new File(dataDir, "values"), properties, config);
258259
this.valueStore = valueStore;
259-
tripleStore = new TripleStore(new File(dataDir, "triples"), config, valueStore);
260+
tripleStore = new TripleStore(new File(dataDir, "triples"), properties, config, valueStore);
260261
mayHaveInferred = tripleStore.hasTriples(false);
261262
initialized = true;
262263
} finally {
@@ -977,7 +978,8 @@ private long removeStatements(long subj, long pred, long obj, boolean explicit,
977978
tripleStore.removeTriplesByContext(subj, pred, obj, contextId, explicit, quad -> {
978979
removeCount[0]++;
979980
for (long id : quad) {
980-
if (id != 0L) {
981+
if (id != 0L && !ValueIds.isInlined(id)) {
982+
// only add references, exclude inlined values
981983
unusedIds.add(id);
982984
}
983985
}

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbStore.java‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,29 +12,26 @@
1212

1313
import java.io.File;
1414
import java.io.IOException;
15-
import java.nio.charset.StandardCharsets;
1615
import java.nio.file.Files;
1716
import java.nio.file.Path;
1817
import java.util.Comparator;
1918
import java.util.concurrent.locks.ReentrantLock;
2019
import java.util.function.Supplier;
2120
import java.util.stream.Stream;
2221

23-
import org.apache.commons.io.FileUtils;
2422
import org.eclipse.rdf4j.collection.factory.api.CollectionFactory;
2523
import org.eclipse.rdf4j.collection.factory.mapdb.MapDb3CollectionFactory;
2624
import org.eclipse.rdf4j.common.annotation.Experimental;
2725
import org.eclipse.rdf4j.common.concurrent.locks.Lock;
2826
import org.eclipse.rdf4j.common.concurrent.locks.LockManager;
29-
import org.eclipse.rdf4j.common.io.MavenUtil;
3027
import org.eclipse.rdf4j.common.transaction.IsolationLevel;
3128
import org.eclipse.rdf4j.common.transaction.IsolationLevels;
3229
import org.eclipse.rdf4j.model.ValueFactory;
3330
import org.eclipse.rdf4j.query.algebra.evaluation.EvaluationStrategy;
3431
import org.eclipse.rdf4j.query.algebra.evaluation.EvaluationStrategyFactory;
3532
import org.eclipse.rdf4j.query.algebra.evaluation.federation.FederatedServiceResolver;
3633
import org.eclipse.rdf4j.query.algebra.evaluation.federation.FederatedServiceResolverClient;
37-
import org.eclipse.rdf4j.query.algebra.evaluation.impl.StrictEvaluationStrategyFactory;
34+
import org.eclipse.rdf4j.query.algebra.evaluation.impl.DefaultEvaluationStrategyFactory;
3835
import org.eclipse.rdf4j.repository.sparql.federation.SPARQLServiceResolver;
3936
import org.eclipse.rdf4j.sail.InterruptedSailException;
4037
import org.eclipse.rdf4j.sail.NotifyingSailConnection;
@@ -62,8 +59,10 @@ public class LmdbStore extends AbstractNotifyingSail implements FederatedService
6259
/*-----------*
6360
* Variables *
6461
*-----------*/
65-
66-
private static final String VERSION = MavenUtil.loadVersion("org.eclipse.rdf4j", "rdf4j-sail-lmdb", "devel");
62+
/**
63+
* The current version of the LMDB store.
64+
*/
65+
static final int VERSION = 2;
6766

6867
/**
6968
* Specifies which triple indexes this lmdb store must use.
@@ -169,7 +168,7 @@ public void setDataDir(File dataDir) {
169168
*/
170169
public synchronized EvaluationStrategyFactory getEvaluationStrategyFactory() {
171170
if (evalStratFactory == null) {
172-
evalStratFactory = new StrictEvaluationStrategyFactory(getFederatedServiceResolver());
171+
evalStratFactory = new DefaultEvaluationStrategyFactory(getFederatedServiceResolver());
173172
}
174173
evalStratFactory.setQuerySolutionCacheThreshold(getIterationCacheSyncThreshold());
175174
evalStratFactory.setTrackResultSize(isTrackResultSize());
@@ -252,18 +251,33 @@ protected void initializeInternal() throws SailException {
252251
logger.debug("Data dir is " + dataDir);
253252

254253
try {
255-
File versionFile = new File(dataDir, "lmdbrdf.ver");
256-
String version = versionFile.exists() ? FileUtils.readFileToString(versionFile, StandardCharsets.UTF_8)
257-
: null;
258-
if (!VERSION.equals(version) && upgradeStore(dataDir, version)) {
259-
FileUtils.writeStringToFile(versionFile, VERSION, StandardCharsets.UTF_8);
254+
StoreProperties properties = new StoreProperties(dataDir);
255+
// ensure that it is an error if an unsupported version of LmdbStore already exists
256+
if (new File(dataDir, "lmdbrdf.ver").exists()) {
257+
throw new SailException("Directory contains data from an older unsupported version of LmdbStore");
258+
}
259+
boolean updateVersion = false;
260+
if (properties.load()) {
261+
if (!String.valueOf(VERSION).equals(properties.getVersion())) {
262+
updateVersion = upgradeStore(dataDir, properties.getVersion());
263+
}
264+
} else {
265+
properties.setVersion(String.valueOf(VERSION));
260266
}
261-
backingStore = new LmdbSailStore(dataDir, config);
267+
268+
backingStore = new LmdbSailStore(dataDir, properties, config);
269+
270+
// update version afer loading and potential internal migration within value and triple store
271+
if (updateVersion) {
272+
properties.setVersion(String.valueOf(VERSION));
273+
}
274+
properties.save();
275+
262276
this.store = new SnapshotSailStore(backingStore, () -> new MemoryOverflowModel() {
263277
@Override
264278
protected LmdbSailStore createSailStore(File dataDir) throws IOException, SailException {
265279
// Model can't fit into memory, use another LmdbSailStore to store delta
266-
LmdbSailStore lmdbSailStore = new LmdbSailStore(dataDir, config);
280+
LmdbSailStore lmdbSailStore = new LmdbSailStore(dataDir, new StoreProperties(), config);
267281
lmdbSailStore.enableMultiThreading = false;
268282
return lmdbSailStore;
269283
}

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbUtil.java‎

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -21,23 +21,18 @@
2121
import static org.lwjgl.util.lmdb.LMDB.MDB_RDONLY;
2222
import static org.lwjgl.util.lmdb.LMDB.MDB_SUCCESS;
2323
import static org.lwjgl.util.lmdb.LMDB.mdb_dbi_open;
24-
import static org.lwjgl.util.lmdb.LMDB.mdb_set_compare;
2524
import static org.lwjgl.util.lmdb.LMDB.mdb_strerror;
2625
import static org.lwjgl.util.lmdb.LMDB.mdb_txn_abort;
2726
import static org.lwjgl.util.lmdb.LMDB.mdb_txn_begin;
2827
import static org.lwjgl.util.lmdb.LMDB.mdb_txn_commit;
2928

3029
import java.io.IOException;
31-
import java.nio.ByteBuffer;
3230
import java.nio.IntBuffer;
33-
import java.util.Comparator;
3431

3532
import org.lwjgl.PointerBuffer;
3633
import org.lwjgl.system.MemoryStack;
3734
import org.lwjgl.system.MemoryUtil;
3835
import org.lwjgl.system.Pointer;
39-
import org.lwjgl.util.lmdb.MDBCmpFuncI;
40-
import org.lwjgl.util.lmdb.MDBVal;
4136
import org.slf4j.Logger;
4237
import org.slf4j.LoggerFactory;
4338

@@ -121,22 +116,16 @@ static <T> T transaction(long env, Transaction<T> transaction) throws IOExceptio
121116
return ret;
122117
}
123118

124-
static int openDatabase(long env, String name, int flags, Comparator<ByteBuffer> comparator) throws IOException {
125-
return transaction(env, (stack, txn) -> {
126-
IntBuffer ip = stack.mallocInt(1);
119+
static int openDatabase(long env, String name, int flags) throws IOException {
120+
return transaction(env, (stack, txn) -> openDatabaseWithTxn(txn, name, flags));
121+
}
127122

123+
static int openDatabaseWithTxn(long txn, String name, int flags) throws IOException {
124+
try (MemoryStack stack = stackPush()) {
125+
IntBuffer ip = stack.mallocInt(1);
128126
E(mdb_dbi_open(txn, name, flags, ip));
129-
int dbi = ip.get(0);
130-
if (comparator != null) {
131-
MDBCmpFuncI cmp = (a, b) -> {
132-
MDBVal aVal = MDBVal.create(a);
133-
MDBVal bVal = MDBVal.create(b);
134-
return comparator.compare(aVal.mv_data(), bVal.mv_data());
135-
};
136-
mdb_set_compare(txn, dbi, cmp);
137-
}
138-
return dbi;
139-
});
127+
return ip.get(0);
128+
}
140129
}
141130

142131
/**
@@ -173,7 +162,6 @@ static boolean requiresResize(long mapSize, long pageSize, long txn, long requir
173162
if (percentageUsed > PERCENTAGE_FULL_TRIGGERS_RESIZE) {
174163
return true;
175164
}
176-
177165
return mapSize - nextPageNo * pageSize < Math.max(requiredSize, MIN_FREE_SPACE);
178166
}
179167

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/PersistentSetFactory.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ class PersistentSetFactory<T extends Serializable> {
7474

7575
dbDir = Files.createTempDirectory(cacheDir.toPath(), "set");
7676
E(mdb_env_open(env, dbDir.toAbsolutePath().toString(), flags, 0664));
77-
this.defaultDbi = openDatabase(env, null, MDB_CREATE, null);
77+
this.defaultDbi = openDatabase(env, null, MDB_CREATE);
7878

7979
MDBStat stat = MDBStat.malloc(stack);
8080
readTransaction(env, (stack2, txn) -> {
@@ -132,7 +132,7 @@ void ensureResize() throws IOException, InterruptedException {
132132

133133
PersistentSet<T> createSet(String name, Function<T, byte[]> writeFunc, Function<ByteBuffer, T> readFunc)
134134
throws IOException {
135-
int dbi = openDatabase(env, name, MDB_CREATE, null);
135+
int dbi = openDatabase(env, name, MDB_CREATE);
136136
return new PersistentSet<>(this, dbi) {
137137
@Override
138138
protected byte[] write(T element) throws IOException {

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/StoreProperties.java‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*******************************************************************************
2+
* Copyright (c) 2026 Eclipse RDF4J contributors.
3+
*
4+
* All rights reserved. This program and the accompanying materials
5+
* are made available under the terms of the Eclipse Distribution License v1.0
6+
* which accompanies this distribution, and is available at
7+
* http://www.eclipse.org/org/documents/edl-v10.php.
8+
*
9+
* SPDX-License-Identifier: BSD-3-Clause
10+
*******************************************************************************/
11+
package org.eclipse.rdf4j.sail.lmdb;
12+
13+
import java.io.File;
14+
import java.io.FileInputStream;
15+
import java.io.FileOutputStream;
16+
import java.io.IOException;
17+
import java.io.InputStream;
18+
import java.io.OutputStream;
19+
import java.util.Optional;
20+
import java.util.Properties;
21+
22+
class StoreProperties {
23+
/**
24+
* The file name for the properties file.
25+
*/
26+
static final String FILE_NAME = "store.properties";
27+
28+
/**
29+
* The key used to store the triple store version in the properties file.
30+
*/
31+
static final String VERSION_KEY = "version";
32+
/**
33+
* The key used to store the triple indexes specification that specifies which triple indexes exist.
34+
*/
35+
static final String INDEXES_KEY = "triple-indexes";
36+
37+
protected final File propertiesFile;
38+
39+
protected String version;
40+
41+
protected String tripleIndexes;
42+
43+
protected boolean loaded;
44+
45+
protected boolean dirty;
46+
47+
StoreProperties() {
48+
this.propertiesFile = null;
49+
}
50+
51+
StoreProperties(File dir) {
52+
this.propertiesFile = new File(dir, FILE_NAME);
53+
}
54+
55+
/**
56+
* Load from properties file.
57+
*
58+
* @return <code>true</code> if loaded from file, else <code>false</code>
59+
*/
60+
boolean load() {
61+
Optional.ofNullable(propertiesFile).filter(File::isFile).ifPresent(file -> {
62+
Properties properties = new Properties();
63+
try (InputStream in = new FileInputStream(file)) {
64+
properties.load(in);
65+
} catch (IOException e) {
66+
throw new IllegalStateException("Unable to load store properties from " + file, e);
67+
}
68+
version = properties.getProperty(VERSION_KEY);
69+
tripleIndexes = properties.getProperty(INDEXES_KEY);
70+
loaded = true;
71+
});
72+
return loaded;
73+
}
74+
75+
/**
76+
* Save to properties file.
77+
*/
78+
void save() {
79+
if (!dirty) {
80+
return;
81+
}
82+
Optional.ofNullable(propertiesFile).ifPresent(file -> {
83+
Properties properties = new Properties();
84+
if (version != null) {
85+
properties.setProperty(VERSION_KEY, version);
86+
}
87+
if (tripleIndexes != null) {
88+
properties.setProperty(INDEXES_KEY, tripleIndexes);
89+
}
90+
File parent = file.getParentFile();
91+
if (parent != null) {
92+
parent.mkdirs();
93+
}
94+
try (OutputStream out = new FileOutputStream(file)) {
95+
properties.store(out, "LmdbStore meta-data");
96+
dirty = false;
97+
} catch (IOException e) {
98+
throw new IllegalStateException("Unable to store properties to " + file, e);
99+
}
100+
});
101+
}
102+
103+
boolean isLoaded() {
104+
return loaded;
105+
}
106+
107+
String getVersion() {
108+
return version;
109+
}
110+
111+
StoreProperties setVersion(String version) {
112+
this.version = version;
113+
this.dirty = true;
114+
return this;
115+
}
116+
117+
String getTripleIndexes() {
118+
return tripleIndexes;
119+
}
120+
121+
StoreProperties setTripleIndexes(String tripleIndexes) {
122+
this.tripleIndexes = tripleIndexes;
123+
this.dirty = true;
124+
return this;
125+
}
126+
}

0 commit comments

Comments
 (0)