Skip to content

Commit 8206d76

Browse files
hmottestadhmottestad
committed
GH-0000 Add config toggle for memory-mapped txn status file
1 parent 37c4f7a commit 8206d76

8 files changed

Lines changed: 129 additions & 11 deletions

File tree

‎core/model-vocabulary/src/main/java/org/eclipse/rdf4j/model/vocabulary/CONFIG.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,12 @@ public static final class Native {
277277
*/
278278
public final static IRI namespaceIDCacheSize = createIRI(NAMESPACE, "native.namespaceIDCacheSize");
279279

280+
/**
281+
* <var>tag:rdf4j.org,2025:config/native.memoryMappedTxnStatusFile</var>
282+
*/
283+
public final static IRI memoryMappedTxnStatusFile = createIRI(NAMESPACE,
284+
"native.memoryMappedTxnStatusFile");
285+
280286
// ValueStore WAL configuration properties
281287
/** <var>tag:rdf4j.org,2023:config/native.walMaxSegmentBytes</var> */
282288
public final static IRI walMaxSegmentBytes = createIRI(NAMESPACE, "native.walMaxSegmentBytes");

‎core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/NativeSailStore.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ class NativeSailStore implements SailStore {
8080

8181
private final ContextStore contextStore;
8282
private final boolean walEnabled;
83+
private final Boolean memoryMappedTxnStatusFileEnabled;
8384

8485
/**
8586
* A lock to control concurrent access by {@link NativeSailSink} to the TripleStore, ValueStore, and NamespaceStore.
@@ -99,7 +100,7 @@ class NativeSailStore implements SailStore {
99100
public NativeSailStore(File dataDir, String tripleIndexes) throws IOException, SailException {
100101
this(dataDir, tripleIndexes, false, ValueStore.VALUE_CACHE_SIZE, ValueStore.VALUE_ID_CACHE_SIZE,
101102
ValueStore.NAMESPACE_CACHE_SIZE, ValueStore.NAMESPACE_ID_CACHE_SIZE,
102-
-1L, -1, -1, null, -1L, -1L, null, false, false, true);
103+
-1L, -1, -1, null, -1L, -1L, null, false, false, true, null);
103104
}
104105

105106
/**
@@ -114,17 +115,19 @@ public NativeSailStore(File dataDir, String tripleIndexes, boolean forceSync, in
114115
throws IOException, SailException {
115116
this(dataDir, tripleIndexes, forceSync, valueCacheSize, valueIDCacheSize, namespaceCacheSize,
116117
namespaceIDCacheSize, walMaxSegmentBytes, walQueueCapacity, walBatchBufferBytes, walSyncPolicy,
117-
walSyncIntervalMillis, walIdlePollIntervalMillis, walDirectoryName, false, false, true);
118+
walSyncIntervalMillis, walIdlePollIntervalMillis, walDirectoryName, false, false, true, null);
118119
}
119120

120121
public NativeSailStore(File dataDir, String tripleIndexes, boolean forceSync, int valueCacheSize,
121122
int valueIDCacheSize, int namespaceCacheSize, int namespaceIDCacheSize, long walMaxSegmentBytes,
122123
int walQueueCapacity, int walBatchBufferBytes,
123124
ValueStoreWalConfig.SyncPolicy walSyncPolicy,
124125
long walSyncIntervalMillis, long walIdlePollIntervalMillis, String walDirectoryName,
125-
boolean walSyncBootstrapOnOpen, boolean walAutoRecoverOnOpen, boolean walEnabled)
126+
boolean walSyncBootstrapOnOpen, boolean walAutoRecoverOnOpen, boolean walEnabled,
127+
Boolean memoryMappedTxnStatusFileEnabled)
126128
throws IOException, SailException {
127129
this.walEnabled = walEnabled;
130+
this.memoryMappedTxnStatusFileEnabled = memoryMappedTxnStatusFileEnabled;
128131
NamespaceStore createdNamespaceStore = null;
129132
ValueStoreWAL createdWal = null;
130133
ValueStore createdValueStore = null;
@@ -171,7 +174,7 @@ public NativeSailStore(File dataDir, String tripleIndexes, boolean forceSync, in
171174
}
172175
createdValueStore = new ValueStore(dataDir, forceSync, valueCacheSize, valueIDCacheSize,
173176
namespaceCacheSize, namespaceIDCacheSize, createdWal);
174-
createdTripleStore = new TripleStore(dataDir, tripleIndexes, forceSync);
177+
createdTripleStore = new TripleStore(dataDir, tripleIndexes, forceSync, memoryMappedTxnStatusFileEnabled);
175178

176179
// Assign fields required by ContextStore before constructing it
177180
namespaceStore = createdNamespaceStore;

‎core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/NativeStore.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,8 @@ protected SailStore createSailStore(File dataDir) throws IOException, SailExcept
144144

145145
private volatile int namespaceIDCacheSize = ValueStore.NAMESPACE_ID_CACHE_SIZE;
146146

147+
private volatile Boolean memoryMappedTxnStatusFileEnabled;
148+
147149
private SailStore store;
148150

149151
// used to decide if store is writable, is true if the store was writable during initialization
@@ -276,6 +278,14 @@ public void setNamespaceIDCacheSize(int namespaceIDCacheSize) {
276278
this.namespaceIDCacheSize = namespaceIDCacheSize;
277279
}
278280

281+
public Boolean getMemoryMappedTxnStatusFileEnabled() {
282+
return memoryMappedTxnStatusFileEnabled;
283+
}
284+
285+
public void setMemoryMappedTxnStatusFileEnabled(Boolean memoryMappedTxnStatusFileEnabled) {
286+
this.memoryMappedTxnStatusFileEnabled = memoryMappedTxnStatusFileEnabled;
287+
}
288+
279289
@Experimental
280290
public void setWalMaxSegmentBytes(long walMaxSegmentBytes) {
281291
this.walMaxSegmentBytes = walMaxSegmentBytes;
@@ -492,7 +502,8 @@ protected void initializeInternal() throws SailException {
492502
walDirectoryName,
493503
walSyncBootstrapOnOpen,
494504
walAutoRecoverOnOpen,
495-
walEnabled);
505+
walEnabled,
506+
memoryMappedTxnStatusFileEnabled);
496507
this.store = new SnapshotSailStore(mainStore, MemoryOverflowIntoNativeStore::new) {
497508

498509
@Override

‎core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/TripleStore.java‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -168,13 +168,18 @@ class TripleStore implements Closeable {
168168
*--------------*/
169169

170170
public TripleStore(File dir, String indexSpecStr) throws IOException, SailException {
171-
this(dir, indexSpecStr, false);
171+
this(dir, indexSpecStr, false, null);
172172
}
173173

174174
public TripleStore(File dir, String indexSpecStr, boolean forceSync) throws IOException, SailException {
175+
this(dir, indexSpecStr, forceSync, null);
176+
}
177+
178+
public TripleStore(File dir, String indexSpecStr, boolean forceSync, Boolean memoryMappedTxnStatusFileEnabled)
179+
throws IOException, SailException {
175180
this.dir = dir;
176181
this.forceSync = forceSync;
177-
this.txnStatusFile = createTxnStatusFile(dir);
182+
this.txnStatusFile = createTxnStatusFile(dir, memoryMappedTxnStatusFileEnabled);
178183

179184
File propFile = new File(dir, PROPERTIES_FILE);
180185

@@ -229,8 +234,12 @@ public TripleStore(File dir, String indexSpecStr, boolean forceSync) throws IOEx
229234
}
230235
}
231236

232-
private static TxnStatusFile createTxnStatusFile(File dir) throws IOException {
233-
if (Boolean.getBoolean(MEMORY_MAPPED_TXN_STATUS_FILE_ENABLED_PROP)) {
237+
private static TxnStatusFile createTxnStatusFile(File dir, Boolean memoryMappedTxnStatusFileEnabled)
238+
throws IOException {
239+
boolean enabled = memoryMappedTxnStatusFileEnabled != null
240+
? memoryMappedTxnStatusFileEnabled
241+
: Boolean.getBoolean(MEMORY_MAPPED_TXN_STATUS_FILE_ENABLED_PROP);
242+
if (enabled) {
234243
return new MemoryMappedTxnStatusFile(dir);
235244
}
236245
return new TxnStatusFile(dir);

‎core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/config/NativeStoreConfig.java‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ public class NativeStoreConfig extends BaseSailConfig {
3737
private int valueIDCacheSize = -1;
3838
private int namespaceCacheSize = -1;
3939
private int namespaceIDCacheSize = -1;
40+
private Boolean memoryMappedTxnStatusFileEnabled;
4041

4142
// WAL: expose max segment bytes via config (optional)
4243
private long walMaxSegmentBytes = -1L;
@@ -124,6 +125,14 @@ public void setNamespaceIDCacheSize(int namespaceIDCacheSize) {
124125
this.namespaceIDCacheSize = namespaceIDCacheSize;
125126
}
126127

128+
public Boolean getMemoryMappedTxnStatusFileEnabled() {
129+
return memoryMappedTxnStatusFileEnabled;
130+
}
131+
132+
public void setMemoryMappedTxnStatusFileEnabled(Boolean memoryMappedTxnStatusFileEnabled) {
133+
this.memoryMappedTxnStatusFileEnabled = memoryMappedTxnStatusFileEnabled;
134+
}
135+
127136
public long getWalMaxSegmentBytes() {
128137
return walMaxSegmentBytes;
129138
}
@@ -231,6 +240,10 @@ public Resource export(Model m) {
231240
if (namespaceIDCacheSize >= 0) {
232241
m.add(implNode, CONFIG.Native.namespaceIDCacheSize, literal(namespaceIDCacheSize));
233242
}
243+
if (memoryMappedTxnStatusFileEnabled != null) {
244+
m.add(implNode, CONFIG.Native.memoryMappedTxnStatusFile,
245+
literal(memoryMappedTxnStatusFileEnabled));
246+
}
234247
// WAL configuration properties
235248
if (walMaxSegmentBytes >= 0) {
236249
m.add(implNode, CONFIG.Native.walMaxSegmentBytes, literal(walMaxSegmentBytes));
@@ -347,8 +360,8 @@ public void parse(Model m, Resource implNode) throws SailConfigException {
347360
}
348361
});
349362

350-
Configurations.getLiteralValue(m, implNode, CONFIG.Native.namespaceIDCacheSize, NAMESPACE_ID_CACHE_SIZE)
351-
.ifPresent(lit -> {
363+
Configurations.getLiteralValue(m, implNode, CONFIG.Native.namespaceIDCacheSize,
364+
NAMESPACE_ID_CACHE_SIZE).ifPresent(lit -> {
352365
try {
353366
setNamespaceIDCacheSize(lit.intValue());
354367
} catch (NumberFormatException e) {
@@ -358,6 +371,16 @@ public void parse(Model m, Resource implNode) throws SailConfigException {
358371
}
359372
});
360373

374+
Configurations.getLiteralValue(m, implNode, CONFIG.Native.memoryMappedTxnStatusFile)
375+
.ifPresent(lit -> {
376+
try {
377+
setMemoryMappedTxnStatusFileEnabled(lit.booleanValue());
378+
} catch (IllegalArgumentException e) {
379+
throw new SailConfigException("Boolean value required for "
380+
+ CONFIG.Native.memoryMappedTxnStatusFile + " property, found " + lit);
381+
}
382+
});
383+
361384
// WAL configuration properties
362385
Configurations.getLiteralValue(m, implNode, CONFIG.Native.walMaxSegmentBytes)
363386
.ifPresent(lit -> {

‎core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/config/NativeStoreFactory.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@ public Sail getSail(SailImplConfig config) throws SailConfigException {
7272
if (nativeConfig.getNamespaceIDCacheSize() >= 0) {
7373
nativeStore.setNamespaceIDCacheSize(nativeConfig.getNamespaceIDCacheSize());
7474
}
75+
if (nativeConfig.getMemoryMappedTxnStatusFileEnabled() != null) {
76+
nativeStore.setMemoryMappedTxnStatusFileEnabled(nativeConfig.getMemoryMappedTxnStatusFileEnabled());
77+
}
7578
if (nativeConfig.getIterationCacheSyncThreshold() > 0) {
7679
nativeStore.setIterationCacheSyncThreshold(nativeConfig.getIterationCacheSyncThreshold());
7780
}

‎core/sail/nativerdf/src/test/java/org/eclipse/rdf4j/sail/nativerdf/NativeStoreTxnStatusConfigTest.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*******************************************************************************
2+
* Copyright (c) 2025 Eclipse RDF4J contributors.
3+
*
4+
* All rights reserved. This program and the accompanying materials
5+
* are made available under the terms of the Eclipse Distribution License v1.0
6+
* which accompanies this distribution, and is available at
7+
* http://www.eclipse.org/org/documents/edl-v10.php.
8+
*
9+
* SPDX-License-Identifier: BSD-3-Clause
10+
*******************************************************************************/
11+
package org.eclipse.rdf4j.sail.nativerdf;
12+
13+
import static org.assertj.core.api.Assertions.assertThat;
14+
15+
import java.io.File;
16+
17+
import org.eclipse.rdf4j.model.IRI;
18+
import org.eclipse.rdf4j.model.ValueFactory;
19+
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
20+
import org.eclipse.rdf4j.repository.Repository;
21+
import org.eclipse.rdf4j.repository.RepositoryConnection;
22+
import org.eclipse.rdf4j.repository.sail.SailRepository;
23+
import org.eclipse.rdf4j.sail.nativerdf.config.NativeStoreConfig;
24+
import org.eclipse.rdf4j.sail.nativerdf.config.NativeStoreFactory;
25+
import org.junit.jupiter.api.AfterEach;
26+
import org.junit.jupiter.api.Test;
27+
import org.junit.jupiter.api.io.TempDir;
28+
29+
class NativeStoreTxnStatusConfigTest {
30+
31+
@TempDir
32+
File dataDir;
33+
34+
@AfterEach
35+
void clearSystemProperty() {
36+
System.clearProperty("org.eclipse.rdf4j.sail.nativerdf.MemoryMappedTxnStatusFile.enabled");
37+
}
38+
39+
@Test
40+
void configEnablesMemoryMappedTxnStatusFile() throws Exception {
41+
NativeStoreConfig cfg = new NativeStoreConfig("spoc");
42+
cfg.setMemoryMappedTxnStatusFileEnabled(true);
43+
44+
NativeStoreFactory factory = new NativeStoreFactory();
45+
NativeStore sail = (NativeStore) factory.getSail(cfg);
46+
sail.setDataDir(dataDir);
47+
48+
Repository repo = new SailRepository(sail);
49+
repo.init();
50+
try (RepositoryConnection conn = repo.getConnection()) {
51+
ValueFactory vf = SimpleValueFactory.getInstance();
52+
IRI p = vf.createIRI("http://example.com/p");
53+
conn.add(vf.createIRI("http://example.com/s"), p, vf.createLiteral("o"));
54+
}
55+
repo.shutDown();
56+
57+
File txnStatusFile = new File(dataDir, TxnStatusFile.FILE_NAME);
58+
assertThat(txnStatusFile).exists();
59+
assertThat(txnStatusFile.length()).isEqualTo(1L);
60+
}
61+
}

‎site/content/documentation/reference/configuration.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,8 @@ Creating more indexes potentially speeds up querying (a lot), but also adds over
251251

252252
The native store automatically creates/drops indexes upon (re)initialization, so the parameter can be adjusted and upon the first refresh of the configuration the native store will change its indexing strategy, without loss of data.
253253

254+
Set `config:native.memoryMappedTxnStatusFile` to `true` to enable the experimental memory-mapped transaction status file. When unset, the store falls back to the legacy file implementation and respects the JVM system property `-Dorg.eclipse.rdf4j.sail.nativerdf.MemoryMappedTxnStatusFile.enabled`.
255+
254256
##### Example configuration
255257

256258
```turtle

0 commit comments

Comments
 (0)