Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,12 @@ public static final class Native {
*/
public final static IRI namespaceIDCacheSize = createIRI(NAMESPACE, "native.namespaceIDCacheSize");

/**
* <var>tag:rdf4j.org,2025:config/native.memoryMappedTxnStatusFile</var>
*/
public final static IRI memoryMappedTxnStatusFile = createIRI(NAMESPACE,
"native.memoryMappedTxnStatusFile");

// ValueStore WAL configuration properties
/** <var>tag:rdf4j.org,2023:config/native.walMaxSegmentBytes</var> */
public final static IRI walMaxSegmentBytes = createIRI(NAMESPACE, "native.walMaxSegmentBytes");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ class NativeSailStore implements SailStore {

private final ContextStore contextStore;
private final boolean walEnabled;
private final Boolean memoryMappedTxnStatusFileEnabled;

/**
* A lock to control concurrent access by {@link NativeSailSink} to the TripleStore, ValueStore, and NamespaceStore.
Expand All @@ -99,7 +100,7 @@ class NativeSailStore implements SailStore {
public NativeSailStore(File dataDir, String tripleIndexes) throws IOException, SailException {
this(dataDir, tripleIndexes, false, ValueStore.VALUE_CACHE_SIZE, ValueStore.VALUE_ID_CACHE_SIZE,
ValueStore.NAMESPACE_CACHE_SIZE, ValueStore.NAMESPACE_ID_CACHE_SIZE,
-1L, -1, -1, null, -1L, -1L, null, false, false, true);
-1L, -1, -1, null, -1L, -1L, null, false, false, true, null);
}

/**
Expand All @@ -114,17 +115,19 @@ public NativeSailStore(File dataDir, String tripleIndexes, boolean forceSync, in
throws IOException, SailException {
this(dataDir, tripleIndexes, forceSync, valueCacheSize, valueIDCacheSize, namespaceCacheSize,
namespaceIDCacheSize, walMaxSegmentBytes, walQueueCapacity, walBatchBufferBytes, walSyncPolicy,
walSyncIntervalMillis, walIdlePollIntervalMillis, walDirectoryName, false, false, true);
walSyncIntervalMillis, walIdlePollIntervalMillis, walDirectoryName, false, false, true, null);
}

public NativeSailStore(File dataDir, String tripleIndexes, boolean forceSync, int valueCacheSize,
int valueIDCacheSize, int namespaceCacheSize, int namespaceIDCacheSize, long walMaxSegmentBytes,
int walQueueCapacity, int walBatchBufferBytes,
ValueStoreWalConfig.SyncPolicy walSyncPolicy,
long walSyncIntervalMillis, long walIdlePollIntervalMillis, String walDirectoryName,
boolean walSyncBootstrapOnOpen, boolean walAutoRecoverOnOpen, boolean walEnabled)
boolean walSyncBootstrapOnOpen, boolean walAutoRecoverOnOpen, boolean walEnabled,
Boolean memoryMappedTxnStatusFileEnabled)
throws IOException, SailException {
this.walEnabled = walEnabled;
this.memoryMappedTxnStatusFileEnabled = memoryMappedTxnStatusFileEnabled;
NamespaceStore createdNamespaceStore = null;
ValueStoreWAL createdWal = null;
ValueStore createdValueStore = null;
Expand Down Expand Up @@ -171,7 +174,7 @@ public NativeSailStore(File dataDir, String tripleIndexes, boolean forceSync, in
}
createdValueStore = new ValueStore(dataDir, forceSync, valueCacheSize, valueIDCacheSize,
namespaceCacheSize, namespaceIDCacheSize, createdWal);
createdTripleStore = new TripleStore(dataDir, tripleIndexes, forceSync);
createdTripleStore = new TripleStore(dataDir, tripleIndexes, forceSync, memoryMappedTxnStatusFileEnabled);

// Assign fields required by ContextStore before constructing it
namespaceStore = createdNamespaceStore;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,8 @@ protected SailStore createSailStore(File dataDir) throws IOException, SailExcept

private volatile int namespaceIDCacheSize = ValueStore.NAMESPACE_ID_CACHE_SIZE;

private volatile Boolean memoryMappedTxnStatusFileEnabled;

private SailStore store;

// used to decide if store is writable, is true if the store was writable during initialization
Expand Down Expand Up @@ -276,6 +278,14 @@ public void setNamespaceIDCacheSize(int namespaceIDCacheSize) {
this.namespaceIDCacheSize = namespaceIDCacheSize;
}

public Boolean getMemoryMappedTxnStatusFileEnabled() {
return memoryMappedTxnStatusFileEnabled;
}

public void setMemoryMappedTxnStatusFileEnabled(Boolean memoryMappedTxnStatusFileEnabled) {
this.memoryMappedTxnStatusFileEnabled = memoryMappedTxnStatusFileEnabled;
}

@Experimental
public void setWalMaxSegmentBytes(long walMaxSegmentBytes) {
this.walMaxSegmentBytes = walMaxSegmentBytes;
Expand Down Expand Up @@ -492,7 +502,8 @@ protected void initializeInternal() throws SailException {
walDirectoryName,
walSyncBootstrapOnOpen,
walAutoRecoverOnOpen,
walEnabled);
walEnabled,
memoryMappedTxnStatusFileEnabled);
this.store = new SnapshotSailStore(mainStore, MemoryOverflowIntoNativeStore::new) {

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,6 @@ class TripleStore implements Closeable {
*/
private static final String INDEXES_KEY = "triple-indexes";

/**
* System property that enables the experimental {@link MemoryMappedTxnStatusFile} implementation instead of the
* default {@link TxnStatusFile}.
*/
private static final String MEMORY_MAPPED_TXN_STATUS_FILE_ENABLED_PROP = "org.eclipse.rdf4j.sail.nativerdf.MemoryMappedTxnStatusFile.enabled";

/**
* The version number for the current triple store.
* <ul>
Expand Down Expand Up @@ -168,13 +162,18 @@ class TripleStore implements Closeable {
*--------------*/

public TripleStore(File dir, String indexSpecStr) throws IOException, SailException {
this(dir, indexSpecStr, false);
this(dir, indexSpecStr, false, null);
}

public TripleStore(File dir, String indexSpecStr, boolean forceSync) throws IOException, SailException {
this(dir, indexSpecStr, forceSync, null);
}

public TripleStore(File dir, String indexSpecStr, boolean forceSync, Boolean memoryMappedTxnStatusFileEnabled)
throws IOException, SailException {
this.dir = dir;
this.forceSync = forceSync;
this.txnStatusFile = createTxnStatusFile(dir);
this.txnStatusFile = createTxnStatusFile(dir, memoryMappedTxnStatusFileEnabled);

File propFile = new File(dir, PROPERTIES_FILE);

Expand Down Expand Up @@ -229,8 +228,10 @@ public TripleStore(File dir, String indexSpecStr, boolean forceSync) throws IOEx
}
}

private static TxnStatusFile createTxnStatusFile(File dir) throws IOException {
if (Boolean.getBoolean(MEMORY_MAPPED_TXN_STATUS_FILE_ENABLED_PROP)) {
private static TxnStatusFile createTxnStatusFile(File dir, Boolean memoryMappedTxnStatusFileEnabled)
throws IOException {
boolean enabled = Boolean.TRUE.equals(memoryMappedTxnStatusFileEnabled);
if (enabled) {
return new MemoryMappedTxnStatusFile(dir);
}
return new TxnStatusFile(dir);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public class NativeStoreConfig extends BaseSailConfig {
private int valueIDCacheSize = -1;
private int namespaceCacheSize = -1;
private int namespaceIDCacheSize = -1;
private Boolean memoryMappedTxnStatusFileEnabled;

// WAL: expose max segment bytes via config (optional)
private long walMaxSegmentBytes = -1L;
Expand Down Expand Up @@ -124,6 +125,14 @@ public void setNamespaceIDCacheSize(int namespaceIDCacheSize) {
this.namespaceIDCacheSize = namespaceIDCacheSize;
}

public Boolean getMemoryMappedTxnStatusFileEnabled() {
return memoryMappedTxnStatusFileEnabled;
}

public void setMemoryMappedTxnStatusFileEnabled(Boolean memoryMappedTxnStatusFileEnabled) {
this.memoryMappedTxnStatusFileEnabled = memoryMappedTxnStatusFileEnabled;
}

public long getWalMaxSegmentBytes() {
return walMaxSegmentBytes;
}
Expand Down Expand Up @@ -231,6 +240,10 @@ public Resource export(Model m) {
if (namespaceIDCacheSize >= 0) {
m.add(implNode, CONFIG.Native.namespaceIDCacheSize, literal(namespaceIDCacheSize));
}
if (memoryMappedTxnStatusFileEnabled != null) {
m.add(implNode, CONFIG.Native.memoryMappedTxnStatusFile,
literal(memoryMappedTxnStatusFileEnabled));
}
// WAL configuration properties
if (walMaxSegmentBytes >= 0) {
m.add(implNode, CONFIG.Native.walMaxSegmentBytes, literal(walMaxSegmentBytes));
Expand Down Expand Up @@ -347,8 +360,8 @@ public void parse(Model m, Resource implNode) throws SailConfigException {
}
});

Configurations.getLiteralValue(m, implNode, CONFIG.Native.namespaceIDCacheSize, NAMESPACE_ID_CACHE_SIZE)
.ifPresent(lit -> {
Configurations.getLiteralValue(m, implNode, CONFIG.Native.namespaceIDCacheSize,
NAMESPACE_ID_CACHE_SIZE).ifPresent(lit -> {
try {
setNamespaceIDCacheSize(lit.intValue());
} catch (NumberFormatException e) {
Expand All @@ -358,6 +371,16 @@ public void parse(Model m, Resource implNode) throws SailConfigException {
}
});

Configurations.getLiteralValue(m, implNode, CONFIG.Native.memoryMappedTxnStatusFile)
.ifPresent(lit -> {
try {
setMemoryMappedTxnStatusFileEnabled(lit.booleanValue());
} catch (IllegalArgumentException e) {
throw new SailConfigException("Boolean value required for "
+ CONFIG.Native.memoryMappedTxnStatusFile + " property, found " + lit);
}
});

// WAL configuration properties
Configurations.getLiteralValue(m, implNode, CONFIG.Native.walMaxSegmentBytes)
.ifPresent(lit -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ public Sail getSail(SailImplConfig config) throws SailConfigException {
if (nativeConfig.getNamespaceIDCacheSize() >= 0) {
nativeStore.setNamespaceIDCacheSize(nativeConfig.getNamespaceIDCacheSize());
}
if (nativeConfig.getMemoryMappedTxnStatusFileEnabled() != null) {
nativeStore.setMemoryMappedTxnStatusFileEnabled(nativeConfig.getMemoryMappedTxnStatusFileEnabled());
}
if (nativeConfig.getIterationCacheSyncThreshold() > 0) {
nativeStore.setIterationCacheSyncThreshold(nativeConfig.getIterationCacheSyncThreshold());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
import org.junit.jupiter.api.io.TempDir;

/**
* Verifies that the implementation used for the transaction status file can be controlled via a system property.
* Verifies that the implementation used for the transaction status file is controlled through configuration rather than
* a JVM system property.
*/
public class MemoryMappedTxnStatusFileConfigTest {

Expand Down Expand Up @@ -52,7 +53,7 @@ public void defaultUsesNioTxnStatusFile() throws Exception {
}

@Test
public void memoryMappedEnabledUsesFixedSizeFile() throws Exception {
public void systemPropertyIsIgnored() throws Exception {
System.setProperty(MEMORY_MAPPED_ENABLED_PROP, "true");

TripleStore tripleStore = new TripleStore(dataDir, "spoc");
Expand All @@ -66,7 +67,7 @@ public void memoryMappedEnabledUsesFixedSizeFile() throws Exception {

File txnStatusFile = new File(dataDir, TxnStatusFile.FILE_NAME);
assertTrue(txnStatusFile.exists(), "Transaction status file should exist");
assertEquals(1L, txnStatusFile.length(),
"Memory-mapped TxnStatusFile keeps a single status byte on disk for NONE status");
assertEquals(0L, txnStatusFile.length(),
"System property does not switch to memory-mapped TxnStatusFile");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/*******************************************************************************
* Copyright (c) 2025 Eclipse RDF4J contributors.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Distribution License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/org/documents/edl-v10.php.
*
* SPDX-License-Identifier: BSD-3-Clause
*******************************************************************************/
package org.eclipse.rdf4j.sail.nativerdf;

import static org.assertj.core.api.Assertions.assertThat;

import java.io.File;
import java.lang.reflect.Field;

import org.eclipse.rdf4j.model.IRI;
import org.eclipse.rdf4j.model.ValueFactory;
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
import org.eclipse.rdf4j.repository.Repository;
import org.eclipse.rdf4j.repository.RepositoryConnection;
import org.eclipse.rdf4j.repository.sail.SailRepository;
import org.eclipse.rdf4j.sail.nativerdf.config.NativeStoreConfig;
import org.eclipse.rdf4j.sail.nativerdf.config.NativeStoreFactory;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

class NativeStoreTxnStatusConfigTest {

@TempDir
File dataDir;

@Test
void configEnablesMemoryMappedTxnStatusFile() throws Exception {
NativeStoreConfig cfg = new NativeStoreConfig("spoc");
cfg.setMemoryMappedTxnStatusFileEnabled(true);

NativeStoreFactory factory = new NativeStoreFactory();
NativeStore sail = (NativeStore) factory.getSail(cfg);
sail.setDataDir(dataDir);

Repository repo = new SailRepository(sail);
repo.init();
assertThat(extractTxnStatusFile(sail)).isInstanceOf(MemoryMappedTxnStatusFile.class);
try (RepositoryConnection conn = repo.getConnection()) {
ValueFactory vf = SimpleValueFactory.getInstance();
IRI p = vf.createIRI("http://example.com/p");
conn.add(vf.createIRI("http://example.com/s"), p, vf.createLiteral("o"));
}
repo.shutDown();

File txnStatusFile = new File(dataDir, TxnStatusFile.FILE_NAME);
assertThat(txnStatusFile).exists();
assertThat(txnStatusFile.length()).isEqualTo(1L);
}

private TxnStatusFile extractTxnStatusFile(NativeStore sail) throws Exception {
NativeSailStore store = (NativeSailStore) sail.getSailStore();

Field tripleStoreField = NativeSailStore.class.getDeclaredField("tripleStore");
tripleStoreField.setAccessible(true);
TripleStore tripleStore = (TripleStore) tripleStoreField.get(store);

Field txnStatusFileField = TripleStore.class.getDeclaredField("txnStatusFile");
txnStatusFileField.setAccessible(true);
return (TxnStatusFile) txnStatusFileField.get(tripleStore);
}
}
2 changes: 2 additions & 0 deletions site/content/documentation/reference/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,8 @@ Creating more indexes potentially speeds up querying (a lot), but also adds over

The native store automatically creates/drops indexes upon (re)initialization, so the parameter can be adjusted and upon the first refresh of the configuration the native store will change its indexing strategy, without loss of data.

Set `config:native.memoryMappedTxnStatusFile` to `true` to enable the experimental memory-mapped transaction status file. When unset, the store falls back to the legacy file implementation.

##### Example configuration

```turtle
Expand Down
Loading