diff --git a/core/repository/api/src/main/resources/org/eclipse/rdf4j/repository/config/s3.ttl b/core/repository/api/src/main/resources/org/eclipse/rdf4j/repository/config/s3.ttl new file mode 100644 index 00000000000..78da5e4c7fc --- /dev/null +++ b/core/repository/api/src/main/resources/org/eclipse/rdf4j/repository/config/s3.ttl @@ -0,0 +1,26 @@ +# +# Configuration template for an S3Store +# +# S3 connection settings (bucket, endpoint, region, credentials) are configured +# at the RDF4J instance level via environment variables or system properties: +# +# RDF4J_S3_BUCKET, RDF4J_S3_ENDPOINT, RDF4J_S3_REGION, +# RDF4J_S3_ACCESS_KEY, RDF4J_S3_SECRET_KEY, RDF4J_S3_FORCE_PATH_STYLE +# +# Each repository uses s3Prefix to partition its data within the shared bucket. +# +@prefix rdfs: . +@prefix config: . +@prefix s3: . + +[] a config:Repository ; + config:rep.id "{%Repository ID|s3%}" ; + rdfs:label "{%Repository title|S3 Store%}" ; + config:rep.impl [ + config:rep.type "openrdf:SailRepository" ; + config:sail.impl [ + config:sail.type "rdf4j:S3Store" ; + s3:s3Prefix "{%S3 Prefix|%}" ; + s3:dataDir "{%Data Directory|%}" + ] + ]. diff --git a/core/sail/pom.xml b/core/sail/pom.xml index c31538770d8..9386f6bd5ac 100644 --- a/core/sail/pom.xml +++ b/core/sail/pom.xml @@ -19,6 +19,7 @@ model shacl lmdb + s3 lucene-api lucene elasticsearch diff --git a/core/sail/s3/docker-compose.yml b/core/sail/s3/docker-compose.yml new file mode 100644 index 00000000000..096bbf7fcb2 --- /dev/null +++ b/core/sail/s3/docker-compose.yml @@ -0,0 +1,32 @@ +services: + minio: + image: minio/minio:latest + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + command: server /data --console-address ":9001" + volumes: + - minio-data:/data + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + + createbucket: + image: minio/mc:latest + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + mc alias set local http://minio:9000 minioadmin minioadmin; + mc mb --ignore-existing local/rdf4j-data; + echo 'Bucket ready'; + " + +volumes: + minio-data: diff --git a/core/sail/s3/pom.xml b/core/sail/s3/pom.xml new file mode 100644 index 00000000000..a7d25931146 --- /dev/null +++ b/core/sail/s3/pom.xml @@ -0,0 +1,111 @@ + + + 4.0.0 + + org.eclipse.rdf4j + rdf4j-sail + 6.0.0-SNAPSHOT + + rdf4j-sail-s3 + RDF4J: S3Store + Sail implementation that stores data on S3-compatible object storage using an LSM-tree. + + + ${project.groupId} + rdf4j-sail-base + ${project.version} + + + ${project.groupId} + rdf4j-queryalgebra-evaluation + ${project.version} + + + ${project.groupId} + rdf4j-queryalgebra-model + ${project.version} + + + ${project.groupId} + rdf4j-query + ${project.version} + + + ${project.groupId} + rdf4j-model + ${project.version} + + + io.minio + minio + 8.5.7 + + + org.slf4j + slf4j-api + + + com.google.guava + guava + + + com.fasterxml.jackson.core + jackson-databind + + + org.apache.parquet + parquet-hadoop + 1.15.2 + + + javax.annotation + javax.annotation-api + + + + + + com.github.ben-manes.caffeine + caffeine + 3.1.8 + + + ${project.groupId} + rdf4j-sail-testsuite + ${project.version} + test + + + ${project.groupId} + rdf4j-repository-testsuite + ${project.version} + test + + + ${project.groupId} + rdf4j-repository-sail + ${project.version} + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + + + org.testcontainers + testcontainers-junit-jupiter + ${testcontainers.version} + test + + + diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/conf/Configuration.java b/core/sail/s3/src/main/java/org/apache/hadoop/conf/Configuration.java new file mode 100644 index 00000000000..7a46f59182e --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -0,0 +1,22 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub for org.apache.hadoop.conf.Configuration. +// Parquet-hadoop references this class in abstract method signatures +// (WriteSupport.init, ParquetWriter.Builder.getWriteSupport). Our code +// overrides the ParquetConfiguration variants instead, so this class is +// never instantiated or used at runtime. It exists only to satisfy the +// JVM class loader. +package org.apache.hadoop.conf; + +public class Configuration { + public Configuration() { + } +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/fs/FileStatus.java b/core/sail/s3/src/main/java/org/apache/hadoop/fs/FileStatus.java new file mode 100644 index 00000000000..5898ef94c1b --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/fs/FileStatus.java @@ -0,0 +1,16 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// Never instantiated at runtime. +package org.apache.hadoop.fs; + +public class FileStatus { +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/fs/FileSystem.java b/core/sail/s3/src/main/java/org/apache/hadoop/fs/FileSystem.java new file mode 100644 index 00000000000..518d1e0fbc4 --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -0,0 +1,16 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// Never instantiated at runtime. +package org.apache.hadoop.fs; + +public abstract class FileSystem { +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/fs/Path.java b/core/sail/s3/src/main/java/org/apache/hadoop/fs/Path.java new file mode 100644 index 00000000000..fe178aab10e --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/fs/Path.java @@ -0,0 +1,16 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// Never instantiated at runtime. +package org.apache.hadoop.fs; + +public class Path { +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/fs/PathFilter.java b/core/sail/s3/src/main/java/org/apache/hadoop/fs/PathFilter.java new file mode 100644 index 00000000000..3aad803c490 --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/fs/PathFilter.java @@ -0,0 +1,16 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// Never instantiated at runtime. +package org.apache.hadoop.fs; + +public interface PathFilter { +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/mapred/JobConf.java b/core/sail/s3/src/main/java/org/apache/hadoop/mapred/JobConf.java new file mode 100644 index 00000000000..126ffd16689 --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/mapred/JobConf.java @@ -0,0 +1,18 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// Never instantiated at runtime. +package org.apache.hadoop.mapred; + +import org.apache.hadoop.conf.Configuration; + +public class JobConf extends Configuration { +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/InputFormat.java b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/InputFormat.java new file mode 100644 index 00000000000..2ac8f746c4e --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/InputFormat.java @@ -0,0 +1,16 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// Never instantiated at runtime. +package org.apache.hadoop.mapreduce; + +public abstract class InputFormat { +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/InputSplit.java b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/InputSplit.java new file mode 100644 index 00000000000..13521f68a59 --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/InputSplit.java @@ -0,0 +1,16 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// Never instantiated at runtime. +package org.apache.hadoop.mapreduce; + +public abstract class InputSplit { +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/Job.java b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/Job.java new file mode 100644 index 00000000000..721f0940068 --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/Job.java @@ -0,0 +1,16 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// Never instantiated at runtime. +package org.apache.hadoop.mapreduce; + +public class Job { +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/JobContext.java b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/JobContext.java new file mode 100644 index 00000000000..909112c5c65 --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/JobContext.java @@ -0,0 +1,16 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// Never instantiated at runtime. +package org.apache.hadoop.mapreduce; + +public interface JobContext { +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/RecordReader.java b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/RecordReader.java new file mode 100644 index 00000000000..f582a259f44 --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/RecordReader.java @@ -0,0 +1,16 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// Never instantiated at runtime. +package org.apache.hadoop.mapreduce; + +public abstract class RecordReader { +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/TaskAttemptContext.java b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/TaskAttemptContext.java new file mode 100644 index 00000000000..942f8ff3ee0 --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/TaskAttemptContext.java @@ -0,0 +1,16 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// Never instantiated at runtime. +package org.apache.hadoop.mapreduce; + +public interface TaskAttemptContext extends JobContext { +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java new file mode 100644 index 00000000000..0bb81f4742f --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java @@ -0,0 +1,19 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// ParquetInputFormat extends this class; loaded when ParquetReadOptions.Builder +// calls ParquetInputFormat.getFilter(). Never used at runtime. +package org.apache.hadoop.mapreduce.lib.input; + +import org.apache.hadoop.mapreduce.InputFormat; + +public abstract class FileInputFormat extends InputFormat { +} diff --git a/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileSplit.java b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileSplit.java new file mode 100644 index 00000000000..ec406809ef1 --- /dev/null +++ b/core/sail/s3/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileSplit.java @@ -0,0 +1,18 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Minimal stub — satisfies JVM class loading for parquet-hadoop. +// Never instantiated at runtime. +package org.apache.hadoop.mapreduce.lib.input; + +import org.apache.hadoop.mapreduce.InputSplit; + +public class FileSplit extends InputSplit { +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3EvaluationStatistics.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3EvaluationStatistics.java new file mode 100644 index 00000000000..aa6255a0d1b --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3EvaluationStatistics.java @@ -0,0 +1,117 @@ +/******************************************************************************* + * Copyright (c) 2024 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import java.util.List; + +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.query.algebra.StatementPattern; +import org.eclipse.rdf4j.query.algebra.Var; +import org.eclipse.rdf4j.query.algebra.evaluation.impl.EvaluationStatistics; +import org.eclipse.rdf4j.sail.s3.storage.Catalog; + +/** + * Evaluation statistics for the S3 SAIL. Uses catalog-level file statistics (row counts and min/max ranges) to estimate + * statement pattern cardinality. + */ +class S3EvaluationStatistics extends EvaluationStatistics { + + private final S3ValueStore valueStore; + private final Catalog catalog; + + S3EvaluationStatistics(S3ValueStore valueStore, Catalog catalog) { + this.valueStore = valueStore; + this.catalog = catalog; + } + + @Override + protected CardinalityCalculator createCardinalityCalculator() { + return new S3CardinalityCalculator(); + } + + protected class S3CardinalityCalculator extends CardinalityCalculator { + + @Override + protected double getCardinality(StatementPattern sp) { + Value subj = getConstantValue(sp.getSubjectVar()); + if (subj != null && !(subj instanceof Resource)) { + subj = null; + } + Value pred = getConstantValue(sp.getPredicateVar()); + if (pred != null && !(pred instanceof IRI)) { + pred = null; + } + Value obj = getConstantValue(sp.getObjectVar()); + Value context = getConstantValue(sp.getContextVar()); + if (context != null && !(context instanceof Resource)) { + context = null; + } + return estimateCardinality((Resource) subj, (IRI) pred, obj, (Resource) context); + } + + private Value getConstantValue(Var var) { + return (var != null) ? var.getValue() : null; + } + } + + private double estimateCardinality(Resource subj, IRI pred, Value obj, Resource context) { + long subjID = S3ValueStore.UNKNOWN_ID; + if (subj != null) { + subjID = valueStore.getId(subj); + if (subjID == S3ValueStore.UNKNOWN_ID) { + return 0; + } + } + + long predID = S3ValueStore.UNKNOWN_ID; + if (pred != null) { + predID = valueStore.getId(pred); + if (predID == S3ValueStore.UNKNOWN_ID) { + return 0; + } + } + + long objID = S3ValueStore.UNKNOWN_ID; + if (obj != null) { + objID = valueStore.getId(obj); + if (objID == S3ValueStore.UNKNOWN_ID) { + return 0; + } + } + + long contextID = S3ValueStore.UNKNOWN_ID; + if (context != null) { + contextID = valueStore.getId(context); + if (contextID == S3ValueStore.UNKNOWN_ID) { + return 0; + } + } + + if (catalog == null) { + return 1000; + } + + // Sum row counts from files whose stats allow matching the pattern, + // then divide by number of sort orders since each triple is stored 3 times + List files = catalog.getFiles(); + long totalMatchingRows = 0; + for (Catalog.ParquetFileInfo file : files) { + if (file.mayContain(subjID, predID, objID, contextID)) { + totalMatchingRows += file.getRowCount(); + } + } + + int numSortOrders = 3; + return (double) totalMatchingRows / numSortOrders; + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3NamespaceStore.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3NamespaceStore.java new file mode 100644 index 00000000000..fbf52a039c4 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3NamespaceStore.java @@ -0,0 +1,97 @@ +/******************************************************************************* + * Copyright (c) 2024 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.eclipse.rdf4j.model.impl.SimpleNamespace; +import org.eclipse.rdf4j.sail.s3.storage.ObjectStore; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * In-memory store for namespace prefix information. All operations are synchronized for thread safety. + */ +class S3NamespaceStore implements Iterable { + + static final String NAMESPACES_KEY = "namespaces/current"; + + private final Map namespacesMap = new LinkedHashMap<>(16); + + public synchronized String getNamespace(String prefix) { + SimpleNamespace namespace = namespacesMap.get(prefix); + return namespace != null ? namespace.getName() : null; + } + + public synchronized void setNamespace(String prefix, String name) { + SimpleNamespace ns = namespacesMap.get(prefix); + if (ns != null) { + if (!ns.getName().equals(name)) { + ns.setName(name); + } + } else { + namespacesMap.put(prefix, new SimpleNamespace(prefix, name)); + } + } + + public synchronized void removeNamespace(String prefix) { + namespacesMap.remove(prefix); + } + + @Override + public synchronized Iterator iterator() { + // return a snapshot to avoid ConcurrentModificationException + return new LinkedHashMap<>(namespacesMap).values().iterator(); + } + + public synchronized void clear() { + namespacesMap.clear(); + } + + @SuppressWarnings("unchecked") + synchronized void deserialize(ObjectStore objectStore, ObjectMapper mapper) { + byte[] data = objectStore.get(NAMESPACES_KEY); + if (data == null) { + return; + } + try { + List> entries = mapper.readValue(data, List.class); + for (Map entry : entries) { + String prefix = entry.get("prefix"); + String name = entry.get("name"); + namespacesMap.put(prefix, new SimpleNamespace(prefix, name)); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to deserialize namespaces", e); + } + } + + synchronized void serialize(ObjectStore objectStore, ObjectMapper mapper) { + try { + List> entries = new ArrayList<>(); + for (SimpleNamespace ns : namespacesMap.values()) { + Map entry = new LinkedHashMap<>(); + entry.put("prefix", ns.getPrefix()); + entry.put("name", ns.getName()); + entries.add(entry); + } + objectStore.put(NAMESPACES_KEY, mapper.writeValueAsBytes(entries)); + } catch (IOException e) { + throw new UncheckedIOException("Failed to serialize namespaces", e); + } + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3SailStore.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3SailStore.java new file mode 100644 index 00000000000..162c1dcc72e --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3SailStore.java @@ -0,0 +1,869 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; + +import org.eclipse.rdf4j.common.iteration.CloseableIteration; +import org.eclipse.rdf4j.common.iteration.CloseableIteratorIteration; +import org.eclipse.rdf4j.common.iteration.EmptyIteration; +import org.eclipse.rdf4j.common.iteration.UnionIteration; +import org.eclipse.rdf4j.common.order.StatementOrder; +import org.eclipse.rdf4j.common.transaction.IsolationLevel; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Namespace; +import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.Statement; +import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.model.ValueFactory; +import org.eclipse.rdf4j.query.algebra.evaluation.impl.EvaluationStatistics; +import org.eclipse.rdf4j.sail.SailException; +import org.eclipse.rdf4j.sail.base.BackingSailSource; +import org.eclipse.rdf4j.sail.base.SailDataset; +import org.eclipse.rdf4j.sail.base.SailSink; +import org.eclipse.rdf4j.sail.base.SailSource; +import org.eclipse.rdf4j.sail.base.SailStore; +import org.eclipse.rdf4j.sail.s3.cache.TieredCache; +import org.eclipse.rdf4j.sail.s3.config.S3StoreConfig; +import org.eclipse.rdf4j.sail.s3.storage.BloomFilter; +import org.eclipse.rdf4j.sail.s3.storage.Catalog; +import org.eclipse.rdf4j.sail.s3.storage.CompactionPolicy; +import org.eclipse.rdf4j.sail.s3.storage.Compactor; +import org.eclipse.rdf4j.sail.s3.storage.FileSystemObjectStore; +import org.eclipse.rdf4j.sail.s3.storage.MemTable; +import org.eclipse.rdf4j.sail.s3.storage.MergeIterator; +import org.eclipse.rdf4j.sail.s3.storage.ObjectStore; +import org.eclipse.rdf4j.sail.s3.storage.ParquetFileBuilder; +import org.eclipse.rdf4j.sail.s3.storage.ParquetQuadSource; +import org.eclipse.rdf4j.sail.s3.storage.ParquetSchemas; +import org.eclipse.rdf4j.sail.s3.storage.QuadEntry; +import org.eclipse.rdf4j.sail.s3.storage.QuadIndex; +import org.eclipse.rdf4j.sail.s3.storage.QuadStats; +import org.eclipse.rdf4j.sail.s3.storage.RawEntrySource; +import org.eclipse.rdf4j.sail.s3.storage.S3ObjectStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * {@link SailStore} implementation that stores RDF quads using Parquet files on S3-compatible object storage with + * stats-based pruning (no predicate partitioning). + * + *

+ * Architecture: single in-memory {@link MemTable} in SPOC order → on flush, write 3 Parquet files per epoch (SPOC, + * OPSC, CSPO sort orders) → multi-tier cache (Caffeine heap + disk) → compaction. + *

+ * + *

+ * When S3 is not configured, operates in pure in-memory mode. + *

+ */ +class S3SailStore implements SailStore { + + private static final Logger logger = LoggerFactory.getLogger(S3SailStore.class); + + private static final QuadIndex SPOC_INDEX = new QuadIndex("spoc"); + private static final QuadIndex CSPO_INDEX = new QuadIndex("cspo"); + private static final List ALL_INDEXES; + + static { + ParquetSchemas.SortOrder[] orders = ParquetSchemas.SortOrder.values(); + List indexes = new ArrayList<>(orders.length); + for (ParquetSchemas.SortOrder order : orders) { + indexes.add(new QuadIndex(order.suffix())); + } + ALL_INDEXES = List.copyOf(indexes); + } + + private final S3ValueStore valueStore; + private final S3NamespaceStore namespaceStore; + + // Single MemTable in SPOC order + private volatile MemTable memTable; + private volatile boolean mayHaveInferred; + + // Persistence fields (null when S3 is not configured) + private final ObjectStore objectStore; + private final ObjectMapper jsonMapper; + private final Catalog catalog; + private final AtomicLong epochCounter; + private final long memTableFlushSize; + private final TieredCache cache; + private final CompactionPolicy compactionPolicy; + private final Compactor compactor; + private final ExecutorService writeExecutor; + private final ExecutorService compactionExecutor; + private volatile CompletableFuture pendingCompaction; + + /** + * A lock to control concurrent access by {@link S3SailSink} to the stores. + */ + private final ReentrantLock sinkStoreAccessLock = new ReentrantLock(); + + S3SailStore(S3StoreConfig config) { + this(config, createObjectStore(config)); + } + + private static ObjectStore createObjectStore(S3StoreConfig config) { + if (config.isS3Configured()) { + return new S3ObjectStore(config.getS3Bucket(), config.getS3Endpoint(), config.getS3Region(), + config.getS3Prefix(), config.getS3AccessKey(), config.getS3SecretKey(), + config.isS3ForcePathStyle()); + } + String dataDir = config.getDataDir(); + if (dataDir != null && !dataDir.isEmpty()) { + return new FileSystemObjectStore(Path.of(dataDir)); + } + return null; // in-memory only + } + + /** + * Package-private constructor for testing with a custom ObjectStore. + */ + S3SailStore(S3StoreConfig config, ObjectStore objectStore) { + this.valueStore = new S3ValueStore(); + this.namespaceStore = new S3NamespaceStore(); + this.objectStore = objectStore; + this.memTableFlushSize = config.getMemTableSize(); + + // Single SPOC index for the MemTable + this.memTable = new MemTable(SPOC_INDEX); + + // Initialize persistence + if (objectStore != null) { + this.jsonMapper = new ObjectMapper(); + this.catalog = Catalog.load(objectStore, jsonMapper); + this.epochCounter = new AtomicLong(catalog.getEpoch() + 1); + + // Initialize cache + Path diskCachePath = config.getDiskCachePath() != null ? Path.of(config.getDiskCachePath()) : null; + this.cache = new TieredCache(config.getMemoryCacheSize(), diskCachePath, + config.getDiskCacheSize(), objectStore); + + this.compactionPolicy = new CompactionPolicy(); + this.compactor = new Compactor(objectStore, cache); + this.writeExecutor = Executors.newFixedThreadPool( + Math.min(ALL_INDEXES.size(), Runtime.getRuntime().availableProcessors())); + this.compactionExecutor = Executors.newSingleThreadExecutor(); + + // Deserialize value store and namespaces + if (catalog.getNextValueId() > 0) { + valueStore.deserialize(objectStore, catalog.getNextValueId()); + } + namespaceStore.deserialize(objectStore, jsonMapper); + } else { + this.jsonMapper = null; + this.catalog = null; + this.epochCounter = null; + this.cache = null; + this.compactionPolicy = null; + this.compactor = null; + this.writeExecutor = null; + this.compactionExecutor = null; + } + } + + @Override + public ValueFactory getValueFactory() { + return valueStore; + } + + @Override + public EvaluationStatistics getEvaluationStatistics() { + return new S3EvaluationStatistics(valueStore, catalog); + } + + @Override + public SailSource getExplicitSailSource() { + return new S3SailSource(true); + } + + @Override + public SailSource getInferredSailSource() { + return new S3SailSource(false); + } + + @Override + public void close() throws SailException { + try { + if (objectStore != null) { + // Await any pending compaction before flushing + CompletableFuture compaction = pendingCompaction; + if (compaction != null) { + compaction.join(); + } + flushToObjectStore(); + if (writeExecutor != null) { + writeExecutor.shutdown(); + } + if (compactionExecutor != null) { + compactionExecutor.shutdown(); + } + if (cache != null) { + cache.close(); + } + objectStore.close(); + } + } catch (IOException e) { + throw new SailException(e); + } + valueStore.close(); + memTable.clear(); + } + + /** + * Flushes active MemTable to Parquet files on the object store. Writes one file per sort order (SPOC, OPSC, CSPO). + */ + private void flushToObjectStore() { + if (objectStore == null) { + return; + } + + if (memTable.size() == 0) { + // No quads to flush — still persist namespaces/values (they may have changed) + valueStore.serialize(objectStore); + namespaceStore.serialize(objectStore, jsonMapper); + return; + } + + long epoch = epochCounter.getAndIncrement(); + + // Freeze active MemTable and swap in fresh one + MemTable frozen = memTable; + frozen.freeze(); + memTable = new MemTable(SPOC_INDEX); + + List allEntries = collectEntries(frozen); + QuadStats stats = QuadStats.fromEntries(allEntries); + writeParquetFiles(epoch, allEntries, stats); + + persistMetadata(epoch); + runCompactionIfNeeded(); + } + + private static List collectEntries(MemTable frozen) { + List entries = new ArrayList<>(frozen.size()); + long[] scratch = new long[4]; + for (Map.Entry entry : frozen.getData().entrySet()) { + frozen.getIndex().keyToQuad(entry.getKey(), scratch); + entries.add(new QuadEntry( + scratch[QuadIndex.SUBJ_IDX], scratch[QuadIndex.PRED_IDX], + scratch[QuadIndex.OBJ_IDX], scratch[QuadIndex.CONTEXT_IDX], + entry.getValue()[0])); + } + return entries; + } + + private void writeParquetFiles(long epoch, List allEntries, QuadStats stats) { + List> futures = new ArrayList<>(ALL_INDEXES.size()); + for (QuadIndex sortIndex : ALL_INDEXES) { + futures.add(CompletableFuture.runAsync(() -> { + String sortSuffix = sortIndex.getFieldSeqString(); + List sorted = new ArrayList<>(allEntries); + sorted.sort(sortIndex.entryComparator()); + + ParquetSchemas.SortOrder sortOrder = ParquetSchemas.SortOrder.fromSuffix(sortSuffix); + byte[] parquetData = ParquetFileBuilder.build(sorted, sortOrder); + + BloomFilter bloom = BloomFilter.buildForEntries(sorted, sortSuffix); + + String s3Key = Catalog.dataKey(0, epoch, sortSuffix); + objectStore.put(s3Key, parquetData); + + if (cache != null) { + cache.writeThrough(s3Key, parquetData); + } + + catalog.addFile(new Catalog.ParquetFileInfo( + s3Key, 0, sortSuffix, sorted.size(), epoch, parquetData.length, stats, bloom)); + }, writeExecutor)); + } + CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)).join(); + } + + private void persistMetadata(long epoch) { + // Save catalog first: if we crash after catalog but before values, + // on restart we have new nextValueId but old values — IDs are gaps (safe). + // The reverse (values first, catalog second) risks ID reuse on crash (corruption). + catalog.setNextValueId(valueStore.getNextId()); + catalog.setEpoch(epoch); + catalog.save(objectStore, jsonMapper, epoch); + valueStore.serialize(objectStore); + namespaceStore.serialize(objectStore, jsonMapper); + } + + /** + * Checks compaction triggers and submits compaction to the background executor if needed. If a compaction is + * already running, skips to avoid queuing multiple compactions. + */ + private void runCompactionIfNeeded() { + if (compactionPolicy == null || compactor == null) { + return; + } + + // Skip if a compaction is already running + CompletableFuture current = pendingCompaction; + if (current != null && !current.isDone()) { + return; + } + + List filesSnapshot = catalog.getFiles(); + boolean needsL0 = compactionPolicy.shouldCompact(filesSnapshot, 0); + boolean needsL1 = compactionPolicy.shouldCompact(filesSnapshot, 1); + + if (!needsL0 && !needsL1) { + return; + } + + pendingCompaction = CompletableFuture.runAsync(() -> { + try { + doCompaction(needsL0, needsL1); + } catch (Exception e) { + logger.error("Background compaction failed", e); + } + }, compactionExecutor); + } + + private void doCompaction(boolean compactL0, boolean compactL1) { + List results = new ArrayList<>(); + + // Snapshot file list under synchronization + List files; + synchronized (catalog) { + files = catalog.getFiles(); + } + + // L0→L1 compaction + if (compactL0) { + List l0Files = CompactionPolicy.filesAtLevel(files, 0); + long compactEpoch = epochCounter.getAndIncrement(); + results.add(compactor.compact(l0Files, 0, 1, compactEpoch, catalog)); + synchronized (catalog) { + files = catalog.getFiles(); + } + } + + // L1→L2 compaction (re-check after L0 compaction may have produced new L1 files) + if (compactL1 || (compactL0 && compactionPolicy.shouldCompact(files, 1))) { + List l1Files = CompactionPolicy.filesAtLevel(files, 1); + long compactEpoch = epochCounter.getAndIncrement(); + results.add(compactor.compact(l1Files, 1, 2, compactEpoch, catalog)); + } + + if (!results.isEmpty()) { + // Save catalog BEFORE deleting old files — crash-safe ordering + synchronized (catalog) { + long epoch = epochCounter.getAndIncrement(); + catalog.setEpoch(epoch); + catalog.save(objectStore, jsonMapper, epoch); + } + + // Now safe to delete old files + for (Compactor.CompactionResult result : results) { + for (String key : result.getDeletedKeys()) { + objectStore.delete(key); + if (cache != null) { + cache.invalidate(key); + } + } + } + } + } + + private boolean hasPersistence() { + return objectStore != null; + } + + /** + * Queries quads using the best available source (merged Parquet + MemTable, or MemTable only). If + * {@code preferredIndex} is non-null, it is used instead of automatic index selection. + */ + private Iterator queryQuads(long s, long p, long o, long c, boolean explicit, + QuadIndex preferredIndex) { + return hasPersistence() + ? createMergedIterator(s, p, o, c, explicit, preferredIndex) + : memTable.scan(s, p, o, c, explicit); + } + + /** + * Resolves a Value to its stored ID. Returns UNKNOWN_ID if the value is null, or the stored ID (which may be + * UNKNOWN_ID if the value is not in the store). + */ + private long resolveValueId(Value value) { + if (value == null) { + return S3ValueStore.UNKNOWN_ID; + } + return valueStore.getId(value); + } + + /** + * Creates a statement iterator for the given pattern using stats-based pruning. + */ + CloseableIteration createStatementIterator( + Resource subj, IRI pred, Value obj, boolean explicit, Resource... contexts) { + + if (!explicit && !mayHaveInferred) { + return new EmptyIteration<>(); + } + + long subjID = resolveValueId(subj); + if (subj != null && subjID == S3ValueStore.UNKNOWN_ID) { + return new EmptyIteration<>(); + } + + long predID = resolveValueId(pred); + if (pred != null && predID == S3ValueStore.UNKNOWN_ID) { + return new EmptyIteration<>(); + } + + long objID = resolveValueId(obj); + if (obj != null && objID == S3ValueStore.UNKNOWN_ID) { + return new EmptyIteration<>(); + } + + List contextIDList = new ArrayList<>(contexts.length == 0 ? 1 : contexts.length); + if (contexts.length == 0) { + contextIDList.add(S3ValueStore.UNKNOWN_ID); + } else { + for (Resource context : contexts) { + if (context == null) { + contextIDList.add(0L); + } else if (!context.isTriple()) { + long contextID = valueStore.getId(context); + if (contextID != S3ValueStore.UNKNOWN_ID) { + contextIDList.add(contextID); + } + } + } + } + + if (contextIDList.isEmpty()) { + return new EmptyIteration<>(); + } + + ArrayList> perContextIterList = new ArrayList<>(contextIDList.size()); + + for (long contextID : contextIDList) { + Iterator quads = queryQuads(subjID, predID, objID, contextID, explicit, null); + perContextIterList.add(new QuadToStatementIteration(quads, valueStore)); + } + + if (perContextIterList.size() == 1) { + return perContextIterList.get(0); + } else { + return new UnionIteration<>(perContextIterList); + } + } + + /** + * Creates a merged iterator across MemTable and Parquet files for a given pattern. Selects the best QuadIndex, + * prunes files using catalog stats, and merges all sources. + */ + private Iterator createMergedIterator(long subjID, long predID, long objID, long contextID, + boolean explicit, QuadIndex preferredIndex) { + + byte expectedFlag = explicit ? MemTable.FLAG_EXPLICIT : MemTable.FLAG_INFERRED; + + // Select best index for the query pattern, or use the preferred index if provided + QuadIndex bestIndex = preferredIndex != null ? preferredIndex + : QuadIndex.getBestIndex(ALL_INDEXES, subjID, predID, objID, contextID); + String sortSuffix = bestIndex.getFieldSeqString(); + + // Build sources: MemTable (newest) + Parquet files (newest epoch first) + List sources = new ArrayList<>(); + + // MemTable source (always newest) — re-encoded in the best index order + sources.add(memTable.asRawSource(bestIndex, subjID, predID, objID, contextID)); + + // Parquet files for the selected sort order + List sortOrderFiles = catalog.getFilesForSortOrder(sortSuffix); + sortOrderFiles = sortOrderFiles.stream() + .sorted(Comparator.comparingLong(Catalog.ParquetFileInfo::getEpoch).reversed()) + .toList(); + + for (Catalog.ParquetFileInfo fileInfo : sortOrderFiles) { + if (!fileInfo.mayContain(subjID, predID, objID, contextID)) { + continue; + } + + byte[] fileData = cache != null ? cache.get(fileInfo.getS3Key()) : objectStore.get(fileInfo.getS3Key()); + if (fileData == null) { + logger.warn("Missing Parquet file: {}", fileInfo.getS3Key()); + continue; + } + + sources.add(new ParquetQuadSource(fileData, bestIndex, subjID, predID, objID, contextID)); + } + + return new MergeIterator(sources, bestIndex, expectedFlag, subjID, predID, objID, contextID); + } + + // ========================================================================= + // Inner classes + // ========================================================================= + + private final class S3SailSource extends BackingSailSource { + + private final boolean explicit; + + S3SailSource(boolean explicit) { + this.explicit = explicit; + } + + @Override + public SailSource fork() { + throw new UnsupportedOperationException("This store does not support multiple datasets"); + } + + @Override + public SailSink sink(IsolationLevel level) throws SailException { + return new S3SailSink(explicit); + } + + @Override + public SailDataset dataset(IsolationLevel level) throws SailException { + return new S3SailDataset(explicit); + } + } + + private final class S3SailSink implements SailSink { + + private final boolean explicit; + + S3SailSink(boolean explicit) { + this.explicit = explicit; + } + + @Override + public void close() { + // no-op + } + + @Override + public void prepare() throws SailException { + // serializable is not supported at this level + } + + @Override + public void flush() throws SailException { + sinkStoreAccessLock.lock(); + try { + flushToObjectStore(); + } finally { + sinkStoreAccessLock.unlock(); + } + } + + @Override + public void setNamespace(String prefix, String name) throws SailException { + sinkStoreAccessLock.lock(); + try { + namespaceStore.setNamespace(prefix, name); + } finally { + sinkStoreAccessLock.unlock(); + } + } + + @Override + public void removeNamespace(String prefix) throws SailException { + sinkStoreAccessLock.lock(); + try { + namespaceStore.removeNamespace(prefix); + } finally { + sinkStoreAccessLock.unlock(); + } + } + + @Override + public void clearNamespaces() throws SailException { + sinkStoreAccessLock.lock(); + try { + namespaceStore.clear(); + } finally { + sinkStoreAccessLock.unlock(); + } + } + + @Override + public void observe(Resource subj, IRI pred, Value obj, Resource... contexts) throws SailException { + // serializable is not supported at this level + } + + @Override + public void clear(Resource... contexts) throws SailException { + removeStatements(null, null, null, explicit, contexts); + } + + @Override + public void approve(Resource subj, IRI pred, Value obj, Resource ctx) throws SailException { + addStatement(subj, pred, obj, explicit, ctx); + } + + @Override + public void approveAll(Set approved, Set approvedContexts) { + sinkStoreAccessLock.lock(); + try { + for (Statement st : approved) { + storeQuad(st.getSubject(), st.getPredicate(), st.getObject(), explicit, st.getContext()); + } + // Size-triggered flush + if (objectStore != null && memTable.approximateSizeInBytes() >= memTableFlushSize) { + flushToObjectStore(); + } + } finally { + sinkStoreAccessLock.unlock(); + } + } + + @Override + public void deprecate(Statement statement) throws SailException { + removeStatements(statement.getSubject(), statement.getPredicate(), statement.getObject(), explicit, + statement.getContext()); + } + + @Override + public boolean deprecateByQuery(Resource subj, IRI pred, Value obj, Resource[] contexts) { + return removeStatements(subj, pred, obj, explicit, contexts) > 0; + } + + @Override + public boolean supportsDeprecateByQuery() { + return true; + } + + private void addStatement(Resource subj, IRI pred, Value obj, boolean explicit, Resource context) { + sinkStoreAccessLock.lock(); + try { + storeQuad(subj, pred, obj, explicit, context); + } finally { + sinkStoreAccessLock.unlock(); + } + } + + private void storeQuad(Resource subj, IRI pred, Value obj, boolean explicit, Resource context) { + long s = valueStore.storeValue(subj); + long p = valueStore.storeValue(pred); + long o = valueStore.storeValue(obj); + long c = context == null ? 0 : valueStore.storeValue(context); + if (!explicit) { + mayHaveInferred = true; + } + memTable.put(s, p, o, c, explicit); + } + + private long removeStatements(Resource subj, IRI pred, Value obj, boolean explicit, Resource... contexts) { + Objects.requireNonNull(contexts, + "contexts argument may not be null; either the value should be cast to Resource or an empty array should be supplied"); + + sinkStoreAccessLock.lock(); + try { + long subjID = resolveValueId(subj); + if (subj != null && subjID == S3ValueStore.UNKNOWN_ID) { + return 0; + } + + long predID = resolveValueId(pred); + if (pred != null && predID == S3ValueStore.UNKNOWN_ID) { + return 0; + } + + long objID = resolveValueId(obj); + if (obj != null && objID == S3ValueStore.UNKNOWN_ID) { + return 0; + } + + final long[] contextIds; + if (contexts.length == 0) { + contextIds = new long[] { S3ValueStore.UNKNOWN_ID }; + } else { + contextIds = new long[contexts.length]; + for (int i = 0; i < contexts.length; i++) { + Resource context = contexts[i]; + if (context == null) { + contextIds[i] = 0; + } else { + long id = valueStore.getId(context); + contextIds[i] = (id != S3ValueStore.UNKNOWN_ID) ? id : Long.MAX_VALUE; + } + } + } + + long removeCount = 0; + for (long contextId : contextIds) { + Iterator iter = queryQuads(subjID, predID, objID, contextId, explicit, null); + + // Buffer results before removing to avoid ConcurrentModificationException + // when the iterator is backed by the MemTable's own map + List toRemove = new ArrayList<>(); + while (iter.hasNext()) { + toRemove.add(iter.next()); + } + for (long[] quad : toRemove) { + memTable.remove(quad[0], quad[1], quad[2], quad[3]); + removeCount++; + } + } + + return removeCount; + } finally { + sinkStoreAccessLock.unlock(); + } + } + } + + private final class S3SailDataset implements SailDataset { + + private final boolean explicit; + + S3SailDataset(boolean explicit) { + this.explicit = explicit; + } + + @Override + public void close() { + // no-op + } + + @Override + public String getNamespace(String prefix) throws SailException { + return namespaceStore.getNamespace(prefix); + } + + @Override + public CloseableIteration getNamespaces() { + return new CloseableIteratorIteration<>(namespaceStore.iterator()); + } + + @Override + public CloseableIteration getContextIDs() throws SailException { + // Use CSPO index where context is the leading field, so context values are grouped + Iterator allQuads = queryQuads(-1, -1, -1, -1, explicit, CSPO_INDEX); + + return new CloseableIteration<>() { + private long lastContextId = Long.MIN_VALUE; + private Resource nextCtx = advance(); + + private Resource advance() { + while (allQuads.hasNext()) { + long[] quad = allQuads.next(); + long ctxId = quad[3]; + if (ctxId != 0 && ctxId != lastContextId) { + lastContextId = ctxId; + Value val = valueStore.getValue(ctxId); + if (val instanceof Resource) { + return (Resource) val; + } + } + } + return null; + } + + @Override + public boolean hasNext() { + return nextCtx != null; + } + + @Override + public Resource next() { + if (nextCtx == null) { + throw new java.util.NoSuchElementException(); + } + Resource result = nextCtx; + nextCtx = advance(); + return result; + } + + @Override + public void close() { + // no-op + } + }; + } + + @Override + public CloseableIteration getStatements(Resource subj, IRI pred, Value obj, + Resource... contexts) throws SailException { + return createStatementIterator(subj, pred, obj, explicit, contexts); + } + + /** + * @throws UnsupportedOperationException always — ordered iteration is not supported + */ + @Override + public CloseableIteration getStatements(StatementOrder statementOrder, Resource subj, + IRI pred, Value obj, Resource... contexts) throws SailException { + throw new UnsupportedOperationException("Ordered iteration is not supported by S3Store"); + } + + @Override + public Set getSupportedOrders(Resource subj, IRI pred, Value obj, Resource... contexts) { + return Set.of(); + } + + @Override + public Comparator getComparator() { + return null; + } + } + + /** + * Converts quad ID arrays from iteration into Statement objects by resolving IDs through the ValueStore. + */ + static final class QuadToStatementIteration implements CloseableIteration { + + private final Iterator quads; + private final S3ValueStore valueStore; + + QuadToStatementIteration(Iterator quads, S3ValueStore valueStore) { + this.quads = quads; + this.valueStore = valueStore; + } + + @Override + public boolean hasNext() { + return quads.hasNext(); + } + + @Override + public Statement next() { + long[] quad = quads.next(); + Resource subj = (Resource) valueStore.getValue(quad[0]); + IRI pred = (IRI) valueStore.getValue(quad[1]); + Value obj = valueStore.getValue(quad[2]); + if (subj == null || pred == null || obj == null) { + // Value ID exists in Parquet but not in value store — can happen after + // crash recovery when catalog was saved but value file was not. + return null; + } + Resource ctx = quad[3] == 0 ? null : (Resource) valueStore.getValue(quad[3]); + return valueStore.createStatement(subj, pred, obj, ctx); + } + + @Override + public void close() { + // no-op + } + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3Store.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3Store.java new file mode 100644 index 00000000000..3e4089d5844 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3Store.java @@ -0,0 +1,230 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import java.util.concurrent.locks.ReentrantLock; + +import org.eclipse.rdf4j.common.annotation.Experimental; +import org.eclipse.rdf4j.common.concurrent.locks.Lock; +import org.eclipse.rdf4j.common.concurrent.locks.LockManager; +import org.eclipse.rdf4j.common.transaction.IsolationLevel; +import org.eclipse.rdf4j.common.transaction.IsolationLevels; +import org.eclipse.rdf4j.model.ValueFactory; +import org.eclipse.rdf4j.model.impl.LinkedHashModel; +import org.eclipse.rdf4j.query.algebra.evaluation.EvaluationStrategyFactory; +import org.eclipse.rdf4j.query.algebra.evaluation.federation.FederatedServiceResolver; +import org.eclipse.rdf4j.query.algebra.evaluation.federation.FederatedServiceResolverClient; +import org.eclipse.rdf4j.query.algebra.evaluation.impl.StrictEvaluationStrategyFactory; +import org.eclipse.rdf4j.repository.sparql.federation.SPARQLServiceResolver; +import org.eclipse.rdf4j.sail.InterruptedSailException; +import org.eclipse.rdf4j.sail.NotifyingSailConnection; +import org.eclipse.rdf4j.sail.SailException; +import org.eclipse.rdf4j.sail.base.SailSource; +import org.eclipse.rdf4j.sail.base.SailStore; +import org.eclipse.rdf4j.sail.base.SnapshotSailStore; +import org.eclipse.rdf4j.sail.helpers.AbstractNotifyingSail; +import org.eclipse.rdf4j.sail.s3.config.S3StoreConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A SAIL implementation that stores RDF data on S3-compatible object storage using an LSM-tree architecture with + * Parquet files, stats-based pruning, and multi-tier caching (Caffeine heap + disk LRU + S3). + * + *

+ * Supports three modes: S3 persistence (bucket configured), local filesystem persistence (dataDir configured), or pure + * in-memory (neither configured). + *

+ * + * @implNote the S3 store is in an experimental state: its existence, signature or behavior may change without warning + * from one release to the next. + */ +@Experimental +public class S3Store extends AbstractNotifyingSail implements FederatedServiceResolverClient { + + private static final Logger logger = LoggerFactory.getLogger(S3Store.class); + + private final S3StoreConfig config; + private SailStore store; + private S3SailStore backingStore; + private EvaluationStrategyFactory evalStratFactory; + + /** + * independent life cycle + */ + private FederatedServiceResolver serviceResolver; + + /** + * dependent life cycle + */ + private SPARQLServiceResolver dependentServiceResolver; + + /** + * Lock manager used to prevent concurrent {@link #getTransactionLock(IsolationLevel)} calls. + */ + private final ReentrantLock txnLockManager = new ReentrantLock(); + + /** + * Holds locks for all isolated transactions. + */ + private final LockManager isolatedLockManager = new LockManager(debugEnabled()); + + /** + * Holds locks for all {@link IsolationLevels#NONE} isolation transactions. + */ + private final LockManager disabledIsolationLockManager = new LockManager(debugEnabled()); + + public S3Store() { + this(new S3StoreConfig()); + } + + public S3Store(S3StoreConfig config) { + super(); + this.config = config; + setSupportedIsolationLevels(IsolationLevels.NONE, IsolationLevels.READ_COMMITTED, + IsolationLevels.SNAPSHOT_READ, IsolationLevels.SNAPSHOT, IsolationLevels.SERIALIZABLE); + setDefaultIsolationLevel(IsolationLevels.SNAPSHOT_READ); + config.getDefaultQueryEvaluationMode().ifPresent(this::setDefaultQueryEvaluationMode); + EvaluationStrategyFactory evalFactory = config.getEvaluationStrategyFactory(); + if (evalFactory != null) { + setEvaluationStrategyFactory(evalFactory); + } + } + + public synchronized EvaluationStrategyFactory getEvaluationStrategyFactory() { + if (evalStratFactory == null) { + evalStratFactory = new StrictEvaluationStrategyFactory(getFederatedServiceResolver()); + } + evalStratFactory.setQuerySolutionCacheThreshold(getIterationCacheSyncThreshold()); + evalStratFactory.setTrackResultSize(isTrackResultSize()); + return evalStratFactory; + } + + public synchronized void setEvaluationStrategyFactory(EvaluationStrategyFactory factory) { + evalStratFactory = factory; + } + + public synchronized FederatedServiceResolver getFederatedServiceResolver() { + if (serviceResolver == null) { + if (dependentServiceResolver == null) { + dependentServiceResolver = new SPARQLServiceResolver(); + } + setFederatedServiceResolver(dependentServiceResolver); + } + return serviceResolver; + } + + @Override + public synchronized void setFederatedServiceResolver(FederatedServiceResolver resolver) { + this.serviceResolver = resolver; + if (resolver != null && evalStratFactory instanceof FederatedServiceResolverClient) { + ((FederatedServiceResolverClient) evalStratFactory).setFederatedServiceResolver(resolver); + } + } + + @Override + protected void initializeInternal() throws SailException { + logger.debug("Initializing S3Store..."); + + try { + backingStore = new S3SailStore(config); + this.store = new SnapshotSailStore(backingStore, LinkedHashModel::new) { + + @Override + public SailSource getExplicitSailSource() { + if (isIsolationDisabled()) { + return backingStore.getExplicitSailSource(); + } else { + return super.getExplicitSailSource(); + } + } + + @Override + public SailSource getInferredSailSource() { + if (isIsolationDisabled()) { + return backingStore.getInferredSailSource(); + } else { + return super.getInferredSailSource(); + } + } + }; + } catch (Exception e) { + throw new SailException(e); + } + + logger.debug("S3Store initialized"); + } + + @Override + protected void shutDownInternal() throws SailException { + logger.debug("Shutting down S3Store..."); + + try { + store.close(); + } finally { + if (dependentServiceResolver != null) { + dependentServiceResolver.shutDown(); + } + } + + logger.debug("S3Store shut down"); + } + + @Override + public boolean isWritable() { + return true; + } + + @Override + protected NotifyingSailConnection getConnectionInternal() throws SailException { + return new S3StoreConnection(this); + } + + @Override + public ValueFactory getValueFactory() { + return store.getValueFactory(); + } + + /** + * This call will block when {@link IsolationLevels#NONE} is provided when there are active transactions with a + * higher isolation and block when a higher isolation is provided when there are active transactions with + * {@link IsolationLevels#NONE} isolation. + */ + Lock getTransactionLock(IsolationLevel level) throws SailException { + txnLockManager.lock(); + try { + if (IsolationLevels.NONE.isCompatibleWith(level)) { + isolatedLockManager.waitForActiveLocks(); + return disabledIsolationLockManager.createLock(level.toString()); + } else { + disabledIsolationLockManager.waitForActiveLocks(); + return isolatedLockManager.createLock(level.toString()); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new InterruptedSailException(e); + } finally { + txnLockManager.unlock(); + } + } + + boolean isIsolationDisabled() { + return disabledIsolationLockManager.isActiveLock(); + } + + SailStore getSailStore() { + return store; + } + + S3SailStore getBackingStore() { + return backingStore; + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3StoreConnection.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3StoreConnection.java new file mode 100644 index 00000000000..4c1d5e46aef --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3StoreConnection.java @@ -0,0 +1,127 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import org.eclipse.rdf4j.common.concurrent.locks.Lock; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.sail.SailException; +import org.eclipse.rdf4j.sail.base.SailSourceConnection; +import org.eclipse.rdf4j.sail.helpers.DefaultSailChangedEvent; + +/** + * Connection to an {@link S3Store}. + */ +public class S3StoreConnection extends SailSourceConnection { + + protected final S3Store s3Store; + + private volatile DefaultSailChangedEvent sailChangedEvent; + + /** + * The transaction lock held by this connection during transactions. + */ + private volatile Lock txnLock; + + protected S3StoreConnection(S3Store sail) { + super(sail, sail.getSailStore(), sail.getEvaluationStrategyFactory()); + this.s3Store = sail; + sailChangedEvent = new DefaultSailChangedEvent(sail); + } + + @Override + protected void startTransactionInternal() throws SailException { + boolean releaseLock = true; + try { + if (txnLock == null || !txnLock.isActive()) { + txnLock = s3Store.getTransactionLock(getTransactionIsolation()); + if (s3Store.isIsolationDisabled()) { + releaseLock = false; + } + } + super.startTransactionInternal(); + } finally { + if (releaseLock && txnLock != null && txnLock.isActive()) { + txnLock.release(); + } + } + } + + @Override + protected void commitInternal() throws SailException { + try { + super.commitInternal(); + } finally { + if (txnLock != null && txnLock.isActive()) { + txnLock.release(); + } + } + + s3Store.notifySailChanged(sailChangedEvent); + + // create a fresh event object + sailChangedEvent = new DefaultSailChangedEvent(s3Store); + } + + @Override + protected void rollbackInternal() throws SailException { + try { + super.rollbackInternal(); + } finally { + if (txnLock != null && txnLock.isActive()) { + txnLock.release(); + } + } + // create a fresh event object + sailChangedEvent = new DefaultSailChangedEvent(s3Store); + } + + @Override + protected void addStatementInternal(Resource subj, IRI pred, Value obj, Resource... contexts) + throws SailException { + sailChangedEvent.setStatementsAdded(true); + } + + @Override + public boolean addInferredStatement(Resource subj, IRI pred, Value obj, Resource... contexts) + throws SailException { + boolean ret = super.addInferredStatement(subj, pred, obj, contexts); + sailChangedEvent.setStatementsAdded(true); + return ret; + } + + @Override + protected void removeStatementsInternal(Resource subj, IRI pred, Value obj, Resource... contexts) + throws SailException { + sailChangedEvent.setStatementsRemoved(true); + } + + @Override + public boolean removeInferredStatement(Resource subj, IRI pred, Value obj, Resource... contexts) + throws SailException { + boolean ret = super.removeInferredStatement(subj, pred, obj, contexts); + sailChangedEvent.setStatementsRemoved(true); + return ret; + } + + @Override + protected void clearInternal(Resource... contexts) throws SailException { + super.clearInternal(contexts); + sailChangedEvent.setStatementsRemoved(true); + } + + @Override + public void clearInferred(Resource... contexts) throws SailException { + super.clearInferred(contexts); + sailChangedEvent.setStatementsRemoved(true); + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3ValueStore.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3ValueStore.java new file mode 100644 index 00000000000..5d56e69af62 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/S3ValueStore.java @@ -0,0 +1,234 @@ +/******************************************************************************* + * Copyright (c) 2024 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.eclipse.rdf4j.model.BNode; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Literal; +import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.model.base.AbstractValueFactory; +import org.eclipse.rdf4j.sail.s3.storage.ObjectStore; +import org.eclipse.rdf4j.sail.s3.storage.Varint; + +/** + * In-memory value store that maps RDF {@link Value} objects to long IDs and vice-versa. Uses {@link ConcurrentHashMap} + * for thread-safe bidirectional lookup. + */ +class S3ValueStore extends AbstractValueFactory { + + static final long UNKNOWN_ID = -1; + static final String VALUES_KEY = "values/current"; + + private static final byte TYPE_IRI = 0; + private static final byte TYPE_LITERAL = 1; + private static final byte TYPE_BNODE = 2; + + private final ConcurrentHashMap valueToId = new ConcurrentHashMap<>(); + private final ConcurrentHashMap idToValue = new ConcurrentHashMap<>(); + private final AtomicLong nextId = new AtomicLong(1); + + /** + * Stores the supplied value and returns the ID assigned to it. If the value already exists, returns the existing + * ID. + * + * @param value the value to store + * @return the ID assigned to the value + */ + public long storeValue(Value value) { + Long existing = valueToId.get(value); + if (existing != null) { + return existing; + } + long id = nextId.getAndIncrement(); + Long previous = valueToId.putIfAbsent(value, id); + if (previous != null) { + // another thread stored it first + return previous; + } + idToValue.put(id, value); + return id; + } + + /** + * Gets the ID for the specified value. + * + * @param value a value + * @return the ID for the value, or {@link #UNKNOWN_ID} if not found + */ + public long getId(Value value) { + Long id = valueToId.get(value); + return id != null ? id : UNKNOWN_ID; + } + + /** + * Gets the value for the specified ID. + * + * @param id a value ID + * @return the value, or {@code null} if not found + */ + public Value getValue(long id) { + return idToValue.get(id); + } + + /** + * Removes all stored values and resets the ID counter. + */ + public void clear() { + valueToId.clear(); + idToValue.clear(); + nextId.set(1); + } + + /** + * Returns the next value ID that would be assigned. + */ + long getNextId() { + return nextId.get(); + } + + /** + * Serializes the value store to the object store. + */ + void serialize(ObjectStore objectStore) { + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(baos); + ByteBuffer buf = ByteBuffer.allocate(9); // reusable scratch for varints + + writeVarint(out, buf, idToValue.size()); + + for (Map.Entry entry : idToValue.entrySet()) { + writeVarint(out, buf, entry.getKey()); + Value val = entry.getValue(); + + if (val instanceof IRI) { + out.writeByte(TYPE_IRI); + writeBytes(out, buf, val.stringValue().getBytes(StandardCharsets.UTF_8)); + } else if (val instanceof Literal) { + out.writeByte(TYPE_LITERAL); + Literal lit = (Literal) val; + writeBytes(out, buf, lit.getLabel().getBytes(StandardCharsets.UTF_8)); + writeBytes(out, buf, lit.getDatatype().stringValue().getBytes(StandardCharsets.UTF_8)); + writeBytes(out, buf, lit.getLanguage().orElse("").getBytes(StandardCharsets.UTF_8)); + } else if (val instanceof BNode) { + out.writeByte(TYPE_BNODE); + writeBytes(out, buf, ((BNode) val).getID().getBytes(StandardCharsets.UTF_8)); + } else { + throw new IllegalStateException("Unsupported value type: " + val.getClass()); + } + } + + out.flush(); + objectStore.put(VALUES_KEY, baos.toByteArray()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Deserializes the value store from the object store. + */ + void deserialize(ObjectStore objectStore, long nextValueId) { + byte[] data = objectStore.get(VALUES_KEY); + if (data == null) { + return; + } + + try { + ByteBuffer bb = ByteBuffer.wrap(data); + long count = Varint.readUnsigned(bb); + + for (long i = 0; i < count; i++) { + long id = Varint.readUnsigned(bb); + int type = bb.get() & 0xFF; + + Value val; + switch (type) { + case TYPE_IRI: { + int len = (int) Varint.readUnsigned(bb); + byte[] payload = new byte[len]; + bb.get(payload); + val = createIRI(new String(payload, StandardCharsets.UTF_8)); + break; + } + case TYPE_LITERAL: { + int labelLen = (int) Varint.readUnsigned(bb); + byte[] labelBytes = new byte[labelLen]; + bb.get(labelBytes); + String label = new String(labelBytes, StandardCharsets.UTF_8); + + int dtLen = (int) Varint.readUnsigned(bb); + byte[] dtBytes = new byte[dtLen]; + bb.get(dtBytes); + String dt = new String(dtBytes, StandardCharsets.UTF_8); + + int langLen = (int) Varint.readUnsigned(bb); + byte[] langBytes = new byte[langLen]; + bb.get(langBytes); + String lang = new String(langBytes, StandardCharsets.UTF_8); + + IRI datatypeIRI = createIRI(dt); + if (!lang.isEmpty()) { + val = createLiteral(label, lang); + } else { + val = createLiteral(label, datatypeIRI); + } + break; + } + case TYPE_BNODE: { + int len = (int) Varint.readUnsigned(bb); + byte[] payload = new byte[len]; + bb.get(payload); + val = createBNode(new String(payload, StandardCharsets.UTF_8)); + break; + } + default: + throw new IllegalStateException("Unknown value type: " + type); + } + + valueToId.put(val, id); + idToValue.put(id, val); + } + + nextId.set(nextValueId); + } catch (Exception e) { + throw new RuntimeException("Failed to deserialize value store", e); + } + } + + /** + * Closes the value store, releasing all resources. + */ + public void close() { + clear(); + } + + private static void writeVarint(DataOutputStream out, ByteBuffer buf, long value) throws IOException { + buf.clear(); + Varint.writeUnsigned(buf, value); + out.write(buf.array(), 0, buf.position()); + } + + private static void writeBytes(DataOutputStream out, ByteBuffer buf, byte[] data) throws IOException { + writeVarint(out, buf, data.length); + out.write(data); + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/cache/L1HeapCache.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/cache/L1HeapCache.java new file mode 100644 index 00000000000..fb16361202d --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/cache/L1HeapCache.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.cache; + +import java.io.Closeable; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +/** + * L1 in-memory cache backed by Caffeine. Caches full file bytes keyed by S3 object key, weighted by byte array length. + */ +public class L1HeapCache implements Closeable { + + private final Cache fileCache; + + public L1HeapCache(long maxWeightBytes) { + this.fileCache = Caffeine.newBuilder() + .maximumWeight(maxWeightBytes) + .weigher((String key, byte[] value) -> value.length) + .recordStats() + .build(); + } + + public byte[] get(String s3Key) { + return fileCache.getIfPresent(s3Key); + } + + public void put(String s3Key, byte[] data) { + fileCache.put(s3Key, data); + } + + public void invalidate(String s3Key) { + fileCache.invalidate(s3Key); + } + + public void invalidateAll() { + fileCache.invalidateAll(); + } + + @Override + public void close() { + fileCache.invalidateAll(); + fileCache.cleanUp(); + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/cache/L2DiskCache.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/cache/L2DiskCache.java new file mode 100644 index 00000000000..a6ed768def9 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/cache/L2DiskCache.java @@ -0,0 +1,215 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.cache; + +import java.io.Closeable; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * L2 disk-based LRU cache that mirrors S3 path structure on local filesystem. Entries are evicted in LRU order when the + * total cache size exceeds the configured maximum. A JSON index file is persisted on close so that cache state survives + * restarts. + */ +public class L2DiskCache implements Closeable { + + private static final Logger logger = LoggerFactory.getLogger(L2DiskCache.class); + private static final String INDEX_FILE = "_cache_index.json"; + + private final Path cacheDir; + private final long maxSizeBytes; + private final AtomicLong currentSizeBytes = new AtomicLong(0); + private final ConcurrentHashMap index = new ConcurrentHashMap<>(); + private final ObjectMapper mapper = new ObjectMapper(); + + public L2DiskCache(Path cacheDir, long maxSizeBytes) { + this.cacheDir = cacheDir; + this.maxSizeBytes = maxSizeBytes; + try { + Files.createDirectories(cacheDir); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + loadIndex(); + } + + public byte[] get(String s3Key) { + CacheEntry entry = index.get(s3Key); + if (entry == null) { + return null; + } + Path filePath = cacheDir.resolve(s3Key); + if (!Files.exists(filePath)) { + index.remove(s3Key); + currentSizeBytes.addAndGet(-entry.sizeBytes); + return null; + } + entry.lastAccessNanos = System.nanoTime(); + try { + return Files.readAllBytes(filePath); + } catch (IOException e) { + logger.warn("Failed to read cache file: {}", filePath, e); + return null; + } + } + + public void put(String s3Key, byte[] data) { + evictIfNeeded(data.length); + Path filePath = cacheDir.resolve(s3Key); + try { + Files.createDirectories(filePath.getParent()); + // Atomic write via temp file + rename + Path tmpFile = filePath.resolveSibling(filePath.getFileName() + ".tmp"); + Files.write(tmpFile, data, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + Files.move(tmpFile, filePath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + logger.warn("Failed to write cache file: {}", filePath, e); + return; + } + + CacheEntry prev = index.put(s3Key, new CacheEntry(data.length, System.nanoTime())); + if (prev != null) { + currentSizeBytes.addAndGet(data.length - prev.sizeBytes); + } else { + currentSizeBytes.addAndGet(data.length); + } + } + + public void remove(String s3Key) { + CacheEntry entry = index.remove(s3Key); + if (entry != null) { + currentSizeBytes.addAndGet(-entry.sizeBytes); + try { + Files.deleteIfExists(cacheDir.resolve(s3Key)); + } catch (IOException e) { + logger.warn("Failed to delete cache file: {}", s3Key, e); + } + } + } + + private synchronized void evictIfNeeded(long incomingSize) { + while (currentSizeBytes.get() + incomingSize > maxSizeBytes && !index.isEmpty()) { + // Find LRU entry + String lruKey = null; + long oldestAccess = Long.MAX_VALUE; + for (var e : index.entrySet()) { + if (e.getValue().lastAccessNanos < oldestAccess) { + oldestAccess = e.getValue().lastAccessNanos; + lruKey = e.getKey(); + } + } + if (lruKey != null) { + remove(lruKey); + } else { + break; + } + } + } + + private void loadIndex() { + Path indexPath = cacheDir.resolve(INDEX_FILE); + if (Files.exists(indexPath)) { + try { + CacheIndex saved = mapper.readValue(indexPath.toFile(), CacheIndex.class); + if (saved.entries != null) { + long totalSize = 0; + for (var e : saved.entries.entrySet()) { + if (Files.exists(cacheDir.resolve(e.getKey()))) { + index.put(e.getKey(), e.getValue()); + totalSize += e.getValue().sizeBytes; + } + } + currentSizeBytes.set(totalSize); + } + return; + } catch (IOException e) { + logger.warn("Failed to load cache index, rebuilding", e); + } + } + rebuildIndex(); + } + + private void rebuildIndex() { + index.clear(); + long totalSize = 0; + try (Stream walk = Files.walk(cacheDir)) { + var iter = walk.filter(Files::isRegularFile) + .filter(p -> !p.getFileName().toString().equals(INDEX_FILE)) + .filter(p -> !p.getFileName().toString().endsWith(".tmp")) + .iterator(); + while (iter.hasNext()) { + Path p = iter.next(); + try { + long size = Files.size(p); + String key = cacheDir.relativize(p).toString(); + index.put(key, new CacheEntry(size, System.nanoTime())); + totalSize += size; + } catch (IOException e) { + // skip unreadable files + } + } + } catch (IOException e) { + logger.warn("Failed to walk cache directory", e); + } + currentSizeBytes.set(totalSize); + } + + public void persistIndex() { + try { + CacheIndex ci = new CacheIndex(); + ci.entries = new ConcurrentHashMap<>(index); + Path indexPath = cacheDir.resolve(INDEX_FILE); + mapper.writeValue(indexPath.toFile(), ci); + } catch (IOException e) { + logger.warn("Failed to persist cache index", e); + } + } + + @Override + public void close() { + persistIndex(); + } + + static class CacheEntry { + @JsonProperty("sizeBytes") + public long sizeBytes; + + @JsonProperty("lastAccessNanos") + public volatile long lastAccessNanos; + + public CacheEntry() { + // for Jackson deserialization + } + + CacheEntry(long sizeBytes, long lastAccessNanos) { + this.sizeBytes = sizeBytes; + this.lastAccessNanos = lastAccessNanos; + } + } + + static class CacheIndex { + @JsonProperty("entries") + public ConcurrentHashMap entries; + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/cache/TieredCache.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/cache/TieredCache.java new file mode 100644 index 00000000000..85b1754aa96 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/cache/TieredCache.java @@ -0,0 +1,95 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.cache; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Path; + +import org.eclipse.rdf4j.sail.s3.storage.ObjectStore; + +/** + * Unified three-tier cache facade: L1 (heap) -> L2 (disk) -> L3 (S3 / ObjectStore). On a cache miss at a given tier, + * data is fetched from the next tier down and promoted into all higher tiers. + */ +public class TieredCache implements Closeable { + + private final L1HeapCache l1; + private final L2DiskCache l2; // nullable when no disk cache path is configured + private final ObjectStore objectStore; + + public TieredCache(long heapCacheSize, Path diskCachePath, long diskCacheSize, ObjectStore objectStore) { + this.l1 = new L1HeapCache(heapCacheSize); + this.l2 = diskCachePath != null ? new L2DiskCache(diskCachePath, diskCacheSize) : null; + this.objectStore = objectStore; + } + + /** + * Get file bytes for the given S3 key. Checks L1 (heap) first, then L2 (disk), then L3 (S3), promoting data into + * higher tiers on a miss. + */ + public byte[] get(String s3Key) { + // L1 + byte[] data = l1.get(s3Key); + if (data != null) { + return data; + } + + // L2 + if (l2 != null) { + data = l2.get(s3Key); + if (data != null) { + l1.put(s3Key, data); // promote to L1 + return data; + } + } + + // L3 (S3) + data = objectStore.get(s3Key); + if (data != null) { + promoteToUpperTiers(s3Key, data); + } + return data; + } + + /** + * Write-through: populate L1 and L2 immediately (e.g., on flush). Does NOT write to S3; the caller handles that + * separately. + */ + public void writeThrough(String s3Key, byte[] data) { + promoteToUpperTiers(s3Key, data); + } + + private void promoteToUpperTiers(String key, byte[] data) { + l1.put(key, data); + if (l2 != null) { + l2.put(key, data); + } + } + + /** + * Invalidate a key from all cache tiers (e.g., after compaction deletes a file). + */ + public void invalidate(String s3Key) { + l1.invalidate(s3Key); + if (l2 != null) { + l2.remove(s3Key); + } + } + + @Override + public void close() throws IOException { + l1.close(); + if (l2 != null) { + l2.close(); + } + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/config/S3StoreConfig.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/config/S3StoreConfig.java new file mode 100644 index 00000000000..397838e9256 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/config/S3StoreConfig.java @@ -0,0 +1,302 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.config; + +import java.util.function.Consumer; + +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Model; +import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.ValueFactory; +import org.eclipse.rdf4j.model.impl.SimpleValueFactory; +import org.eclipse.rdf4j.model.util.ModelException; +import org.eclipse.rdf4j.model.util.Models; +import org.eclipse.rdf4j.sail.base.config.BaseSailConfig; +import org.eclipse.rdf4j.sail.config.SailConfigException; + +/** + * Configuration for S3-backed SAIL store. + */ +public class S3StoreConfig extends BaseSailConfig { + + /** + * The default memtable size (64 MiB). + */ + public static final long DEFAULT_MEM_TABLE_SIZE = 67_108_864; + + /** + * The default memory cache size (256 MiB). + */ + public static final long DEFAULT_MEMORY_CACHE_SIZE = 268_435_456; + + /** + * The default disk cache size (10 GiB). + */ + public static final long DEFAULT_DISK_CACHE_SIZE = 10_737_418_240L; + + private long memTableSize = -1; + + private long memoryCacheSize = -1; + + private long diskCacheSize = -1; + + private String diskCachePath; + + private String s3Bucket; + + private String s3Endpoint; + + private String s3Region; + + private String s3Prefix; + + private String s3AccessKey; + + private String s3SecretKey; + + private Boolean s3ForcePathStyle; + + private String dataDir; + + /*--------------* + * Constructors * + *--------------*/ + + public S3StoreConfig() { + super(S3StoreFactory.SAIL_TYPE); + } + + /*---------* + * Methods * + *---------*/ + + /** + * Resolves a configuration value from (in order): environment variable, system property, or null. + * + *

+ * S3 connection settings are shared at the RDF4J instance level so that multiple S3 SAIL repositories can share the + * same bucket. Each repository differentiates itself via {@code s3Prefix}. + *

+ * + *

+ * Environment variables: {@code RDF4J_S3_BUCKET}, {@code RDF4J_S3_ENDPOINT}, {@code RDF4J_S3_REGION}, + * {@code RDF4J_S3_ACCESS_KEY}, {@code RDF4J_S3_SECRET_KEY}, {@code RDF4J_S3_FORCE_PATH_STYLE}. + *

+ */ + private static String resolveEnv(String envVar, String sysProp) { + String val = System.getenv(envVar); + if (val != null && !val.isEmpty()) { + return val; + } + val = System.getProperty(sysProp); + if (val != null && !val.isEmpty()) { + return val; + } + return null; + } + + private static String resolveField(String field, String envVar, String sysProp) { + return field != null ? field : resolveEnv(envVar, sysProp); + } + + public long getMemTableSize() { + return memTableSize >= 0 ? memTableSize : DEFAULT_MEM_TABLE_SIZE; + } + + public S3StoreConfig setMemTableSize(long memTableSize) { + this.memTableSize = memTableSize; + return this; + } + + public long getMemoryCacheSize() { + return memoryCacheSize >= 0 ? memoryCacheSize : DEFAULT_MEMORY_CACHE_SIZE; + } + + public S3StoreConfig setMemoryCacheSize(long memoryCacheSize) { + this.memoryCacheSize = memoryCacheSize; + return this; + } + + public long getDiskCacheSize() { + return diskCacheSize >= 0 ? diskCacheSize : DEFAULT_DISK_CACHE_SIZE; + } + + public S3StoreConfig setDiskCacheSize(long diskCacheSize) { + this.diskCacheSize = diskCacheSize; + return this; + } + + public String getDiskCachePath() { + return diskCachePath; + } + + public S3StoreConfig setDiskCachePath(String diskCachePath) { + this.diskCachePath = diskCachePath; + return this; + } + + public String getS3Bucket() { + return resolveField(s3Bucket, "RDF4J_S3_BUCKET", "rdf4j.s3.bucket"); + } + + public S3StoreConfig setS3Bucket(String s3Bucket) { + this.s3Bucket = s3Bucket; + return this; + } + + public String getS3Endpoint() { + return resolveField(s3Endpoint, "RDF4J_S3_ENDPOINT", "rdf4j.s3.endpoint"); + } + + public S3StoreConfig setS3Endpoint(String s3Endpoint) { + this.s3Endpoint = s3Endpoint; + return this; + } + + public String getS3Region() { + String resolved = resolveField(s3Region, "RDF4J_S3_REGION", "rdf4j.s3.region"); + return resolved != null ? resolved : "us-east-1"; + } + + public S3StoreConfig setS3Region(String s3Region) { + this.s3Region = s3Region; + return this; + } + + public String getS3Prefix() { + return s3Prefix != null ? s3Prefix : ""; + } + + public S3StoreConfig setS3Prefix(String s3Prefix) { + this.s3Prefix = s3Prefix; + return this; + } + + public String getS3AccessKey() { + return resolveField(s3AccessKey, "RDF4J_S3_ACCESS_KEY", "rdf4j.s3.accessKey"); + } + + public S3StoreConfig setS3AccessKey(String s3AccessKey) { + this.s3AccessKey = s3AccessKey; + return this; + } + + public String getS3SecretKey() { + return resolveField(s3SecretKey, "RDF4J_S3_SECRET_KEY", "rdf4j.s3.secretKey"); + } + + public S3StoreConfig setS3SecretKey(String s3SecretKey) { + this.s3SecretKey = s3SecretKey; + return this; + } + + public boolean isS3ForcePathStyle() { + if (s3ForcePathStyle != null) { + return s3ForcePathStyle; + } + String env = resolveEnv("RDF4J_S3_FORCE_PATH_STYLE", "rdf4j.s3.forcePathStyle"); + return env == null || Boolean.parseBoolean(env); + } + + public S3StoreConfig setS3ForcePathStyle(boolean s3ForcePathStyle) { + this.s3ForcePathStyle = s3ForcePathStyle; + return this; + } + + public boolean isS3Configured() { + return getS3Bucket() != null && !getS3Bucket().isEmpty(); + } + + public String getDataDir() { + return resolveField(dataDir, "RDF4J_S3_DATA_DIR", "rdf4j.s3.dataDir"); + } + + public S3StoreConfig setDataDir(String dataDir) { + this.dataDir = dataDir; + return this; + } + + @Override + public Resource export(Model m) { + Resource implNode = super.export(m); + ValueFactory vf = SimpleValueFactory.getInstance(); + + m.setNamespace("s3", S3StoreSchema.NAMESPACE); + exportLong(m, implNode, vf, S3StoreSchema.MEM_TABLE_SIZE, memTableSize); + exportLong(m, implNode, vf, S3StoreSchema.MEMORY_CACHE_SIZE, memoryCacheSize); + exportLong(m, implNode, vf, S3StoreSchema.DISK_CACHE_SIZE, diskCacheSize); + exportString(m, implNode, vf, S3StoreSchema.DISK_CACHE_PATH, diskCachePath); + exportString(m, implNode, vf, S3StoreSchema.S3_BUCKET, s3Bucket); + exportString(m, implNode, vf, S3StoreSchema.S3_ENDPOINT, s3Endpoint); + exportString(m, implNode, vf, S3StoreSchema.S3_REGION, s3Region); + exportString(m, implNode, vf, S3StoreSchema.S3_PREFIX, s3Prefix); + exportString(m, implNode, vf, S3StoreSchema.S3_ACCESS_KEY, s3AccessKey); + exportString(m, implNode, vf, S3StoreSchema.S3_SECRET_KEY, s3SecretKey); + if (s3ForcePathStyle != null) { + m.add(implNode, S3StoreSchema.S3_FORCE_PATH_STYLE, vf.createLiteral(s3ForcePathStyle)); + } + exportString(m, implNode, vf, S3StoreSchema.DATA_DIR, dataDir); + return implNode; + } + + private static void exportString(Model m, Resource node, ValueFactory vf, IRI prop, String value) { + if (value != null) { + m.add(node, prop, vf.createLiteral(value)); + } + } + + private static void exportLong(Model m, Resource node, ValueFactory vf, IRI prop, long value) { + if (value >= 0) { + m.add(node, prop, vf.createLiteral(value)); + } + } + + @Override + public void parse(Model m, Resource implNode) throws SailConfigException { + super.parse(m, implNode); + + try { + parseLong(m, implNode, S3StoreSchema.MEM_TABLE_SIZE, this::setMemTableSize); + parseLong(m, implNode, S3StoreSchema.MEMORY_CACHE_SIZE, this::setMemoryCacheSize); + parseLong(m, implNode, S3StoreSchema.DISK_CACHE_SIZE, this::setDiskCacheSize); + parseString(m, implNode, S3StoreSchema.DISK_CACHE_PATH, this::setDiskCachePath); + parseString(m, implNode, S3StoreSchema.S3_BUCKET, this::setS3Bucket); + parseString(m, implNode, S3StoreSchema.S3_ENDPOINT, this::setS3Endpoint); + parseString(m, implNode, S3StoreSchema.S3_REGION, this::setS3Region); + parseString(m, implNode, S3StoreSchema.S3_PREFIX, this::setS3Prefix); + parseString(m, implNode, S3StoreSchema.S3_ACCESS_KEY, this::setS3AccessKey); + parseString(m, implNode, S3StoreSchema.S3_SECRET_KEY, this::setS3SecretKey); + Models.objectLiteral(m.getStatements(implNode, S3StoreSchema.S3_FORCE_PATH_STYLE, null)) + .ifPresent(lit -> setS3ForcePathStyle(lit.booleanValue())); + parseString(m, implNode, S3StoreSchema.DATA_DIR, this::setDataDir); + } catch (ModelException e) { + throw new SailConfigException(e.getMessage(), e); + } + } + + private static void parseString(Model m, Resource node, IRI prop, Consumer setter) { + Models.objectLiteral(m.getStatements(node, prop, null)) + .ifPresent(lit -> setter.accept(lit.getLabel())); + } + + private static void parseLong(Model m, Resource node, IRI prop, Consumer setter) { + Models.objectLiteral(m.getStatements(node, prop, null)) + .ifPresent(lit -> { + try { + setter.accept(lit.longValue()); + } catch (NumberFormatException e) { + throw new SailConfigException( + "Long value required for " + prop + " property, found " + lit); + } + }); + } + +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/config/S3StoreFactory.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/config/S3StoreFactory.java new file mode 100644 index 00000000000..ab043b9fd61 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/config/S3StoreFactory.java @@ -0,0 +1,56 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.config; + +import org.eclipse.rdf4j.sail.Sail; +import org.eclipse.rdf4j.sail.config.SailConfigException; +import org.eclipse.rdf4j.sail.config.SailFactory; +import org.eclipse.rdf4j.sail.config.SailImplConfig; +import org.eclipse.rdf4j.sail.s3.S3Store; + +/** + * A {@link SailFactory} that creates {@link S3Store}s based on RDF configuration data. + */ +public class S3StoreFactory implements SailFactory { + + /** + * The type of repositories that are created by this factory. + * + * @see SailFactory#getSailType() + */ + public static final String SAIL_TYPE = "rdf4j:S3Store"; + + /** + * Returns the Sail's type: rdf4j:S3Store. + */ + @Override + public String getSailType() { + return SAIL_TYPE; + } + + @Override + public SailImplConfig getConfig() { + return new S3StoreConfig(); + } + + @Override + public Sail getSail(SailImplConfig config) throws SailConfigException { + if (!SAIL_TYPE.equals(config.getType())) { + throw new SailConfigException("Invalid Sail type: " + config.getType()); + } + + if (config instanceof S3StoreConfig) { + return new S3Store((S3StoreConfig) config); + } + throw new SailConfigException( + "Expected S3StoreConfig but got " + config.getClass().getName()); + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/config/S3StoreSchema.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/config/S3StoreSchema.java new file mode 100644 index 00000000000..27420f9dad0 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/config/S3StoreSchema.java @@ -0,0 +1,81 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.config; + +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.ValueFactory; +import org.eclipse.rdf4j.model.impl.SimpleValueFactory; + +/** + * Defines constants for the S3Store schema which is used by {@link S3StoreFactory}s to initialize S3Stores. + */ +public class S3StoreSchema { + + /** + * The S3Store schema namespace (http://rdf4j.org/config/sail/s3#). + */ + public static final String NAMESPACE = "http://rdf4j.org/config/sail/s3#"; + + /** + * http://rdf4j.org/config/sail/s3#memTableSize + */ + public final static IRI MEM_TABLE_SIZE; + + /** + * http://rdf4j.org/config/sail/s3#memoryCacheSize + */ + public final static IRI MEMORY_CACHE_SIZE; + + /** + * http://rdf4j.org/config/sail/s3#diskCacheSize + */ + public final static IRI DISK_CACHE_SIZE; + + /** + * http://rdf4j.org/config/sail/s3#diskCachePath + */ + public final static IRI DISK_CACHE_PATH; + + public final static IRI S3_BUCKET; + + public final static IRI S3_ENDPOINT; + + public final static IRI S3_REGION; + + public final static IRI S3_PREFIX; + + public final static IRI S3_ACCESS_KEY; + + public final static IRI S3_SECRET_KEY; + + public final static IRI S3_FORCE_PATH_STYLE; + + /** + * http://rdf4j.org/config/sail/s3#dataDir + */ + public final static IRI DATA_DIR; + + static { + ValueFactory factory = SimpleValueFactory.getInstance(); + MEM_TABLE_SIZE = factory.createIRI(NAMESPACE, "memTableSize"); + MEMORY_CACHE_SIZE = factory.createIRI(NAMESPACE, "memoryCacheSize"); + DISK_CACHE_SIZE = factory.createIRI(NAMESPACE, "diskCacheSize"); + DISK_CACHE_PATH = factory.createIRI(NAMESPACE, "diskCachePath"); + S3_BUCKET = factory.createIRI(NAMESPACE, "s3Bucket"); + S3_ENDPOINT = factory.createIRI(NAMESPACE, "s3Endpoint"); + S3_REGION = factory.createIRI(NAMESPACE, "s3Region"); + S3_PREFIX = factory.createIRI(NAMESPACE, "s3Prefix"); + S3_ACCESS_KEY = factory.createIRI(NAMESPACE, "s3AccessKey"); + S3_SECRET_KEY = factory.createIRI(NAMESPACE, "s3SecretKey"); + S3_FORCE_PATH_STYLE = factory.createIRI(NAMESPACE, "s3ForcePathStyle"); + DATA_DIR = factory.createIRI(NAMESPACE, "dataDir"); + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/BloomFilter.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/BloomFilter.java new file mode 100644 index 00000000000..7a92af97964 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/BloomFilter.java @@ -0,0 +1,180 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.List; + +/** + * A simple bit-array bloom filter for long values. Uses two independent hash functions derived from a single + * murmur3-style hash to set/test bits. + * + *

+ * Each Parquet file gets one bloom filter keyed on the leading component of the file's sort order (e.g. subject + * IDs for SPOC files, object IDs for OPSC files, context IDs for CSPO files). + *

+ */ +public final class BloomFilter { + + private static final int MIN_BITS = 64; + private final long[] bits; + private final int numBits; + private final int numHashFunctions; + + /** + * Creates a bloom filter sized for the expected number of insertions and false positive probability. + * + * @param expectedInsertions expected number of distinct elements + * @param fpp desired false positive probability (e.g. 0.01 for 1%) + */ + public BloomFilter(int expectedInsertions, double fpp) { + if (expectedInsertions <= 0) { + expectedInsertions = 1; + } + this.numBits = Math.max(MIN_BITS, optimalNumBits(expectedInsertions, fpp)); + this.numHashFunctions = optimalNumHashFunctions(expectedInsertions, numBits); + this.bits = new long[(numBits + 63) >>> 6]; + } + + private BloomFilter(long[] bits, int numBits, int numHashFunctions) { + this.bits = bits; + this.numBits = numBits; + this.numHashFunctions = numHashFunctions; + } + + /** + * Adds a value to the bloom filter. + */ + public void add(long value) { + long hash1 = murmurHash(value); + long hash2 = murmurHash(value ^ 0x9E3779B97F4A7C15L); + for (int i = 0; i < numHashFunctions; i++) { + int bit = (int) (((hash1 + (long) i * hash2) & Long.MAX_VALUE) % numBits); + bits[bit >>> 6] |= 1L << (bit & 63); + } + } + + /** + * Tests whether a value might be in the set. + * + * @return {@code true} if the value might be present; {@code false} if it is definitely not present + */ + public boolean mightContain(long value) { + long hash1 = murmurHash(value); + long hash2 = murmurHash(value ^ 0x9E3779B97F4A7C15L); + for (int i = 0; i < numHashFunctions; i++) { + int bit = (int) (((hash1 + (long) i * hash2) & Long.MAX_VALUE) % numBits); + if ((bits[bit >>> 6] & (1L << (bit & 63))) == 0) { + return false; + } + } + return true; + } + + /** + * Builds a bloom filter for the leading component of the given sort order. + */ + public static BloomFilter buildForEntries(List entries, String sortSuffix) { + BloomFilter bloom = new BloomFilter(Math.max(1, entries.size()), 0.01); + for (QuadEntry entry : entries) { + bloom.add(leadingComponent(entry, sortSuffix)); + } + return bloom; + } + + /** + * Extracts the leading component value from a quad entry based on the sort order suffix. + */ + static long leadingComponent(QuadEntry entry, String sortSuffix) { + switch (sortSuffix.charAt(0)) { + case 's': + return entry.subject; + case 'o': + return entry.object; + case 'c': + return entry.context; + case 'p': + return entry.predicate; + default: + return entry.subject; + } + } + + /** + * Extracts the leading component value from raw quad IDs based on the sort order suffix. + */ + static long leadingComponent(long s, long p, long o, long c, String sortOrder) { + if (sortOrder == null) { + return -1; + } + switch (sortOrder.charAt(0)) { + case 's': + return s; + case 'p': + return p; + case 'o': + return o; + case 'c': + return c; + default: + return -1; + } + } + + /** + * Serializes this bloom filter to a Base64-encoded string for JSON storage. + */ + public String toBase64() { + // Format: [numBits (4 bytes)] [numHashFunctions (4 bytes)] [bits array (8 bytes each)] + ByteBuffer buf = ByteBuffer.allocate(8 + bits.length * 8); + buf.putInt(numBits); + buf.putInt(numHashFunctions); + for (long word : bits) { + buf.putLong(word); + } + return Base64.getEncoder().encodeToString(buf.array()); + } + + /** + * Deserializes a bloom filter from a Base64-encoded string. + */ + public static BloomFilter fromBase64(String encoded) { + ByteBuffer buf = ByteBuffer.wrap(Base64.getDecoder().decode(encoded)); + int numBits = buf.getInt(); + int numHash = buf.getInt(); + int arrayLen = buf.remaining() / 8; + long[] bits = new long[arrayLen]; + for (int i = 0; i < arrayLen; i++) { + bits[i] = buf.getLong(); + } + return new BloomFilter(bits, numBits, numHash); + } + + private static long murmurHash(long value) { + long h = value; + h ^= h >>> 33; + h *= 0xFF51AFD7ED558CCDL; + h ^= h >>> 33; + h *= 0xC4CEB9FE1A85EC53L; + h ^= h >>> 33; + return h; + } + + private static int optimalNumBits(int n, double fpp) { + return (int) (-n * Math.log(fpp) / (Math.log(2) * Math.log(2))); + } + + private static int optimalNumHashFunctions(int n, int m) { + return Math.max(1, (int) Math.round((double) m / n * Math.log(2))); + } + +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ByteArrayInputFile.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ByteArrayInputFile.java new file mode 100644 index 00000000000..355dbb544df --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ByteArrayInputFile.java @@ -0,0 +1,142 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.parquet.io.InputFile; +import org.apache.parquet.io.SeekableInputStream; + +/** + * An {@link InputFile} implementation that reads Parquet data from an in-memory byte array. This avoids any dependency + * on Hadoop's file system abstraction. + */ +public class ByteArrayInputFile implements InputFile { + + private final byte[] data; + + /** + * Creates a new input file backed by the given byte array. + * + * @param data the Parquet file content + */ + public ByteArrayInputFile(byte[] data) { + this.data = data; + } + + @Override + public long getLength() { + return data.length; + } + + @Override + public SeekableInputStream newStream() { + return new ByteArraySeekableInputStream(data); + } + + /** + * A {@link SeekableInputStream} backed by a byte array. + */ + private static class ByteArraySeekableInputStream extends SeekableInputStream { + + private final byte[] data; + private int pos; + + ByteArraySeekableInputStream(byte[] data) { + this.data = data; + this.pos = 0; + } + + @Override + public int read() throws IOException { + if (pos >= data.length) { + return -1; + } + return data[pos++] & 0xFF; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (pos >= data.length) { + return -1; + } + int available = data.length - pos; + int toRead = Math.min(len, available); + System.arraycopy(data, pos, b, off, toRead); + pos += toRead; + return toRead; + } + + @Override + public long getPos() throws IOException { + return pos; + } + + @Override + public void seek(long newPos) throws IOException { + if (newPos < 0 || newPos > data.length) { + throw new IOException("Seek position " + newPos + " is out of range [0, " + data.length + "]"); + } + this.pos = (int) newPos; + } + + @Override + public void readFully(byte[] bytes) throws IOException { + readFully(bytes, 0, bytes.length); + } + + @Override + public void readFully(byte[] bytes, int start, int len) throws IOException { + int available = data.length - pos; + if (available < len) { + throw new EOFException( + "Reached end of stream: needed " + len + " bytes but only " + available + " available"); + } + System.arraycopy(data, pos, bytes, start, len); + pos += len; + } + + @Override + public int read(ByteBuffer buf) throws IOException { + int len = buf.remaining(); + if (len == 0) { + return 0; + } + int available = data.length - pos; + if (available <= 0) { + return -1; + } + int toRead = Math.min(len, available); + buf.put(data, pos, toRead); + pos += toRead; + return toRead; + } + + @Override + public void readFully(ByteBuffer buf) throws IOException { + int len = buf.remaining(); + int available = data.length - pos; + if (available < len) { + throw new EOFException( + "Reached end of stream: needed " + len + " bytes but only " + available + " available"); + } + buf.put(data, pos, len); + pos += len; + } + + @Override + public int available() { + return data.length - pos; + } + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ByteArrayOutputFile.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ByteArrayOutputFile.java new file mode 100644 index 00000000000..f2fca2287fc --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ByteArrayOutputFile.java @@ -0,0 +1,105 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import org.apache.parquet.io.OutputFile; +import org.apache.parquet.io.PositionOutputStream; + +/** + * An {@link OutputFile} implementation that writes Parquet data to an in-memory byte array. This avoids any dependency + * on Hadoop's file system abstraction. + * + *

+ * After writing is complete, call {@link #toByteArray()} to retrieve the serialized Parquet bytes. + */ +public class ByteArrayOutputFile implements OutputFile { + + private ByteArrayOutputStream baos; + + @Override + public PositionOutputStream create(long blockSizeHint) throws IOException { + baos = new ByteArrayOutputStream(); + return new ByteArrayPositionOutputStream(baos); + } + + @Override + public PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException { + return create(blockSizeHint); + } + + @Override + public boolean supportsBlockSize() { + return false; + } + + @Override + public long defaultBlockSize() { + return 0; + } + + /** + * Returns the bytes written to this output file. + * + * @return the Parquet file content as a byte array + * @throws IllegalStateException if no data has been written yet + */ + public byte[] toByteArray() { + if (baos == null) { + throw new IllegalStateException("No data has been written"); + } + return baos.toByteArray(); + } + + /** + * A {@link PositionOutputStream} backed by a {@link ByteArrayOutputStream}. + */ + private static class ByteArrayPositionOutputStream extends PositionOutputStream { + + private final ByteArrayOutputStream baos; + + ByteArrayPositionOutputStream(ByteArrayOutputStream baos) { + this.baos = baos; + } + + @Override + public long getPos() { + return baos.size(); + } + + @Override + public void write(int b) throws IOException { + baos.write(b); + } + + @Override + public void write(byte[] b) throws IOException { + baos.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + baos.write(b, off, len); + } + + @Override + public void flush() throws IOException { + baos.flush(); + } + + @Override + public void close() throws IOException { + baos.close(); + } + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/Catalog.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/Catalog.java new file mode 100644 index 00000000000..64f29ab510a --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/Catalog.java @@ -0,0 +1,447 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * JSON-serialized catalog tracking Parquet files with per-file statistics. + * + *

+ * All quads are stored in flat files (no predicate partitioning). Each file has statistics including min/max for all + * four quad components (subject, predicate, object, context). + * + *

S3 Layout

+ * + * 
+ * catalog/current             -> plain text "v{epoch}.json"
+ * catalog/v{epoch}.json       -> JSON catalog
+ *
+ * + *

JSON Structure

+ * + * 
+ * {
+ *   "version": 3,
+ *   "epoch": 42,
+ *   "nextValueId": 12345,
+ *   "files": [ { file info... } ]
+ * }
+ *
+ */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class Catalog { + + static final String CATALOG_POINTER_KEY = "catalog/current"; + private static final String CATALOG_DIR = "catalog/"; + + @JsonProperty("version") + private int version = 3; + + @JsonProperty("epoch") + private long epoch; + + @JsonProperty("nextValueId") + private long nextValueId; + + @JsonProperty("files") + private volatile List files = new ArrayList<>(); + + public Catalog() { + } + + public int getVersion() { + return version; + } + + public void setVersion(int version) { + this.version = version; + } + + public long getEpoch() { + return epoch; + } + + public void setEpoch(long epoch) { + this.epoch = epoch; + } + + public long getNextValueId() { + return nextValueId; + } + + public void setNextValueId(long nextValueId) { + this.nextValueId = nextValueId; + } + + public List getFiles() { + return files; + } + + public void setFiles(List files) { + this.files = new ArrayList<>(files); + } + + /** + * Loads the catalog from the object store. + * + *

+ * Reads the {@code catalog/current} pointer to find the active catalog version, then parses the corresponding JSON + * file. Returns an empty catalog if no pointer or catalog file exists. + * + * @param store the object store to read from + * @param mapper the Jackson ObjectMapper for JSON parsing + * @return the loaded catalog, or an empty catalog if none exists + */ + public static Catalog load(ObjectStore store, ObjectMapper mapper) { + byte[] pointer = store.get(CATALOG_POINTER_KEY); + if (pointer == null) { + return new Catalog(); + } + String catalogKey = CATALOG_DIR + new String(pointer, StandardCharsets.UTF_8).trim(); + byte[] json = store.get(catalogKey); + if (json == null) { + return new Catalog(); + } + try { + return mapper.readValue(json, Catalog.class); + } catch (IOException e) { + throw new UncheckedIOException("Failed to parse catalog", e); + } + } + + /** + * Saves this catalog to the object store. + * + *

+ * Writes the catalog JSON to {@code catalog/v{epoch}.json} and updates the {@code catalog/current} pointer. The + * epoch field is set to the given value before saving. + * + * @param store the object store to write to + * @param mapper the Jackson ObjectMapper for JSON serialization + * @param epoch the epoch number for this catalog version + */ + public void save(ObjectStore store, ObjectMapper mapper, long epoch) { + this.epoch = epoch; + try { + String versionedKey = "v" + epoch + ".json"; + byte[] json = mapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(this); + store.put(CATALOG_DIR + versionedKey, json); + store.put(CATALOG_POINTER_KEY, versionedKey.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new UncheckedIOException("Failed to save catalog", e); + } + } + + /** + * Adds a Parquet file to the catalog. Copy-on-write for thread safety. + * + * @param info the file info to add + */ + public void addFile(ParquetFileInfo info) { + List updated = new ArrayList<>(files); + updated.add(info); + files = updated; + } + + /** + * Removes Parquet files by their S3 keys. Copy-on-write for thread safety. + * + * @param s3Keys the set of S3 keys to remove + */ + public void removeFiles(Set s3Keys) { + List updated = new ArrayList<>(files); + updated.removeIf(f -> s3Keys.contains(f.getS3Key())); + files = updated; + } + + /** + * Returns all files for the given sort order. Reads from a volatile snapshot so it is safe to call without external + * synchronization. + * + * @param sortOrder the sort order suffix (e.g. "spoc", "opsc", "cspo") + * @return list of files matching the sort order + */ + public List getFilesForSortOrder(String sortOrder) { + List snapshot = files; + List result = new ArrayList<>(); + for (ParquetFileInfo f : snapshot) { + if (sortOrder.equals(f.getSortOrder())) { + result.add(f); + } + } + return result; + } + + /** + * Generates the S3 key for a data file at the given level, epoch, and sort suffix. + */ + public static String dataKey(int level, long epoch, String sortSuffix) { + return "data/L" + level + "-" + String.format("%05d", epoch) + "-" + sortSuffix + ".parquet"; + } + + /** + * Metadata about a single Parquet file in the catalog, including its location, sort order, size, and min/max + * statistics for subject, predicate, object, and context columns. + */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class ParquetFileInfo { + + @JsonProperty("s3Key") + private String s3Key; + + @JsonProperty("level") + private int level; + + @JsonProperty("sortOrder") + private String sortOrder; + + @JsonProperty("rowCount") + private long rowCount; + + @JsonProperty("epoch") + private long epoch; + + @JsonProperty("sizeBytes") + private long sizeBytes; + + @JsonProperty("minSubject") + private long minSubject; + + @JsonProperty("maxSubject") + private long maxSubject; + + @JsonProperty("minPredicate") + private long minPredicate; + + @JsonProperty("maxPredicate") + private long maxPredicate; + + @JsonProperty("minObject") + private long minObject; + + @JsonProperty("maxObject") + private long maxObject; + + @JsonProperty("minContext") + private long minContext; + + @JsonProperty("maxContext") + private long maxContext; + + @JsonProperty("bloomFilter") + private String bloomFilterBase64; + + private transient BloomFilter bloomFilter; + + public ParquetFileInfo() { + } + + public ParquetFileInfo(String s3Key, int level, String sortOrder, long rowCount, + long epoch, long sizeBytes, QuadStats stats) { + this(s3Key, level, sortOrder, rowCount, epoch, sizeBytes, + stats.minSubject, stats.maxSubject, stats.minPredicate, stats.maxPredicate, + stats.minObject, stats.maxObject, stats.minContext, stats.maxContext); + } + + public ParquetFileInfo(String s3Key, int level, String sortOrder, long rowCount, + long epoch, long sizeBytes, QuadStats stats, BloomFilter bloomFilter) { + this(s3Key, level, sortOrder, rowCount, epoch, sizeBytes, stats); + setBloomFilter(bloomFilter); + } + + ParquetFileInfo(String s3Key, int level, String sortOrder, long rowCount, + long epoch, long sizeBytes, + long minSubject, long maxSubject, + long minPredicate, long maxPredicate, + long minObject, long maxObject, + long minContext, long maxContext) { + this.s3Key = s3Key; + this.level = level; + this.sortOrder = sortOrder; + this.rowCount = rowCount; + this.epoch = epoch; + this.sizeBytes = sizeBytes; + this.minSubject = minSubject; + this.maxSubject = maxSubject; + this.minPredicate = minPredicate; + this.maxPredicate = maxPredicate; + this.minObject = minObject; + this.maxObject = maxObject; + this.minContext = minContext; + this.maxContext = maxContext; + } + + public String getS3Key() { + return s3Key; + } + + public void setS3Key(String s3Key) { + this.s3Key = s3Key; + } + + public int getLevel() { + return level; + } + + public void setLevel(int level) { + this.level = level; + } + + public String getSortOrder() { + return sortOrder; + } + + public void setSortOrder(String sortOrder) { + this.sortOrder = sortOrder; + } + + public long getRowCount() { + return rowCount; + } + + public void setRowCount(long rowCount) { + this.rowCount = rowCount; + } + + public long getEpoch() { + return epoch; + } + + public void setEpoch(long epoch) { + this.epoch = epoch; + } + + public long getSizeBytes() { + return sizeBytes; + } + + public void setSizeBytes(long sizeBytes) { + this.sizeBytes = sizeBytes; + } + + public long getMinSubject() { + return minSubject; + } + + public void setMinSubject(long minSubject) { + this.minSubject = minSubject; + } + + public long getMaxSubject() { + return maxSubject; + } + + public void setMaxSubject(long maxSubject) { + this.maxSubject = maxSubject; + } + + public long getMinPredicate() { + return minPredicate; + } + + public void setMinPredicate(long minPredicate) { + this.minPredicate = minPredicate; + } + + public long getMaxPredicate() { + return maxPredicate; + } + + public void setMaxPredicate(long maxPredicate) { + this.maxPredicate = maxPredicate; + } + + public long getMinObject() { + return minObject; + } + + public void setMinObject(long minObject) { + this.minObject = minObject; + } + + public long getMaxObject() { + return maxObject; + } + + public void setMaxObject(long maxObject) { + this.maxObject = maxObject; + } + + public long getMinContext() { + return minContext; + } + + public void setMinContext(long minContext) { + this.minContext = minContext; + } + + public long getMaxContext() { + return maxContext; + } + + public void setMaxContext(long maxContext) { + this.maxContext = maxContext; + } + + @JsonIgnore + public BloomFilter getBloomFilter() { + if (bloomFilter == null && bloomFilterBase64 != null) { + bloomFilter = BloomFilter.fromBase64(bloomFilterBase64); + } + return bloomFilter; + } + + public void setBloomFilter(BloomFilter filter) { + this.bloomFilter = filter; + this.bloomFilterBase64 = filter != null ? filter.toBase64() : null; + } + + /** + * Tests whether this file's statistics allow it to contain a quad matching the given pattern. Bound components + * (>= 0) are checked against the file's min/max range; unbound components (< 0) are wildcards. Also checks the + * bloom filter for the leading component if available. + */ + public boolean mayContain(long s, long p, long o, long c) { + if (s >= 0 && (s < minSubject || s > maxSubject)) { + return false; + } + if (p >= 0 && (p < minPredicate || p > maxPredicate)) { + return false; + } + if (o >= 0 && (o < minObject || o > maxObject)) { + return false; + } + if (c >= 0 && (c < minContext || c > maxContext)) { + return false; + } + // Check bloom filter for the leading component + BloomFilter bf = getBloomFilter(); + if (bf != null) { + long leadingVal = BloomFilter.leadingComponent(s, p, o, c, sortOrder); + if (leadingVal >= 0 && !bf.mightContain(leadingVal)) { + return false; + } + } + return true; + } + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/CompactionPolicy.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/CompactionPolicy.java new file mode 100644 index 00000000000..6b300a13174 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/CompactionPolicy.java @@ -0,0 +1,73 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Determines when compaction should be triggered. Counts distinct epochs at each level in the flat file catalog and + * compares against configurable thresholds. + */ +public class CompactionPolicy { + + /** Default number of L0 epochs before triggering L0→L1 compaction. */ + public static final int DEFAULT_L0_THRESHOLD = 8; + + /** Default number of L1 epochs before triggering L1→L2 compaction. */ + public static final int DEFAULT_L1_THRESHOLD = 4; + + private final int l0Threshold; + private final int l1Threshold; + + public CompactionPolicy() { + this(DEFAULT_L0_THRESHOLD, DEFAULT_L1_THRESHOLD); + } + + public CompactionPolicy(int l0Threshold, int l1Threshold) { + this.l0Threshold = l0Threshold; + this.l1Threshold = l1Threshold; + } + + /** + * Checks if compaction should run at the given level. + * + * @param files all catalog files + * @param level the source level (0 or 1) + * @return true if the number of distinct epochs at that level >= threshold + */ + public boolean shouldCompact(List files, int level) { + int threshold = level == 0 ? l0Threshold : l1Threshold; + return countEpochsAtLevel(files, level) >= threshold; + } + + private static int countEpochsAtLevel(List files, int level) { + Set epochs = new HashSet<>(); + for (Catalog.ParquetFileInfo f : files) { + if (f.getLevel() == level) { + epochs.add(f.getEpoch()); + } + } + return epochs.size(); + } + + /** + * Returns the files at the given level. + * + * @param files all catalog files + * @param level the target level (0, 1, or 2) + * @return files at that level + */ + public static List filesAtLevel(List files, int level) { + return files.stream().filter(f -> f.getLevel() == level).toList(); + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/Compactor.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/Compactor.java new file mode 100644 index 00000000000..81d551a36b2 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/Compactor.java @@ -0,0 +1,210 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.TreeMap; + +import org.eclipse.rdf4j.sail.s3.cache.TieredCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Performs merge compaction on Parquet files. Merges files at a source level into one set of files at the target level, + * per sort order. + * + *

    + *
  • L0→L1: merge all L0 files per sort order, tombstones preserved
    • + *
  • L1→L2: merge all L1 files per sort order, tombstones suppressed (L2 = highest level)
    • + *
+ */ +public class Compactor { + + private static final Logger logger = LoggerFactory.getLogger(Compactor.class); + private static final ParquetSchemas.SortOrder[] SORT_ORDERS = ParquetSchemas.SortOrder.values(); + + private final ObjectStore objectStore; + private final TieredCache cache; + + public Compactor(ObjectStore objectStore, TieredCache cache) { + this.objectStore = objectStore; + this.cache = cache; + } + + /** + * Compacts files at the source level into a single set of files at the target level. + * + * @param sourceFiles all files at the source level + * @param sourceLevel the source level (0 or 1) + * @param targetLevel the target level (1 or 2) + * @param epoch the epoch for the new compacted files + * @param catalog the catalog to update + * @return result containing new files created and old files removed + */ + public CompactionResult compact(List sourceFiles, + int sourceLevel, int targetLevel, long epoch, Catalog catalog) { + + boolean suppressTombstones = (targetLevel == 2); + List newFiles = new ArrayList<>(); + Set oldKeys = new HashSet<>(); + + for (ParquetSchemas.SortOrder sortOrder : SORT_ORDERS) { + String suffix = sortOrder.suffix(); + QuadIndex quadIndex = new QuadIndex(suffix); + + // Collect source files for this sort order, ordered newest-first (highest epoch first) + List sortOrderFiles = sourceFiles.stream() + .filter(f -> suffix.equals(f.getSortOrder())) + .sorted(Comparator.comparingLong(Catalog.ParquetFileInfo::getEpoch).reversed()) + .toList(); + + if (sortOrderFiles.isEmpty()) { + continue; + } + + // Collect old keys for cleanup + for (Catalog.ParquetFileInfo f : sortOrderFiles) { + oldKeys.add(f.getS3Key()); + } + + // Build merge sources from Parquet files (newest first) + List sources = new ArrayList<>(); + for (Catalog.ParquetFileInfo fileInfo : sortOrderFiles) { + byte[] fileData = cache != null ? cache.get(fileInfo.getS3Key()) : objectStore.get(fileInfo.getS3Key()); + if (fileData == null) { + logger.warn("Missing Parquet file during compaction: {}", fileInfo.getS3Key()); + continue; + } + sources.add(new ParquetQuadSource(fileData, quadIndex)); + } + + if (sources.isEmpty()) { + continue; + } + + // Merge and collect entries + List merged = mergeEntries(sources, quadIndex, suppressTombstones); + + if (merged.isEmpty()) { + continue; + } + + // Write merged Parquet file + String s3Key = Catalog.dataKey(targetLevel, epoch, suffix); + + byte[] parquetData = ParquetFileBuilder.build(merged, sortOrder); + + objectStore.put(s3Key, parquetData); + if (cache != null) { + cache.writeThrough(s3Key, parquetData); + } + + BloomFilter bloom = BloomFilter.buildForEntries(merged, suffix); + + QuadStats stats = QuadStats.fromEntries(merged); + newFiles.add(new Catalog.ParquetFileInfo(s3Key, targetLevel, suffix, merged.size(), + epoch, parquetData.length, stats, bloom)); + } + + // Update catalog in memory: remove old files, add new ones. + // Physical deletion of old files is deferred to the caller, after the catalog is saved, + // to prevent data loss if the process crashes between deletion and catalog save. + catalog.removeFiles(oldKeys); + for (Catalog.ParquetFileInfo newFile : newFiles) { + catalog.addFile(newFile); + } + + logger.info("Compacted L{}→L{}: {} files merged into {} files", + sourceLevel, targetLevel, oldKeys.size(), newFiles.size()); + + return new CompactionResult(newFiles, oldKeys); + } + + private List mergeEntries(List sources, QuadIndex quadIndex, + boolean suppressTombstones) { + // Sources are ordered newest-first, so for dedup, first occurrence wins + TreeMap deduped = new TreeMap<>(); + for (RawEntrySource source : sources) { + while (source.hasNext()) { + byte[] key = source.peekKey(); + byte flag = source.peekFlag(); + CompactKey ck = new CompactKey(key); + if (!deduped.containsKey(ck)) { + long[] quad = new long[4]; + quadIndex.keyToQuad(key, quad); + if (!suppressTombstones || flag != MemTable.FLAG_TOMBSTONE) { + deduped.put(ck, new QuadEntry( + quad[QuadIndex.SUBJ_IDX], quad[QuadIndex.PRED_IDX], + quad[QuadIndex.OBJ_IDX], quad[QuadIndex.CONTEXT_IDX], flag)); + } + } + source.advance(); + } + } + + return new ArrayList<>(deduped.values()); + } + + private static class CompactKey implements Comparable { + final byte[] key; + + CompactKey(byte[] key) { + this.key = key.clone(); + } + + @Override + public int compareTo(CompactKey other) { + return Arrays.compareUnsigned(this.key, other.key); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CompactKey)) { + return false; + } + return Arrays.equals(key, ((CompactKey) o).key); + } + + @Override + public int hashCode() { + return Arrays.hashCode(key); + } + } + + /** + * Result of a compaction operation. + */ + public static class CompactionResult { + private final List newFiles; + private final Set deletedKeys; + + public CompactionResult(List newFiles, Set deletedKeys) { + this.newFiles = newFiles; + this.deletedKeys = deletedKeys; + } + + public List getNewFiles() { + return newFiles; + } + + public Set getDeletedKeys() { + return deletedKeys; + } + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/FileSystemObjectStore.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/FileSystemObjectStore.java new file mode 100644 index 00000000000..fa4cb4e96c2 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/FileSystemObjectStore.java @@ -0,0 +1,125 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +/** + * {@link ObjectStore} implementation backed by the local filesystem. Stores each key as a file under the configured + * root directory, creating subdirectories as needed. + */ +public class FileSystemObjectStore implements ObjectStore { + + private final Path root; + + public FileSystemObjectStore(Path root) { + this.root = root; + } + + private Path resolve(String key) { + return root.resolve(key); + } + + @Override + public void put(String key, byte[] data) { + try { + Path target = resolve(key); + Files.createDirectories(target.getParent()); + // Atomic write via temp file + rename to prevent corrupt files on crash + Path tmp = target.resolveSibling(target.getFileName() + ".tmp"); + Files.write(tmp, data, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + Files.move(tmp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public byte[] get(String key) { + try { + Path target = resolve(key); + if (!Files.exists(target)) { + return null; + } + return Files.readAllBytes(target); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public byte[] getRange(String key, long offset, long length) { + Path target = resolve(key); + if (!Files.exists(target)) { + return null; + } + try (RandomAccessFile raf = new RandomAccessFile(target.toFile(), "r")) { + long fileLen = raf.length(); + int start = (int) Math.min(offset, fileLen); + int readLen = (int) Math.min(length, fileLen - start); + if (readLen <= 0) { + return new byte[0]; + } + raf.seek(start); + byte[] buf = new byte[readLen]; + raf.readFully(buf); + return buf; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void delete(String key) { + try { + Path target = resolve(key); + Files.deleteIfExists(target); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List list(String subPrefix) { + List result = new ArrayList<>(); + Path prefixPath = resolve(subPrefix); + Path searchDir = Files.isDirectory(prefixPath) ? prefixPath : prefixPath.getParent(); + if (searchDir == null || !Files.exists(searchDir)) { + return result; + } + try (Stream walk = Files.walk(searchDir)) { + walk.filter(Files::isRegularFile) + .forEach(p -> { + String relative = root.relativize(p).toString(); + if (relative.startsWith(subPrefix)) { + result.add(relative); + } + }); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return result; + } + + @Override + public void close() { + // no-op + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/MemTable.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/MemTable.java new file mode 100644 index 00000000000..3fc87d3ab5b --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/MemTable.java @@ -0,0 +1,387 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * In-memory sorted store for quads using a {@link ConcurrentSkipListMap}. Stores quads as varint-encoded byte[] keys + * (in the order defined by a {@link QuadIndex}) with a 1-byte flag value: + *
    + *
  • {@code 0x01} = explicit
    • + *
  • {@code 0x02} = inferred
    • + *
  • {@code 0x00} = tombstone (deleted)
    • + *
+ * + *

+ * The unsigned byte comparison on keys preserves the varint lexicographic ordering, which in turn preserves the numeric + * ordering of the encoded IDs. + *

+ */ +public class MemTable { + + public static final byte FLAG_TOMBSTONE = 0x00; + public static final byte FLAG_EXPLICIT = 0x01; + public static final byte FLAG_INFERRED = 0x02; + + private static final byte[] VALUE_EXPLICIT = new byte[] { FLAG_EXPLICIT }; + private static final byte[] VALUE_INFERRED = new byte[] { FLAG_INFERRED }; + private static final byte[] VALUE_TOMBSTONE = new byte[] { FLAG_TOMBSTONE }; + + /** Estimated overhead per skip-list entry: key array header + value array header + node overhead. */ + private static final int ENTRY_OVERHEAD = 16 + 16 + 64; + + private final QuadIndex index; + private final ConcurrentSkipListMap data; + private final AtomicBoolean frozen = new AtomicBoolean(false); + private final AtomicLong estimatedBytes = new AtomicLong(); + + /** + * Creates a new MemTable backed by the given index for key encoding. + * + * @param index the QuadIndex that determines key encoding order + */ + public MemTable(QuadIndex index) { + this.index = index; + this.data = new ConcurrentSkipListMap<>(Arrays::compareUnsigned); + } + + /** + * Stores a quad in the table. + * + * @param s subject ID + * @param p predicate ID + * @param o object ID + * @param c context ID + * @param explicit true for explicit, false for inferred + * @throws IllegalStateException if the table is frozen + */ + public void put(long s, long p, long o, long c, boolean explicit) { + checkNotFrozen(); + byte[] key = index.toKeyBytes(s, p, o, c); + byte[] prev = data.put(key, explicit ? VALUE_EXPLICIT : VALUE_INFERRED); + if (prev == null) { + estimatedBytes.addAndGet(key.length + 1L + ENTRY_OVERHEAD); + } + } + + /** + * Removes a quad by writing a tombstone. + * + * @param s subject ID + * @param p predicate ID + * @param o object ID + * @param c context ID + * @throws IllegalStateException if the table is frozen + */ + public void remove(long s, long p, long o, long c) { + checkNotFrozen(); + byte[] key = index.toKeyBytes(s, p, o, c); + byte[] prev = data.put(key, VALUE_TOMBSTONE); + if (prev == null) { + estimatedBytes.addAndGet(key.length + 1L + ENTRY_OVERHEAD); + } + } + + /** + * Checks if a quad exists (is not a tombstone). + * + * @param s subject ID + * @param p predicate ID + * @param o object ID + * @param c context ID + * @param explicit true to check explicit, false for inferred + * @return true if the quad exists with the matching flag + */ + public boolean get(long s, long p, long o, long c, boolean explicit) { + byte[] key = index.toKeyBytes(s, p, o, c); + byte[] value = data.get(key); + if (value == null || value[0] == FLAG_TOMBSTONE) { + return false; + } + return explicit ? value[0] == FLAG_EXPLICIT : value[0] == FLAG_INFERRED; + } + + /** + * Returns an iterator over matching quads using range scan. Bound components (>= 0) form a prefix; unbound + * components (-1) are wildcards. + * + * @param s subject ID, or -1 for wildcard + * @param p predicate ID, or -1 for wildcard + * @param o object ID, or -1 for wildcard + * @param c context ID, or -1 for wildcard + * @param explicit true for explicit, false for inferred + * @return an iterator over matching quads as long[4] arrays in SPOC order + */ + public Iterator scan(long s, long p, long o, long c, boolean explicit) { + byte[] minKey = index.getMinKeyBytes(s, p, o, c); + byte[] maxKey = index.getMaxKeyBytes(s, p, o, c); + byte expectedFlag = explicit ? FLAG_EXPLICIT : FLAG_INFERRED; + + ConcurrentNavigableMap range = data.subMap(minKey, true, maxKey, true); + + return new ScanIterator(range, index, expectedFlag, s, p, o, c); + } + + /** + * Returns the number of entries in the table (including tombstones). + */ + public int size() { + return data.size(); + } + + /** + * Returns a rough estimate of memory consumption in bytes. O(1) — maintained incrementally on put/remove. + */ + public long approximateSizeInBytes() { + return estimatedBytes.get(); + } + + /** + * Freezes this table in place, preventing further writes. Does not create a copy. + * + * @return this MemTable, now frozen + */ + public MemTable freeze() { + frozen.set(true); + return this; + } + + /** + * Returns whether this table is frozen (immutable). + */ + public boolean isFrozen() { + return frozen.get(); + } + + /** + * Clears all entries from the table. + * + * @throws IllegalStateException if the table is frozen + */ + public void clear() { + checkNotFrozen(); + data.clear(); + estimatedBytes.set(0); + } + + /** + * Returns the QuadIndex used by this table. + */ + public QuadIndex getIndex() { + return index; + } + + /** + * Returns an unmodifiable view of the underlying data map. + */ + public Map getData() { + return Collections.unmodifiableMap(data); + } + + /** + * Returns a {@link RawEntrySource} over the given key range using this table's native index. Includes tombstones + * (no flag filtering). Used by {@link MergeIterator}. + */ + public RawEntrySource asRawSource(long s, long p, long o, long c) { + byte[] minKey = index.getMinKeyBytes(s, p, o, c); + byte[] maxKey = index.getMaxKeyBytes(s, p, o, c); + ConcurrentNavigableMap range = data.subMap(minKey, true, maxKey, true); + return new RawSourceImpl(range); + } + + /** + * Returns a {@link RawEntrySource} with keys re-encoded in the specified target index order. When the target index + * matches this table's native index, delegates to {@link #asRawSource(long, long, long, long)} directly. + * + * @param targetIndex the desired key encoding order + * @param s subject filter, or -1 for wildcard + * @param p predicate filter, or -1 for wildcard + * @param o object filter, or -1 for wildcard + * @param c context filter, or -1 for wildcard + * @return a RawEntrySource with keys in the target index order + */ + public RawEntrySource asRawSource(QuadIndex targetIndex, long s, long p, long o, long c) { + if (targetIndex.getFieldSeqString().equals(index.getFieldSeqString())) { + return asRawSource(s, p, o, c); + } + + // Scan all matching entries, re-encode in target order, sort + byte[] minKey = index.getMinKeyBytes(s, p, o, c); + byte[] maxKey = index.getMaxKeyBytes(s, p, o, c); + ConcurrentNavigableMap range = data.subMap(minKey, true, maxKey, true); + + List entries = new ArrayList<>(); + long[] quad = new long[4]; + for (Map.Entry entry : range.entrySet()) { + index.keyToQuad(entry.getKey(), quad); + if (!QuadIndex.matches(quad, s, p, o, c)) { + continue; + } + byte[] newKey = targetIndex.toKeyBytes( + quad[QuadIndex.SUBJ_IDX], quad[QuadIndex.PRED_IDX], + quad[QuadIndex.OBJ_IDX], quad[QuadIndex.CONTEXT_IDX]); + entries.add(new ReorderedEntry(newKey, entry.getValue()[0])); + } + + entries.sort((a, b) -> Arrays.compareUnsigned(a.key, b.key)); + return new ReorderedRawSource(entries); + } + + private static class ReorderedEntry { + final byte[] key; + final byte flag; + + ReorderedEntry(byte[] key, byte flag) { + this.key = key; + this.flag = flag; + } + } + + private static class ReorderedRawSource implements RawEntrySource { + private final List entries; + private int pos; + + ReorderedRawSource(List entries) { + this.entries = entries; + this.pos = 0; + } + + @Override + public boolean hasNext() { + return pos < entries.size(); + } + + @Override + public byte[] peekKey() { + return entries.get(pos).key; + } + + @Override + public byte peekFlag() { + return entries.get(pos).flag; + } + + @Override + public void advance() { + pos++; + } + } + + private static class RawSourceImpl implements RawEntrySource { + private final Iterator> delegate; + private Map.Entry current; + + RawSourceImpl(ConcurrentNavigableMap range) { + this.delegate = range.entrySet().iterator(); + if (delegate.hasNext()) { + current = delegate.next(); + } + } + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public byte[] peekKey() { + return current.getKey(); + } + + @Override + public byte peekFlag() { + return current.getValue()[0]; + } + + @Override + public void advance() { + if (delegate.hasNext()) { + current = delegate.next(); + } else { + current = null; + } + } + } + + private void checkNotFrozen() { + if (frozen.get()) { + throw new IllegalStateException("MemTable is frozen and cannot accept writes"); + } + } + + /** + * Iterator that filters range scan results by flag value and pattern match. Skips tombstones and entries where + * bound components don't match the query pattern. Returns quads in SPOC order. + */ + private static class ScanIterator implements Iterator { + private final Iterator> delegate; + private final QuadIndex quadIndex; + private final byte expectedFlag; + private final long patternS, patternP, patternO, patternC; + private long[] next; + + ScanIterator(ConcurrentNavigableMap range, QuadIndex quadIndex, byte expectedFlag, + long s, long p, long o, long c) { + this.delegate = range.entrySet().iterator(); + this.quadIndex = quadIndex; + this.expectedFlag = expectedFlag; + this.patternS = s; + this.patternP = p; + this.patternO = o; + this.patternC = c; + advance(); + } + + private void advance() { + next = null; + while (delegate.hasNext()) { + Map.Entry entry = delegate.next(); + byte flag = entry.getValue()[0]; + if (flag != expectedFlag) { + continue; + } + long[] quad = new long[4]; + quadIndex.keyToQuad(entry.getKey(), quad); + if (!QuadIndex.matches(quad, patternS, patternP, patternO, patternC)) { + continue; + } + next = quad; + return; + } + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public long[] next() { + if (next == null) { + throw new NoSuchElementException(); + } + long[] result = next; + advance(); + return result; + } + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/MergeIterator.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/MergeIterator.java new file mode 100644 index 00000000000..7b64d529919 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/MergeIterator.java @@ -0,0 +1,151 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +/** + * K-way merge iterator over multiple {@link RawEntrySource} instances ordered newest-to-oldest. Deduplicates entries + * with the same key (newest wins), suppresses tombstones, and filters by expected flag and pattern match. + */ +public class MergeIterator implements Iterator { + + private final QuadIndex quadIndex; + private final byte expectedFlag; + private final long patternS, patternP, patternO, patternC; + private final PriorityQueue heap; + private final List allSources; + private long[] next; + + /** + * @param sources list of sources ordered newest-to-oldest (index 0 = newest) + * @param quadIndex the quad index for decoding keys + * @param expectedFlag the flag to match (FLAG_EXPLICIT or FLAG_INFERRED) + * @param s subject pattern, or -1 for wildcard + * @param p predicate pattern, or -1 for wildcard + * @param o object pattern, or -1 for wildcard + * @param c context pattern, or -1 for wildcard + */ + public MergeIterator(List sources, QuadIndex quadIndex, byte expectedFlag, + long s, long p, long o, long c) { + this.quadIndex = quadIndex; + this.expectedFlag = expectedFlag; + this.patternS = s; + this.patternP = p; + this.patternO = o; + this.patternC = c; + this.heap = new PriorityQueue<>(); + this.allSources = sources; + + for (int i = 0; i < sources.size(); i++) { + RawEntrySource src = sources.get(i); + if (src.hasNext()) { + heap.add(new SourceCursor(src, i)); + } + } + + advance(); + } + + private void advance() { + next = null; + while (!heap.isEmpty()) { + // Pop minimum key + SourceCursor min = heap.poll(); + byte[] winningKey = min.source.peekKey().clone(); + byte winningFlag = min.source.peekFlag(); + + // Advance the winning source + min.source.advance(); + if (min.source.hasNext()) { + heap.add(min); + } + + // Drain all sources with the same key (deduplication) + while (!heap.isEmpty() && Arrays.compareUnsigned(heap.peek().source.peekKey(), winningKey) == 0) { + SourceCursor dup = heap.poll(); + dup.source.advance(); + if (dup.source.hasNext()) { + heap.add(dup); + } + } + + // Tombstone suppression + if (winningFlag == MemTable.FLAG_TOMBSTONE) { + continue; + } + + // Flag filter + if (winningFlag != expectedFlag) { + continue; + } + + // Decode key and verify pattern + long[] quad = new long[4]; + quadIndex.keyToQuad(winningKey, quad); + + if (!QuadIndex.matches(quad, patternS, patternP, patternO, patternC)) { + continue; + } + + next = quad; + return; + } + + // Heap exhausted — close all sources + closeSources(); + } + + private void closeSources() { + for (RawEntrySource source : allSources) { + source.close(); + } + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public long[] next() { + if (next == null) { + throw new NoSuchElementException(); + } + long[] result = next; + advance(); + return result; + } + + private static class SourceCursor implements Comparable { + final RawEntrySource source; + final int sourceIndex; // lower = newer + + SourceCursor(RawEntrySource source, int sourceIndex) { + this.source = source; + this.sourceIndex = sourceIndex; + } + + @Override + public int compareTo(SourceCursor other) { + int keyCmp = Arrays.compareUnsigned(this.source.peekKey(), other.source.peekKey()); + if (keyCmp != 0) { + return keyCmp; + } + // Ties broken by source index: lower = newer = wins (poll first) + return Integer.compare(this.sourceIndex, other.sourceIndex); + } + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ObjectStore.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ObjectStore.java new file mode 100644 index 00000000000..b62924d4554 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ObjectStore.java @@ -0,0 +1,30 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.io.Closeable; +import java.util.List; + +/** + * Abstraction over object storage (S3-compatible or filesystem). + */ +public interface ObjectStore extends Closeable { + + void put(String key, byte[] data); + + byte[] get(String key); + + byte[] getRange(String key, long offset, long length); + + void delete(String key); + + List list(String subPrefix); +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ParquetFileBuilder.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ParquetFileBuilder.java new file mode 100644 index 00000000000..330c441c908 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ParquetFileBuilder.java @@ -0,0 +1,200 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.conf.ParquetConfiguration; +import org.apache.parquet.conf.PlainParquetConfiguration; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.api.WriteSupport; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.io.OutputFile; +import org.apache.parquet.io.api.RecordConsumer; +import org.apache.parquet.schema.MessageType; + +/** + * Writes quad entries to a Parquet file in memory and returns the serialized bytes. + * + *

+ * This builder uses Parquet's {@link OutputFile} API to avoid Hadoop filesystem dependencies. Entries should already be + * sorted by the caller according to the specified {@link ParquetSchemas.SortOrder}. + * + *

+ * Example usage: + * + * 

+ * List<QuadEntry> entries = ...;
+ * byte[] parquetBytes = ParquetFileBuilder.build(entries, SortOrder.SPOC);
+ *
+ */ +public final class ParquetFileBuilder { + + /** Default row group size: 8 MiB. */ + private static final int DEFAULT_ROW_GROUP_SIZE = 8 * 1024 * 1024; + + /** Default page size: 64 KiB. */ + private static final int DEFAULT_PAGE_SIZE = 64 * 1024; + + private ParquetFileBuilder() { + // utility class + } + + /** + * Builds a Parquet file from the given entries using default settings. + * + *

+ * Uses {@link ParquetSchemas#QUAD_SCHEMA}, 8 MiB row group size, and 64 KiB page size. + * + * @param entries the quad entries to write (must already be sorted) + * @param sortOrder the sort order of the entries + * @return the serialized Parquet file as a byte array + */ + public static byte[] build(List entries, ParquetSchemas.SortOrder sortOrder) { + return build(entries, ParquetSchemas.QUAD_SCHEMA, sortOrder, + DEFAULT_ROW_GROUP_SIZE, DEFAULT_PAGE_SIZE); + } + + /** + * Builds a Parquet file from the given entries with full control over parameters. + * + * @param entries the quad entries to write (must already be sorted) + * @param schema the Parquet schema to use + * @param sortOrder the sort order of the entries + * @param rowGroupSize the row group size in bytes + * @param pageSize the page size in bytes + * @return the serialized Parquet file as a byte array + */ + public static byte[] build(List entries, MessageType schema, + ParquetSchemas.SortOrder sortOrder, + int rowGroupSize, int pageSize) { + try { + ByteArrayOutputFile outputFile = new ByteArrayOutputFile(); + + try (ParquetWriter writer = new QuadEntryWriterBuilder(outputFile, schema) + .withConf(new PlainParquetConfiguration()) + .withCodecFactory(SimpleCodecFactory.INSTANCE) + .withCompressionCodec(CompressionCodecName.ZSTD) + .withRowGroupSize(rowGroupSize) + .withPageSize(pageSize) + .withDictionaryEncoding(true) + .build()) { + for (QuadEntry entry : entries) { + writer.write(entry); + } + } + + return outputFile.toByteArray(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to build Parquet file", e); + } + } + + /** + * Custom {@link WriteSupport} that writes {@link QuadEntry} records to Parquet. + */ + private static class QuadEntryWriteSupport extends WriteSupport { + + private final MessageType schema; + private RecordConsumer recordConsumer; + + QuadEntryWriteSupport(MessageType schema) { + this.schema = schema; + } + + @Override + public WriteContext init(Configuration configuration) { + return new WriteContext(schema, new HashMap<>()); + } + + @Override + public WriteContext init(ParquetConfiguration configuration) { + return new WriteContext(schema, new HashMap<>()); + } + + @Override + public void prepareForWrite(RecordConsumer recordConsumer) { + this.recordConsumer = recordConsumer; + } + + @Override + public void write(QuadEntry entry) { + recordConsumer.startMessage(); + + int fieldIndex = 0; + + // subject + recordConsumer.startField(ParquetSchemas.COL_SUBJECT, fieldIndex); + recordConsumer.addLong(entry.subject); + recordConsumer.endField(ParquetSchemas.COL_SUBJECT, fieldIndex); + fieldIndex++; + + // predicate + recordConsumer.startField(ParquetSchemas.COL_PREDICATE, fieldIndex); + recordConsumer.addLong(entry.predicate); + recordConsumer.endField(ParquetSchemas.COL_PREDICATE, fieldIndex); + fieldIndex++; + + // object + recordConsumer.startField(ParquetSchemas.COL_OBJECT, fieldIndex); + recordConsumer.addLong(entry.object); + recordConsumer.endField(ParquetSchemas.COL_OBJECT, fieldIndex); + fieldIndex++; + + // context + recordConsumer.startField(ParquetSchemas.COL_CONTEXT, fieldIndex); + recordConsumer.addLong(entry.context); + recordConsumer.endField(ParquetSchemas.COL_CONTEXT, fieldIndex); + fieldIndex++; + + // flag + recordConsumer.startField(ParquetSchemas.COL_FLAG, fieldIndex); + recordConsumer.addInteger(entry.flag); + recordConsumer.endField(ParquetSchemas.COL_FLAG, fieldIndex); + + recordConsumer.endMessage(); + } + } + + /** + * Builder for creating a {@link ParquetWriter} that writes {@link QuadEntry} records. Uses + * {@link PlainParquetConfiguration} to avoid Hadoop runtime dependencies. + */ + private static class QuadEntryWriterBuilder + extends ParquetWriter.Builder { + + private final MessageType schema; + + QuadEntryWriterBuilder(OutputFile file, MessageType schema) { + super(file); + this.schema = schema; + } + + @Override + protected QuadEntryWriterBuilder self() { + return this; + } + + @Override + protected WriteSupport getWriteSupport(Configuration conf) { + return new QuadEntryWriteSupport(schema); + } + + @Override + protected WriteSupport getWriteSupport(ParquetConfiguration conf) { + return new QuadEntryWriteSupport(schema); + } + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ParquetQuadSource.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ParquetQuadSource.java new file mode 100644 index 00000000000..aa3881090f1 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ParquetQuadSource.java @@ -0,0 +1,224 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; + +import org.apache.parquet.ParquetReadOptions; +import org.apache.parquet.column.page.PageReadStore; +import org.apache.parquet.column.statistics.LongStatistics; +import org.apache.parquet.column.statistics.Statistics; +import org.apache.parquet.conf.PlainParquetConfiguration; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.convert.GroupRecordConverter; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.io.ColumnIOFactory; +import org.apache.parquet.io.MessageColumnIO; +import org.apache.parquet.io.RecordReader; +import org.apache.parquet.schema.MessageType; + +/** + * A {@link RawEntrySource} that streams entries from an in-memory Parquet file. Entries are read one row group at a + * time, and rows within each row group are read lazily. + * + *

+ * The key format encodes all four quad components (s, p, o, c) as varints in the order defined by the + * {@link QuadIndex}. + *

+ */ +public class ParquetQuadSource implements RawEntrySource { + + private final ParquetFileReader reader; + private final MessageType schema; + private final MessageColumnIO columnIO; + private final QuadIndex quadIndex; + private final long filterS, filterP, filterO, filterC; + private final List rowGroups; + private int rowGroupIndex; + + private RecordReader recordReader; + private long remainingRows; + private byte[] nextKey; + private byte nextFlag; + private boolean closed; + + /** + * Creates a streaming source from Parquet file bytes. + */ + public ParquetQuadSource(byte[] parquetData, QuadIndex quadIndex) { + this(parquetData, quadIndex, -1, -1, -1, -1); + } + + /** + * Creates a streaming source from Parquet file bytes with filtering. + */ + public ParquetQuadSource(byte[] parquetData, QuadIndex quadIndex, + long subject, long predicate, long object, long context) { + this.quadIndex = quadIndex; + this.filterS = subject; + this.filterP = predicate; + this.filterO = object; + this.filterC = context; + + try { + ByteArrayInputFile inputFile = new ByteArrayInputFile(parquetData); + this.reader = ParquetFileReader.open(inputFile, + new ParquetReadOptions.Builder(new PlainParquetConfiguration()) + .withCodecFactory(SimpleCodecFactory.INSTANCE) + .build()); + this.schema = reader.getFooter().getFileMetaData().getSchema(); + this.columnIO = new ColumnIOFactory().getColumnIO(schema); + this.rowGroups = reader.getRowGroups(); + this.rowGroupIndex = 0; + this.remainingRows = 0; + this.recordReader = null; + + // Buffer the first matching entry + advanceToNext(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to open Parquet file for streaming", e); + } + } + + @Override + public boolean hasNext() { + return nextKey != null; + } + + @Override + public byte[] peekKey() { + return nextKey; + } + + @Override + public byte peekFlag() { + return nextFlag; + } + + @Override + public void advance() { + advanceToNext(); + } + + @Override + public void close() { + if (!closed) { + closed = true; + try { + reader.close(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to close Parquet reader", e); + } + } + } + + private void advanceToNext() { + nextKey = null; + try { + while (true) { + // Read rows from current row group + while (remainingRows > 0) { + remainingRows--; + Group group = recordReader.read(); + long subject = group.getLong(ParquetSchemas.COL_SUBJECT, 0); + long predicate = group.getLong(ParquetSchemas.COL_PREDICATE, 0); + long object = group.getLong(ParquetSchemas.COL_OBJECT, 0); + long context = group.getLong(ParquetSchemas.COL_CONTEXT, 0); + int flag = group.getInteger(ParquetSchemas.COL_FLAG, 0); + + long[] quad = { subject, predicate, object, context }; + if (!QuadIndex.matches(quad, filterS, filterP, filterO, filterC)) { + continue; + } + + nextKey = quadIndex.toKeyBytes(subject, predicate, object, context); + nextFlag = (byte) flag; + return; + } + + // Move to next row group + if (!loadNextRowGroup()) { + return; + } + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to read Parquet row", e); + } + } + + private boolean loadNextRowGroup() throws IOException { + while (rowGroupIndex < rowGroups.size()) { + BlockMetaData block = rowGroups.get(rowGroupIndex); + rowGroupIndex++; + + // Row group filtering: check column statistics + if (!rowGroupMayMatch(block)) { + reader.skipNextRowGroup(); + continue; + } + + PageReadStore pages = reader.readNextRowGroup(); + if (pages == null) { + continue; + } + + remainingRows = pages.getRowCount(); + recordReader = columnIO.getRecordReader(pages, new GroupRecordConverter(schema)); + return true; + } + return false; + } + + /** + * Checks whether a row group's column statistics allow a match against the current filter. If a bound filter value + * falls outside a column's [min, max] range, the entire row group can be skipped. + */ + private boolean rowGroupMayMatch(BlockMetaData block) { + for (ColumnChunkMetaData col : block.getColumns()) { + Statistics stats = col.getStatistics(); + if (stats == null || stats.isEmpty() || !stats.hasNonNullValue()) { + continue; + } + if (!(stats instanceof LongStatistics)) { + continue; + } + LongStatistics longStats = (LongStatistics) stats; + long min = longStats.getMin(); + long max = longStats.getMax(); + + String colName = col.getPath().toDotString(); + long filterVal = getFilterForColumn(colName); + if (filterVal >= 0 && (filterVal < min || filterVal > max)) { + return false; + } + } + return true; + } + + private long getFilterForColumn(String colName) { + switch (colName) { + case ParquetSchemas.COL_SUBJECT: + return filterS; + case ParquetSchemas.COL_PREDICATE: + return filterP; + case ParquetSchemas.COL_OBJECT: + return filterO; + case ParquetSchemas.COL_CONTEXT: + return filterC; + default: + return -1; + } + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ParquetSchemas.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ParquetSchemas.java new file mode 100644 index 00000000000..817b12c952b --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/ParquetSchemas.java @@ -0,0 +1,103 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Types; + +/** + * Parquet schema definitions for quad storage. + * + *

+ * All files use {@link #QUAD_SCHEMA} with 5 columns (subject, predicate, object, context, flag). Three sort orders + * determine the key encoding: SPOC (subject-leading), OPSC (object-leading), and CSPO (context-leading). + */ +public final class ParquetSchemas { + + /** Column name for subject ID. */ + public static final String COL_SUBJECT = "subject"; + + /** Column name for predicate ID. */ + public static final String COL_PREDICATE = "predicate"; + + /** Column name for object ID. */ + public static final String COL_OBJECT = "object"; + + /** Column name for context (named graph) ID. */ + public static final String COL_CONTEXT = "context"; + + /** Column name for the entry flag (e.g. insert vs tombstone). */ + public static final String COL_FLAG = "flag"; + + /** + * Schema for all Parquet files. Includes all 5 columns: subject, predicate, object, context, flag. + */ + public static final MessageType QUAD_SCHEMA = Types.buildMessage() + .required(PrimitiveTypeName.INT64) + .named(COL_SUBJECT) + .required(PrimitiveTypeName.INT64) + .named(COL_PREDICATE) + .required(PrimitiveTypeName.INT64) + .named(COL_OBJECT) + .required(PrimitiveTypeName.INT64) + .named(COL_CONTEXT) + .required(PrimitiveTypeName.INT32) + .named(COL_FLAG) + .named("quad"); + + /** + * Sort orders for quad entries within a Parquet file. + */ + public enum SortOrder { + /** Subject-Predicate-Object-Context ordering. */ + SPOC("spoc"), + /** Object-Predicate-Subject-Context ordering. */ + OPSC("opsc"), + /** Context-Subject-Predicate-Object ordering. */ + CSPO("cspo"); + + private final String suffix; + + SortOrder(String suffix) { + this.suffix = suffix; + } + + /** + * Returns the file-name suffix for this sort order. + * + * @return the suffix string + */ + public String suffix() { + return suffix; + } + + /** + * Returns the SortOrder for the given suffix string. + * + * @param suffix the suffix (e.g. "spoc", "opsc", "cspo") + * @return the matching SortOrder + * @throws IllegalArgumentException if no match found + */ + public static SortOrder fromSuffix(String suffix) { + for (SortOrder so : values()) { + if (so.suffix.equals(suffix)) { + return so; + } + } + throw new IllegalArgumentException("Unknown sort order suffix: " + suffix); + } + } + + private ParquetSchemas() { + // utility class + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/QuadEntry.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/QuadEntry.java new file mode 100644 index 00000000000..63ee6dd9c37 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/QuadEntry.java @@ -0,0 +1,30 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +/** + * A quad entry with subject, predicate, object, context value IDs and a flag byte. + */ +public final class QuadEntry { + public final long subject; + public final long predicate; + public final long object; + public final long context; + public final byte flag; + + public QuadEntry(long subject, long predicate, long object, long context, byte flag) { + this.subject = subject; + this.predicate = predicate; + this.object = object; + this.context = context; + this.flag = flag; + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/QuadIndex.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/QuadIndex.java new file mode 100644 index 00000000000..4f4662e93ca --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/QuadIndex.java @@ -0,0 +1,262 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.nio.ByteBuffer; +import java.util.Comparator; +import java.util.List; +import java.util.function.ToLongFunction; + +/** + * Manages index permutations for quad (S, P, O, C) storage. Each QuadIndex defines a field ordering (e.g. "spoc", + * "posc") and provides methods to encode/decode keys in that order, compute pattern scores for query optimization, and + * construct range scan boundaries. + * + *

+ * Based on the TripleStore.TripleIndex pattern from the LMDB SAIL module. + *

+ */ +public class QuadIndex { + + public static final int SUBJ_IDX = 0; + public static final int PRED_IDX = 1; + public static final int OBJ_IDX = 2; + public static final int CONTEXT_IDX = 3; + + static final int MAX_KEY_LENGTH = 4 * 9; // 4 varints, max 9 bytes each + + private final char[] fieldSeq; + private final String fieldSeqString; + private final int[] indexMap; + + /** + * Creates a new QuadIndex with the given field sequence. + * + * @param fieldSeq a 4-character string consisting of 's', 'p', 'o', 'c' in any order + * @throws IllegalArgumentException if the field sequence is invalid + */ + public QuadIndex(String fieldSeq) { + if (fieldSeq == null || fieldSeq.length() != 4) { + throw new IllegalArgumentException("Field sequence must be exactly 4 characters: " + fieldSeq); + } + this.fieldSeq = fieldSeq.toCharArray(); + this.fieldSeqString = fieldSeq; + this.indexMap = buildIndexMap(this.fieldSeq); + } + + /** + * Returns the field sequence as a String. + */ + public String getFieldSeqString() { + return fieldSeqString; + } + + /** + * Determines the 'score' of this index on the supplied pattern. The higher the score, the better the index is + * suited for matching the pattern. Score equals the number of leading bound components. Lowest score is 0, meaning + * a sequential scan. + * + * @param subj subject ID, or -1 for wildcard + * @param pred predicate ID, or -1 for wildcard + * @param obj object ID, or -1 for wildcard + * @param context context ID, or -1 for wildcard + * @return pattern score (0-4) + */ + public int getPatternScore(long subj, long pred, long obj, long context) { + long[] values = { subj, pred, obj, context }; + int score = 0; + for (int idx : indexMap) { + if (values[idx] >= 0) { + score++; + } else { + return score; + } + } + return score; + } + + /** + * Encodes a quad as a byte array key in index order. + * + * @param subj subject ID + * @param pred predicate ID + * @param obj object ID + * @param context context ID + * @return encoded byte array key + */ + public byte[] toKeyBytes(long subj, long pred, long obj, long context) { + long[] values = { subj, pred, obj, context }; + int length = Varint.calcListLengthUnsigned( + values[indexMap[0]], values[indexMap[1]], + values[indexMap[2]], values[indexMap[3]]); + ByteBuffer bb = ByteBuffer.allocate(length); + for (int idx : indexMap) { + Varint.writeUnsigned(bb, values[idx]); + } + return bb.array(); + } + + /** + * Reads a key back to quad values in SPOC order. + * + * @param key buffer positioned at the start of the key + * @param quad array of length 4 to receive [subj, pred, obj, context] + */ + public void keyToQuad(ByteBuffer key, long[] quad) { + Varint.readQuadUnsigned(key, indexMap, quad); + } + + /** + * Reads a key from a byte array back to quad values in SPOC order. + * + * @param key byte array containing the encoded key + * @param quad array of length 4 to receive [subj, pred, obj, context] + */ + public void keyToQuad(byte[] key, long[] quad) { + ByteBuffer bb = ByteBuffer.wrap(key); + Varint.readQuadUnsigned(bb, indexMap, quad); + } + + /** + * Constructs the minimum key as a byte array for a range scan. Unbound or zero-valued components become 0. + */ + public byte[] getMinKeyBytes(long subj, long pred, long obj, long context) { + return toKeyBytes( + subj <= 0 ? 0 : subj, + pred <= 0 ? 0 : pred, + obj <= 0 ? 0 : obj, + context <= 0 ? 0 : context); + } + + /** + * Constructs the maximum key as a byte array for a range scan. Unbound components (negative) become Long.MAX_VALUE. + */ + public byte[] getMaxKeyBytes(long subj, long pred, long obj, long context) { + return toKeyBytes( + subj < 0 ? Long.MAX_VALUE : subj, + pred < 0 ? Long.MAX_VALUE : pred, + obj < 0 ? Long.MAX_VALUE : obj, + context < 0 ? Long.MAX_VALUE : context); + } + + /** + * Finds the best index from the given list for a query pattern by choosing the index with the highest pattern + * score. + * + * @param indexes list of available indexes + * @param subj subject ID, or -1 for wildcard + * @param pred predicate ID, or -1 for wildcard + * @param obj object ID, or -1 for wildcard + * @param context context ID, or -1 for wildcard + * @return the best matching index + */ + public static QuadIndex getBestIndex(List indexes, long subj, long pred, long obj, long context) { + int bestScore = -1; + QuadIndex bestIndex = null; + + for (QuadIndex index : indexes) { + int score = index.getPatternScore(subj, pred, obj, context); + if (score > bestScore) { + bestScore = score; + bestIndex = index; + } + } + + return bestIndex; + } + + /** + * Tests whether a decoded quad matches the given pattern. Unbound components (< 0) are treated as wildcards. + * + * @param quad a long[4] array in SPOC order + * @param s subject pattern, or -1 for wildcard + * @param p predicate pattern, or -1 for wildcard + * @param o object pattern, or -1 for wildcard + * @param c context pattern, or -1 for wildcard + * @return true if all bound components match + */ + public static boolean matches(long[] quad, long s, long p, long o, long c) { + return (s < 0 || quad[SUBJ_IDX] == s) + && (p < 0 || quad[PRED_IDX] == p) + && (o < 0 || quad[OBJ_IDX] == o) + && (c < 0 || quad[CONTEXT_IDX] == c); + } + + /** + * Returns a comparator that orders {@link QuadEntry} objects according to this index's field sequence. Field + * extractors are precomputed at construction time for efficient sorting. + */ + public Comparator entryComparator() { + ToLongFunction e0 = extractorFor(indexMap[0]); + ToLongFunction e1 = extractorFor(indexMap[1]); + ToLongFunction e2 = extractorFor(indexMap[2]); + ToLongFunction e3 = extractorFor(indexMap[3]); + return (a, b) -> { + int cmp = Long.compare(e0.applyAsLong(a), e0.applyAsLong(b)); + if (cmp != 0) { + return cmp; + } + cmp = Long.compare(e1.applyAsLong(a), e1.applyAsLong(b)); + if (cmp != 0) { + return cmp; + } + cmp = Long.compare(e2.applyAsLong(a), e2.applyAsLong(b)); + if (cmp != 0) { + return cmp; + } + return Long.compare(e3.applyAsLong(a), e3.applyAsLong(b)); + }; + } + + @Override + public String toString() { + return fieldSeqString; + } + + private static ToLongFunction extractorFor(int componentIndex) { + switch (componentIndex) { + case SUBJ_IDX: + return e -> e.subject; + case PRED_IDX: + return e -> e.predicate; + case OBJ_IDX: + return e -> e.object; + case CONTEXT_IDX: + return e -> e.context; + default: + throw new IllegalArgumentException("Invalid component index: " + componentIndex); + } + } + + private static int[] buildIndexMap(char[] fieldSeq) { + int[] indexes = new int[fieldSeq.length]; + for (int i = 0; i < fieldSeq.length; i++) { + switch (fieldSeq[i]) { + case 's': + indexes[i] = SUBJ_IDX; + break; + case 'p': + indexes[i] = PRED_IDX; + break; + case 'o': + indexes[i] = OBJ_IDX; + break; + case 'c': + indexes[i] = CONTEXT_IDX; + break; + default: + throw new IllegalArgumentException( + "Invalid character '" + fieldSeq[i] + "' in field sequence: " + new String(fieldSeq)); + } + } + return indexes; + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/QuadStats.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/QuadStats.java new file mode 100644 index 00000000000..3652c5595ae --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/QuadStats.java @@ -0,0 +1,74 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.util.List; + +/** + * Min/max statistics for all four quad components (subject, predicate, object, context). + */ +public final class QuadStats { + + public final long minSubject, maxSubject; + public final long minPredicate, maxPredicate; + public final long minObject, maxObject; + public final long minContext, maxContext; + + public QuadStats(long minSubject, long maxSubject, + long minPredicate, long maxPredicate, + long minObject, long maxObject, + long minContext, long maxContext) { + this.minSubject = minSubject; + this.maxSubject = maxSubject; + this.minPredicate = minPredicate; + this.maxPredicate = maxPredicate; + this.minObject = minObject; + this.maxObject = maxObject; + this.minContext = minContext; + this.maxContext = maxContext; + } + + /** + * Computes min/max stats from a list of QuadEntry objects. Tombstones are excluded so that deleted entries do not + * inflate the range statistics used for pruning. + */ + public static QuadStats fromEntries(List entries) { + Accumulator acc = new Accumulator(); + for (QuadEntry e : entries) { + if (e.flag != MemTable.FLAG_TOMBSTONE) { + acc.add(e.subject, e.predicate, e.object, e.context); + } + } + return acc.build(); + } + + private static class Accumulator { + long minS = Long.MAX_VALUE, maxS = Long.MIN_VALUE; + long minP = Long.MAX_VALUE, maxP = Long.MIN_VALUE; + long minO = Long.MAX_VALUE, maxO = Long.MIN_VALUE; + long minC = Long.MAX_VALUE, maxC = Long.MIN_VALUE; + + void add(long s, long p, long o, long c) { + minS = Math.min(minS, s); + maxS = Math.max(maxS, s); + minP = Math.min(minP, p); + maxP = Math.max(maxP, p); + minO = Math.min(minO, o); + maxO = Math.max(maxO, o); + minC = Math.min(minC, c); + maxC = Math.max(maxC, c); + } + + QuadStats build() { + return new QuadStats(minS, maxS, minP, maxP, minO, maxO, minC, maxC); + } + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/RawEntrySource.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/RawEntrySource.java new file mode 100644 index 00000000000..5d5346a6546 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/RawEntrySource.java @@ -0,0 +1,29 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +/** + * A source of raw key/flag entries for merge iterators. {@link MemTable} and {@link ParquetQuadSource} expose this + * interface over a key range. + */ +public interface RawEntrySource { + + boolean hasNext(); + + byte[] peekKey(); + + byte peekFlag(); + + void advance(); + + default void close() { + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/S3ObjectStore.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/S3ObjectStore.java new file mode 100644 index 00000000000..cf3ca3812b5 --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/S3ObjectStore.java @@ -0,0 +1,146 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; + +import io.minio.GetObjectArgs; +import io.minio.ListObjectsArgs; +import io.minio.MinioClient; +import io.minio.PutObjectArgs; +import io.minio.RemoveObjectArgs; +import io.minio.Result; +import io.minio.errors.ErrorResponseException; +import io.minio.messages.Item; + +/** + * {@link ObjectStore} implementation backed by an S3-compatible service via MinIO client. + */ +public class S3ObjectStore implements ObjectStore { + + private final MinioClient client; + private final String bucket; + private final String prefix; + + public S3ObjectStore(String bucket, String endpoint, String region, String prefix, + String accessKey, String secretKey, boolean forcePathStyle) { + this.bucket = bucket; + if (prefix == null || prefix.isEmpty()) { + this.prefix = ""; + } else if (prefix.endsWith("/")) { + this.prefix = prefix; + } else { + this.prefix = prefix + "/"; + } + + this.client = MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region(region) + .build(); + } + + private String resolve(String key) { + return prefix + key; + } + + @Override + public void put(String key, byte[] data) { + try { + ByteArrayInputStream bais = new ByteArrayInputStream(data); + client.putObject(PutObjectArgs.builder() + .bucket(bucket) + .object(resolve(key)) + .stream(bais, data.length, -1) + .build()); + } catch (Exception e) { + throw new UncheckedIOException(new IOException("Failed to put " + key, e)); + } + } + + @Override + public byte[] get(String key) { + return executeGet(GetObjectArgs.builder() + .bucket(bucket) + .object(resolve(key)) + .build(), key); + } + + @Override + public byte[] getRange(String key, long offset, long length) { + return executeGet(GetObjectArgs.builder() + .bucket(bucket) + .object(resolve(key)) + .offset(offset) + .length(length) + .build(), key); + } + + private byte[] executeGet(GetObjectArgs args, String key) { + try (InputStream is = client.getObject(args)) { + return is.readAllBytes(); + } catch (ErrorResponseException e) { + if ("NoSuchKey".equals(e.errorResponse().code())) { + return null; + } + throw new UncheckedIOException(new IOException("Failed to get " + key, e)); + } catch (Exception e) { + throw new UncheckedIOException(new IOException("Failed to get " + key, e)); + } + } + + @Override + public void delete(String key) { + try { + client.removeObject(RemoveObjectArgs.builder() + .bucket(bucket) + .object(resolve(key)) + .build()); + } catch (Exception e) { + throw new UncheckedIOException(new IOException("Failed to delete " + key, e)); + } + } + + @Override + public List list(String subPrefix) { + List keys = new ArrayList<>(); + String fullPrefix = resolve(subPrefix); + Iterable> results = client.listObjects(ListObjectsArgs.builder() + .bucket(bucket) + .prefix(fullPrefix) + .recursive(true) + .build()); + try { + for (Result result : results) { + String objectKey = result.get().objectName(); + // Strip the store prefix to return relative keys + if (objectKey.startsWith(prefix)) { + keys.add(objectKey.substring(prefix.length())); + } else { + keys.add(objectKey); + } + } + } catch (Exception e) { + throw new UncheckedIOException(new IOException("Failed to list " + subPrefix, e)); + } + return keys; + } + + @Override + public void close() { + // MinioClient doesn't need explicit close + } +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/SimpleCodecFactory.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/SimpleCodecFactory.java new file mode 100644 index 00000000000..8f098c419ff --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/SimpleCodecFactory.java @@ -0,0 +1,138 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.compression.CompressionCodecFactory; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; + +import com.github.luben.zstd.Zstd; + +/** + * A lightweight {@link CompressionCodecFactory} that handles ZSTD and UNCOMPRESSED codecs without any Hadoop + * dependencies. Uses {@code zstd-jni} directly for ZSTD compression/decompression. + */ +final class SimpleCodecFactory implements CompressionCodecFactory { + + static final SimpleCodecFactory INSTANCE = new SimpleCodecFactory(); + + private SimpleCodecFactory() { + } + + @Override + public BytesInputCompressor getCompressor(CompressionCodecName codec) { + switch (codec) { + case ZSTD: + return ZSTD_COMPRESSOR; + case UNCOMPRESSED: + return NOOP_COMPRESSOR; + default: + throw new UnsupportedOperationException("Unsupported compression codec: " + codec); + } + } + + @Override + public BytesInputDecompressor getDecompressor(CompressionCodecName codec) { + switch (codec) { + case ZSTD: + return ZSTD_DECOMPRESSOR; + case UNCOMPRESSED: + return NOOP_DECOMPRESSOR; + default: + throw new UnsupportedOperationException("Unsupported compression codec: " + codec); + } + } + + @Override + public void release() { + // no resources to release + } + + private static final BytesInputCompressor ZSTD_COMPRESSOR = new BytesInputCompressor() { + @Override + public BytesInput compress(BytesInput bytes) throws IOException { + byte[] input = bytes.toByteArray(); + byte[] compressed = Zstd.compress(input); + return BytesInput.from(compressed); + } + + @Override + public CompressionCodecName getCodecName() { + return CompressionCodecName.ZSTD; + } + + @Override + public void release() { + } + }; + + private static final BytesInputDecompressor ZSTD_DECOMPRESSOR = new BytesInputDecompressor() { + @Override + public BytesInput decompress(BytesInput bytes, int uncompressedSize) throws IOException { + byte[] input = bytes.toByteArray(); + byte[] decompressed = new byte[uncompressedSize]; + Zstd.decompress(decompressed, input); + return BytesInput.from(decompressed); + } + + @Override + public void decompress(ByteBuffer input, int compressedSize, ByteBuffer output, int uncompressedSize) + throws IOException { + byte[] compressedBytes = new byte[compressedSize]; + input.get(compressedBytes); + byte[] decompressed = new byte[uncompressedSize]; + Zstd.decompress(decompressed, compressedBytes); + output.put(decompressed); + } + + @Override + public void release() { + } + }; + + private static final BytesInputCompressor NOOP_COMPRESSOR = new BytesInputCompressor() { + @Override + public BytesInput compress(BytesInput bytes) throws IOException { + return bytes; + } + + @Override + public CompressionCodecName getCodecName() { + return CompressionCodecName.UNCOMPRESSED; + } + + @Override + public void release() { + } + }; + + private static final BytesInputDecompressor NOOP_DECOMPRESSOR = new BytesInputDecompressor() { + @Override + public BytesInput decompress(BytesInput bytes, int uncompressedSize) throws IOException { + return bytes; + } + + @Override + public void decompress(ByteBuffer input, int compressedSize, ByteBuffer output, int uncompressedSize) + throws IOException { + byte[] data = new byte[compressedSize]; + input.get(data); + output.put(data); + } + + @Override + public void release() { + } + }; +} diff --git a/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/Varint.java b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/Varint.java new file mode 100644 index 00000000000..69988c616fc --- /dev/null +++ b/core/sail/s3/src/main/java/org/eclipse/rdf4j/sail/s3/storage/Varint.java @@ -0,0 +1,406 @@ +/******************************************************************************* + * Copyright (c) 2021 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * Encodes and decodes unsigned values using variable-length encoding. + * + *

+ * Uses the variable-length encoding of SQLite which + * preserves lexicographic ordering: smaller values use fewer bytes, and lexicographic byte order matches numeric order. + *

+ * + *

+ * Adapted from the LMDB Varint implementation with LMDB-specific dependencies removed (no SignificantBytesBE, no + * GroupMatcher). All reads use heap-based byte-by-byte decoding. + *

+ */ +public final class Varint { + + static final byte[] ENCODED_LONG_MAX = new byte[] { + (byte) 0xFF, // header: 8 payload bytes + 0x7F, // MSB of Long.MAX_VALUE + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF + }; + + static final byte[] ENCODED_LONG_MAX_QUAD = new byte[] { + (byte) 0xFF, 0x7F, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, + (byte) 0xFF, + (byte) 0xFF, 0x7F, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, + (byte) 0xFF, + (byte) 0xFF, 0x7F, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, + (byte) 0xFF, + (byte) 0xFF, 0x7F, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, + (byte) 0xFF + }; + + static final byte[] ALL_ZERO_QUAD = new byte[] { 0, 0, 0, 0 }; + + private Varint() { + } + + /** + * Encodes a value using the variable-length encoding of SQLite. + * + *

+ * The encoding has the following properties: + *

    + *
  1. Smaller (and more common) values use fewer bytes.
    2. + *
  2. The length of any varint can be determined by looking at just the first byte.
    3. + *
  3. Lexicographical and numeric ordering for varints are the same.
    4. + *
+ *

+ * + * @param bb buffer for writing bytes + * @param value value to encode + */ + public static void writeUnsigned(final ByteBuffer bb, final long value) { + if (value == Long.MAX_VALUE) { + final ByteOrder prev = bb.order(); + if (prev != ByteOrder.BIG_ENDIAN) { + bb.order(ByteOrder.BIG_ENDIAN); + } + try { + bb.put(ENCODED_LONG_MAX); + } finally { + if (prev != ByteOrder.BIG_ENDIAN) { + bb.order(prev); + } + } + return; + } + + if (value <= 240) { + bb.put((byte) value); + } else if (value <= 2287) { + long v = value - 240; + final ByteOrder prev = bb.order(); + if (prev != ByteOrder.BIG_ENDIAN) { + bb.order(ByteOrder.BIG_ENDIAN); + } + try { + int hi = (int) (v >>> 8) + 241; + int lo = (int) (v & 0xFF); + bb.putShort((short) ((hi << 8) | lo)); + } finally { + if (prev != ByteOrder.BIG_ENDIAN) { + bb.order(prev); + } + } + } else if (value <= 67823) { + long v = value - 2288; + bb.put((byte) 249); + final ByteOrder prev = bb.order(); + if (prev != ByteOrder.BIG_ENDIAN) { + bb.order(ByteOrder.BIG_ENDIAN); + } + try { + bb.putShort((short) v); + } finally { + if (prev != ByteOrder.BIG_ENDIAN) { + bb.order(prev); + } + } + } else { + int bytes = descriptor(value) + 1; + bb.put((byte) (250 + (bytes - 3))); + writeSignificantBits(bb, value, bytes); + } + } + + private static void writeSignificantBits(ByteBuffer bb, long value, int bytes) { + final ByteOrder prev = bb.order(); + if (prev != ByteOrder.BIG_ENDIAN) { + bb.order(ByteOrder.BIG_ENDIAN); + } + try { + int i = bytes; + + if ((i & 1) != 0) { + bb.put((byte) (value >>> ((i - 1) * 8))); + i--; + } + + if (i == 8) { + bb.putLong(value); + return; + } + + if (i >= 4) { + int shift = (i - 4) * 8; + bb.putInt((int) (value >>> shift)); + i -= 4; + } + + while (i >= 2) { + int shift = (i - 2) * 8; + bb.putShort((short) (value >>> shift)); + i -= 2; + } + } finally { + if (prev != ByteOrder.BIG_ENDIAN) { + bb.order(prev); + } + } + } + + /** + * Calculates required length in bytes to encode the given long value. + * + * @param value the value + * @return length in bytes + */ + public static int calcLengthUnsigned(long value) { + if (value <= 240) { + return 1; + } else if (value <= 2287) { + return 2; + } else if (value <= 67823) { + return 3; + } else { + int bytes = descriptor(value) + 1; + return 1 + bytes; + } + } + + /** + * Calculates required length in bytes to encode a list of four long values. + * + * @param a first value + * @param b second value + * @param c third value + * @param d fourth value + * @return length in bytes + */ + public static int calcListLengthUnsigned(long a, long b, long c, long d) { + return calcLengthUnsigned(a) + calcLengthUnsigned(b) + calcLengthUnsigned(c) + calcLengthUnsigned(d); + } + + /** + * The number of bytes required to represent the given number minus one. + */ + private static byte descriptor(long value) { + return value == 0 ? 0 : (byte) (7 - Long.numberOfLeadingZeros(value) / 8); + } + + /** Lookup: lead byte (0..255) -> number of additional bytes (0..8). */ + private static final byte[] VARINT_EXTRA_BYTES = buildVarintExtraBytes(); + + private static byte[] buildVarintExtraBytes() { + final byte[] t = new byte[256]; + for (int i = 0; i <= 240; i++) { + t[i] = 0; + } + for (int i = 241; i <= 248; i++) { + t[i] = 1; + } + t[249] = 2; + for (int i = 250; i <= 255; i++) { + t[i] = (byte) (i - 247); + } + return t; + } + + /** + * Decodes a value using SQLite's variable-length integer encoding. + * + * @param bb buffer for reading bytes + * @return decoded value + * @throws IllegalArgumentException if encoded varint is longer than 9 bytes + */ + public static long readUnsigned(ByteBuffer bb) throws IllegalArgumentException { + int a0 = bb.get() & 0xFF; + + if (a0 <= 240) { + return a0; + } else if (a0 <= 248) { + int a1 = bb.get() & 0xFF; + return 240L + ((long) (a0 - 241) << 8) + a1; + } else if (a0 == 249) { + int a1 = bb.get() & 0xFF; + int a2 = bb.get() & 0xFF; + return 2288L + ((long) a1 << 8) + a2; + } else { + int bytes = a0 - 250 + 3; + return readSignificantBits(bb, bytes); + } + } + + /** + * Skips over a single varint in the buffer. + * + * @param bb buffer to advance + */ + public static void skipUnsigned(ByteBuffer bb) { + final int a0 = bb.get() & 0xFF; + if (a0 <= 240) { + return; + } + final int extra = VARINT_EXTRA_BYTES[a0]; + bb.position(bb.position() + extra); + } + + /** + * Decodes a value at an absolute position without advancing the buffer position. + * + * @param bb buffer for reading bytes + * @param pos absolute position in the buffer + * @return decoded value + */ + public static long readUnsigned(ByteBuffer bb, int pos) throws IllegalArgumentException { + int a0 = bb.get(pos) & 0xFF; + if (a0 <= 240) { + return a0; + } else if (a0 <= 248) { + int a1 = bb.get(pos + 1) & 0xFF; + return 240 + 256 * (a0 - 241) + a1; + } else if (a0 == 249) { + int a1 = bb.get(pos + 1) & 0xFF; + int a2 = bb.get(pos + 2) & 0xFF; + return 2288 + 256 * a1 + a2; + } else { + int bytes = a0 - 250 + 3; + return readSignificantBitsAbsolute(bb, pos + 1, bytes); + } + } + + private static final int[] FIRST_TO_LENGTH = buildFirstToLength(); + + private static int[] buildFirstToLength() { + int[] t = new int[256]; + for (int i = 0; i <= 240; i++) { + t[i] = 1; + } + for (int i = 241; i <= 248; i++) { + t[i] = 2; + } + t[249] = 3; + for (int i = 250; i <= 255; i++) { + t[i] = i - 246; + } + return t; + } + + /** + * Determines length of an encoded varint value by inspecting the first byte. + * + * @param a0 first byte of varint value + * @return total length in bytes + */ + public static int firstToLength(byte a0) { + return FIRST_TO_LENGTH[a0 & 0xFF]; + } + + /** + * Decodes a single element of a list of variable-length long values from a buffer. + * + * @param bb buffer for reading bytes + * @param index the element's index + * @return the decoded value + */ + public static long readListElementUnsigned(ByteBuffer bb, int index) { + int pos = 0; + for (int i = 0; i < index; i++) { + pos += firstToLength(bb.get(pos)); + } + return readUnsigned(bb, pos); + } + + /** + * Encodes multiple values using variable-length encoding into the given buffer. + * + * @param bb buffer for writing bytes + * @param values array with values to write + */ + public static void writeListUnsigned(final ByteBuffer bb, final long[] values) { + for (long value : values) { + writeUnsigned(bb, value); + } + } + + /** + * Decodes multiple values using variable-length encoding from the given buffer. + * + * @param bb buffer for reading bytes + * @param values array for the result values + */ + public static void readListUnsigned(ByteBuffer bb, long[] values) { + for (int i = 0; i < values.length; i++) { + values[i] = readUnsigned(bb); + } + } + + /** + * Decodes exactly 4 values (a quad) from the given buffer. + * + * @param bb buffer for reading bytes + * @param values array of length 4 for the result values + */ + public static void readQuadUnsigned(ByteBuffer bb, long[] values) { + values[0] = readUnsigned(bb); + values[1] = readUnsigned(bb); + values[2] = readUnsigned(bb); + values[3] = readUnsigned(bb); + } + + /** + * Decodes multiple values using variable-length encoding, placing each value into the position specified by the + * index map. + * + * @param bb buffer for reading bytes + * @param indexMap map for indexes of values within values array + * @param values array for the result values + */ + public static void readListUnsigned(ByteBuffer bb, int[] indexMap, long[] values) { + for (int i = 0; i < values.length; i++) { + values[indexMap[i]] = readUnsigned(bb); + } + } + + /** + * Decodes exactly 4 values (a quad) from the given buffer, placing each value at the index specified by the map. + * + * @param bb buffer for reading bytes + * @param indexMap map for indexes of values within values array + * @param values array of length 4 for the result values + */ + public static void readQuadUnsigned(ByteBuffer bb, int[] indexMap, long[] values) { + values[indexMap[0]] = readUnsigned(bb); + values[indexMap[1]] = readUnsigned(bb); + values[indexMap[2]] = readUnsigned(bb); + values[indexMap[3]] = readUnsigned(bb); + } + + /** + * Reads n significant bytes from the buffer in big-endian order (byte-by-byte, heap-safe). + */ + private static long readSignificantBits(ByteBuffer bb, int n) { + long value = 0; + for (int i = 0; i < n; i++) { + value = (value << 8) | (bb.get() & 0xFFL); + } + return value; + } + + /** + * Reads n significant bytes from the buffer at an absolute position in big-endian order. + */ + private static long readSignificantBitsAbsolute(ByteBuffer bb, int pos, int bytes) { + long value = 0; + for (int i = 0; i < bytes; i++) { + value = (value << 8) | (bb.get(pos + i) & 0xFFL); + } + return value; + } +} diff --git a/core/sail/s3/src/main/resources/META-INF/services/org.eclipse.rdf4j.sail.config.SailFactory b/core/sail/s3/src/main/resources/META-INF/services/org.eclipse.rdf4j.sail.config.SailFactory new file mode 100644 index 00000000000..599d4243971 --- /dev/null +++ b/core/sail/s3/src/main/resources/META-INF/services/org.eclipse.rdf4j.sail.config.SailFactory @@ -0,0 +1 @@ +org.eclipse.rdf4j.sail.s3.config.S3StoreFactory diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3EvaluationStrategyTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3EvaluationStrategyTest.java new file mode 100644 index 00000000000..ba4920ac6b8 --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3EvaluationStrategyTest.java @@ -0,0 +1,23 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import org.eclipse.rdf4j.sail.base.config.BaseSailConfig; +import org.eclipse.rdf4j.sail.s3.config.S3StoreConfig; +import org.eclipse.rdf4j.testsuite.sail.EvaluationStrategyTest; + +public class S3EvaluationStrategyTest extends EvaluationStrategyTest { + + @Override + protected BaseSailConfig getBaseSailConfig() { + return new S3StoreConfig(); + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3PersistenceMinioIT.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3PersistenceMinioIT.java new file mode 100644 index 00000000000..62c7ecaaaf2 --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3PersistenceMinioIT.java @@ -0,0 +1,115 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.eclipse.rdf4j.common.iteration.CloseableIteration; +import org.eclipse.rdf4j.common.transaction.IsolationLevels; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Statement; +import org.eclipse.rdf4j.model.ValueFactory; +import org.eclipse.rdf4j.model.impl.SimpleValueFactory; +import org.eclipse.rdf4j.sail.s3.config.S3StoreConfig; +import org.eclipse.rdf4j.sail.s3.storage.S3ObjectStore; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import io.minio.BucketExistsArgs; +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; + +/** + * Integration test for S3 persistence using a real MinIO container via Testcontainers. Suffixed IT so it runs with + * {@code mvn verify} (Failsafe), not {@code mvn test}. + */ +@Testcontainers +class S3PersistenceMinioIT { + + private static final String BUCKET = "test-bucket"; + private static final String ACCESS_KEY = "minioadmin"; + private static final String SECRET_KEY = "minioadmin"; + private static final ValueFactory VF = SimpleValueFactory.getInstance(); + + @Container + static final GenericContainer MINIO = new GenericContainer<>("minio/minio:latest") + .withExposedPorts(9000) + .withEnv("MINIO_ROOT_USER", ACCESS_KEY) + .withEnv("MINIO_ROOT_PASSWORD", SECRET_KEY) + .withCommand("server", "/data"); + + private static String endpoint; + + @BeforeAll + static void createBucket() throws Exception { + endpoint = "http://" + MINIO.getHost() + ":" + MINIO.getMappedPort(9000); + MinioClient client = MinioClient.builder() + .endpoint(endpoint) + .credentials(ACCESS_KEY, SECRET_KEY) + .build(); + if (!client.bucketExists(BucketExistsArgs.builder().bucket(BUCKET).build())) { + client.makeBucket(MakeBucketArgs.builder().bucket(BUCKET).build()); + } + } + + private S3ObjectStore createStore(String prefix) { + return new S3ObjectStore(BUCKET, endpoint, "us-east-1", prefix, ACCESS_KEY, SECRET_KEY, true); + } + + @Test + void writeFlushShutdownRestart() throws Exception { + String prefix = "test-" + System.nanoTime() + "/"; + + IRI s = VF.createIRI("http://example.org/s1"); + IRI p = VF.createIRI("http://example.org/p1"); + IRI o = VF.createIRI("http://example.org/o1"); + + // Write and flush + { + S3ObjectStore objectStore = createStore(prefix); + S3StoreConfig config = new S3StoreConfig(); + S3SailStore sailStore = new S3SailStore(config, objectStore); + + var source = sailStore.getExplicitSailSource(); + var sink = source.sink(IsolationLevels.NONE); + sink.approve(s, p, o, null); + sink.flush(); + sailStore.close(); + } + + // Restart and verify + { + S3ObjectStore objectStore = createStore(prefix); + S3StoreConfig config = new S3StoreConfig(); + S3SailStore sailStore = new S3SailStore(config, objectStore); + + var source = sailStore.getExplicitSailSource(); + var dataset = source.dataset(IsolationLevels.NONE); + + CloseableIteration iter = dataset.getStatements(null, null, null); + assertTrue(iter.hasNext()); + Statement stmt = iter.next(); + assertEquals(s.stringValue(), stmt.getSubject().stringValue()); + assertEquals(p.stringValue(), stmt.getPredicate().stringValue()); + assertEquals(o.stringValue(), stmt.getObject().stringValue()); + assertFalse(iter.hasNext()); + + iter.close(); + dataset.close(); + sailStore.close(); + } + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3PersistenceTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3PersistenceTest.java new file mode 100644 index 00000000000..768da6278d0 --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3PersistenceTest.java @@ -0,0 +1,372 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import org.eclipse.rdf4j.common.iteration.CloseableIteration; +import org.eclipse.rdf4j.common.transaction.IsolationLevels; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Statement; +import org.eclipse.rdf4j.model.ValueFactory; +import org.eclipse.rdf4j.model.impl.SimpleValueFactory; +import org.eclipse.rdf4j.sail.s3.config.S3StoreConfig; +import org.eclipse.rdf4j.sail.s3.storage.FileSystemObjectStore; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class S3PersistenceTest { + + @TempDir + Path tempDir; + + private static final ValueFactory VF = SimpleValueFactory.getInstance(); + + @Test + void writeFlushShutdownRestart_quadsReadable() throws Exception { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + S3StoreConfig config = new S3StoreConfig(); + + IRI s = VF.createIRI("http://example.org/s1"); + IRI p = VF.createIRI("http://example.org/p1"); + IRI o = VF.createIRI("http://example.org/o1"); + IRI ctx = VF.createIRI("http://example.org/g1"); + + // Phase 1: Write and flush + { + S3SailStore sailStore = new S3SailStore(config, store); + ValueFactory svf = sailStore.getValueFactory(); + + // Add statements using sink + var source = sailStore.getExplicitSailSource(); + var sink = source.sink(IsolationLevels.NONE); + + // We need to resolve values through the sail's value factory + S3ValueStore vs = (S3ValueStore) svf; + long sId = vs.storeValue(s); + long pId = vs.storeValue(p); + long oId = vs.storeValue(o); + long ctxId = vs.storeValue(ctx); + + sink.approve(s, p, o, ctx); + sink.flush(); + sailStore.close(); + } + + // Phase 2: Restart and verify + { + S3SailStore sailStore = new S3SailStore(config, store); + ValueFactory svf = sailStore.getValueFactory(); + + var source = sailStore.getExplicitSailSource(); + var dataset = source.dataset(IsolationLevels.NONE); + + CloseableIteration iter = dataset.getStatements(null, null, null); + assertTrue(iter.hasNext(), "Should have at least one statement after restart"); + + Statement stmt = iter.next(); + assertEquals(s.stringValue(), stmt.getSubject().stringValue()); + assertEquals(p.stringValue(), stmt.getPredicate().stringValue()); + assertEquals(o.stringValue(), stmt.getObject().stringValue()); + assertEquals(ctx.stringValue(), stmt.getContext().stringValue()); + + assertFalse(iter.hasNext(), "Should have exactly one statement"); + iter.close(); + dataset.close(); + sailStore.close(); + } + } + + @Test + void multipleFlushes_allDataReadable() throws Exception { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + S3StoreConfig config = new S3StoreConfig(); + + IRI s1 = VF.createIRI("http://example.org/s1"); + IRI s2 = VF.createIRI("http://example.org/s2"); + IRI p = VF.createIRI("http://example.org/p"); + IRI o = VF.createIRI("http://example.org/o"); + + // Write, flush, write more, flush again + { + S3SailStore sailStore = new S3SailStore(config, store); + + var source = sailStore.getExplicitSailSource(); + var sink = source.sink(IsolationLevels.NONE); + + sink.approve(s1, p, o, null); + sink.flush(); + + sink.approve(s2, p, o, null); + sink.flush(); + + sailStore.close(); + } + + // Restart and verify both statements exist + { + S3SailStore sailStore = new S3SailStore(config, store); + + var source = sailStore.getExplicitSailSource(); + var dataset = source.dataset(IsolationLevels.NONE); + + CloseableIteration iter = dataset.getStatements(null, null, null); + int count = 0; + while (iter.hasNext()) { + iter.next(); + count++; + } + assertEquals(2, count, "Should have 2 statements after restart"); + + iter.close(); + dataset.close(); + sailStore.close(); + } + } + + @Test + void deleteAndRestart_deletedQuadsGone() throws Exception { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + S3StoreConfig config = new S3StoreConfig(); + + IRI s1 = VF.createIRI("http://example.org/s1"); + IRI s2 = VF.createIRI("http://example.org/s2"); + IRI p = VF.createIRI("http://example.org/p"); + IRI o = VF.createIRI("http://example.org/o"); + + // Write two statements, delete one, flush + { + S3SailStore sailStore = new S3SailStore(config, store); + + var source = sailStore.getExplicitSailSource(); + var sink = source.sink(IsolationLevels.NONE); + + sink.approve(s1, p, o, null); + sink.approve(s2, p, o, null); + sink.flush(); + + // Now delete s1 + Statement toDeprecate = VF.createStatement(s1, p, o); + sink.deprecate(toDeprecate); + sink.flush(); + + sailStore.close(); + } + + // Restart and verify only s2 remains + { + S3SailStore sailStore = new S3SailStore(config, store); + + var source = sailStore.getExplicitSailSource(); + var dataset = source.dataset(IsolationLevels.NONE); + + CloseableIteration iter = dataset.getStatements(null, null, null); + assertTrue(iter.hasNext()); + Statement stmt = iter.next(); + assertEquals(s2.stringValue(), stmt.getSubject().stringValue()); + assertFalse(iter.hasNext()); + + iter.close(); + dataset.close(); + sailStore.close(); + } + } + + @Test + void multiplePredicates_allQueriesWork() throws Exception { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + S3StoreConfig config = new S3StoreConfig(); + + IRI s1 = VF.createIRI("http://example.org/s1"); + IRI s2 = VF.createIRI("http://example.org/s2"); + IRI p1 = VF.createIRI("http://example.org/name"); + IRI p2 = VF.createIRI("http://example.org/age"); + IRI o1 = VF.createIRI("http://example.org/Alice"); + IRI o2 = VF.createIRI("http://example.org/30"); + + // Write data with multiple predicates, flush, restart + { + S3SailStore sailStore = new S3SailStore(config, store); + var source = sailStore.getExplicitSailSource(); + var sink = source.sink(IsolationLevels.NONE); + + sink.approve(s1, p1, o1, null); + sink.approve(s1, p2, o2, null); + sink.approve(s2, p1, o2, null); + sink.flush(); + sailStore.close(); + } + + // Restart and verify queries + { + S3SailStore sailStore = new S3SailStore(config, store); + var source = sailStore.getExplicitSailSource(); + var dataset = source.dataset(IsolationLevels.NONE); + + // All statements + List all = drain(dataset.getStatements(null, null, null)); + assertEquals(3, all.size()); + + // By predicate (p1) + List byP1 = drain(dataset.getStatements(null, p1, null)); + assertEquals(2, byP1.size()); + for (Statement st : byP1) { + assertEquals(p1.stringValue(), st.getPredicate().stringValue()); + } + + // By predicate (p2) + List byP2 = drain(dataset.getStatements(null, p2, null)); + assertEquals(1, byP2.size()); + assertEquals(p2.stringValue(), byP2.get(0).getPredicate().stringValue()); + + // By subject + List byS1 = drain(dataset.getStatements(s1, null, null)); + assertEquals(2, byS1.size()); + + // By subject + predicate + List byS1P1 = drain(dataset.getStatements(s1, p1, null)); + assertEquals(1, byS1P1.size()); + + // By object + List byO2 = drain(dataset.getStatements(null, null, o2)); + assertEquals(2, byO2.size()); + + dataset.close(); + sailStore.close(); + } + } + + @Test + void fileLayout_flatDataDirectory() throws Exception { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + S3StoreConfig config = new S3StoreConfig(); + + IRI s = VF.createIRI("http://example.org/s1"); + IRI p = VF.createIRI("http://example.org/p1"); + IRI o = VF.createIRI("http://example.org/o1"); + + { + S3SailStore sailStore = new S3SailStore(config, store); + var source = sailStore.getExplicitSailSource(); + var sink = source.sink(IsolationLevels.NONE); + sink.approve(s, p, o, null); + sink.flush(); + sailStore.close(); + } + + // Verify flat file paths (no predicates/ directory) + List dataFiles = store.list("data/"); + assertFalse(dataFiles.isEmpty(), "Should have data files"); + for (String key : dataFiles) { + assertFalse(key.contains("predicates/"), "Should not have predicate partitions: " + key); + assertTrue(key.startsWith("data/L0-"), "Should start with data/L0-: " + key); + assertTrue(key.endsWith(".parquet"), "Should end with .parquet: " + key); + } + + // Should have 3 files (one per sort order) + assertEquals(3, dataFiles.size(), "Should have 3 files (spoc, opsc, cspo)"); + + // Check sort orders are present + List suffixes = dataFiles.stream() + .map(k -> k.substring(k.lastIndexOf('-') + 1, k.lastIndexOf('.'))) + .collect(Collectors.toList()); + assertTrue(suffixes.contains("spoc"), "Missing spoc file"); + assertTrue(suffixes.contains("opsc"), "Missing opsc file"); + assertTrue(suffixes.contains("cspo"), "Missing cspo file"); + } + + @Test + void contextQuery_afterRestart() throws Exception { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + S3StoreConfig config = new S3StoreConfig(); + + IRI s = VF.createIRI("http://example.org/s1"); + IRI p = VF.createIRI("http://example.org/p1"); + IRI o = VF.createIRI("http://example.org/o1"); + IRI g1 = VF.createIRI("http://example.org/graph1"); + IRI g2 = VF.createIRI("http://example.org/graph2"); + + { + S3SailStore sailStore = new S3SailStore(config, store); + var source = sailStore.getExplicitSailSource(); + var sink = source.sink(IsolationLevels.NONE); + sink.approve(s, p, o, g1); + sink.approve(s, p, o, g2); + sink.flush(); + sailStore.close(); + } + + { + S3SailStore sailStore = new S3SailStore(config, store); + var source = sailStore.getExplicitSailSource(); + var dataset = source.dataset(IsolationLevels.NONE); + + // Query by context g1 + List byG1 = drain( + dataset.getStatements(null, null, null, new org.eclipse.rdf4j.model.Resource[] { g1 })); + assertEquals(1, byG1.size()); + assertEquals(g1.stringValue(), byG1.get(0).getContext().stringValue()); + + // Query all + List all = drain(dataset.getStatements(null, null, null)); + assertEquals(2, all.size()); + + dataset.close(); + sailStore.close(); + } + } + + private List drain(CloseableIteration iter) { + List result = new ArrayList<>(); + while (iter.hasNext()) { + result.add(iter.next()); + } + iter.close(); + return result; + } + + @Test + void namespacePersistence() throws Exception { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + S3StoreConfig config = new S3StoreConfig(); + + // Set a namespace + { + S3SailStore sailStore = new S3SailStore(config, store); + + var source = sailStore.getExplicitSailSource(); + var sink = source.sink(IsolationLevels.NONE); + sink.setNamespace("ex", "http://example.org/"); + sink.flush(); + sailStore.close(); + } + + // Restart and verify namespace persists + { + S3SailStore sailStore = new S3SailStore(config, store); + + var source = sailStore.getExplicitSailSource(); + var dataset = source.dataset(IsolationLevels.NONE); + + assertEquals("http://example.org/", dataset.getNamespace("ex")); + + dataset.close(); + sailStore.close(); + } + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3SparqlOrderByTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3SparqlOrderByTest.java new file mode 100644 index 00000000000..3f3736c072d --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3SparqlOrderByTest.java @@ -0,0 +1,24 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import org.eclipse.rdf4j.repository.Repository; +import org.eclipse.rdf4j.repository.sail.SailRepository; +import org.eclipse.rdf4j.sail.s3.config.S3StoreConfig; +import org.eclipse.rdf4j.testsuite.repository.SparqlOrderByTest; + +public class S3SparqlOrderByTest extends SparqlOrderByTest { + + @Override + protected Repository newRepository() { + return new SailRepository(new S3Store(new S3StoreConfig())); + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3StoreConnectionTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3StoreConnectionTest.java new file mode 100644 index 00000000000..ec7a292aa5d --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3StoreConnectionTest.java @@ -0,0 +1,26 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import java.io.File; + +import org.eclipse.rdf4j.repository.Repository; +import org.eclipse.rdf4j.repository.sail.SailRepository; +import org.eclipse.rdf4j.sail.s3.config.S3StoreConfig; +import org.eclipse.rdf4j.testsuite.repository.RepositoryConnectionTest; + +public class S3StoreConnectionTest extends RepositoryConnectionTest { + + @Override + protected Repository createRepository(File dataDir) { + return new SailRepository(new S3Store(new S3StoreConfig())); + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3StoreIsolationLevelTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3StoreIsolationLevelTest.java new file mode 100644 index 00000000000..a37fa581dec --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3StoreIsolationLevelTest.java @@ -0,0 +1,24 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import org.eclipse.rdf4j.sail.NotifyingSail; +import org.eclipse.rdf4j.sail.SailException; +import org.eclipse.rdf4j.sail.s3.config.S3StoreConfig; +import org.eclipse.rdf4j.testsuite.sail.SailIsolationLevelTest; + +public class S3StoreIsolationLevelTest extends SailIsolationLevelTest { + + @Override + protected NotifyingSail createSail() throws SailException { + return new S3Store(new S3StoreConfig()); + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3StoreRepositoryTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3StoreRepositoryTest.java new file mode 100644 index 00000000000..33c138afef5 --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3StoreRepositoryTest.java @@ -0,0 +1,24 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import org.eclipse.rdf4j.repository.Repository; +import org.eclipse.rdf4j.repository.sail.SailRepository; +import org.eclipse.rdf4j.sail.s3.config.S3StoreConfig; +import org.eclipse.rdf4j.testsuite.repository.RepositoryTest; + +public class S3StoreRepositoryTest extends RepositoryTest { + + @Override + protected Repository createRepository() { + return new SailRepository(new S3Store(new S3StoreConfig())); + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3StoreTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3StoreTest.java new file mode 100644 index 00000000000..4e624b1ec50 --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3StoreTest.java @@ -0,0 +1,26 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import org.eclipse.rdf4j.sail.NotifyingSail; +import org.eclipse.rdf4j.sail.SailException; +import org.eclipse.rdf4j.sail.s3.config.S3StoreConfig; +import org.eclipse.rdf4j.testsuite.sail.RDFNotifyingStoreTest; + +public class S3StoreTest extends RDFNotifyingStoreTest { + + @Override + protected NotifyingSail createSail() throws SailException { + NotifyingSail sail = new S3Store(new S3StoreConfig()); + sail.init(); + return sail; + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3ValueStoreSerializationTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3ValueStoreSerializationTest.java new file mode 100644 index 00000000000..6158ca991f8 --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/S3ValueStoreSerializationTest.java @@ -0,0 +1,136 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; + +import org.eclipse.rdf4j.model.BNode; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Literal; +import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.sail.s3.storage.FileSystemObjectStore; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class S3ValueStoreSerializationTest { + + @TempDir + Path tempDir; + + @Test + void roundTrip_iri() { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + S3ValueStore vs = new S3ValueStore(); + + IRI iri = vs.createIRI("http://example.org/test"); + long id = vs.storeValue(iri); + + vs.serialize(store); + + S3ValueStore vs2 = new S3ValueStore(); + vs2.deserialize(store, vs.getNextId()); + + Value restored = vs2.getValue(id); + assertNotNull(restored); + assertTrue(restored instanceof IRI); + assertEquals("http://example.org/test", restored.stringValue()); + assertEquals(id, vs2.getId(iri)); + } + + @Test + void roundTrip_literal() { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + S3ValueStore vs = new S3ValueStore(); + + Literal lit = vs.createLiteral("hello world"); + long id = vs.storeValue(lit); + + vs.serialize(store); + + S3ValueStore vs2 = new S3ValueStore(); + vs2.deserialize(store, vs.getNextId()); + + Value restored = vs2.getValue(id); + assertNotNull(restored); + assertTrue(restored instanceof Literal); + assertEquals("hello world", ((Literal) restored).getLabel()); + } + + @Test + void roundTrip_literalWithLanguage() { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + S3ValueStore vs = new S3ValueStore(); + + Literal lit = vs.createLiteral("bonjour", "fr"); + long id = vs.storeValue(lit); + + vs.serialize(store); + + S3ValueStore vs2 = new S3ValueStore(); + vs2.deserialize(store, vs.getNextId()); + + Value restored = vs2.getValue(id); + assertNotNull(restored); + assertTrue(restored instanceof Literal); + assertEquals("bonjour", ((Literal) restored).getLabel()); + assertTrue(((Literal) restored).getLanguage().isPresent()); + assertEquals("fr", ((Literal) restored).getLanguage().get()); + } + + @Test + void roundTrip_bnode() { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + S3ValueStore vs = new S3ValueStore(); + + BNode bnode = vs.createBNode("node123"); + long id = vs.storeValue(bnode); + + vs.serialize(store); + + S3ValueStore vs2 = new S3ValueStore(); + vs2.deserialize(store, vs.getNextId()); + + Value restored = vs2.getValue(id); + assertNotNull(restored); + assertTrue(restored instanceof BNode); + assertEquals("node123", ((BNode) restored).getID()); + } + + @Test + void roundTrip_multipleValues() { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + S3ValueStore vs = new S3ValueStore(); + + IRI iri1 = vs.createIRI("http://example.org/s1"); + IRI iri2 = vs.createIRI("http://example.org/p1"); + Literal lit = vs.createLiteral("value"); + BNode bnode = vs.createBNode("b1"); + + long id1 = vs.storeValue(iri1); + long id2 = vs.storeValue(iri2); + long id3 = vs.storeValue(lit); + long id4 = vs.storeValue(bnode); + + vs.serialize(store); + + S3ValueStore vs2 = new S3ValueStore(); + vs2.deserialize(store, vs.getNextId()); + + assertEquals(iri1, vs2.getValue(id1)); + assertEquals(iri2, vs2.getValue(id2)); + assertEquals(lit.getLabel(), ((Literal) vs2.getValue(id3)).getLabel()); + assertEquals(bnode.getID(), ((BNode) vs2.getValue(id4)).getID()); + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/CatalogTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/CatalogTest.java new file mode 100644 index 00000000000..8945c04e406 --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/CatalogTest.java @@ -0,0 +1,151 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Tests for {@link Catalog} v3 — flat file list with per-file predicate statistics. + */ +class CatalogTest { + + @TempDir + Path tempDir; + + private final ObjectMapper mapper = new ObjectMapper(); + + @Test + void newCatalog_version3() { + Catalog catalog = new Catalog(); + assertEquals(3, catalog.getVersion()); + } + + @Test + void addFile_appearsInFileList() { + Catalog catalog = new Catalog(); + Catalog.ParquetFileInfo info = makeFileInfo("data/L0-00001-spoc.parquet", "spoc", 0, 1); + catalog.addFile(info); + + assertEquals(1, catalog.getFiles().size()); + assertEquals("data/L0-00001-spoc.parquet", catalog.getFiles().get(0).getS3Key()); + } + + @Test + void removeFiles_removesMatchingKeys() { + Catalog catalog = new Catalog(); + catalog.addFile(makeFileInfo("data/L0-00001-spoc.parquet", "spoc", 0, 1)); + catalog.addFile(makeFileInfo("data/L0-00001-opsc.parquet", "opsc", 0, 1)); + catalog.addFile(makeFileInfo("data/L0-00002-spoc.parquet", "spoc", 0, 2)); + + catalog.removeFiles(Set.of("data/L0-00001-spoc.parquet", "data/L0-00001-opsc.parquet")); + + assertEquals(1, catalog.getFiles().size()); + assertEquals("data/L0-00002-spoc.parquet", catalog.getFiles().get(0).getS3Key()); + } + + @Test + void getFilesForSortOrder_filtersCorrectly() { + Catalog catalog = new Catalog(); + catalog.addFile(makeFileInfo("data/L0-00001-spoc.parquet", "spoc", 0, 1)); + catalog.addFile(makeFileInfo("data/L0-00001-opsc.parquet", "opsc", 0, 1)); + catalog.addFile(makeFileInfo("data/L0-00001-cspo.parquet", "cspo", 0, 1)); + catalog.addFile(makeFileInfo("data/L0-00002-spoc.parquet", "spoc", 0, 2)); + + List spocFiles = catalog.getFilesForSortOrder("spoc"); + assertEquals(2, spocFiles.size()); + + List opscFiles = catalog.getFilesForSortOrder("opsc"); + assertEquals(1, opscFiles.size()); + + List cspoFiles = catalog.getFilesForSortOrder("cspo"); + assertEquals(1, cspoFiles.size()); + } + + @Test + void saveAndLoad_roundTrip() { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + Catalog catalog = new Catalog(); + catalog.setNextValueId(42); + catalog.addFile(makeFileInfo("data/L0-00001-spoc.parquet", "spoc", 0, 1)); + catalog.addFile(makeFileInfo("data/L0-00001-opsc.parquet", "opsc", 0, 1)); + catalog.save(store, mapper, 5); + + Catalog loaded = Catalog.load(store, mapper); + + assertEquals(3, loaded.getVersion()); + assertEquals(5, loaded.getEpoch()); + assertEquals(42, loaded.getNextValueId()); + assertEquals(2, loaded.getFiles().size()); + } + + @Test + void parquetFileInfo_predicateStats() { + Catalog.ParquetFileInfo info = new Catalog.ParquetFileInfo( + "data/L0-00001-spoc.parquet", 0, "spoc", 100, 1, 4096, + 1, 50, // subject + 10, 20, // predicate + 5, 40, // object + 0, 99 // context + ); + + assertEquals(10, info.getMinPredicate()); + assertEquals(20, info.getMaxPredicate()); + assertEquals(1, info.getMinSubject()); + assertEquals(50, info.getMaxSubject()); + } + + @Test + void saveAndLoad_preservesPredicateStats() { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + Catalog catalog = new Catalog(); + catalog.addFile(new Catalog.ParquetFileInfo( + "data/L0-00001-spoc.parquet", 0, "spoc", 100, 1, 4096, + 1, 50, 10, 20, 5, 40, 0, 99)); + catalog.save(store, mapper, 1); + + Catalog loaded = Catalog.load(store, mapper); + Catalog.ParquetFileInfo info = loaded.getFiles().get(0); + + assertEquals(10, info.getMinPredicate()); + assertEquals(20, info.getMaxPredicate()); + assertEquals(1, info.getMinSubject()); + assertEquals(50, info.getMaxSubject()); + assertEquals(5, info.getMinObject()); + assertEquals(40, info.getMaxObject()); + assertEquals(0, info.getMinContext()); + assertEquals(99, info.getMaxContext()); + } + + @Test + void loadEmpty_returnsDefaultCatalog() { + FileSystemObjectStore store = new FileSystemObjectStore(tempDir); + Catalog loaded = Catalog.load(store, mapper); + + assertEquals(3, loaded.getVersion()); + assertEquals(0, loaded.getEpoch()); + assertTrue(loaded.getFiles().isEmpty()); + } + + private Catalog.ParquetFileInfo makeFileInfo(String s3Key, String sortOrder, int level, long epoch) { + return new Catalog.ParquetFileInfo(s3Key, level, sortOrder, 10, epoch, 1024, + 1, 100, 1, 100, 1, 100, 0, 100); + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/MemTableReorderTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/MemTableReorderTest.java new file mode 100644 index 00000000000..86e060f6c99 --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/MemTableReorderTest.java @@ -0,0 +1,166 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link MemTable#asRawSource(QuadIndex, long, long, long, long)} — re-encoding keys from native SPOC order + * into a different target index order. + */ +class MemTableReorderTest { + + private final QuadIndex spoc = new QuadIndex("spoc"); + private final QuadIndex opsc = new QuadIndex("opsc"); + private final QuadIndex cspo = new QuadIndex("cspo"); + + @Test + void sameIndex_delegatesToNativeSource() { + MemTable mt = new MemTable(spoc); + mt.put(10, 20, 30, 40, true); + mt.put(1, 2, 3, 4, true); + + RawEntrySource source = mt.asRawSource(spoc, -1, -1, -1, -1); + List results = drain(source, spoc); + + assertEquals(2, results.size()); + // SPOC order: (1,2,3,4) before (10,20,30,40) + assertArrayEquals(new long[] { 1, 2, 3, 4 }, results.get(0)); + assertArrayEquals(new long[] { 10, 20, 30, 40 }, results.get(1)); + } + + @Test + void reorderToOPSC_sortsByObjectFirst() { + MemTable mt = new MemTable(spoc); + mt.put(1, 2, 30, 4, true); // object=30 + mt.put(5, 6, 10, 8, true); // object=10 + mt.put(9, 10, 20, 12, true); // object=20 + + RawEntrySource source = mt.asRawSource(opsc, -1, -1, -1, -1); + List results = drain(source, opsc); + + assertEquals(3, results.size()); + // OPSC order: sorted by object: 10, 20, 30 + assertEquals(10, results.get(0)[QuadIndex.OBJ_IDX]); + assertEquals(20, results.get(1)[QuadIndex.OBJ_IDX]); + assertEquals(30, results.get(2)[QuadIndex.OBJ_IDX]); + } + + @Test + void reorderToCSPO_sortsByContextFirst() { + MemTable mt = new MemTable(spoc); + mt.put(1, 2, 3, 30, true); // context=30 + mt.put(4, 5, 6, 10, true); // context=10 + mt.put(7, 8, 9, 20, true); // context=20 + + RawEntrySource source = mt.asRawSource(cspo, -1, -1, -1, -1); + List results = drain(source, cspo); + + assertEquals(3, results.size()); + // CSPO order: sorted by context: 10, 20, 30 + assertEquals(10, results.get(0)[QuadIndex.CONTEXT_IDX]); + assertEquals(20, results.get(1)[QuadIndex.CONTEXT_IDX]); + assertEquals(30, results.get(2)[QuadIndex.CONTEXT_IDX]); + } + + @Test + void reorderedSource_preservesAllComponents() { + MemTable mt = new MemTable(spoc); + mt.put(11, 22, 33, 44, true); + + RawEntrySource source = mt.asRawSource(opsc, -1, -1, -1, -1); + List results = drain(source, opsc); + + assertEquals(1, results.size()); + assertEquals(11, results.get(0)[QuadIndex.SUBJ_IDX]); + assertEquals(22, results.get(0)[QuadIndex.PRED_IDX]); + assertEquals(33, results.get(0)[QuadIndex.OBJ_IDX]); + assertEquals(44, results.get(0)[QuadIndex.CONTEXT_IDX]); + } + + @Test + void reorderedSource_appliesSubjectFilter() { + MemTable mt = new MemTable(spoc); + mt.put(1, 2, 3, 0, true); + mt.put(5, 6, 7, 0, true); + + RawEntrySource source = mt.asRawSource(opsc, 1, -1, -1, -1); + List results = drain(source, opsc); + + assertEquals(1, results.size()); + assertEquals(1, results.get(0)[QuadIndex.SUBJ_IDX]); + } + + @Test + void reorderedSource_appliesPredicateFilter() { + MemTable mt = new MemTable(spoc); + mt.put(1, 10, 3, 0, true); + mt.put(2, 20, 4, 0, true); + mt.put(3, 10, 5, 0, true); + + RawEntrySource source = mt.asRawSource(cspo, -1, 10, -1, -1); + List results = drain(source, cspo); + + assertEquals(2, results.size()); + for (long[] q : results) { + assertEquals(10, q[QuadIndex.PRED_IDX]); + } + } + + @Test + void reorderedSource_includesAliveAndTombstones() { + MemTable mt = new MemTable(spoc); + mt.put(1, 2, 3, 0, true); + mt.remove(5, 6, 7, 0); + + RawEntrySource source = mt.asRawSource(opsc, -1, -1, -1, -1); + + int count = 0; + boolean foundTombstone = false; + while (source.hasNext()) { + if (source.peekFlag() == MemTable.FLAG_TOMBSTONE) { + foundTombstone = true; + } + source.advance(); + count++; + } + + assertEquals(2, count); + assertTrue(foundTombstone, "RawEntrySource should include tombstones"); + } + + @Test + void reorderedSource_emptyTable_returnsEmpty() { + MemTable mt = new MemTable(spoc); + + RawEntrySource source = mt.asRawSource(opsc, -1, -1, -1, -1); + assertFalse(source.hasNext()); + } + + private List drain(RawEntrySource source, QuadIndex decodeIndex) { + List result = new ArrayList<>(); + while (source.hasNext()) { + long[] quad = new long[4]; + decodeIndex.keyToQuad(source.peekKey(), quad); + result.add(quad); + source.advance(); + } + return result; + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/MergeIteratorTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/MergeIteratorTest.java new file mode 100644 index 00000000000..6702130c0af --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/MergeIteratorTest.java @@ -0,0 +1,176 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import org.junit.jupiter.api.Test; + +class MergeIteratorTest { + + private final QuadIndex spoc = new QuadIndex("spoc"); + + @Test + void newerSourceWins() { + // Newer MemTable overrides older SSTable + MemTable newer = new MemTable(spoc); + newer.put(1, 2, 3, 0, true); // explicit in newer + + MemTable older = new MemTable(spoc); + older.put(1, 2, 3, 0, false); // inferred in older + + List sources = Arrays.asList( + newer.asRawSource(-1, -1, -1, -1), + older.asRawSource(-1, -1, -1, -1)); + + MergeIterator iter = new MergeIterator(sources, spoc, MemTable.FLAG_EXPLICIT, -1, -1, -1, -1); + List results = toList(iter); + assertEquals(1, results.size()); + assertArrayEquals(new long[] { 1, 2, 3, 0 }, results.get(0)); + } + + @Test + void tombstoneSuppression() { + MemTable newer = new MemTable(spoc); + newer.remove(1, 2, 3, 0); // tombstone + + MemTable older = new MemTable(spoc); + older.put(1, 2, 3, 0, true); // explicit + + List sources = Arrays.asList( + newer.asRawSource(-1, -1, -1, -1), + older.asRawSource(-1, -1, -1, -1)); + + MergeIterator iter = new MergeIterator(sources, spoc, MemTable.FLAG_EXPLICIT, -1, -1, -1, -1); + List results = toList(iter); + assertEquals(0, results.size()); + } + + @Test + void multiSourceMerge() { + MemTable m1 = new MemTable(spoc); + m1.put(1, 2, 3, 0, true); + m1.put(3, 4, 5, 0, true); + + MemTable m2 = new MemTable(spoc); + m2.put(2, 3, 4, 0, true); + m2.put(4, 5, 6, 0, true); + + List sources = Arrays.asList( + m1.asRawSource(-1, -1, -1, -1), + m2.asRawSource(-1, -1, -1, -1)); + + MergeIterator iter = new MergeIterator(sources, spoc, MemTable.FLAG_EXPLICIT, -1, -1, -1, -1); + List results = toList(iter); + assertEquals(4, results.size()); + // Should be sorted by key (SPOC order) + assertEquals(1, results.get(0)[0]); + assertEquals(2, results.get(1)[0]); + assertEquals(3, results.get(2)[0]); + assertEquals(4, results.get(3)[0]); + } + + @Test + void emptySource() { + MemTable empty = new MemTable(spoc); + MemTable withData = new MemTable(spoc); + withData.put(1, 2, 3, 0, true); + + List sources = Arrays.asList( + empty.asRawSource(-1, -1, -1, -1), + withData.asRawSource(-1, -1, -1, -1)); + + MergeIterator iter = new MergeIterator(sources, spoc, MemTable.FLAG_EXPLICIT, -1, -1, -1, -1); + List results = toList(iter); + assertEquals(1, results.size()); + } + + @Test + void allEmptySources() { + MemTable empty1 = new MemTable(spoc); + MemTable empty2 = new MemTable(spoc); + + List sources = Arrays.asList( + empty1.asRawSource(-1, -1, -1, -1), + empty2.asRawSource(-1, -1, -1, -1)); + + MergeIterator iter = new MergeIterator(sources, spoc, MemTable.FLAG_EXPLICIT, -1, -1, -1, -1); + assertFalse(iter.hasNext()); + } + + @Test + void patternFilter() { + MemTable m1 = new MemTable(spoc); + m1.put(1, 2, 3, 0, true); + m1.put(1, 2, 4, 0, true); + m1.put(2, 3, 4, 0, true); + + List sources = List.of(m1.asRawSource(1, -1, -1, -1)); + + MergeIterator iter = new MergeIterator(sources, spoc, MemTable.FLAG_EXPLICIT, 1, -1, -1, -1); + List results = toList(iter); + assertEquals(2, results.size()); + } + + @Test + void mergeMemTableWithOlderSource() { + // MemTable (newer) + older MemTable source + MemTable memTable = new MemTable(spoc); + memTable.put(1, 2, 3, 0, true); + + MemTable olderData = new MemTable(spoc); + olderData.put(2, 3, 4, 0, true); + olderData.put(4, 5, 6, 0, true); + + List sources = Arrays.asList( + memTable.asRawSource(-1, -1, -1, -1), + olderData.asRawSource(-1, -1, -1, -1)); + + MergeIterator iter = new MergeIterator(sources, spoc, MemTable.FLAG_EXPLICIT, -1, -1, -1, -1); + List results = toList(iter); + assertEquals(3, results.size()); + } + + @Test + void tombstoneInNewerShadowsOlder() { + // Older source has a value, newer MemTable deletes it + MemTable olderData = new MemTable(spoc); + olderData.put(1, 2, 3, 0, true); + olderData.put(4, 5, 6, 0, true); + + MemTable memTable = new MemTable(spoc); + memTable.remove(1, 2, 3, 0); // tombstone shadows older entry + + List sources = Arrays.asList( + memTable.asRawSource(-1, -1, -1, -1), + olderData.asRawSource(-1, -1, -1, -1)); + + MergeIterator iter = new MergeIterator(sources, spoc, MemTable.FLAG_EXPLICIT, -1, -1, -1, -1); + List results = toList(iter); + assertEquals(1, results.size()); + assertArrayEquals(new long[] { 4, 5, 6, 0 }, results.get(0)); + } + + private List toList(Iterator iter) { + List list = new ArrayList<>(); + while (iter.hasNext()) { + list.add(iter.next()); + } + return list; + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/ParquetRoundTripTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/ParquetRoundTripTest.java new file mode 100644 index 00000000000..3b6f24e5741 --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/ParquetRoundTripTest.java @@ -0,0 +1,201 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +/** + * Tests for Parquet write/read round-trips using {@link ParquetFileBuilder} and {@link ParquetQuadSource} with the + * {@link ParquetSchemas#QUAD_SCHEMA} (5 columns, 4-varint keys). + */ +class ParquetRoundTripTest { + + @Test + void roundTrip_spocOrder_allFieldsPreserved() { + QuadIndex spoc = new QuadIndex("spoc"); + List entries = List.of( + new QuadEntry(1, 2, 3, 4, MemTable.FLAG_EXPLICIT), + new QuadEntry(5, 6, 7, 8, MemTable.FLAG_INFERRED), + new QuadEntry(9, 10, 11, 0, MemTable.FLAG_TOMBSTONE)); + + byte[] parquetData = ParquetFileBuilder.build(entries, ParquetSchemas.SortOrder.SPOC); + ParquetQuadSource source = new ParquetQuadSource(parquetData, spoc); + + List results = drainWithFlags(source, spoc); + assertEquals(3, results.size()); + + // First entry: (1,2,3,4) FLAG_EXPLICIT + assertArrayEquals(new long[] { 1, 2, 3, 4, MemTable.FLAG_EXPLICIT }, results.get(0)); + // Second entry: (5,6,7,8) FLAG_INFERRED + assertArrayEquals(new long[] { 5, 6, 7, 8, MemTable.FLAG_INFERRED }, results.get(1)); + // Third entry: (9,10,11,0) FLAG_TOMBSTONE + assertArrayEquals(new long[] { 9, 10, 11, 0, MemTable.FLAG_TOMBSTONE }, results.get(2)); + } + + @Test + void roundTrip_opscOrder_keysSortedByObject() { + QuadIndex opsc = new QuadIndex("opsc"); + // Written sorted in OPSC order (by object: 10, 20, 30) + List entries = List.of( + new QuadEntry(100, 200, 10, 0, MemTable.FLAG_EXPLICIT), + new QuadEntry(300, 400, 20, 0, MemTable.FLAG_EXPLICIT), + new QuadEntry(500, 600, 30, 0, MemTable.FLAG_EXPLICIT)); + + byte[] parquetData = ParquetFileBuilder.build(entries, ParquetSchemas.SortOrder.OPSC); + ParquetQuadSource source = new ParquetQuadSource(parquetData, opsc); + + List results = drain(source, opsc); + assertEquals(3, results.size()); + // Keys should be in OPSC order (object first) + assertEquals(10, results.get(0)[QuadIndex.OBJ_IDX]); + assertEquals(20, results.get(1)[QuadIndex.OBJ_IDX]); + assertEquals(30, results.get(2)[QuadIndex.OBJ_IDX]); + } + + @Test + void roundTrip_cspoOrder_keysSortedByContext() { + QuadIndex cspo = new QuadIndex("cspo"); + // Written sorted in CSPO order (by context: 5, 10, 15) + List entries = List.of( + new QuadEntry(1, 2, 3, 5, MemTable.FLAG_EXPLICIT), + new QuadEntry(4, 5, 6, 10, MemTable.FLAG_EXPLICIT), + new QuadEntry(7, 8, 9, 15, MemTable.FLAG_EXPLICIT)); + + byte[] parquetData = ParquetFileBuilder.build(entries, ParquetSchemas.SortOrder.CSPO); + ParquetQuadSource source = new ParquetQuadSource(parquetData, cspo); + + List results = drain(source, cspo); + assertEquals(3, results.size()); + assertEquals(5, results.get(0)[QuadIndex.CONTEXT_IDX]); + assertEquals(10, results.get(1)[QuadIndex.CONTEXT_IDX]); + assertEquals(15, results.get(2)[QuadIndex.CONTEXT_IDX]); + } + + @Test + void roundTrip_filterBySubject() { + QuadIndex spoc = new QuadIndex("spoc"); + List entries = List.of( + new QuadEntry(1, 2, 3, 0, MemTable.FLAG_EXPLICIT), + new QuadEntry(5, 6, 7, 0, MemTable.FLAG_EXPLICIT), + new QuadEntry(10, 11, 12, 0, MemTable.FLAG_EXPLICIT)); + + byte[] parquetData = ParquetFileBuilder.build(entries, ParquetSchemas.SortOrder.SPOC); + ParquetQuadSource source = new ParquetQuadSource(parquetData, spoc, 5, -1, -1, -1); + + List results = drain(source, spoc); + assertEquals(1, results.size()); + assertEquals(5, results.get(0)[QuadIndex.SUBJ_IDX]); + } + + @Test + void roundTrip_filterByPredicate() { + QuadIndex spoc = new QuadIndex("spoc"); + List entries = List.of( + new QuadEntry(1, 10, 3, 0, MemTable.FLAG_EXPLICIT), + new QuadEntry(2, 20, 4, 0, MemTable.FLAG_EXPLICIT), + new QuadEntry(3, 10, 5, 0, MemTable.FLAG_EXPLICIT)); + + byte[] parquetData = ParquetFileBuilder.build(entries, ParquetSchemas.SortOrder.SPOC); + ParquetQuadSource source = new ParquetQuadSource(parquetData, spoc, -1, 10, -1, -1); + + List results = drain(source, spoc); + assertEquals(2, results.size()); + for (long[] q : results) { + assertEquals(10, q[QuadIndex.PRED_IDX]); + } + } + + @Test + void roundTrip_filterByMultipleComponents() { + QuadIndex spoc = new QuadIndex("spoc"); + List entries = List.of( + new QuadEntry(1, 2, 3, 4, MemTable.FLAG_EXPLICIT), + new QuadEntry(1, 2, 99, 4, MemTable.FLAG_EXPLICIT), + new QuadEntry(1, 99, 3, 4, MemTable.FLAG_EXPLICIT)); + + byte[] parquetData = ParquetFileBuilder.build(entries, ParquetSchemas.SortOrder.SPOC); + ParquetQuadSource source = new ParquetQuadSource(parquetData, spoc, 1, 2, 3, 4); + + List results = drain(source, spoc); + assertEquals(1, results.size()); + assertArrayEquals(new long[] { 1, 2, 3, 4 }, results.get(0)); + } + + @Test + void roundTrip_emptyFile() { + QuadIndex spoc = new QuadIndex("spoc"); + byte[] parquetData = ParquetFileBuilder.build(List.of(), ParquetSchemas.SortOrder.SPOC); + ParquetQuadSource source = new ParquetQuadSource(parquetData, spoc); + assertFalse(source.hasNext()); + } + + @Test + void mergeIterator_acrossParquetSources() { + QuadIndex spoc = new QuadIndex("spoc"); + + // File 1: newer epoch + List file1 = List.of( + new QuadEntry(1, 2, 3, 0, MemTable.FLAG_EXPLICIT), + new QuadEntry(5, 6, 7, 0, MemTable.FLAG_EXPLICIT)); + byte[] data1 = ParquetFileBuilder.build(file1, ParquetSchemas.SortOrder.SPOC); + + // File 2: older epoch, overlaps on (1,2,3,0) + List file2 = List.of( + new QuadEntry(1, 2, 3, 0, MemTable.FLAG_INFERRED), + new QuadEntry(10, 11, 12, 0, MemTable.FLAG_EXPLICIT)); + byte[] data2 = ParquetFileBuilder.build(file2, ParquetSchemas.SortOrder.SPOC); + + List sources = List.of( + new ParquetQuadSource(data1, spoc), + new ParquetQuadSource(data2, spoc)); + + MergeIterator iter = new MergeIterator(sources, spoc, MemTable.FLAG_EXPLICIT, -1, -1, -1, -1); + List results = new ArrayList<>(); + while (iter.hasNext()) { + results.add(iter.next()); + } + + // (1,2,3,0) from newer file is explicit → included + // (5,6,7,0) explicit → included + // (10,11,12,0) explicit → included + assertEquals(3, results.size()); + } + + private List drain(ParquetQuadSource source, QuadIndex decodeIndex) { + List result = new ArrayList<>(); + while (source.hasNext()) { + long[] quad = new long[4]; + decodeIndex.keyToQuad(source.peekKey(), quad); + result.add(quad); + source.advance(); + } + return result; + } + + private List drainWithFlags(ParquetQuadSource source, QuadIndex decodeIndex) { + List result = new ArrayList<>(); + while (source.hasNext()) { + long[] quad = new long[4]; + decodeIndex.keyToQuad(source.peekKey(), quad); + long[] withFlag = new long[] { quad[0], quad[1], quad[2], quad[3], source.peekFlag() }; + result.add(withFlag); + source.advance(); + } + return result; + } +} diff --git a/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/QuadIndexSelectionTest.java b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/QuadIndexSelectionTest.java new file mode 100644 index 00000000000..eb25cff98fd --- /dev/null +++ b/core/sail/s3/src/test/java/org/eclipse/rdf4j/sail/s3/storage/QuadIndexSelectionTest.java @@ -0,0 +1,106 @@ +/******************************************************************************* + * Copyright (c) 2025 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.s3.storage; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link QuadIndex#getBestIndex(List, long, long, long, long)} — ensures the best sort order is selected for + * different query patterns. + */ +class QuadIndexSelectionTest { + + private static final QuadIndex SPOC = new QuadIndex("spoc"); + private static final QuadIndex OPSC = new QuadIndex("opsc"); + private static final QuadIndex CSPO = new QuadIndex("cspo"); + private static final List ALL = List.of(SPOC, OPSC, CSPO); + + @Test + void subjectBound_selectsSPOC() { + QuadIndex best = QuadIndex.getBestIndex(ALL, 1, -1, -1, -1); + assertEquals("spoc", best.getFieldSeqString()); + } + + @Test + void objectBound_selectsOPSC() { + QuadIndex best = QuadIndex.getBestIndex(ALL, -1, -1, 1, -1); + assertEquals("opsc", best.getFieldSeqString()); + } + + @Test + void contextBound_selectsCSPO() { + QuadIndex best = QuadIndex.getBestIndex(ALL, -1, -1, -1, 1); + assertEquals("cspo", best.getFieldSeqString()); + } + + @Test + void subjectAndPredicateBound_selectsSPOC() { + QuadIndex best = QuadIndex.getBestIndex(ALL, 1, 2, -1, -1); + assertEquals("spoc", best.getFieldSeqString()); + } + + @Test + void allBound_selectsSPOC() { + QuadIndex best = QuadIndex.getBestIndex(ALL, 1, 2, 3, 4); + assertEquals("spoc", best.getFieldSeqString()); + } + + @Test + void noneBound_selectsSPOC_asDefault() { + QuadIndex best = QuadIndex.getBestIndex(ALL, -1, -1, -1, -1); + // All have score 0; SPOC is first in the list so it wins ties + assertEquals("spoc", best.getFieldSeqString()); + } + + @Test + void predicateOnlyBound_selectsSPOC() { + // Predicate is second in SPOC, second in OPSC, third in CSPO — all have score 0 + // SPOC wins as first in list + QuadIndex best = QuadIndex.getBestIndex(ALL, -1, 5, -1, -1); + assertEquals("spoc", best.getFieldSeqString()); + } + + @Test + void objectAndPredicate_selectsOPSC() { + // OPSC: o(bound)=1, p(bound)=2, score=2 + // SPOC: s(unbound)=0, score=0 + // CSPO: c(unbound)=0, score=0 + QuadIndex best = QuadIndex.getBestIndex(ALL, -1, 5, 10, -1); + assertEquals("opsc", best.getFieldSeqString()); + } + + @Test + void contextAndSubject_selectsCSPO() { + // CSPO: c(bound)=1, s(bound)=2, score=2 + // SPOC: s(bound)=1, p(unbound), score=1 + // OPSC: o(unbound), score=0 + QuadIndex best = QuadIndex.getBestIndex(ALL, 1, -1, -1, 5); + assertEquals("cspo", best.getFieldSeqString()); + } + + @Test + void patternScore_countsLeadingBound() { + assertEquals(4, SPOC.getPatternScore(1, 2, 3, 4)); + assertEquals(2, SPOC.getPatternScore(1, 2, -1, -1)); + assertEquals(1, SPOC.getPatternScore(1, -1, -1, -1)); + assertEquals(0, SPOC.getPatternScore(-1, 2, 3, 4)); // s unbound → 0 + + assertEquals(2, OPSC.getPatternScore(-1, 2, 3, -1)); // o=3 bound, p=2 bound → 2 + assertEquals(0, OPSC.getPatternScore(1, -1, -1, -1)); // o unbound → 0 + + assertEquals(1, CSPO.getPatternScore(-1, -1, -1, 5)); // c=5 bound → 1 + assertEquals(3, CSPO.getPatternScore(1, 2, -1, 5)); // c=5, s=1, p=2 → 3 + } +} diff --git a/tools/workbench/src/main/webapp/transformations/create-s3.xsl b/tools/workbench/src/main/webapp/transformations/create-s3.xsl new file mode 100644 index 00000000000..a83c5e32ec1 --- /dev/null +++ b/tools/workbench/src/main/webapp/transformations/create-s3.xsl @@ -0,0 +1,82 @@ + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ + + +
+ + + +
S3 Prefix + +
Data Directory + +
+ + +
+
+ + + + diff --git a/tools/workbench/src/main/webapp/transformations/create.xsl b/tools/workbench/src/main/webapp/transformations/create.xsl index d2ced21ed14..08705d11ab9 100644 --- a/tools/workbench/src/main/webapp/transformations/create.xsl +++ b/tools/workbench/src/main/webapp/transformations/create.xsl @@ -74,6 +74,7 @@ +