Skip to content

Commit 1d600f6

Browse files
kenwenzelkenwenzel
committed
GH-4950 LMDB: support inlined values in value store
1 parent 65611d0 commit 1d600f6

3 files changed

Lines changed: 63 additions & 30 deletions

File tree

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -767,7 +767,8 @@ private long removeStatements(long subj, long pred, long obj, boolean explicit,
767767
tripleStore.removeTriplesByContext(subj, pred, obj, contextId, explicit, quad -> {
768768
removeCount[0]++;
769769
for (long id : quad) {
770-
if (id != 0L) {
770+
if (id != 0L && !ValueIds.isInlined(id)) {
771+
// only add references, exclude inlined values
771772
unusedIds.add(id);
772773
}
773774
}

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java‎

Lines changed: 56 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@
7676
import org.eclipse.rdf4j.model.util.Literals;
7777
import org.eclipse.rdf4j.sail.lmdb.LmdbUtil.Transaction;
7878
import org.eclipse.rdf4j.sail.lmdb.config.LmdbStoreConfig;
79+
import org.eclipse.rdf4j.sail.lmdb.inlined.Values;
7980
import org.eclipse.rdf4j.sail.lmdb.model.LmdbBNode;
8081
import org.eclipse.rdf4j.sail.lmdb.model.LmdbIRI;
8182
import org.eclipse.rdf4j.sail.lmdb.model.LmdbLiteral;
@@ -513,6 +514,10 @@ public LmdbValue getLazyValue(long id) throws IOException {
513514
resultValue = new LmdbBNode(lazyRevision, id);
514515
break;
515516
default:
517+
if (ValueIds.isInlined(id)) {
518+
resultValue = new LmdbLiteral(lazyRevision, id);
519+
break;
520+
}
516521
throw new IOException("Unsupported value with id type: " + idType);
517522
}
518523
// Store value in cache
@@ -540,6 +545,12 @@ public LmdbValue getValue(long id) throws IOException {
540545
LmdbValue resultValue = cachedValue(cacheID);
541546

542547
if (resultValue == null) {
548+
// unpack inlined values if possible
549+
if (ValueIds.isInlined(id)) {
550+
Literal unpacked = Values.unpackLiteral(id, this);
551+
return new LmdbLiteral(revision, unpacked.getLabel(), unpacked.getDatatype(), id);
552+
}
553+
543554
// Value not in cache, fetch it from file
544555
byte[] data = getData(id);
545556

@@ -564,6 +575,13 @@ public LmdbValue getValue(long id) throws IOException {
564575
* @return <code>true</code> if value could be successfully resolved, else <code>false</code>
565576
*/
566577
public boolean resolveValue(long id, LmdbValue value) {
578+
// unpack inlined values if possible
579+
if (ValueIds.isInlined(id)) {
580+
Literal unpacked = Values.unpackLiteral(id, this);
581+
((LmdbLiteral) value).setLabel(unpacked.getLabel());
582+
((LmdbLiteral) value).setDatatype(unpacked.getDatatype());
583+
return true;
584+
}
567585
try {
568586
byte[] data = getData(id);
569587
if (data != null) {
@@ -903,13 +921,10 @@ public long getId(Value value) throws IOException {
903921
public long getId(Value value, boolean create) throws IOException {
904922
// Try to get the internal ID from the value itself
905923
boolean isOwnValue = isOwnValue(value);
906-
907924
if (isOwnValue) {
908925
LmdbValue lmdbValue = (LmdbValue) value;
909-
910926
if (revisionIsCurrent(lmdbValue)) {
911927
long id = lmdbValue.getInternalID();
912-
913928
if (id != LmdbValue.UNKNOWN_ID) {
914929
return id;
915930
}
@@ -926,42 +941,57 @@ public long getId(Value value, boolean create) throws IOException {
926941

927942
if (cachedID != null) {
928943
long id = cachedID;
929-
930944
if (isOwnValue) {
931945
// Store id in value for fast access in any consecutive calls
932946
((LmdbValue) value).setInternalID(id, revision);
933947
}
934-
935948
return id;
936949
}
937950

938-
// ID not cached, search in file
939-
byte[] data = value2data(value, create);
940-
if (data == null && value instanceof Literal) {
941-
data = literal2legacy((Literal) value);
951+
long id = LmdbValue.UNKNOWN_ID;
952+
if (value instanceof Literal) {
953+
// inline value into id if possible
954+
try {
955+
long packedId = Values.packLiteral((Literal) value);
956+
if (packedId != 0L) {
957+
Literal unpacked = Values.unpackLiteral(packedId, this);
958+
if (unpacked.equals(value)) {
959+
id = packedId;
960+
}
961+
}
962+
} catch (IllegalArgumentException e) {
963+
// ignore, invalid literal
964+
}
942965
}
943966

944-
if (data != null) {
945-
long id = findId(data, create);
946-
if (id != LmdbValue.UNKNOWN_ID) {
947-
if (isOwnValue) {
948-
// Store id in value for fast access in any consecutive calls
949-
((LmdbValue) value).setInternalID(id, revision);
950-
// Store id in cache
951-
valueIDCache.put((LmdbValue) value, id);
952-
} else {
953-
// Store id in cache
954-
LmdbValue nv = getLmdbValue(value);
955-
nv.setInternalID(id, revision);
967+
if (id == LmdbValue.UNKNOWN_ID) {
968+
// not inlined or ID not cached, search in index
969+
byte[] data = value2data(value, create);
970+
if (data == null && value instanceof Literal) {
971+
data = literal2legacy((Literal) value);
972+
}
956973

957-
if (nv.isIRI() && isCommonVocabulary(((IRI) nv))) {
958-
commonVocabulary.put(value, id);
959-
}
974+
if (data != null) {
975+
id = findId(data, create);
976+
}
977+
}
960978

961-
valueIDCache.put(nv, id);
979+
if (id != LmdbValue.UNKNOWN_ID) {
980+
if (isOwnValue) {
981+
// Store id in value for fast access in any consecutive calls
982+
((LmdbValue) value).setInternalID(id, revision);
983+
// Store id in cache
984+
valueIDCache.put((LmdbValue) value, id);
985+
} else {
986+
// Store id in cache
987+
LmdbValue nv = getLmdbValue(value);
988+
nv.setInternalID(id, revision);
989+
990+
if (nv.isIRI() && isCommonVocabulary(((IRI) nv))) {
991+
commonVocabulary.put(value, id);
962992
}
993+
valueIDCache.put(nv, id);
963994
}
964-
965995
return id;
966996
}
967997
} finally {

‎core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/ValueStoreTest.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,11 +152,12 @@ public void testGcValuesAfterRestart() throws Exception {
152152

153153
@Test
154154
public void testGcDatatypes() throws Exception {
155-
IRI[] types = new IRI[] { XSD.STRING, XSD.INTEGER, XSD.DOUBLE, XSD.DECIMAL, XSD.FLOAT };
155+
IRI[] types = new IRI[] { XSD.STRING, XSD.INTEGER, XSD.LONG, XSD.DECIMAL };
156156
LmdbValue values[] = new LmdbValue[types.length];
157157
valueStore.startTransaction(true);
158158
for (int i = 0; i < values.length; i++) {
159-
values[i] = valueStore.createLiteral("123", types[i]);
159+
// use a value that is large enough to not being inlined
160+
values[i] = valueStore.createLiteral(Long.toString(Long.MAX_VALUE - 1), types[i]);
160161
valueStore.storeValue(values[i]);
161162
}
162163
valueStore.commit();
@@ -198,7 +199,8 @@ public void testGcDatatypes() throws Exception {
198199
public void testGcURIs() throws Exception {
199200
for (boolean storeAndGcUri : List.of(false, true)) {
200201
valueStore.startTransaction(true);
201-
LmdbLiteral literal = valueStore.createLiteral("123", XSD.STRING);
202+
// use a value that is large enough to not being inlined
203+
LmdbLiteral literal = valueStore.createLiteral("123".repeat(5), XSD.STRING);
202204
valueStore.storeValue(literal);
203205
if (storeAndGcUri) {
204206
valueStore.storeValue(XSD.STRING);

0 commit comments

Comments
 (0)