Skip to content

Commit 40501f5

Browse files
kenwenzelkenwenzel
committed
GH-4950 LMDB: support inlined values in value store
1 parent 35b8555 commit 40501f5

3 files changed

Lines changed: 63 additions & 30 deletions

File tree

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -774,7 +774,8 @@ private long removeStatements(long subj, long pred, long obj, boolean explicit,
774774
tripleStore.removeTriplesByContext(subj, pred, obj, contextId, explicit, quad -> {
775775
removeCount[0]++;
776776
for (long id : quad) {
777-
if (id != 0L) {
777+
if (id != 0L && !ValueIds.isInlined(id)) {
778+
// only add references, exclude inlined values
778779
unusedIds.add(id);
779780
}
780781
}

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java‎

Lines changed: 56 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@
8484
import org.eclipse.rdf4j.sail.SailException;
8585
import org.eclipse.rdf4j.sail.lmdb.LmdbUtil.Transaction;
8686
import org.eclipse.rdf4j.sail.lmdb.config.LmdbStoreConfig;
87+
import org.eclipse.rdf4j.sail.lmdb.inlined.Values;
8788
import org.eclipse.rdf4j.sail.lmdb.model.LmdbBNode;
8889
import org.eclipse.rdf4j.sail.lmdb.model.LmdbIRI;
8990
import org.eclipse.rdf4j.sail.lmdb.model.LmdbLiteral;
@@ -576,6 +577,10 @@ public LmdbValue getLazyValue(long id) throws IOException {
576577
resultValue = new LmdbBNode(lazyRevision, id);
577578
break;
578579
default:
580+
if (ValueIds.isInlined(id)) {
581+
resultValue = new LmdbLiteral(lazyRevision, id);
582+
break;
583+
}
579584
throw new IOException("Unsupported value with id type: " + idType);
580585
}
581586
// Store value in cache
@@ -602,6 +607,12 @@ public LmdbValue getValue(long id) throws IOException {
602607
LmdbValue resultValue = cachedValue(id);
603608

604609
if (resultValue == null) {
610+
// unpack inlined values if possible
611+
if (ValueIds.isInlined(id)) {
612+
Literal unpacked = Values.unpackLiteral(id, this);
613+
return new LmdbLiteral(revision, unpacked.getLabel(), unpacked.getDatatype(), id);
614+
}
615+
605616
// Value not in cache, fetch it from file
606617
byte[] data = getData(id);
607618

@@ -626,6 +637,13 @@ public LmdbValue getValue(long id) throws IOException {
626637
* @return <code>true</code> if value could be successfully resolved, else <code>false</code>
627638
*/
628639
public boolean resolveValue(long id, LmdbValue value) {
640+
// unpack inlined values if possible
641+
if (ValueIds.isInlined(id)) {
642+
Literal unpacked = Values.unpackLiteral(id, this);
643+
((LmdbLiteral) value).setLabel(unpacked.getLabel());
644+
((LmdbLiteral) value).setDatatype(unpacked.getDatatype());
645+
return true;
646+
}
629647
// Try to get from cache
630648
LmdbValue cached = cachedValue(id);
631649
if (cached != null && this.getRevision().getRevisionId() == cached.getValueStoreRevision().getRevisionId()) {
@@ -975,13 +993,10 @@ public long getId(Value value) throws IOException {
975993
public long getId(Value value, boolean create) throws IOException {
976994
// Try to get the internal ID from the value itself
977995
boolean isOwnValue = isOwnValue(value);
978-
979996
if (isOwnValue) {
980997
LmdbValue lmdbValue = (LmdbValue) value;
981-
982998
if (revisionIsCurrent(lmdbValue)) {
983999
long id = lmdbValue.getInternalID();
984-
9851000
if (id != LmdbValue.UNKNOWN_ID) {
9861001
return id;
9871002
}
@@ -998,42 +1013,57 @@ public long getId(Value value, boolean create) throws IOException {
9981013

9991014
if (cachedID != null) {
10001015
long id = cachedID;
1001-
10021016
if (isOwnValue) {
10031017
// Store id in value for fast access in any consecutive calls
10041018
((LmdbValue) value).setInternalID(id, revision);
10051019
}
1006-
10071020
return id;
10081021
}
10091022

1010-
// ID not cached, search in file
1011-
byte[] data = value2data(value, create);
1012-
if (data == null && value instanceof Literal) {
1013-
data = literal2legacy((Literal) value);
1023+
long id = LmdbValue.UNKNOWN_ID;
1024+
if (value instanceof Literal) {
1025+
// inline value into id if possible
1026+
try {
1027+
long packedId = Values.packLiteral((Literal) value);
1028+
if (packedId != 0L) {
1029+
Literal unpacked = Values.unpackLiteral(packedId, this);
1030+
if (unpacked.equals(value)) {
1031+
id = packedId;
1032+
}
1033+
}
1034+
} catch (IllegalArgumentException e) {
1035+
// ignore, invalid literal
1036+
}
10141037
}
10151038

1016-
if (data != null) {
1017-
long id = findId(data, create);
1018-
if (id != LmdbValue.UNKNOWN_ID) {
1019-
if (isOwnValue) {
1020-
// Store id in value for fast access in any consecutive calls
1021-
((LmdbValue) value).setInternalID(id, revision);
1022-
// Store id in cache
1023-
valueIDCache.put((LmdbValue) value, id);
1024-
} else {
1025-
// Store id in cache
1026-
LmdbValue nv = getLmdbValue(value);
1027-
nv.setInternalID(id, revision);
1039+
if (id == LmdbValue.UNKNOWN_ID) {
1040+
// not inlined or ID not cached, search in index
1041+
byte[] data = value2data(value, create);
1042+
if (data == null && value instanceof Literal) {
1043+
data = literal2legacy((Literal) value);
1044+
}
10281045

1029-
if (nv.isIRI() && isCommonVocabulary(((IRI) nv))) {
1030-
commonVocabulary.put(value, id);
1031-
}
1046+
if (data != null) {
1047+
id = findId(data, create);
1048+
}
1049+
}
10321050

1033-
valueIDCache.put(nv, id);
1051+
if (id != LmdbValue.UNKNOWN_ID) {
1052+
if (isOwnValue) {
1053+
// Store id in value for fast access in any consecutive calls
1054+
((LmdbValue) value).setInternalID(id, revision);
1055+
// Store id in cache
1056+
valueIDCache.put((LmdbValue) value, id);
1057+
} else {
1058+
// Store id in cache
1059+
LmdbValue nv = getLmdbValue(value);
1060+
nv.setInternalID(id, revision);
1061+
1062+
if (nv.isIRI() && isCommonVocabulary(((IRI) nv))) {
1063+
commonVocabulary.put(value, id);
10341064
}
1065+
valueIDCache.put(nv, id);
10351066
}
1036-
10371067
return id;
10381068
}
10391069
} finally {

‎core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/ValueStoreTest.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,11 +152,12 @@ public void testGcValuesAfterRestart() throws Exception {
152152

153153
@Test
154154
public void testGcDatatypes() throws Exception {
155-
IRI[] types = new IRI[] { XSD.STRING, XSD.INTEGER, XSD.DOUBLE, XSD.DECIMAL, XSD.FLOAT };
155+
IRI[] types = new IRI[] { XSD.STRING, XSD.INTEGER, XSD.LONG, XSD.DECIMAL };
156156
LmdbValue values[] = new LmdbValue[types.length];
157157
valueStore.startTransaction(true);
158158
for (int i = 0; i < values.length; i++) {
159-
values[i] = valueStore.createLiteral("123", types[i]);
159+
// use a value that is large enough to not being inlined
160+
values[i] = valueStore.createLiteral(Long.toString(Long.MAX_VALUE - 1), types[i]);
160161
valueStore.storeValue(values[i]);
161162
}
162163
valueStore.commit();
@@ -198,7 +199,8 @@ public void testGcDatatypes() throws Exception {
198199
public void testGcURIs() throws Exception {
199200
for (boolean storeAndGcUri : List.of(false, true)) {
200201
valueStore.startTransaction(true);
201-
LmdbLiteral literal = valueStore.createLiteral("123", XSD.STRING);
202+
// use a value that is large enough to not being inlined
203+
LmdbLiteral literal = valueStore.createLiteral("123".repeat(5), XSD.STRING);
202204
valueStore.storeValue(literal);
203205
if (storeAndGcUri) {
204206
valueStore.storeValue(XSD.STRING);

0 commit comments

Comments
 (0)