Skip to content

Commit 0963f6f

Browse files
hmottestadhmottestad
committed
GH-5153 benchmarks
1 parent d05b511 commit 0963f6f

6 files changed

Lines changed: 390 additions & 3 deletions

File tree

‎core/sail/shacl/src/test/java/org/eclipse/rdf4j/sail/shacl/benchmark/BenchmarkConfigs.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,25 @@ public class BenchmarkConfigs {
2424

2525
public static List<List<Statement>> generateStatements(StatementCreator statementCreator) {
2626

27+
return generateStatements(NUMBER_OF_TRANSACTIONS, STATEMENTS_PER_TRANSACTION, NUMBER_OF_EMPTY_TRANSACTIONS,
28+
statementCreator);
29+
}
30+
31+
public static List<List<Statement>> generateStatements(int numberOfTransactions, int statementsPerTransaction,
32+
int numberOfEmptyTransactions, StatementCreator statementCreator) {
33+
2734
List<List<Statement>> allStatements = new ArrayList<>();
2835

29-
for (int j = 0; j < BenchmarkConfigs.NUMBER_OF_TRANSACTIONS; j++) {
36+
for (int j = 0; j < numberOfTransactions; j++) {
3037
List<Statement> statements = new ArrayList<>();
3138
allStatements.add(statements);
32-
for (int i = 0; i < BenchmarkConfigs.STATEMENTS_PER_TRANSACTION; i++) {
39+
for (int i = 0; i < statementsPerTransaction; i++) {
3340

3441
statementCreator.createStatement(statements, i, j);
3542
}
3643
}
3744

38-
for (int j = 0; j < BenchmarkConfigs.NUMBER_OF_EMPTY_TRANSACTIONS; j++) {
45+
for (int j = 0; j < numberOfEmptyTransactions; j++) {
3946
List<Statement> statements = new ArrayList<>();
4047
allStatements.add(statements);
4148
}

‎core/sail/shacl/src/test/java/org/eclipse/rdf4j/sail/shacl/benchmark/MaxCountSparqlBenchmarkEmpty.java‎

Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,300 @@
1+
/*******************************************************************************
2+
* Copyright (c) 2024 Eclipse RDF4J contributors.
3+
*
4+
* All rights reserved. This program and the accompanying materials
5+
* are made available under the terms of the Eclipse Distribution License v1.0
6+
* which accompanies this distribution, and is available at
7+
* http://www.eclipse.org/org/documents/edl-v10.php.
8+
*
9+
* SPDX-License-Identifier: BSD-3-Clause
10+
******************************************************************************/
11+
12+
package org.eclipse.rdf4j.sail.shacl.benchmark;
13+
14+
import java.util.List;
15+
import java.util.concurrent.TimeUnit;
16+
17+
import org.eclipse.rdf4j.model.IRI;
18+
import org.eclipse.rdf4j.model.Statement;
19+
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
20+
import org.eclipse.rdf4j.model.vocabulary.RDF;
21+
import org.eclipse.rdf4j.model.vocabulary.RDFS;
22+
import org.eclipse.rdf4j.repository.RepositoryException;
23+
import org.eclipse.rdf4j.repository.sail.SailRepository;
24+
import org.eclipse.rdf4j.repository.sail.SailRepositoryConnection;
25+
import org.eclipse.rdf4j.sail.shacl.ShaclSail;
26+
import org.eclipse.rdf4j.sail.shacl.ShaclSailConnection;
27+
import org.eclipse.rdf4j.sail.shacl.ShaclSailValidationException;
28+
import org.eclipse.rdf4j.sail.shacl.Utils;
29+
import org.eclipse.rdf4j.sail.shacl.ast.constraintcomponents.MaxCountConstraintComponent;
30+
import org.openjdk.jmh.annotations.Benchmark;
31+
import org.openjdk.jmh.annotations.BenchmarkMode;
32+
import org.openjdk.jmh.annotations.Fork;
33+
import org.openjdk.jmh.annotations.Level;
34+
import org.openjdk.jmh.annotations.Measurement;
35+
import org.openjdk.jmh.annotations.Mode;
36+
import org.openjdk.jmh.annotations.OutputTimeUnit;
37+
import org.openjdk.jmh.annotations.Param;
38+
import org.openjdk.jmh.annotations.Scope;
39+
import org.openjdk.jmh.annotations.Setup;
40+
import org.openjdk.jmh.annotations.State;
41+
import org.openjdk.jmh.annotations.TearDown;
42+
import org.openjdk.jmh.annotations.Warmup;
43+
import org.slf4j.LoggerFactory;
44+
45+
import ch.qos.logback.classic.Logger;
46+
47+
/**
48+
* @author Håvard Ottestad
49+
*/
50+
@State(Scope.Benchmark)
51+
@Warmup(iterations = 5)
52+
@BenchmarkMode({ Mode.AverageTime })
53+
@Fork(value = 1, jvmArgs = { "-Xmx2G", "-Xms2G" })
54+
@Measurement(iterations = 5)
55+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
56+
public class MaxCountSparqlBenchmarkEmpty {
57+
58+
@Param({ "1", "2", "3", "4" })
59+
public int MAX_COUNT = 1;
60+
61+
@Param({ "manyInvalidStatements", "mostlyValidStatements" })
62+
public String statementList;
63+
64+
private static List<List<Statement>> manyInvalidStatements;
65+
private static List<List<Statement>> mostlyValidStatements1;
66+
private static List<List<Statement>> mostlyValidStatements2;
67+
private static List<List<Statement>> mostlyValidStatements3;
68+
private static List<List<Statement>> mostlyValidStatements4;
69+
70+
static {
71+
fillData();
72+
}
73+
74+
@Setup(Level.Trial)
75+
public void setUp() throws InterruptedException {
76+
77+
Logger root = (Logger) LoggerFactory.getLogger(ShaclSailConnection.class.getName());
78+
root.setLevel(ch.qos.logback.classic.Level.INFO);
79+
80+
System.gc();
81+
Thread.sleep(100);
82+
}
83+
84+
private static void fillData() {
85+
SimpleValueFactory vf = SimpleValueFactory.getInstance();
86+
87+
manyInvalidStatements = BenchmarkConfigs.generateStatements(1, 10, 0, ((statements, i, j) -> {
88+
IRI iri = vf.createIRI("http://example.com/invalid_" + i + "_" + j);
89+
90+
statements.add(vf.createStatement(iri, RDF.TYPE, RDFS.RESOURCE));
91+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label1" + i)));
92+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label2" + i)));
93+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label3" + i)));
94+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label4" + i)));
95+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label5" + i)));
96+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label6" + "_" + i + "_" + j)));
97+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label7" + "_" + i + "_" + j)));
98+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label8" + "_" + i + "_" + j)));
99+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label9" + "_" + i + "_" + j)));
100+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label10" + "_" + i + "_" + j)));
101+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label11" + "_" + i + "_" + j)));
102+
103+
for (int i2 = 0; i2 < 1000; i2++) {
104+
IRI validIri = vf.createIRI("http://example.com/valid" + i2 + "_" + i + "_" + j);
105+
statements.add(vf.createStatement(validIri, RDF.TYPE, RDFS.RESOURCE));
106+
statements.add(
107+
vf.createStatement(validIri, RDFS.LABEL, vf.createLiteral("label1" + i2 + "_" + i + "_" + j)));
108+
}
109+
110+
}));
111+
112+
mostlyValidStatements1 = BenchmarkConfigs.generateStatements(1, 1, 0, ((statements, i, j) -> {
113+
IRI iri = vf.createIRI("http://example.com/invalid_" + i + "_" + j);
114+
115+
statements.add(vf.createStatement(iri, RDF.TYPE, RDFS.RESOURCE));
116+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label1" + i)));
117+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label2" + i)));
118+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label3" + i)));
119+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label4" + i)));
120+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label5" + i)));
121+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label6" + "_" + i + "_" + j)));
122+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label7" + "_" + i + "_" + j)));
123+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label8" + "_" + i + "_" + j)));
124+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label9" + "_" + i + "_" + j)));
125+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label10" + "_" + i + "_" + j)));
126+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label11" + "_" + i + "_" + j)));
127+
128+
for (int i2 = 0; i2 < 10000; i2++) {
129+
IRI validIri = vf.createIRI("http://example.com/valid" + i2 + "_" + i + "_" + j);
130+
statements.add(vf.createStatement(validIri, RDF.TYPE, RDFS.RESOURCE));
131+
statements.add(
132+
vf.createStatement(validIri, RDFS.LABEL, vf.createLiteral("label1" + i2 + "_" + i + "_" + j)));
133+
}
134+
135+
}));
136+
137+
mostlyValidStatements2 = BenchmarkConfigs.generateStatements(1, 1, 0, ((statements, i, j) -> {
138+
IRI iri = vf.createIRI("http://example.com/invalid_" + i + "_" + j);
139+
140+
statements.add(vf.createStatement(iri, RDF.TYPE, RDFS.RESOURCE));
141+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label1" + i)));
142+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label2" + i)));
143+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label3" + i)));
144+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label4" + i)));
145+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label5" + i)));
146+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label6" + "_" + i + "_" + j)));
147+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label7" + "_" + i + "_" + j)));
148+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label8" + "_" + i + "_" + j)));
149+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label9" + "_" + i + "_" + j)));
150+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label10" + "_" + i + "_" + j)));
151+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label11" + "_" + i + "_" + j)));
152+
153+
for (int i2 = 0; i2 < 10000; i2++) {
154+
IRI validIri = vf.createIRI("http://example.com/valid" + i2 + "_" + i + "_" + j);
155+
statements.add(vf.createStatement(validIri, RDF.TYPE, RDFS.RESOURCE));
156+
statements.add(
157+
vf.createStatement(validIri, RDFS.LABEL, vf.createLiteral("label1" + i2 + "_" + i + "_" + j)));
158+
statements.add(
159+
vf.createStatement(validIri, RDFS.LABEL, vf.createLiteral("label2" + i2 + "_" + i + "_" + j)));
160+
}
161+
162+
}));
163+
164+
mostlyValidStatements3 = BenchmarkConfigs.generateStatements(1, 1, 0, ((statements, i, j) -> {
165+
IRI iri = vf.createIRI("http://example.com/invalid_" + i + "_" + j);
166+
167+
statements.add(vf.createStatement(iri, RDF.TYPE, RDFS.RESOURCE));
168+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label1" + i)));
169+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label2" + i)));
170+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label3" + i)));
171+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label4" + i)));
172+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label5" + i)));
173+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label6" + "_" + i + "_" + j)));
174+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label7" + "_" + i + "_" + j)));
175+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label8" + "_" + i + "_" + j)));
176+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label9" + "_" + i + "_" + j)));
177+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label10" + "_" + i + "_" + j)));
178+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label11" + "_" + i + "_" + j)));
179+
180+
for (int i2 = 0; i2 < 10000; i2++) {
181+
IRI validIri = vf.createIRI("http://example.com/valid" + i2 + "_" + i + "_" + j);
182+
statements.add(vf.createStatement(validIri, RDF.TYPE, RDFS.RESOURCE));
183+
statements.add(
184+
vf.createStatement(validIri, RDFS.LABEL, vf.createLiteral("label1" + i2 + "_" + i + "_" + j)));
185+
statements.add(
186+
vf.createStatement(validIri, RDFS.LABEL, vf.createLiteral("label2" + i2 + "_" + i + "_" + j)));
187+
statements.add(
188+
vf.createStatement(validIri, RDFS.LABEL, vf.createLiteral("label3" + i2 + "_" + i + "_" + j)));
189+
}
190+
191+
}));
192+
193+
mostlyValidStatements4 = BenchmarkConfigs.generateStatements(1, 1, 0, ((statements, i, j) -> {
194+
IRI iri = vf.createIRI("http://example.com/invalid_" + i + "_" + j);
195+
196+
statements.add(vf.createStatement(iri, RDF.TYPE, RDFS.RESOURCE));
197+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label1" + i)));
198+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label2" + i)));
199+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label3" + i)));
200+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label4" + i)));
201+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label5" + i)));
202+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label6" + "_" + i + "_" + j)));
203+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label7" + "_" + i + "_" + j)));
204+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label8" + "_" + i + "_" + j)));
205+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label9" + "_" + i + "_" + j)));
206+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label10" + "_" + i + "_" + j)));
207+
statements.add(vf.createStatement(iri, RDFS.LABEL, vf.createLiteral("label11" + "_" + i + "_" + j)));
208+
209+
for (int i2 = 0; i2 < 10000; i2++) {
210+
IRI validIri = vf.createIRI("http://example.com/valid" + i2 + "_" + i + "_" + j);
211+
statements.add(vf.createStatement(validIri, RDF.TYPE, RDFS.RESOURCE));
212+
statements.add(
213+
vf.createStatement(validIri, RDFS.LABEL, vf.createLiteral("label1" + i2 + "_" + i + "_" + j)));
214+
statements.add(
215+
vf.createStatement(validIri, RDFS.LABEL, vf.createLiteral("label2" + i2 + "_" + i + "_" + j)));
216+
statements.add(
217+
vf.createStatement(validIri, RDFS.LABEL, vf.createLiteral("label3" + i2 + "_" + i + "_" + j)));
218+
statements.add(
219+
vf.createStatement(validIri, RDFS.LABEL, vf.createLiteral("label4" + i2 + "_" + i + "_" + j)));
220+
}
221+
222+
}));
223+
}
224+
225+
@TearDown(Level.Trial)
226+
public void tearDown() {
227+
MaxCountConstraintComponent.SPARQL_VALIDATION_APPROACH_LIMIT = 5;
228+
}
229+
230+
@Benchmark
231+
public void shaclBulkSparql() throws Exception {
232+
233+
MaxCountConstraintComponent.SPARQL_VALIDATION_APPROACH_LIMIT = 10;
234+
235+
SailRepository repository = new SailRepository(
236+
Utils.getInitializedShaclSail("shaclMaxCountBenchmark" + MAX_COUNT + ".trig"));
237+
238+
try (SailRepositoryConnection connection = repository.getConnection()) {
239+
connection.begin(ShaclSail.TransactionSettings.ValidationApproach.Bulk);
240+
for (List<Statement> statements : getStatements()) {
241+
connection.add(statements);
242+
}
243+
try {
244+
connection.commit();
245+
} catch (RepositoryException e) {
246+
if (!(e.getCause() instanceof ShaclSailValidationException)) {
247+
throw e;
248+
}
249+
}
250+
}
251+
repository.shutDown();
252+
253+
}
254+
255+
private List<List<Statement>> getStatements() {
256+
if (statementList.equals("mostlyValidStatements")) {
257+
statementList = "mostlyValidStatements" + MAX_COUNT;
258+
}
259+
260+
switch (statementList) {
261+
case "manyInvalidStatements":
262+
return manyInvalidStatements;
263+
case "mostlyValidStatements1":
264+
return mostlyValidStatements1;
265+
case "mostlyValidStatements2":
266+
return mostlyValidStatements2;
267+
case "mostlyValidStatements3":
268+
return mostlyValidStatements3;
269+
case "mostlyValidStatements4":
270+
return mostlyValidStatements4;
271+
}
272+
throw new IllegalStateException();
273+
}
274+
275+
@Benchmark
276+
public void shaclBulkNonSparql() throws Exception {
277+
278+
MaxCountConstraintComponent.SPARQL_VALIDATION_APPROACH_LIMIT = 0;
279+
280+
SailRepository repository = new SailRepository(
281+
Utils.getInitializedShaclSail("shaclMaxCountBenchmark" + MAX_COUNT + ".trig"));
282+
283+
try (SailRepositoryConnection connection = repository.getConnection()) {
284+
connection.begin(ShaclSail.TransactionSettings.ValidationApproach.Bulk);
285+
for (List<Statement> statements : getStatements()) {
286+
connection.add(statements);
287+
}
288+
try {
289+
connection.commit();
290+
} catch (RepositoryException e) {
291+
if (!(e.getCause() instanceof ShaclSailValidationException)) {
292+
throw e;
293+
}
294+
}
295+
}
296+
repository.shutDown();
297+
298+
}
299+
300+
}

‎core/sail/shacl/src/test/resources/shaclMaxCountBenchmark1.trig‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
@base <http://example.com/ns> .
2+
@prefix ex: <http://example.com/ns#> .
3+
@prefix owl: <http://www.w3.org/2002/07/owl#> .
4+
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
5+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
6+
@prefix sh: <http://www.w3.org/ns/shacl#> .
7+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
8+
@prefix rdf4j: <http://rdf4j.org/schema/rdf4j#> .
9+
10+
rdf4j:SHACLShapeGraph {
11+
ex:PersonShape
12+
a sh:NodeShape ;
13+
sh:targetClass rdfs:Resource ;
14+
sh:property ex:PersonShapeProperty .
15+
16+
17+
ex:PersonShapeProperty
18+
sh:path rdfs:label ;
19+
sh:maxCount 1 .
20+
}

‎core/sail/shacl/src/test/resources/shaclMaxCountBenchmark2.trig‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
@base <http://example.com/ns> .
2+
@prefix ex: <http://example.com/ns#> .
3+
@prefix owl: <http://www.w3.org/2002/07/owl#> .
4+
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
5+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
6+
@prefix sh: <http://www.w3.org/ns/shacl#> .
7+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
8+
@prefix rdf4j: <http://rdf4j.org/schema/rdf4j#> .
9+
10+
rdf4j:SHACLShapeGraph {
11+
ex:PersonShape
12+
a sh:NodeShape ;
13+
sh:targetClass rdfs:Resource ;
14+
sh:property ex:PersonShapeProperty .
15+
16+
17+
ex:PersonShapeProperty
18+
sh:path rdfs:label ;
19+
sh:maxCount 2 .
20+
}

‎core/sail/shacl/src/test/resources/shaclMaxCountBenchmark3.trig‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
@base <http://example.com/ns> .
2+
@prefix ex: <http://example.com/ns#> .
3+
@prefix owl: <http://www.w3.org/2002/07/owl#> .
4+
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
5+
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
6+
@prefix sh: <http://www.w3.org/ns/shacl#> .
7+
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
8+
@prefix rdf4j: <http://rdf4j.org/schema/rdf4j#> .
9+
10+
rdf4j:SHACLShapeGraph {
11+
ex:PersonShape
12+
a sh:NodeShape ;
13+
sh:targetClass rdfs:Resource ;
14+
sh:property ex:PersonShapeProperty .
15+
16+
17+
ex:PersonShapeProperty
18+
sh:path rdfs:label ;
19+
sh:maxCount 3 .
20+
}

0 commit comments

Comments
 (0)