diff --git a/core/common/iterator/src/main/java/org/eclipse/rdf4j/common/iteration/DistinctIteration.java b/core/common/iterator/src/main/java/org/eclipse/rdf4j/common/iteration/DistinctIteration.java index 0f9215ac837..1d27e8550f2 100644 --- a/core/common/iterator/src/main/java/org/eclipse/rdf4j/common/iteration/DistinctIteration.java +++ b/core/common/iterator/src/main/java/org/eclipse/rdf4j/common/iteration/DistinctIteration.java @@ -8,9 +8,11 @@ * * SPDX-License-Identifier: BSD-3-Clause *******************************************************************************/ - +// Some portions generated by Codex package org.eclipse.rdf4j.common.iteration; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.HashSet; import java.util.Set; import java.util.function.Supplier; @@ -20,6 +22,8 @@ */ public class DistinctIteration extends FilterIteration { + private static final String OPERATOR_NAME = "DISTINCT"; + /*-----------* * Variables * *-----------*/ @@ -27,7 +31,7 @@ public class DistinctIteration extends FilterIteration { /** * The elements that have already been returned. */ - private final Set excludeSet; + private Set excludeSet; /*--------------* * Constructors * @@ -76,6 +80,15 @@ public DistinctIteration(CloseableIteration iter, Supplier> */ @Override protected boolean accept(E object) { + try { + QueryExecutionContextBridge.throwIfHeavyOperatorExecutionDisabled(OPERATOR_NAME); + QueryExecutionContextBridge.markHeavy(OPERATOR_NAME); + QueryExecutionContextBridge.checkpoint(OPERATOR_NAME); + } catch (Throwable t) { + excludeSet = null; + throw t; + } + if (inExcludeSet(object)) { // object has already been returned return false; @@ -87,7 +100,7 @@ protected boolean accept(E object) { @Override protected void handleClose() { - + excludeSet = null; } /** @@ -104,4 +117,86 @@ private boolean inExcludeSet(E object) { protected boolean add(E object) { return excludeSet.add(object); } + + private static final class QueryExecutionContextBridge { + + private static final String QUERY_EXECUTION_CONTEXT_CLASS = "org.eclipse.rdf4j.http.client.QueryExecutionContext"; + + private static volatile boolean initialized; + private static volatile Method markHeavyMethod; + private static volatile Method checkpointMethod; + private static volatile Method throwIfHeavyOperatorExecutionDisabledMethod; + + private QueryExecutionContextBridge() { + } + + private static void markHeavy(String operator) { + Method method = getMethod(true); + if (method != null) { + invoke(method, operator); + } + } + + private static void checkpoint(String operator) { + Method method = getMethod(false); + if (method != null) { + invoke(method, operator); + } + } + + private static void throwIfHeavyOperatorExecutionDisabled(String operator) { + if (!initialized) { + initialize(); + } + if (throwIfHeavyOperatorExecutionDisabledMethod != null) { + invoke(throwIfHeavyOperatorExecutionDisabledMethod, operator); + } + } + + private static Method getMethod(boolean markHeavy) { + if (!initialized) { + initialize(); + } + return markHeavy ? markHeavyMethod : checkpointMethod; + } + + private static synchronized void initialize() { + if (initialized) { + return; + } + try { + Class contextType = Class.forName(QUERY_EXECUTION_CONTEXT_CLASS); + markHeavyMethod = contextType.getMethod("markHeavy", String.class); + checkpointMethod = contextType.getMethod("checkpoint", String.class); + throwIfHeavyOperatorExecutionDisabledMethod = contextType.getMethod( + "throwIfHeavyOperatorExecutionDisabled", String.class); + } catch (ClassNotFoundException | NoSuchMethodException e) { + markHeavyMethod = null; + checkpointMethod = null; + throwIfHeavyOperatorExecutionDisabledMethod = null; + } + initialized = true; + } + + private static void invoke(Method method, String operator) { + try { + method.invoke(null, operator); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Unable to access query execution context bridge", e); + } catch (InvocationTargetException e) { + throw propagate(e.getCause()); + } + } + + private static RuntimeException propagate(Throwable throwable) { + if (throwable instanceof RuntimeException) { + throw (RuntimeException) throwable; + } + if (throwable instanceof Error) { + throw (Error) throwable; + } + throw new IllegalStateException("Unexpected checked exception from query execution context bridge", + throwable); + } + } } diff --git a/core/common/iterator/src/test/java/org/eclipse/rdf4j/common/iteration/DistinctIterationTest.java b/core/common/iterator/src/test/java/org/eclipse/rdf4j/common/iteration/DistinctIterationTest.java new file mode 100644 index 00000000000..640025e63fe --- /dev/null +++ b/core/common/iterator/src/test/java/org/eclipse/rdf4j/common/iteration/DistinctIterationTest.java @@ -0,0 +1,56 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex +package org.eclipse.rdf4j.common.iteration; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.eclipse.rdf4j.http.client.QueryExecutionContext; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +class DistinctIterationTest { + + @AfterEach + void tearDown() { + QueryExecutionContext.reset(); + } + + @Test + void shouldConsultQueryExecutionContextWhenFilteringDistinctValues() { + QueryExecutionContext.failOnCheckpoint(new RuntimeException("breaker checkpoint")); + + DistinctIteration iteration = new DistinctIteration<>( + new CloseableIteratorIteration<>(List.of(1, 2, 3).iterator()), java.util.HashSet::new); + + RuntimeException exception = assertThrows(RuntimeException.class, iteration::hasNext); + assertEquals("breaker checkpoint", exception.getMessage()); + assertTrue(QueryExecutionContext.getMarkHeavyCalls() > 0); + assertTrue(QueryExecutionContext.getCheckpointCalls() > 0); + } + + @Test + void shouldAbortDistinctIterationWhenHeavyOperatorExecutionIsDisabled() { + QueryExecutionContext.disableHeavyOperatorExecution(new RuntimeException("critical-breaker-stop")); + + DistinctIteration iteration = new DistinctIteration<>( + new CloseableIteratorIteration<>(List.of(1, 2, 3).iterator()), java.util.HashSet::new); + + RuntimeException exception = assertThrows(RuntimeException.class, iteration::hasNext); + assertEquals("critical-breaker-stop", exception.getMessage()); + assertEquals(0, QueryExecutionContext.getMarkHeavyCalls()); + assertEquals(0, QueryExecutionContext.getCheckpointCalls()); + } +} diff --git a/core/common/iterator/src/test/java/org/eclipse/rdf4j/http/client/QueryExecutionContext.java b/core/common/iterator/src/test/java/org/eclipse/rdf4j/http/client/QueryExecutionContext.java new file mode 100644 index 00000000000..d2b693c786e --- /dev/null +++ b/core/common/iterator/src/test/java/org/eclipse/rdf4j/http/client/QueryExecutionContext.java @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex +package org.eclipse.rdf4j.http.client; + +import java.util.concurrent.atomic.AtomicInteger; + +public final class QueryExecutionContext { + + private static final AtomicInteger MARK_HEAVY_CALLS = new AtomicInteger(); + private static final AtomicInteger CHECKPOINT_CALLS = new AtomicInteger(); + + private static volatile boolean heavyOperatorExecutionEnabled = true; + private static volatile RuntimeException checkpointFailure; + private static volatile RuntimeException heavyOperatorExecutionFailure; + + private QueryExecutionContext() { + } + + public static void markHeavy(String operator) { + MARK_HEAVY_CALLS.incrementAndGet(); + } + + public static void checkpoint(String operator) { + CHECKPOINT_CALLS.incrementAndGet(); + if (checkpointFailure != null) { + throw checkpointFailure; + } + } + + public static void throwIfHeavyOperatorExecutionDisabled(String operator) { + if (!heavyOperatorExecutionEnabled) { + throw heavyOperatorExecutionFailure != null ? heavyOperatorExecutionFailure + : new RuntimeException("heavy operator execution disabled: " + operator); + } + } + + public static void failOnCheckpoint(RuntimeException runtimeException) { + checkpointFailure = runtimeException; + } + + public static void disableHeavyOperatorExecution(RuntimeException runtimeException) { + heavyOperatorExecutionEnabled = false; + heavyOperatorExecutionFailure = runtimeException; + } + + public static void enableHeavyOperatorExecution() { + heavyOperatorExecutionEnabled = true; + heavyOperatorExecutionFailure = null; + } + + public static int getMarkHeavyCalls() { + return MARK_HEAVY_CALLS.get(); + } + + public static int getCheckpointCalls() { + return CHECKPOINT_CALLS.get(); + } + + public static void reset() { + MARK_HEAVY_CALLS.set(0); + CHECKPOINT_CALLS.set(0); + heavyOperatorExecutionEnabled = true; + checkpointFailure = null; + heavyOperatorExecutionFailure = null; + } +} diff --git a/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryCircuitBreaker.java b/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryCircuitBreaker.java new file mode 100644 index 00000000000..5429c3c283c --- /dev/null +++ b/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryCircuitBreaker.java @@ -0,0 +1,896 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex +package org.eclipse.rdf4j.http.client; + +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.eclipse.rdf4j.query.QueryInterruptedException; +import org.eclipse.rdf4j.repository.RepositoryConnection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * JVM-global query pressure circuit breaker. + */ +public final class QueryCircuitBreaker { + + public static final String ENABLED_PROPERTY = "rdf4j.query.breaker.enabled"; + public static final String WARN_GC_MS_PROPERTY = "rdf4j.query.breaker.warn.gc.ms"; + public static final String HIGH_GC_MS_PROPERTY = "rdf4j.query.breaker.high.gc.ms"; + public static final String CRITICAL_GC_MS_PROPERTY = "rdf4j.query.breaker.critical.gc.ms"; + public static final String WARN_FREE_MB_PROPERTY = "rdf4j.query.breaker.warn.free.mb"; + public static final String HIGH_FREE_MB_PROPERTY = "rdf4j.query.breaker.high.free.mb"; + public static final String CRITICAL_FREE_MB_PROPERTY = "rdf4j.query.breaker.critical.free.mb"; + public static final String WARN_ADMISSION_DELAY_MS_PROPERTY = "rdf4j.query.breaker.warn.admission.delay.ms"; + public static final String CHECKPOINT_DELAY_MS_PROPERTY = "rdf4j.query.breaker.checkpoint.delay.ms"; + public static final String CANCEL_COOLDOWN_MS_PROPERTY = "rdf4j.query.breaker.cancel.cooldown.ms"; + public static final String RETRY_AFTER_SECONDS_PROPERTY = "rdf4j.query.breaker.retry.after.seconds"; + + private static final Logger LOGGER = LoggerFactory.getLogger(QueryCircuitBreaker.class); + private static final long DEFAULT_RECOVERY_COOLDOWN_MS = 1000; + private static final long DEFAULT_GC_MONITOR_POLL_MS = 250; + private static final long CHECKPOINT_GC_BASE_INTERVAL_MS = 1000; + private static final int CHECKPOINT_GC_MAX_INTERVAL_STEPS = 10; + private static final long CHECKPOINT_GC_RESET_AFTER_MS = 30_000; + private static final long HIGH_MEMORY_GC_INTERVAL_MS = 5000; + private static final long CRITICAL_MEMORY_GC_INTERVAL_MS = 1000; + private static final long THROTTLE_WARNING_INTERVAL_MS = 60_000; + private static final String GC_MONITOR_THREAD_NAME = "rdf4j-query-breaker-gc-monitor"; + private static final QueryCircuitBreaker INSTANCE = new QueryCircuitBreaker(new QueryPressureMonitor(), + Configuration::fromSystemProperties, System::currentTimeMillis, Thread::sleep, + System::gc, DEFAULT_RECOVERY_COOLDOWN_MS, DEFAULT_GC_MONITOR_POLL_MS, true); + + private final QueryPressureMonitor pressureMonitor; + private final Supplier configurationSupplier; + private final LongSupplier clock; + private final Sleeper sleeper; + private final GcInvoker gcInvoker; + private final long recoveryCooldownMs; + private final long gcMonitorPollMs; + private final ConcurrentMap activeHandles = new ConcurrentHashMap<>(); + private final AtomicLong handleSequence = new AtomicLong(); + private final AtomicLong rejectCount = new AtomicLong(); + private final AtomicLong cancelCount = new AtomicLong(); + private final AtomicLong lastEntryThrottleWarningAt = new AtomicLong(Long.MIN_VALUE); + private final AtomicLong lastRunningThrottleWarningAt = new AtomicLong(Long.MIN_VALUE); + + private volatile QueryPressureState currentState = QueryPressureState.NORMAL; + private volatile Transition lastTransition = Transition.initial(); + private volatile long lastCancelAt = Long.MIN_VALUE; + private volatile QueryPressureState lastMonitorMemoryState = QueryPressureState.NORMAL; + private volatile long lastMonitorGcAt = Long.MIN_VALUE; + private long lastCheckpointGcAt = Long.MIN_VALUE; + private int checkpointGcIntervalStep; + + public static QueryCircuitBreaker getInstance() { + return INSTANCE; + } + + public QueryCircuitBreaker(QueryPressureMonitor pressureMonitor) { + this(pressureMonitor, Configuration::fromSystemProperties, System::currentTimeMillis, Thread::sleep, + System::gc, DEFAULT_RECOVERY_COOLDOWN_MS, DEFAULT_GC_MONITOR_POLL_MS, true); + } + + QueryCircuitBreaker(QueryPressureMonitor pressureMonitor, Supplier configurationSupplier, + LongSupplier clock, Sleeper sleeper, long recoveryCooldownMs) { + this(pressureMonitor, configurationSupplier, clock, sleeper, System::gc, recoveryCooldownMs, + DEFAULT_GC_MONITOR_POLL_MS, false); + } + + QueryCircuitBreaker(QueryPressureMonitor pressureMonitor, Supplier configurationSupplier, + LongSupplier clock, Sleeper sleeper, GcInvoker gcInvoker, long recoveryCooldownMs, long gcMonitorPollMs, + boolean startGcMonitorThread) { + this.pressureMonitor = Objects.requireNonNull(pressureMonitor, "Pressure monitor was null"); + this.configurationSupplier = Objects.requireNonNull(configurationSupplier, "Configuration supplier was null"); + this.clock = Objects.requireNonNull(clock, "Clock was null"); + this.sleeper = Objects.requireNonNull(sleeper, "Sleeper was null"); + this.gcInvoker = Objects.requireNonNull(gcInvoker, "GC invoker was null"); + this.recoveryCooldownMs = recoveryCooldownMs; + this.gcMonitorPollMs = gcMonitorPollMs; + if (startGcMonitorThread) { + startGcMonitorThread(); + } + } + + public QueryCircuitBreakerHandle register(QueryCircuitBreakerHandle.Source source, String repositoryId, + String queryText) { + return register(source, repositoryId, queryText, null); + } + + public QueryCircuitBreakerHandle register(QueryCircuitBreakerHandle.Source source, String repositoryId, + String queryText, Runnable remoteCancel) { + String executionId = source.name().toLowerCase(Locale.ROOT) + "-" + handleSequence.incrementAndGet(); + QueryCircuitBreakerHandle handle = new QueryCircuitBreakerHandle(executionId, source, + normalizeRepositoryId(repositoryId), hashQuery(queryText), clock.getAsLong(), remoteCancel); + activeHandles.put(handle.getExecutionId(), handle); + return handle; + } + + public T execute(QueryCircuitBreakerHandle handle, + RepositoryConnection repositoryConnection, + QueryOperation operation) throws X, QueryInterruptedException { + Objects.requireNonNull(handle, "Handle was null"); + Objects.requireNonNull(operation, "Operation was null"); + + handle.attachCurrentThread(repositoryConnection); + beforeExecution(handle); + try (QueryExecutionContext.Activation ignored = QueryExecutionContext.activate(handle)) { + return operation.execute(); + } + } + + public void beforeExecution(QueryCircuitBreakerHandle handle) throws QueryInterruptedException { + Configuration configuration = configurationSupplier.get(); + QueryPressureMonitor.Snapshot snapshot = pressureMonitor.sample(); + QueryPressureState state = refreshState("admission", configuration, snapshot); + if (!configuration.isEnabled()) { + return; + } + + warnAboutNewQueryThrottlingIfNeeded(state, configuration, snapshot); + + if (state == QueryPressureState.WARN && configuration.getWarnAdmissionDelayMs() > 0) { + delay(configuration.getWarnAdmissionDelayMs(), state, configuration, false); + } + + if (state.rejectsNewQueries()) { + System.gc(); + rejectCount.incrementAndGet(); + if (state == QueryPressureState.CRITICAL) { + cancelOneHeavyQueryIfNeeded(configuration, snapshot, "critical-admission"); + } + throw CircuitBreakerException.rejected(state, configuration.getRetryAfterSeconds(), + buildPressureMessage("Query rejected by global memory circuit breaker", snapshot, state)); + } + } + + public void markHeavy(QueryCircuitBreakerHandle handle, String operator) { + if (handle == null) { + return; + } + handle.markHeavy(operator, clock.getAsLong()); + } + + int throttleCount = 0; + + public void checkpoint(QueryCircuitBreakerHandle handle, String operator) throws QueryInterruptedException { + if (handle == null) { + return; + } + + Configuration configuration = configurationSupplier.get(); + handle.markHeavy(operator, clock.getAsLong()); + if (!configuration.isEnabled()) { + return; + } + + QueryPressureMonitor.Snapshot snapshot = pressureMonitor.sample(); + QueryPressureState state = refreshState("checkpoint:" + operator, configuration, snapshot); + warnAboutRunningQueryThrottlingIfNeeded(state, snapshot); + if (shouldRequestCheckpointGc(state, configuration, snapshot)) { + maybeRunCheckpointGc(); + } + if (state == QueryPressureState.CRITICAL) { + cancelOneHeavyQueryIfNeeded(configuration, snapshot, "critical-checkpoint:" + operator); + } + if (handle.isCancelRequested()) { + throw CircuitBreakerException.cancelled(handle.getCancellationState(), configuration.getRetryAfterSeconds(), + handle.getCancellationReason()); + } + if (state.throttlesRunningQueries() && configuration.getCheckpointDelayMs() > 0 + && (throttleCount++) % 256 == 0) { + delay(configuration.getCheckpointDelayMs(), state, configuration, true); + } + } + + public void complete(QueryCircuitBreakerHandle handle) { + if (handle == null) { + return; + } + activeHandles.remove(handle.getExecutionId(), handle); + handle.finish(); + } + + public StatusSnapshot snapshotStatus() { + Configuration configuration = configurationSupplier.get(); + QueryPressureMonitor.Snapshot snapshot = pressureMonitor.sample(); + QueryPressureState state = refreshState("status", configuration, snapshot); + List activeQueries = activeHandles.values() + .stream() + .sorted(Comparator.comparingLong(QueryCircuitBreakerHandle::getStartTimeMillis)) + .map(handle -> new ActiveQueryStatus(handle.getExecutionId(), handle.getSource().name(), + handle.getRepositoryId(), handle.getQueryHash(), handle.getStartTimeMillis(), + handle.getLastHeavyCheckpointMillis(), handle.getLastHeavyOperator(), + handle.isCancelRequested())) + .collect(Collectors.toList()); + Transition transition = lastTransition; + return new StatusSnapshot(configuration.isEnabled(), state.name(), snapshot.getFreeMemoryMb(), + snapshot.getRollingGcMs(), configuration, activeQueries, rejectCount.get(), cancelCount.get(), + transition.getTimestampMillis(), transition.getReason()); + } + + void refreshHeavyOperatorExecutionState() { + Configuration configuration = configurationSupplier.get(); + QueryPressureMonitor.Snapshot snapshot = pressureMonitor.sample(); + refreshState("activate", configuration, snapshot); + } + + public static CircuitBreakerException asCircuitBreakerException(Throwable throwable) { + if (throwable instanceof CircuitBreakerException) { + return (CircuitBreakerException) throwable; + } + return null; + } + + private void delay(long delayMs, QueryPressureState state, Configuration configuration, boolean checkpointDelay) + throws QueryInterruptedException { + try { + sleeper.sleep(delayMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw CircuitBreakerException.cancelled(state, configuration.getRetryAfterSeconds(), + checkpointDelay ? "Query interrupted during breaker checkpoint delay" + : "Query interrupted during breaker admission delay", + e); + } + } + + private QueryPressureState refreshState(String reason, Configuration configuration, + QueryPressureMonitor.Snapshot snapshot) { + QueryPressureState next = configuration.isEnabled() ? determineState(configuration, snapshot) + : QueryPressureState.NORMAL; + QueryPressureState previous = currentState; + long now = clock.getAsLong(); + + if (next.ordinal() < previous.ordinal() && now - lastTransition.getTimestampMillis() < recoveryCooldownMs) { + next = previous; + } + + if (next != previous) { + currentState = next; + lastTransition = new Transition(now, reason, previous, next); + LOGGER.info( + "Query circuit breaker transition previous={} current={} freeMb={} rollingGcMs={} reason={}", + previous, next, snapshot.getFreeMemoryMb(), snapshot.getRollingGcMs(), reason); + if (!isCheckpointReason(reason) && shouldRequestTransitionGc(currentState, configuration, snapshot)) { + System.gc(); + } + } + if (!throttlesNewQueriesAtEntry(currentState, configuration)) { + lastEntryThrottleWarningAt.set(Long.MIN_VALUE); + } + if (!currentState.throttlesRunningQueries()) { + lastRunningThrottleWarningAt.set(Long.MIN_VALUE); + } + QueryExecutionContext.setIgnoreCheckpointStride(currentState.throttlesRunningQueries()); + QueryExecutionContext.setHeavyOperatorExecutionEnabled(currentState != QueryPressureState.CRITICAL); + return currentState; + } + + private void warnAboutNewQueryThrottlingIfNeeded(QueryPressureState state, Configuration configuration, + QueryPressureMonitor.Snapshot snapshot) { + if (!throttlesNewQueriesAtEntry(state, configuration) + || !shouldLogThrottleWarning(lastEntryThrottleWarningAt)) { + return; + } + LOGGER.warn( + "Query circuit breaker is throttling new queries at entry state={} action={} freeMb={} rollingGcMs={}", + state, state.rejectsNewQueries() ? "reject" : "delay", snapshot.getFreeMemoryMb(), + snapshot.getRollingGcMs()); + } + + private void warnAboutRunningQueryThrottlingIfNeeded(QueryPressureState state, + QueryPressureMonitor.Snapshot snapshot) { + if (!state.throttlesRunningQueries() || !shouldLogThrottleWarning(lastRunningThrottleWarningAt)) { + return; + } + LOGGER.warn("Query circuit breaker is throttling running queries state={} freeMb={} rollingGcMs={}", state, + snapshot.getFreeMemoryMb(), snapshot.getRollingGcMs()); + } + + private boolean throttlesNewQueriesAtEntry(QueryPressureState state, Configuration configuration) { + return (state == QueryPressureState.WARN && configuration.getWarnAdmissionDelayMs() > 0) + || state.rejectsNewQueries(); + } + + private boolean isCheckpointReason(String reason) { + return reason.startsWith("checkpoint:"); + } + + private boolean shouldRequestTransitionGc(QueryPressureState state, Configuration configuration, + QueryPressureMonitor.Snapshot snapshot) { + return state == QueryPressureState.HIGH + || (state == QueryPressureState.WARN + && determineFreeMemoryState(configuration, + snapshot.getFreeMemoryMb()) == QueryPressureState.WARN); + } + + private boolean shouldRequestCheckpointGc(QueryPressureState state, Configuration configuration, + QueryPressureMonitor.Snapshot snapshot) { + return state == QueryPressureState.CRITICAL || shouldRequestTransitionGc(state, configuration, snapshot); + } + + private void maybeRunCheckpointGc() { + long now = clock.getAsLong(); + if (!shouldRunCheckpointGc(now)) { + return; + } + gcInvoker.runGc(); + } + + private synchronized boolean shouldRunCheckpointGc(long now) { + if (lastCheckpointGcAt == Long.MIN_VALUE || now - lastCheckpointGcAt > CHECKPOINT_GC_RESET_AFTER_MS) { + lastCheckpointGcAt = now; + checkpointGcIntervalStep = 1; + return true; + } + + long requiredDelayMs = checkpointGcIntervalStep * CHECKPOINT_GC_BASE_INTERVAL_MS; + if (now - lastCheckpointGcAt < requiredDelayMs) { + return false; + } + + lastCheckpointGcAt = now; + checkpointGcIntervalStep = Math.min(checkpointGcIntervalStep + 1, CHECKPOINT_GC_MAX_INTERVAL_STEPS); + return true; + } + + private boolean shouldLogThrottleWarning(AtomicLong lastWarningAt) { + long now = clock.getAsLong(); + while (true) { + long previous = lastWarningAt.get(); + if (previous != Long.MIN_VALUE && now - previous < THROTTLE_WARNING_INTERVAL_MS) { + return false; + } + if (lastWarningAt.compareAndSet(previous, now)) { + return true; + } + } + } + + private QueryPressureState determineState(Configuration configuration, QueryPressureMonitor.Snapshot snapshot) { + QueryPressureState gcState = determineGcState(configuration, snapshot.getRollingGcMs()); + QueryPressureState freeMemoryState = determineFreeMemoryState(configuration, snapshot.getFreeMemoryMb()); + return max(freeMemoryState, capGcState(gcState, freeMemoryState)); + } + + void runGcMonitorCycle() { + Configuration configuration = configurationSupplier.get(); + QueryPressureMonitor.Snapshot snapshot = pressureMonitor.sample(); + refreshState("monitor", configuration, snapshot); + + QueryPressureState freeMemoryState = configuration.isEnabled() + ? determineFreeMemoryState(configuration, snapshot.getFreeMemoryMb()) + : QueryPressureState.NORMAL; + if (freeMemoryState != lastMonitorMemoryState) { + lastMonitorMemoryState = freeMemoryState; + lastMonitorGcAt = Long.MIN_VALUE; + } + + long intervalMs = gcMonitorIntervalMs(freeMemoryState); + if (intervalMs < 0) { + return; + } + + long now = clock.getAsLong(); + if (lastMonitorGcAt != Long.MIN_VALUE && now - lastMonitorGcAt < intervalMs) { + return; + } + + lastMonitorGcAt = now; + LOGGER.info("Query circuit breaker requesting System.gc() monitorState={} freeMb={} rollingGcMs={}", + freeMemoryState, snapshot.getFreeMemoryMb(), snapshot.getRollingGcMs()); + gcInvoker.runGc(); + } + + private void startGcMonitorThread() { + Thread gcMonitorThread = new Thread(this::runGcMonitorLoop, GC_MONITOR_THREAD_NAME); + gcMonitorThread.setDaemon(true); + gcMonitorThread.start(); + } + + private void runGcMonitorLoop() { + while (!Thread.currentThread().isInterrupted()) { + try { + runGcMonitorCycle(); + Thread.sleep(gcMonitorPollMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } catch (RuntimeException e) { + LOGGER.warn("Query circuit breaker GC monitor loop failed", e); + } + } + } + + private long gcMonitorIntervalMs(QueryPressureState freeMemoryState) { + if (freeMemoryState == QueryPressureState.CRITICAL) { + return CRITICAL_MEMORY_GC_INTERVAL_MS; + } + if (freeMemoryState == QueryPressureState.HIGH) { + return HIGH_MEMORY_GC_INTERVAL_MS; + } + return -1; + } + + private QueryPressureState determineGcState(Configuration configuration, int rollingGcMs) { + if (rollingGcMs >= configuration.getCriticalGcMs()) { + return QueryPressureState.CRITICAL; + } + if (rollingGcMs >= configuration.getHighGcMs()) { + return QueryPressureState.HIGH; + } + if (rollingGcMs >= configuration.getWarnGcMs()) { + return QueryPressureState.WARN; + } + return QueryPressureState.NORMAL; + } + + private QueryPressureState determineFreeMemoryState(Configuration configuration, int freeMemoryMb) { + if (freeMemoryMb <= configuration.getCriticalFreeMb()) { + return QueryPressureState.CRITICAL; + } + if (freeMemoryMb <= configuration.getHighFreeMb()) { + return QueryPressureState.HIGH; + } + if (freeMemoryMb <= configuration.getWarnFreeMb()) { + return QueryPressureState.WARN; + } + return QueryPressureState.NORMAL; + } + + private QueryPressureState capGcState(QueryPressureState gcState, QueryPressureState freeMemoryState) { + switch (freeMemoryState) { + case NORMAL: + return min(gcState, QueryPressureState.WARN); + case WARN: + return min(gcState, QueryPressureState.HIGH); + case HIGH: + case CRITICAL: + default: + return gcState; + } + } + + private QueryPressureState min(QueryPressureState left, QueryPressureState right) { + return left.ordinal() <= right.ordinal() ? left : right; + } + + private QueryPressureState max(QueryPressureState left, QueryPressureState right) { + return left.ordinal() >= right.ordinal() ? left : right; + } + + private synchronized void cancelOneHeavyQueryIfNeeded(Configuration configuration, + QueryPressureMonitor.Snapshot snapshot, + String reason) { + long now = clock.getAsLong(); + if (lastCancelAt != Long.MIN_VALUE && now - lastCancelAt < configuration.getCancelCooldownMs()) { + return; + } + + QueryCircuitBreakerHandle candidate = activeHandles.values() + .stream() + .filter(QueryCircuitBreakerHandle::isActive) + .filter(QueryCircuitBreakerHandle::hasHeavyCheckpoint) + .filter(handle -> !handle.isCancelRequested()) + .max(Comparator.comparingLong(QueryCircuitBreakerHandle::getLastHeavyCheckpointMillis)) + .orElse(null); + + if (candidate == null) { + LOGGER.warn( + "Query circuit breaker found no heavy query to cancel state={} freeMb={} rollingGcMs={} reason={}", + currentState, snapshot.getFreeMemoryMb(), snapshot.getRollingGcMs(), reason); + return; + } + + String cancelReason = buildPressureMessage("Query cancelled by global memory circuit breaker", snapshot, + QueryPressureState.CRITICAL); + if (!candidate.requestCancel(QueryPressureState.CRITICAL, cancelReason)) { + return; + } + + lastCancelAt = now; + cancelCount.incrementAndGet(); + LOGGER.warn( + "Query circuit breaker cancelled executionId={} source={} repositoryId={} queryHash={} lastHeavyOperator={} reason={}", + candidate.getExecutionId(), candidate.getSource(), candidate.getRepositoryId(), + candidate.getQueryHash(), + candidate.getLastHeavyOperator(), reason); + } + + void interruptForCriticalHeavyOperator(QueryCircuitBreakerHandle handle, String operator) + throws QueryInterruptedException { + if (handle == null) { + return; + } + + Configuration configuration = configurationSupplier.get(); + QueryPressureMonitor.Snapshot snapshot = pressureMonitor.sample(); + handle.markHeavy(operator, clock.getAsLong()); + + String cancelReason = buildPressureMessage("Query cancelled by global memory circuit breaker", snapshot, + QueryPressureState.CRITICAL); + if (handle.requestCancel(QueryPressureState.CRITICAL, cancelReason)) { + cancelCount.incrementAndGet(); + } + + String effectiveReason = handle.getCancellationReason() != null ? handle.getCancellationReason() : cancelReason; + throw CircuitBreakerException.cancelled(handle.getCancellationState(), configuration.getRetryAfterSeconds(), + effectiveReason); + } + + private String buildPressureMessage(String prefix, QueryPressureMonitor.Snapshot snapshot, + QueryPressureState state) { + return prefix + " (state=" + state + ", freeMb=" + snapshot.getFreeMemoryMb() + ", rollingGcMs=" + + snapshot.getRollingGcMs() + ")"; + } + + private String normalizeRepositoryId(String repositoryId) { + if (repositoryId == null || repositoryId.isBlank()) { + return "unknown-repository"; + } + return repositoryId.trim(); + } + + private String hashQuery(String queryText) { + return Integer.toHexString(Objects.toString(queryText, "").hashCode()); + } + + @FunctionalInterface + public interface QueryOperation { + T execute() throws X; + } + + @FunctionalInterface + interface Sleeper { + void sleep(long millis) throws InterruptedException; + } + + @FunctionalInterface + interface GcInvoker { + void runGc(); + } + + public static final class CircuitBreakerException extends QueryInterruptedException { + private static final long serialVersionUID = -5525396489058918736L; + + private final QueryPressureState state; + private final int retryAfterSeconds; + private final boolean rejected; + private final boolean cancelled; + + private CircuitBreakerException(String message, QueryPressureState state, int retryAfterSeconds, + boolean rejected, boolean cancelled, Throwable cause) { + super(message, cause); + this.state = state; + this.retryAfterSeconds = retryAfterSeconds; + this.rejected = rejected; + this.cancelled = cancelled; + } + + static CircuitBreakerException rejected(QueryPressureState state, int retryAfterSeconds, String message) { + return new CircuitBreakerException(message, state, retryAfterSeconds, true, false, null); + } + + static CircuitBreakerException cancelled(QueryPressureState state, int retryAfterSeconds, String message) { + return new CircuitBreakerException(message, state, retryAfterSeconds, false, true, null); + } + + static CircuitBreakerException cancelled(QueryPressureState state, int retryAfterSeconds, String message, + Throwable cause) { + return new CircuitBreakerException(message, state, retryAfterSeconds, false, true, cause); + } + + public QueryPressureState getState() { + return state; + } + + public int getRetryAfterSeconds() { + return retryAfterSeconds; + } + + public boolean isRejected() { + return rejected; + } + + public boolean isCancelled() { + return cancelled; + } + } + + public static final class StatusSnapshot { + private final boolean enabled; + private final String state; + private final int freeMemoryMb; + private final int rollingGcMs; + private final Configuration configuration; + private final List activeQueries; + private final long rejectCount; + private final long cancelCount; + private final long lastTransitionTimestamp; + private final String lastTransitionReason; + + private StatusSnapshot(boolean enabled, String state, int freeMemoryMb, int rollingGcMs, + Configuration configuration, List activeQueries, long rejectCount, long cancelCount, + long lastTransitionTimestamp, String lastTransitionReason) { + this.enabled = enabled; + this.state = state; + this.freeMemoryMb = freeMemoryMb; + this.rollingGcMs = rollingGcMs; + this.configuration = configuration; + this.activeQueries = activeQueries; + this.rejectCount = rejectCount; + this.cancelCount = cancelCount; + this.lastTransitionTimestamp = lastTransitionTimestamp; + this.lastTransitionReason = lastTransitionReason; + } + + public boolean isEnabled() { + return enabled; + } + + public String getState() { + return state; + } + + public int getFreeMemoryMb() { + return freeMemoryMb; + } + + public int getRollingGcMs() { + return rollingGcMs; + } + + public Configuration getConfiguration() { + return configuration; + } + + public int getActiveQueryCount() { + return activeQueries.size(); + } + + public List getActiveQueries() { + return activeQueries; + } + + public long getRejectCount() { + return rejectCount; + } + + public long getCancelCount() { + return cancelCount; + } + + public long getLastTransitionTimestamp() { + return lastTransitionTimestamp; + } + + public String getLastTransitionReason() { + return lastTransitionReason; + } + } + + public static final class ActiveQueryStatus { + private final String executionId; + private final String source; + private final String repositoryId; + private final String queryHash; + private final long startTimeMillis; + private final long lastHeavyCheckpointMillis; + private final String lastHeavyOperator; + private final boolean cancelRequested; + + private ActiveQueryStatus(String executionId, String source, String repositoryId, String queryHash, + long startTimeMillis, long lastHeavyCheckpointMillis, String lastHeavyOperator, + boolean cancelRequested) { + this.executionId = executionId; + this.source = source; + this.repositoryId = repositoryId; + this.queryHash = queryHash; + this.startTimeMillis = startTimeMillis; + this.lastHeavyCheckpointMillis = lastHeavyCheckpointMillis; + this.lastHeavyOperator = lastHeavyOperator; + this.cancelRequested = cancelRequested; + } + + public String getExecutionId() { + return executionId; + } + + public String getSource() { + return source; + } + + public String getRepositoryId() { + return repositoryId; + } + + public String getQueryHash() { + return queryHash; + } + + public long getStartTimeMillis() { + return startTimeMillis; + } + + public long getLastHeavyCheckpointMillis() { + return lastHeavyCheckpointMillis; + } + + public String getLastHeavyOperator() { + return lastHeavyOperator; + } + + public boolean isCancelRequested() { + return cancelRequested; + } + } + + public static final class Configuration { + private static final long BYTES_PER_MB = 1024L * 1024L; + + private final boolean enabled; + private final int warnGcMs; + private final int highGcMs; + private final int criticalGcMs; + private final int warnFreeMb; + private final int highFreeMb; + private final int criticalFreeMb; + private final int warnAdmissionDelayMs; + private final int checkpointDelayMs; + private final int cancelCooldownMs; + private final int retryAfterSeconds; + + private Configuration(boolean enabled, int warnGcMs, int highGcMs, int criticalGcMs, int warnFreeMb, + int highFreeMb, int criticalFreeMb, int warnAdmissionDelayMs, int checkpointDelayMs, + int cancelCooldownMs, int retryAfterSeconds) { + this.enabled = enabled; + this.warnGcMs = warnGcMs; + this.highGcMs = highGcMs; + this.criticalGcMs = criticalGcMs; + this.warnFreeMb = warnFreeMb; + this.highFreeMb = highFreeMb; + this.criticalFreeMb = criticalFreeMb; + this.warnAdmissionDelayMs = warnAdmissionDelayMs; + this.checkpointDelayMs = checkpointDelayMs; + this.cancelCooldownMs = cancelCooldownMs; + this.retryAfterSeconds = retryAfterSeconds; + } + + static Configuration fromSystemProperties() { + return new Configuration(Boolean.parseBoolean(System.getProperty(ENABLED_PROPERTY, "false")), + getIntProperty(WARN_GC_MS_PROPERTY, 400), + getIntProperty(HIGH_GC_MS_PROPERTY, 600), + getIntProperty(CRITICAL_GC_MS_PROPERTY, 800), + getIntProperty(WARN_FREE_MB_PROPERTY, defaultWarnFreeMb()), + getIntProperty(HIGH_FREE_MB_PROPERTY, defaultHighFreeMb()), + getIntProperty(CRITICAL_FREE_MB_PROPERTY, 96), + getIntProperty(WARN_ADMISSION_DELAY_MS_PROPERTY, 50), + getIntProperty(CHECKPOINT_DELAY_MS_PROPERTY, 1), + getIntProperty(CANCEL_COOLDOWN_MS_PROPERTY, 1000), + getIntProperty(RETRY_AFTER_SECONDS_PROPERTY, 3)); + } + + private static int defaultWarnFreeMb() { + return Math.max(256, percentOfMaxMemoryMb(2)); + } + + private static int defaultHighFreeMb() { + return Math.max(128, percentOfMaxMemoryMb(1)); + } + + private static int percentOfMaxMemoryMb(int percent) { + long scaledBytes = Runtime.getRuntime().maxMemory() * percent; + long divisor = 100L * BYTES_PER_MB; + return (int) ((scaledBytes + divisor - 1) / divisor); + } + + private static int getIntProperty(String propertyName, int defaultValue) { + String rawValue = System.getProperty(propertyName); + if (rawValue == null || rawValue.isBlank()) { + return defaultValue; + } + try { + return Integer.parseInt(rawValue.trim()); + } catch (NumberFormatException e) { + LOGGER.warn("Invalid integer value '{}' for property {}. Falling back to {}", rawValue, propertyName, + defaultValue); + return defaultValue; + } + } + + public boolean isEnabled() { + return enabled; + } + + public int getWarnGcMs() { + return warnGcMs; + } + + public int getHighGcMs() { + return highGcMs; + } + + public int getCriticalGcMs() { + return criticalGcMs; + } + + public int getWarnFreeMb() { + return warnFreeMb; + } + + public int getHighFreeMb() { + return highFreeMb; + } + + public int getCriticalFreeMb() { + return criticalFreeMb; + } + + public int getWarnAdmissionDelayMs() { + return warnAdmissionDelayMs; + } + + public int getCheckpointDelayMs() { + return checkpointDelayMs; + } + + public int getCancelCooldownMs() { + return cancelCooldownMs; + } + + public int getRetryAfterSeconds() { + return retryAfterSeconds; + } + } + + private static final class Transition { + private final long timestampMillis; + private final String reason; + private final QueryPressureState previousState; + private final QueryPressureState nextState; + + private Transition(long timestampMillis, String reason, QueryPressureState previousState, + QueryPressureState nextState) { + this.timestampMillis = timestampMillis; + this.reason = reason; + this.previousState = previousState; + this.nextState = nextState; + } + + private static Transition initial() { + return new Transition(0L, "initial", QueryPressureState.NORMAL, QueryPressureState.NORMAL); + } + + private long getTimestampMillis() { + return timestampMillis; + } + + private String getReason() { + return reason; + } + + @Override + public String toString() { + return previousState + "->" + nextState + "@" + timestampMillis + ":" + reason; + } + } +} diff --git a/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryCircuitBreakerHandle.java b/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryCircuitBreakerHandle.java new file mode 100644 index 00000000000..4df1154aab9 --- /dev/null +++ b/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryCircuitBreakerHandle.java @@ -0,0 +1,171 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex +package org.eclipse.rdf4j.http.client; + +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import org.eclipse.rdf4j.repository.RepositoryConnection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * One active query tracked by the global breaker. + */ +public final class QueryCircuitBreakerHandle { + + public enum Source { + SERVER, + TX, + WORKBENCH + } + + private static final Logger LOGGER = LoggerFactory.getLogger(QueryCircuitBreakerHandle.class); + + private final String executionId; + private final Source source; + private final String repositoryId; + private final String queryHash; + private final long startTimeMillis; + private final Runnable remoteCancel; + private final AtomicBoolean active = new AtomicBoolean(true); + private final AtomicBoolean cancelRequested = new AtomicBoolean(false); + private final AtomicLong lastHeavyCheckpointMillis = new AtomicLong(-1); + + private volatile Thread workerThread; + private volatile RepositoryConnection repositoryConnection; + private volatile String lastHeavyOperator; + private volatile QueryPressureState cancellationState = QueryPressureState.NORMAL; + private volatile String cancellationReason; + + QueryCircuitBreakerHandle(String executionId, Source source, String repositoryId, String queryHash, + long startTimeMillis, Runnable remoteCancel) { + this.executionId = Objects.requireNonNull(executionId, "Execution id was null"); + this.source = Objects.requireNonNull(source, "Source was null"); + this.repositoryId = Objects.requireNonNull(repositoryId, "Repository id was null"); + this.queryHash = Objects.requireNonNull(queryHash, "Query hash was null"); + this.startTimeMillis = startTimeMillis; + this.remoteCancel = remoteCancel; + } + + public String getExecutionId() { + return executionId; + } + + public Source getSource() { + return source; + } + + public String getRepositoryId() { + return repositoryId; + } + + public String getQueryHash() { + return queryHash; + } + + public long getStartTimeMillis() { + return startTimeMillis; + } + + public boolean isActive() { + return active.get(); + } + + public boolean isCancelRequested() { + return cancelRequested.get(); + } + + public long getLastHeavyCheckpointMillis() { + return lastHeavyCheckpointMillis.get(); + } + + public String getLastHeavyOperator() { + return lastHeavyOperator; + } + + public QueryPressureState getCancellationState() { + return cancellationState; + } + + public String getCancellationReason() { + return cancellationReason; + } + + public boolean hasHeavyCheckpoint() { + return lastHeavyCheckpointMillis.get() > 0; + } + + public void attachCurrentThread(RepositoryConnection repositoryConnection) { + attach(Thread.currentThread(), repositoryConnection); + } + + void attach(Thread workerThread, RepositoryConnection repositoryConnection) { + if (workerThread != null) { + this.workerThread = workerThread; + } + if (repositoryConnection != null) { + this.repositoryConnection = repositoryConnection; + } + } + + void markHeavy(String operator, long checkpointTimeMillis) { + lastHeavyOperator = operator; + lastHeavyCheckpointMillis.set(checkpointTimeMillis); + } + + boolean requestCancel(QueryPressureState state, String reason) { + if (!active.get() || !cancelRequested.compareAndSet(false, true)) { + return false; + } + + cancellationState = state; + cancellationReason = reason; + + Thread activeWorker = workerThread; + if (activeWorker != null) { + activeWorker.interrupt(); + } + + closeConnection(); + runRemoteCancel(); + return true; + } + + void finish() { + active.set(false); + } + + private void closeConnection() { + RepositoryConnection connection = repositoryConnection; + if (connection == null) { + return; + } + try { + connection.close(); + } catch (Exception e) { + LOGGER.debug("Error while closing breaker-tracked repository connection", e); + } + } + + private void runRemoteCancel() { + if (remoteCancel == null) { + return; + } + try { + remoteCancel.run(); + } catch (Exception e) { + LOGGER.debug("Error while forwarding breaker-triggered cancellation", e); + } + } +} diff --git a/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryExecutionContext.java b/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryExecutionContext.java new file mode 100644 index 00000000000..dfce2b562e8 --- /dev/null +++ b/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryExecutionContext.java @@ -0,0 +1,117 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex +package org.eclipse.rdf4j.http.client; + +import java.util.Objects; + +import org.eclipse.rdf4j.query.QueryInterruptedException; + +/** + * Thread-local execution context for the currently active breaker-tracked query. + */ +public final class QueryExecutionContext { + + private static boolean ignoreCheckpointStride = false; + private static final int CHECKPOINT_STRIDE = 1024; + private static final int CHECKPOINT_MASK = CHECKPOINT_STRIDE - 1; + private static final ThreadLocal CURRENT = new ThreadLocal<>(); + private static boolean heavyOperatorExecutionEnabled = true; + private static int checkpointCalls; + + private QueryExecutionContext() { + } + + public static Activation activate(QueryCircuitBreakerHandle handle) { + QueryCircuitBreakerHandle normalizedHandle = Objects.requireNonNull(handle, "Handle was null"); + State previous = CURRENT.get(); + State next = new State(normalizedHandle, previous); + normalizedHandle.attachCurrentThread(null); + CURRENT.set(next); + QueryCircuitBreaker.getInstance().refreshHeavyOperatorExecutionState(); + return () -> { + State current = CURRENT.get(); + if (current != next) { + return; + } + if (previous == null) { + CURRENT.remove(); + } else { + CURRENT.set(previous); + } + }; + } + + public static QueryCircuitBreakerHandle getHandle() { + State state = CURRENT.get(); + return state == null ? null : state.handle; + } + + public static void markHeavy(String operator) { + QueryCircuitBreakerHandle handle = getHandle(); + if (handle != null) { + QueryCircuitBreaker.getInstance().markHeavy(handle, operator); + } + } + + public static void checkpoint(String operator) throws QueryInterruptedException { + if (!shouldCheckpoint()) + return; + State state = CURRENT.get(); + if (state != null) { + QueryCircuitBreaker.getInstance().checkpoint(state.handle, operator); + } + } + + public static void throwIfHeavyOperatorExecutionDisabled(String operator) throws QueryInterruptedException { + if (heavyOperatorExecutionEnabled) { + return; + } + + State state = CURRENT.get(); + + if (state == null) { + return; + } + + QueryCircuitBreaker.getInstance().interruptForCriticalHeavyOperator(state.handle, operator); + } + + static void setHeavyOperatorExecutionEnabled(boolean heavyOperatorExecutionEnabled) { + QueryExecutionContext.heavyOperatorExecutionEnabled = heavyOperatorExecutionEnabled; + } + + static void setIgnoreCheckpointStride(boolean ignoreCheckpointStride) { + QueryExecutionContext.ignoreCheckpointStride = ignoreCheckpointStride; + } + + public interface Activation extends AutoCloseable { + @Override + void close(); + } + + private static boolean shouldCheckpoint() { + if (ignoreCheckpointStride) + return true; + return ((++checkpointCalls) & CHECKPOINT_MASK) == 0; + } + + private static final class State { + private final QueryCircuitBreakerHandle handle; + private final State previous; + + private State(QueryCircuitBreakerHandle handle, State previous) { + this.handle = handle; + this.previous = previous; + } + + } +} diff --git a/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryPressureMonitor.java b/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryPressureMonitor.java new file mode 100644 index 00000000000..e9b7f43b799 --- /dev/null +++ b/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryPressureMonitor.java @@ -0,0 +1,158 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex +package org.eclipse.rdf4j.http.client; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.util.Arrays; +import java.util.List; +import java.util.function.LongSupplier; + +import javax.management.NotificationEmitter; +import javax.management.openmbean.CompositeData; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.sun.management.GarbageCollectionNotificationInfo; + +/** + * Tracks JVM pressure using rolling garbage collection duration and currently available heap. + */ +public final class QueryPressureMonitor { + + private static final Logger LOGGER = LoggerFactory.getLogger(QueryPressureMonitor.class); + private static final Runtime RUNTIME = Runtime.getRuntime(); + private static final int BUCKET_MS = 100; + private static final int BUCKET_COUNT = 10; + + private final int[] gcLoadBuckets = new int[BUCKET_COUNT]; + private final LongSupplier freeMemoryMbSupplier; + private final LongSupplier clock; + private final Object lock = new Object(); + + private long lastAbsoluteBucket = Long.MIN_VALUE; + + public QueryPressureMonitor() { + this(QueryPressureMonitor::getSystemFreeMemoryMb, System::currentTimeMillis, true); + } + + QueryPressureMonitor(LongSupplier freeMemoryMbSupplier, LongSupplier clock, boolean installGcListeners) { + this.freeMemoryMbSupplier = freeMemoryMbSupplier; + this.clock = clock; + if (installGcListeners) { + installGcListeners(); + } + } + + Snapshot sample() { + synchronized (lock) { + long now = clock.getAsLong(); + rotateBuckets(now); + long rollingGcMs = 0; + for (int gcLoadBucket : gcLoadBuckets) { + rollingGcMs += gcLoadBucket; + } + return new Snapshot((int) Math.min(Integer.MAX_VALUE, rollingGcMs), + (int) Math.min(Integer.MAX_VALUE, Math.max(0, freeMemoryMbSupplier.getAsLong()))); + } + } + + void recordGcPause(long durationMs) { + if (durationMs <= 0) { + return; + } + + synchronized (lock) { + long now = clock.getAsLong(); + rotateBuckets(now); + int bucket = currentBucket(now); + long updated = (long) gcLoadBuckets[bucket] + durationMs; + gcLoadBuckets[bucket] = (int) Math.min(Integer.MAX_VALUE, updated); + } + } + + private void installGcListeners() { + List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); + for (GarbageCollectorMXBean gcBean : gcBeans) { + if (!(gcBean instanceof NotificationEmitter)) { + continue; + } + + NotificationEmitter emitter = (NotificationEmitter) gcBean; + emitter.addNotificationListener((notification, handback) -> { + try { + GarbageCollectionNotificationInfo gcNotificationInfo = GarbageCollectionNotificationInfo + .from((CompositeData) notification.getUserData()); + if (gcNotificationInfo != null && gcNotificationInfo.getGcInfo() != null) { + recordGcPause(gcNotificationInfo.getGcInfo().getDuration()); + } + } catch (RuntimeException e) { + LOGGER.debug("Unable to record GC notification for query pressure monitor", e); + } + }, null, null); + } + } + + private void rotateBuckets(long now) { + long absoluteBucket = now / BUCKET_MS; + if (lastAbsoluteBucket == Long.MIN_VALUE) { + lastAbsoluteBucket = absoluteBucket; + return; + } + + long advance = absoluteBucket - lastAbsoluteBucket; + if (advance <= 0) { + return; + } + + if (advance >= BUCKET_COUNT) { + Arrays.fill(gcLoadBuckets, 0); + } else { + for (long i = 1; i <= advance; i++) { + gcLoadBuckets[(int) ((lastAbsoluteBucket + i) % BUCKET_COUNT)] = 0; + } + } + lastAbsoluteBucket = absoluteBucket; + } + + private int currentBucket(long now) { + return (int) ((now / BUCKET_MS) % BUCKET_COUNT); + } + + private static long getSystemFreeMemoryMb() { + long maxMemory = RUNTIME.maxMemory(); + long totalMemory = RUNTIME.totalMemory(); + long freeMemory = RUNTIME.freeMemory(); + long usedMemory = totalMemory - freeMemory; + long freeToAllocateMemory = maxMemory - usedMemory; + return freeToAllocateMemory / (1024 * 1024); + } + + static final class Snapshot { + private final int rollingGcMs; + private final int freeMemoryMb; + + private Snapshot(int rollingGcMs, int freeMemoryMb) { + this.rollingGcMs = rollingGcMs; + this.freeMemoryMb = freeMemoryMb; + } + + int getRollingGcMs() { + return rollingGcMs; + } + + int getFreeMemoryMb() { + return freeMemoryMb; + } + } +} diff --git a/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryPressureState.java b/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryPressureState.java new file mode 100644 index 00000000000..7756b5d8143 --- /dev/null +++ b/core/http/client/src/main/java/org/eclipse/rdf4j/http/client/QueryPressureState.java @@ -0,0 +1,30 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex +package org.eclipse.rdf4j.http.client; + +/** + * Global query memory pressure levels. + */ +public enum QueryPressureState { + NORMAL, + WARN, + HIGH, + CRITICAL; + + public boolean throttlesRunningQueries() { + return this == HIGH || this == CRITICAL; + } + + public boolean rejectsNewQueries() { + return throttlesRunningQueries(); + } +} diff --git a/core/http/client/src/test/java/org/eclipse/rdf4j/http/client/QueryCircuitBreakerLoggingTest.java b/core/http/client/src/test/java/org/eclipse/rdf4j/http/client/QueryCircuitBreakerLoggingTest.java new file mode 100644 index 00000000000..17c4acd8490 --- /dev/null +++ b/core/http/client/src/test/java/org/eclipse/rdf4j/http/client/QueryCircuitBreakerLoggingTest.java @@ -0,0 +1,176 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex +package org.eclipse.rdf4j.http.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Constructor; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; + +class QueryCircuitBreakerLoggingTest { + + private static final Constructor CONFIGURATION_CONSTRUCTOR = configurationCtor(); + private static final Logger LOGGER = (Logger) LoggerFactory.getLogger(QueryCircuitBreaker.class); + private static final String ENTRY_THROTTLE_WARNING = "Query circuit breaker is throttling new queries at entry"; + private static final String RUNNING_THROTTLE_WARNING = "Query circuit breaker is throttling running queries"; + + private ListAppender listAppender; + + @BeforeEach + void attachAppender() { + listAppender = new ListAppender<>(); + listAppender.start(); + LOGGER.addAppender(listAppender); + } + + @AfterEach + void detachAppender() { + LOGGER.detachAppender(listAppender); + listAppender.stop(); + } + + @Test + void shouldWarnWhenThrottlingNewQueriesAtEntryOncePerMinute() { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 10, 1000, + 7), 1000); + List handles = new ArrayList<>(); + + try { + fixture.freeMemoryMb.set(350); + QueryCircuitBreakerHandle warnHandle = registerHandle(breaker, handles, "warn-entry-1"); + breaker.beforeExecution(warnHandle); + + assertEquals(1, warningCount(ENTRY_THROTTLE_WARNING)); + assertTrue(lastWarning(ENTRY_THROTTLE_WARNING).contains("action=delay")); + + QueryCircuitBreakerHandle secondWarnHandle = registerHandle(breaker, handles, "warn-entry-2"); + breaker.beforeExecution(secondWarnHandle); + assertEquals(1, warningCount(ENTRY_THROTTLE_WARNING)); + + fixture.clock.addAndGet(60_000); + QueryCircuitBreakerHandle thirdWarnHandle = registerHandle(breaker, handles, "warn-entry-3"); + breaker.beforeExecution(thirdWarnHandle); + assertEquals(2, warningCount(ENTRY_THROTTLE_WARNING)); + + fixture.clock.addAndGet(60_000); + fixture.freeMemoryMb.set(250); + QueryCircuitBreakerHandle highHandle = registerHandle(breaker, handles, "high-entry"); + QueryCircuitBreaker.CircuitBreakerException exception = assertThrows( + QueryCircuitBreaker.CircuitBreakerException.class, () -> breaker.beforeExecution(highHandle)); + + assertTrue(exception.isRejected()); + assertEquals(3, warningCount(ENTRY_THROTTLE_WARNING)); + assertTrue(lastWarning(ENTRY_THROTTLE_WARNING).contains("action=reject")); + } finally { + handles.forEach(breaker::complete); + } + } + + @Test + void shouldWarnWhenThrottlingRunningQueriesOncePerMinute() { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 10, 1000, + 7), 1000); + QueryCircuitBreakerHandle handle = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "running-query"); + + try { + fixture.freeMemoryMb.set(250); + breaker.checkpoint(handle, "JOIN"); + assertEquals(1, warningCount(RUNNING_THROTTLE_WARNING)); + + breaker.checkpoint(handle, "JOIN"); + assertEquals(1, warningCount(RUNNING_THROTTLE_WARNING)); + + fixture.clock.addAndGet(60_000); + breaker.checkpoint(handle, "JOIN"); + assertEquals(2, warningCount(RUNNING_THROTTLE_WARNING)); + } finally { + breaker.complete(handle); + } + } + + private int warningCount(String messageFragment) { + return (int) listAppender.list.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(ILoggingEvent::getFormattedMessage) + .filter(message -> message.contains(messageFragment)) + .count(); + } + + private String lastWarning(String messageFragment) { + return listAppender.list.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(ILoggingEvent::getFormattedMessage) + .filter(message -> message.contains(messageFragment)) + .reduce((first, second) -> second) + .orElseThrow(); + } + + private QueryCircuitBreakerHandle registerHandle(QueryCircuitBreaker breaker, + List handles, String queryText) { + QueryCircuitBreakerHandle handle = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", queryText); + handles.add(handle); + return handle; + } + + private static QueryCircuitBreaker.Configuration configuration(boolean enabled, int warnGcMs, int highGcMs, + int criticalGcMs, int warnFreeMb, int highFreeMb, int criticalFreeMb, int warnAdmissionDelayMs, + int checkpointDelayMs, int cancelCooldownMs, int retryAfterSeconds) { + try { + return CONFIGURATION_CONSTRUCTOR.newInstance(enabled, warnGcMs, highGcMs, criticalGcMs, warnFreeMb, + highFreeMb, criticalFreeMb, warnAdmissionDelayMs, checkpointDelayMs, cancelCooldownMs, + retryAfterSeconds); + } catch (Exception e) { + throw new IllegalStateException("Unable to construct query breaker configuration", e); + } + } + + private static Constructor configurationCtor() { + try { + Constructor constructor = QueryCircuitBreaker.Configuration.class + .getDeclaredConstructor(boolean.class, int.class, int.class, int.class, int.class, int.class, + int.class, int.class, int.class, int.class, int.class); + constructor.setAccessible(true); + return constructor; + } catch (Exception e) { + throw new IllegalStateException("Unable to access query breaker configuration constructor", e); + } + } + + private static final class Fixture { + private final AtomicLong clock = new AtomicLong(); + private final AtomicLong freeMemoryMb = new AtomicLong(1024); + private final QueryPressureMonitor monitor = new QueryPressureMonitor(freeMemoryMb::get, clock::get, false); + private final List sleeps = new ArrayList<>(); + + private QueryCircuitBreaker breaker(QueryCircuitBreaker.Configuration configuration, long recoveryCooldownMs) { + return new QueryCircuitBreaker(monitor, () -> configuration, clock::get, sleeps::add, + recoveryCooldownMs); + } + } +} diff --git a/core/http/client/src/test/java/org/eclipse/rdf4j/http/client/QueryCircuitBreakerTest.java b/core/http/client/src/test/java/org/eclipse/rdf4j/http/client/QueryCircuitBreakerTest.java new file mode 100644 index 00000000000..e7aa65421c4 --- /dev/null +++ b/core/http/client/src/test/java/org/eclipse/rdf4j/http/client/QueryCircuitBreakerTest.java @@ -0,0 +1,610 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex +package org.eclipse.rdf4j.http.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.Test; + +class QueryCircuitBreakerTest { + + private static final Constructor CONFIGURATION_CONSTRUCTOR = configurationCtor(); + + @Test + void shouldStartBackgroundMonitorThreadForHighMemoryPressure() throws Exception { + PropertiesScope properties = new PropertiesScope() + .with(QueryCircuitBreaker.ENABLED_PROPERTY, "true") + .with(QueryCircuitBreaker.WARN_GC_MS_PROPERTY, "1000") + .with(QueryCircuitBreaker.HIGH_GC_MS_PROPERTY, "2000") + .with(QueryCircuitBreaker.CRITICAL_GC_MS_PROPERTY, "3000") + .with(QueryCircuitBreaker.WARN_FREE_MB_PROPERTY, "400") + .with(QueryCircuitBreaker.HIGH_FREE_MB_PROPERTY, "300") + .with(QueryCircuitBreaker.CRITICAL_FREE_MB_PROPERTY, "200"); + Set existingThreadIds = threadIds("rdf4j-query-breaker-gc-monitor"); + Thread monitorThread = null; + + try { + properties.apply(); + AtomicLong freeMemoryMb = new AtomicLong(250); + QueryCircuitBreaker breaker = new QueryCircuitBreaker( + new QueryPressureMonitor(freeMemoryMb::get, System::currentTimeMillis, false)); + + monitorThread = waitForNewThread("rdf4j-query-breaker-gc-monitor", existingThreadIds, 2500); + + assertNotNull(monitorThread); + assertTrue(monitorThread.isDaemon()); + assertEquals(QueryPressureState.HIGH, waitForCurrentState(breaker, 2500)); + } finally { + properties.restore(); + if (monitorThread != null) { + monitorThread.interrupt(); + monitorThread.join(1000); + } + } + } + + @Test + void shouldDeriveDefaultFreeMemoryThresholdsFromMaxHeap() throws Exception { + Thresholds thresholds = readDefaultThresholdsFromFreshJvm("12801m"); + + assertEquals(257, thresholds.warnFreeMb); + assertEquals(129, thresholds.highFreeMb); + } + + @Test + void shouldThrottleExecutionContextCheckpointToEvery1024Calls() throws Exception { + PropertiesScope properties = new PropertiesScope().with(QueryCircuitBreaker.ENABLED_PROPERTY, "false"); + QueryCircuitBreaker breaker = QueryCircuitBreaker.getInstance(); + QueryCircuitBreakerHandle handle = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "checkpoint-throttle"); + + try { + properties.apply(); + try (QueryExecutionContext.Activation ignored = QueryExecutionContext.activate(handle)) { + for (int i = 0; i < 1023; i++) { + QueryExecutionContext.checkpoint("JOIN"); + } + assertEquals(-1L, handle.getLastHeavyCheckpointMillis()); + + QueryExecutionContext.checkpoint("JOIN"); + assertTrue(handle.getLastHeavyCheckpointMillis() > 0); + } + } finally { + properties.restore(); + breaker.complete(handle); + } + } + + @Test + void shouldDisableHeavyOperatorExecutionOnlyWhileCritical() throws Exception { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 10, 0, + 7), 0); + + assertTrue(readHeavyOperatorExecutionEnabled()); + + fixture.freeMemoryMb.set(150); + assertEquals("CRITICAL", breaker.snapshotStatus().getState()); + assertFalse(readHeavyOperatorExecutionEnabled()); + + fixture.freeMemoryMb.set(1024); + assertEquals("NORMAL", breaker.snapshotStatus().getState()); + assertTrue(readHeavyOperatorExecutionEnabled()); + } + + @Test + void shouldIgnoreCheckpointStrideOnlyAtHighOrAbove() throws Exception { + PropertiesScope properties = new PropertiesScope() + .with(QueryCircuitBreaker.ENABLED_PROPERTY, "true") + .with(QueryCircuitBreaker.WARN_GC_MS_PROPERTY, Integer.toString(Integer.MAX_VALUE)) + .with(QueryCircuitBreaker.HIGH_GC_MS_PROPERTY, Integer.toString(Integer.MAX_VALUE)) + .with(QueryCircuitBreaker.CRITICAL_GC_MS_PROPERTY, Integer.toString(Integer.MAX_VALUE)) + .with(QueryCircuitBreaker.WARN_FREE_MB_PROPERTY, Integer.toString(Integer.MAX_VALUE)) + .with(QueryCircuitBreaker.HIGH_FREE_MB_PROPERTY, "0") + .with(QueryCircuitBreaker.CRITICAL_FREE_MB_PROPERTY, "0") + .with(QueryCircuitBreaker.CHECKPOINT_DELAY_MS_PROPERTY, "0"); + QueryCircuitBreaker breaker = QueryCircuitBreaker.getInstance(); + QueryCircuitBreakerHandle handle = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "warn-checkpoint-stride"); + + try { + properties.apply(); + resetCheckpointCalls(); + try (QueryExecutionContext.Activation ignored = QueryExecutionContext.activate(handle)) { + for (int i = 0; i < 1023; i++) { + QueryExecutionContext.checkpoint("JOIN"); + } + assertEquals(-1L, handle.getLastHeavyCheckpointMillis()); + + QueryExecutionContext.checkpoint("JOIN"); + assertTrue(handle.getLastHeavyCheckpointMillis() > 0); + } + } finally { + properties.restore(); + breaker.complete(handle); + resetCheckpointCalls(); + } + } + + private static void retryAssertion(Runnable assertion) throws InterruptedException { + final int retries = 100; + for (int i = 0; i < retries; i++) { + try { + assertion.run(); + break; + } catch (AssertionError e) { + if (i == retries - 1) { + throw e; + } + Thread.sleep(100); + } + } + } + + @Test + void shouldTransitionAcrossPressureLevelsUsingFreeMemoryThresholds() { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 10, 1000, + 7), 1000); + + assertEquals("NORMAL", breaker.snapshotStatus().getState()); + + fixture.freeMemoryMb.set(350); + assertEquals("WARN", breaker.snapshotStatus().getState()); + + fixture.freeMemoryMb.set(250); + assertEquals("HIGH", breaker.snapshotStatus().getState()); + + fixture.freeMemoryMb.set(150); + assertEquals("CRITICAL", breaker.snapshotStatus().getState()); + } + + @Test + void shouldRequireLowFreeMemoryBeforeEscalatingBeyondWarnFromGcPressure() { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 10, 1000, + 7), 1000); + + fixture.monitor.recordGcPause(120); + assertEquals("WARN", breaker.snapshotStatus().getState()); + + fixture.clock.addAndGet(100); + fixture.monitor.recordGcPause(120); + assertEquals("WARN", breaker.snapshotStatus().getState()); + + fixture.freeMemoryMb.set(350); + assertEquals("HIGH", breaker.snapshotStatus().getState()); + + fixture.clock.addAndGet(100); + fixture.monitor.recordGcPause(120); + assertEquals("HIGH", breaker.snapshotStatus().getState()); + + fixture.freeMemoryMb.set(250); + assertEquals("CRITICAL", breaker.snapshotStatus().getState()); + } + + @Test + void shouldRunMonitorGcEveryFiveSecondsInHighMemoryMode() { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 10, 1000, + 7), 1000, () -> fixture.gcInvocations.add(fixture.clock.get())); + + fixture.freeMemoryMb.set(250); + breaker.runGcMonitorCycle(); + assertEquals(List.of(0L), fixture.gcInvocations); + + fixture.clock.set(4999); + breaker.runGcMonitorCycle(); + assertEquals(List.of(0L), fixture.gcInvocations); + + fixture.clock.set(5000); + breaker.runGcMonitorCycle(); + assertEquals(List.of(0L, 5000L), fixture.gcInvocations); + } + + @Test + void shouldRunMonitorGcEverySecondInCriticalMemoryMode() { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 10, 1000, + 7), 1000, () -> fixture.gcInvocations.add(fixture.clock.get())); + + fixture.freeMemoryMb.set(150); + breaker.runGcMonitorCycle(); + assertEquals(List.of(0L), fixture.gcInvocations); + + fixture.clock.set(999); + breaker.runGcMonitorCycle(); + assertEquals(List.of(0L), fixture.gcInvocations); + + fixture.clock.set(1000); + breaker.runGcMonitorCycle(); + assertEquals(List.of(0L, 1000L), fixture.gcInvocations); + } + + @Test + void shouldBackOffCheckpointGcRequestsAndResetAfterLongPause() { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 0, 1000, + 7), 0, () -> fixture.gcInvocations.add(fixture.clock.get())); + QueryCircuitBreakerHandle handle = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "checkpoint-gc-backoff"); + + fixture.freeMemoryMb.set(250); + breaker.checkpoint(handle, "JOIN"); + assertEquals(List.of(0L), fixture.gcInvocations); + + fixture.clock.set(999); + breaker.checkpoint(handle, "JOIN"); + assertEquals(List.of(0L), fixture.gcInvocations); + + fixture.clock.set(1000); + breaker.checkpoint(handle, "JOIN"); + assertEquals(List.of(0L, 1000L), fixture.gcInvocations); + + fixture.clock.set(2999); + breaker.checkpoint(handle, "JOIN"); + assertEquals(List.of(0L, 1000L), fixture.gcInvocations); + + fixture.clock.set(3000); + breaker.checkpoint(handle, "JOIN"); + assertEquals(List.of(0L, 1000L, 3000L), fixture.gcInvocations); + + fixture.clock.set(6000); + breaker.checkpoint(handle, "JOIN"); + assertEquals(List.of(0L, 1000L, 3000L, 6000L), fixture.gcInvocations); + + fixture.clock.set(36001); + breaker.checkpoint(handle, "JOIN"); + assertEquals(List.of(0L, 1000L, 3000L, 6000L, 36001L), fixture.gcInvocations); + + fixture.clock.set(37000); + breaker.checkpoint(handle, "JOIN"); + assertEquals(List.of(0L, 1000L, 3000L, 6000L, 36001L), fixture.gcInvocations); + + fixture.clock.set(37001); + breaker.checkpoint(handle, "JOIN"); + assertEquals(List.of(0L, 1000L, 3000L, 6000L, 36001L, 37001L), fixture.gcInvocations); + + breaker.complete(handle); + } + + @Test + void shouldHoldCriticalStateUntilRecoveryCooldownExpires() { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 10, 1000, + 7), 1000); + + fixture.freeMemoryMb.set(150); + assertEquals("CRITICAL", breaker.snapshotStatus().getState()); + + fixture.freeMemoryMb.set(1024); + fixture.clock.addAndGet(500); + assertEquals("CRITICAL", breaker.snapshotStatus().getState()); + + fixture.clock.addAndGet(600); + assertEquals("NORMAL", breaker.snapshotStatus().getState()); + } + + @Test + void shouldThrottleCheckpointsOnlyAtHighOrAbove() { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 10, 1000, + 7), 1000); + QueryCircuitBreakerHandle handle = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "checkpoint-query"); + + fixture.freeMemoryMb.set(350); + breaker.checkpoint(handle, "JOIN"); + assertEquals(List.of(), fixture.sleeps); + + fixture.freeMemoryMb.set(250); + breaker.checkpoint(handle, "JOIN"); + assertEquals(List.of(10L), fixture.sleeps); + + breaker.complete(handle); + } + + @Test + void shouldDelayWarnAdmissionAndRejectHighAdmission() { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 10, 1000, + 7), 1000); + + QueryCircuitBreakerHandle warnHandle = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "warn-query"); + fixture.freeMemoryMb.set(350); + breaker.beforeExecution(warnHandle); + + assertEquals(List.of(25L), fixture.sleeps); + + QueryCircuitBreakerHandle highHandle = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "high-query"); + fixture.freeMemoryMb.set(250); + QueryCircuitBreaker.CircuitBreakerException exception = assertThrows( + QueryCircuitBreaker.CircuitBreakerException.class, () -> breaker.beforeExecution(highHandle)); + + assertTrue(exception.isRejected()); + assertEquals(QueryPressureState.HIGH, exception.getState()); + assertEquals(7, exception.getRetryAfterSeconds()); + assertEquals(1, breaker.snapshotStatus().getRejectCount()); + assertEquals("admission", breaker.snapshotStatus().getLastTransitionReason()); + + breaker.complete(warnHandle); + breaker.complete(highHandle); + } + + @Test + void shouldActivateAndCleanupThreadLocalExecutionContext() { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 10, 1000, + 7), 1000); + QueryCircuitBreakerHandle handle = breaker.register(QueryCircuitBreakerHandle.Source.WORKBENCH, "repo", + "select * where { ?s ?p ?o }"); + + assertNull(QueryExecutionContext.getHandle()); + + String executionId = breaker.execute(handle, null, () -> { + assertSame(handle, QueryExecutionContext.getHandle()); + return handle.getExecutionId(); + }); + + assertEquals(handle.getExecutionId(), executionId); + assertNull(QueryExecutionContext.getHandle()); + + breaker.complete(handle); + } + + @Test + void shouldCancelOnlyOneHeavyQueryPerCooldownWindow() { + Fixture fixture = new Fixture(); + QueryCircuitBreaker breaker = fixture.breaker(configuration(true, 100, 200, 300, 400, 300, 200, 25, 10, 1000, + 7), 1000); + QueryCircuitBreakerHandle firstHeavy = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "first"); + QueryCircuitBreakerHandle secondHeavy = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "second"); + + fixture.clock.set(10); + breaker.markHeavy(firstHeavy, "GROUP_BY"); + fixture.clock.set(20); + breaker.markHeavy(secondHeavy, "ORDER_BY"); + + fixture.freeMemoryMb.set(150); + + QueryCircuitBreakerHandle firstAdmission = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "admission-1"); + assertThrows(QueryCircuitBreaker.CircuitBreakerException.class, () -> breaker.beforeExecution(firstAdmission)); + assertTrue(secondHeavy.isCancelRequested()); + assertFalse(firstHeavy.isCancelRequested()); + assertEquals(1, breaker.snapshotStatus().getCancelCount()); + + QueryCircuitBreakerHandle secondAdmission = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "admission-2"); + assertThrows(QueryCircuitBreaker.CircuitBreakerException.class, () -> breaker.beforeExecution(secondAdmission)); + assertFalse(firstHeavy.isCancelRequested()); + assertEquals(1, breaker.snapshotStatus().getCancelCount()); + + fixture.clock.addAndGet(1001); + QueryCircuitBreakerHandle thirdAdmission = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "admission-3"); + assertThrows(QueryCircuitBreaker.CircuitBreakerException.class, () -> breaker.beforeExecution(thirdAdmission)); + assertTrue(firstHeavy.isCancelRequested()); + assertEquals(2, breaker.snapshotStatus().getCancelCount()); + + breaker.complete(firstHeavy); + breaker.complete(secondHeavy); + breaker.complete(firstAdmission); + breaker.complete(secondAdmission); + breaker.complete(thirdAdmission); + } + + private static QueryCircuitBreaker.Configuration configuration(boolean enabled, int warnGcMs, int highGcMs, + int criticalGcMs, int warnFreeMb, int highFreeMb, int criticalFreeMb, int warnAdmissionDelayMs, + int checkpointDelayMs, int cancelCooldownMs, int retryAfterSeconds) { + try { + return CONFIGURATION_CONSTRUCTOR.newInstance(enabled, warnGcMs, highGcMs, criticalGcMs, warnFreeMb, + highFreeMb, criticalFreeMb, warnAdmissionDelayMs, checkpointDelayMs, cancelCooldownMs, + retryAfterSeconds); + } catch (Exception e) { + throw new IllegalStateException("Unable to construct query breaker configuration", e); + } + } + + private static Constructor configurationCtor() { + try { + Constructor constructor = QueryCircuitBreaker.Configuration.class + .getDeclaredConstructor(boolean.class, int.class, int.class, int.class, int.class, int.class, + int.class, int.class, int.class, int.class, int.class); + constructor.setAccessible(true); + return constructor; + } catch (Exception e) { + throw new IllegalStateException("Unable to access query breaker configuration constructor", e); + } + } + + private static boolean readHeavyOperatorExecutionEnabled() throws Exception { + Field field = QueryExecutionContext.class.getDeclaredField("heavyOperatorExecutionEnabled"); + field.setAccessible(true); + return field.getBoolean(null); + } + + private static boolean readIgnoreCheckpointStride() { + Field field = null; + try { + field = QueryExecutionContext.class.getDeclaredField("ignoreCheckpointStride"); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + field.setAccessible(true); + try { + return field.getBoolean(null); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + private static void resetCheckpointCalls() throws Exception { + Field field = QueryExecutionContext.class.getDeclaredField("checkpointCalls"); + field.setAccessible(true); + field.setInt(null, 0); + } + + private static Set threadIds(String threadName) { + Set ids = new HashSet<>(); + for (Thread thread : Thread.getAllStackTraces().keySet()) { + if (threadName.equals(thread.getName())) { + ids.add(thread.getId()); + } + } + return ids; + } + + private static Thread waitForNewThread(String threadName, Set existingThreadIds, long timeoutMs) + throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMs; + while (System.currentTimeMillis() < deadline) { + for (Thread thread : Thread.getAllStackTraces().keySet()) { + if (threadName.equals(thread.getName()) && !existingThreadIds.contains(thread.getId())) { + return thread; + } + } + Thread.sleep(25); + } + return null; + } + + private static QueryPressureState waitForCurrentState(QueryCircuitBreaker breaker, long timeoutMs) + throws Exception { + Field currentState = QueryCircuitBreaker.class.getDeclaredField("currentState"); + currentState.setAccessible(true); + long deadline = System.currentTimeMillis() + timeoutMs; + QueryPressureState state = (QueryPressureState) currentState.get(breaker); + while (System.currentTimeMillis() < deadline && state == QueryPressureState.NORMAL) { + Thread.sleep(25); + state = (QueryPressureState) currentState.get(breaker); + } + return state; + } + + private static Thresholds readDefaultThresholdsFromFreshJvm(String maxHeap) throws Exception { + String classpath = System.getProperty("java.class.path"); + String javaExecutable = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + Process process = new ProcessBuilder(javaExecutable, "-Xmx" + maxHeap, + "-D" + QueryCircuitBreaker.WARN_FREE_MB_PROPERTY + "=", + "-D" + QueryCircuitBreaker.HIGH_FREE_MB_PROPERTY + "=", + "-cp", classpath, DefaultThresholdProbe.class.getName()) + .redirectErrorStream(true) + .start(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8).trim(); + int exitCode = process.waitFor(); + assertEquals(0, exitCode, output); + + String[] parts = output.split(","); + return new Thresholds(Integer.parseInt(parts[0]), Integer.parseInt(parts[1])); + } + + private static final class PropertiesScope { + private final List keys = new ArrayList<>(); + private final List previousValues = new ArrayList<>(); + private final List nextValues = new ArrayList<>(); + + private PropertiesScope with(String key, String value) { + keys.add(key); + previousValues.add(System.getProperty(key)); + nextValues.add(value); + return this; + } + + private PropertiesScope without(String key) { + keys.add(key); + previousValues.add(System.getProperty(key)); + nextValues.add(null); + return this; + } + + private void apply() { + for (int i = 0; i < keys.size(); i++) { + String nextValue = nextValues.get(i); + if (nextValue == null) { + System.clearProperty(keys.get(i)); + } else { + System.setProperty(keys.get(i), nextValue); + } + } + } + + private void restore() { + for (int i = 0; i < keys.size(); i++) { + String previousValue = previousValues.get(i); + if (previousValue == null) { + System.clearProperty(keys.get(i)); + } else { + System.setProperty(keys.get(i), previousValue); + } + } + } + } + + private static final class Fixture { + private final AtomicLong clock = new AtomicLong(); + private final AtomicLong freeMemoryMb = new AtomicLong(1024); + private final QueryPressureMonitor monitor = new QueryPressureMonitor(freeMemoryMb::get, clock::get, false); + private final List gcInvocations = new ArrayList<>(); + private final List sleeps = new ArrayList<>(); + + private QueryCircuitBreaker breaker(QueryCircuitBreaker.Configuration configuration, long recoveryCooldownMs) { + return new QueryCircuitBreaker(monitor, () -> configuration, clock::get, sleeps::add, + recoveryCooldownMs); + } + + private QueryCircuitBreaker breaker(QueryCircuitBreaker.Configuration configuration, long recoveryCooldownMs, + QueryCircuitBreaker.GcInvoker gcInvoker) { + return new QueryCircuitBreaker(monitor, () -> configuration, clock::get, sleeps::add, gcInvoker, + recoveryCooldownMs, 1, false); + } + } + + public static final class DefaultThresholdProbe { + public static void main(String[] args) { + QueryCircuitBreaker.Configuration configuration = QueryCircuitBreaker.Configuration.fromSystemProperties(); + System.out.print(configuration.getWarnFreeMb()); + System.out.print(","); + System.out.print(configuration.getHighFreeMb()); + } + } + + private static final class Thresholds { + private final int warnFreeMb; + private final int highFreeMb; + + private Thresholds(int warnFreeMb, int highFreeMb) { + this.warnFreeMb = warnFreeMb; + this.highFreeMb = highFreeMb; + } + } +} diff --git a/core/http/client/src/test/java/org/eclipse/rdf4j/http/client/QueryPressureMonitorTest.java b/core/http/client/src/test/java/org/eclipse/rdf4j/http/client/QueryPressureMonitorTest.java new file mode 100644 index 00000000000..19a6687e110 --- /dev/null +++ b/core/http/client/src/test/java/org/eclipse/rdf4j/http/client/QueryPressureMonitorTest.java @@ -0,0 +1,37 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex +package org.eclipse.rdf4j.http.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.Test; + +class QueryPressureMonitorTest { + + @Test + void shouldOnlyCountGcPauseInsideRollingOneSecondWindow() { + AtomicLong clock = new AtomicLong(); + QueryPressureMonitor monitor = new QueryPressureMonitor(() -> 1024, clock::get, false); + + monitor.recordGcPause(40); + clock.addAndGet(100); + monitor.recordGcPause(60); + + assertEquals(100, monitor.sample().getRollingGcMs()); + + clock.addAndGet(1000); + + assertEquals(0, monitor.sample().getRollingGcMs()); + } +} diff --git a/core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/GroupIterator.java b/core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/GroupIterator.java index 8d83907246b..05539de3d49 100644 --- a/core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/GroupIterator.java +++ b/core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/GroupIterator.java @@ -34,6 +34,7 @@ import org.eclipse.rdf4j.common.iteration.AbstractCloseableIteratorIteration; import org.eclipse.rdf4j.common.iteration.CloseableIteration; import org.eclipse.rdf4j.common.transaction.QueryEvaluationMode; +import org.eclipse.rdf4j.http.client.QueryExecutionContext; import org.eclipse.rdf4j.model.Literal; import org.eclipse.rdf4j.model.Value; import org.eclipse.rdf4j.model.ValueFactory; @@ -84,6 +85,9 @@ */ public class GroupIterator extends AbstractCloseableIteratorIteration { + private static final int BUILD_CHECKPOINT_INTERVAL = 128; + private static final String OPERATOR_NAME = "GROUP_BY"; + /*-----------* * Constants * *-----------*/ @@ -274,6 +278,9 @@ private BiConsumer makeBindSolution( private Collection buildEntries(List> aggregates) throws QueryEvaluationException { + QueryExecutionContext.throwIfHeavyOperatorExecutionDisabled(OPERATOR_NAME); + QueryExecutionContext.markHeavy(OPERATOR_NAME); + QueryExecutionContext.checkpoint(OPERATOR_NAME + "_START"); // store the arguments' iterator so it can be closed while building entries this.argumentsIter = arguments.evaluate(parentBindings); try (var iter = argumentsIter) { @@ -297,9 +304,16 @@ private Collection buildEntries(List entries = cf.createGroupByMap(); // Make an optimized hash function valid during this query evaluation step. ToIntFunction hashMaker = cf.hashOfBindingSetFuntion(getValues); - while (!isClosed() && iter.hasNext()) { + while (!isClosed()) { + QueryExecutionContext.throwIfHeavyOperatorExecutionDisabled(OPERATOR_NAME); + if (!iter.hasNext()) { + break; + } BindingSet sol = iter.next(); inputRows++; + if ((inputRows & (BUILD_CHECKPOINT_INTERVAL - 1)) == 0) { + QueryExecutionContext.checkpoint(OPERATOR_NAME + "_BUILD"); + } // The binding set key will be constant BindingSetKey key = cf.createBindingSetKey(sol, getValues, hashMaker); Entry entry = entries.get(key); diff --git a/core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/OrderIterator.java b/core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/OrderIterator.java index 43ad68233b4..e85fe629689 100644 --- a/core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/OrderIterator.java +++ b/core/queryalgebra/evaluation/src/main/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/OrderIterator.java @@ -8,6 +8,7 @@ * * SPDX-License-Identifier: BSD-3-Clause *******************************************************************************/ +// Some portions generated by Codex package org.eclipse.rdf4j.query.algebra.evaluation.iterator; import java.io.Closeable; @@ -34,6 +35,7 @@ import org.eclipse.rdf4j.common.iteration.CloseableIteratorIteration; import org.eclipse.rdf4j.common.iteration.DelayedIteration; import org.eclipse.rdf4j.common.iteration.LimitIteration; +import org.eclipse.rdf4j.http.client.QueryExecutionContext; import org.eclipse.rdf4j.query.BindingSet; import org.eclipse.rdf4j.query.QueryEvaluationException; @@ -45,6 +47,9 @@ */ public class OrderIterator extends DelayedIteration { + private static final int INPUT_CHECKPOINT_INTERVAL = 128; + private static final String OPERATOR_NAME = "ORDER_BY"; + /*-----------* * Variables * *-----------*/ @@ -93,6 +98,9 @@ public OrderIterator(CloseableIteration iter, Comparator @Override protected CloseableIteration createIteration() throws QueryEvaluationException { + QueryExecutionContext.throwIfHeavyOperatorExecutionDisabled(OPERATOR_NAME); + QueryExecutionContext.markHeavy(OPERATOR_NAME); + QueryExecutionContext.checkpoint(OPERATOR_NAME + "_START"); BindingSet threshold = null; List list = new LinkedList<>(); long inputRowsRead = 0; @@ -102,7 +110,9 @@ protected CloseableIteration createIteration() throws QueryEvaluatio int syncThreshold = (int) Math.min(iterationSyncThreshold, Integer.MAX_VALUE); try { while (iter.hasNext()) { + QueryExecutionContext.throwIfHeavyOperatorExecutionDisabled(OPERATOR_NAME); if (list.size() >= syncThreshold && list.size() < limit) { + QueryExecutionContext.checkpoint(OPERATOR_NAME + "_SPILL"); SerializedQueue queue = new SerializedQueue<>("orderiter"); sort(list).forEach(queue::add); serialized.add(queue); @@ -127,6 +137,9 @@ protected CloseableIteration createIteration() throws QueryEvaluatio } BindingSet next = iter.next(); inputRowsRead++; + if ((inputRowsRead & (INPUT_CHECKPOINT_INTERVAL - 1)) == 0) { + QueryExecutionContext.checkpoint(OPERATOR_NAME + "_INPUT"); + } onInputRowRead(next); if (threshold == null || comparator.compare(next, threshold) < 0) { list.add(next); @@ -174,6 +187,8 @@ protected void decrement(int amount) throws QueryEvaluationException { } private Stream sort(Collection collection) { + QueryExecutionContext.throwIfHeavyOperatorExecutionDisabled(OPERATOR_NAME); + QueryExecutionContext.checkpoint(OPERATOR_NAME + "_SORT"); BindingSet[] array = collection.toArray(new BindingSet[collection.size()]); Arrays.parallelSort(array, comparator); Stream stream = Stream.of(array); diff --git a/core/queryalgebra/evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/GroupIteratorTest.java b/core/queryalgebra/evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/GroupIteratorTest.java index 9010f061938..cef52c13a4e 100644 --- a/core/queryalgebra/evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/GroupIteratorTest.java +++ b/core/queryalgebra/evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/GroupIteratorTest.java @@ -16,14 +16,18 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiFunction; @@ -31,6 +35,10 @@ import java.util.function.Predicate; import org.eclipse.rdf4j.common.iteration.LookAheadIteration; +import org.eclipse.rdf4j.http.client.QueryCircuitBreaker; +import org.eclipse.rdf4j.http.client.QueryCircuitBreakerHandle; +import org.eclipse.rdf4j.http.client.QueryExecutionContext; +import org.eclipse.rdf4j.http.client.QueryPressureState; import org.eclipse.rdf4j.model.Literal; import org.eclipse.rdf4j.model.Value; import org.eclipse.rdf4j.model.ValueFactory; @@ -40,6 +48,7 @@ import org.eclipse.rdf4j.model.vocabulary.XSD; import org.eclipse.rdf4j.query.BindingSet; import org.eclipse.rdf4j.query.QueryEvaluationException; +import org.eclipse.rdf4j.query.QueryInterruptedException; import org.eclipse.rdf4j.query.algebra.AggregateFunctionCall; import org.eclipse.rdf4j.query.algebra.Avg; import org.eclipse.rdf4j.query.algebra.BindingSetAssignment; @@ -85,6 +94,10 @@ public class GroupIteratorTest { private static final BindingSetAssignment EMPTY_ASSIGNMENT = new BindingSetAssignment(); private static final BindingSetAssignment NONEMPTY_ASSIGNMENT = new BindingSetAssignment(); private static final AggregateFunctionFactory AGGREGATE_FUNCTION_FACTORY = new FakeAggregateFunctionFactory(); + private static final String BREAKER_ENABLED = "rdf4j.query.breaker.enabled"; + private static final String BREAKER_WARN_FREE_MB = "rdf4j.query.breaker.warn.free.mb"; + private static final String BREAKER_HIGH_FREE_MB = "rdf4j.query.breaker.high.free.mb"; + private static final String BREAKER_CRITICAL_FREE_MB = "rdf4j.query.breaker.critical.free.mb"; @BeforeAll public static void init() { @@ -430,6 +443,124 @@ protected void handleClose() { } } + @Test + public void testCircuitBreakerInterruptsGroupIterator() throws Exception { + Group group = new Group(NONEMPTY_ASSIGNMENT); + group.addGroupBindingName("a"); + + QueryCircuitBreaker breaker = QueryCircuitBreaker.getInstance(); + withCriticalBreakerProperties(() -> { + QueryCircuitBreakerHandle handle = breaker.register(QueryCircuitBreakerHandle.Source.WORKBENCH, "test", + "select * where { ?s ?p ?o } group by ?a"); + try (QueryExecutionContext.Activation ignored = QueryExecutionContext.activate(handle); + GroupIterator groupIterator = new GroupIterator(EVALUATOR, group, EmptyBindingSet.getInstance(), + CONTEXT)) { + assertThatExceptionOfType(QueryInterruptedException.class).isThrownBy(groupIterator::next); + } finally { + breaker.complete(handle); + } + }); + } + + private void withCriticalBreakerProperties(ThrowingRunnable action) throws Exception { + String previousEnabled = System.getProperty(BREAKER_ENABLED); + String previousWarnFreeMb = System.getProperty(BREAKER_WARN_FREE_MB); + String previousHighFreeMb = System.getProperty(BREAKER_HIGH_FREE_MB); + String previousCriticalFreeMb = System.getProperty(BREAKER_CRITICAL_FREE_MB); + try { + resetGlobalBreaker(); + System.setProperty(BREAKER_ENABLED, "true"); + System.setProperty(BREAKER_WARN_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + System.setProperty(BREAKER_HIGH_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + System.setProperty(BREAKER_CRITICAL_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + action.run(); + } finally { + restoreProperty(BREAKER_ENABLED, previousEnabled); + restoreProperty(BREAKER_WARN_FREE_MB, previousWarnFreeMb); + restoreProperty(BREAKER_HIGH_FREE_MB, previousHighFreeMb); + restoreProperty(BREAKER_CRITICAL_FREE_MB, previousCriticalFreeMb); + resetGlobalBreaker(); + } + } + + @Test + public void testHeavyOperatorKillSwitchInterruptsGroupIteratorWithoutBreakerCheckpoint() throws Exception { + String previousEnabled = System.getProperty(BREAKER_ENABLED); + Field heavyOperatorExecutionEnabled = QueryExecutionContext.class + .getDeclaredField("heavyOperatorExecutionEnabled"); + heavyOperatorExecutionEnabled.setAccessible(true); + boolean previousHeavyOperatorExecutionEnabled = heavyOperatorExecutionEnabled.getBoolean(null); + + Group group = new Group(NONEMPTY_ASSIGNMENT); + group.addGroupBindingName("a"); + + QueryCircuitBreaker breaker = QueryCircuitBreaker.getInstance(); + QueryCircuitBreakerHandle handle = breaker.register(QueryCircuitBreakerHandle.Source.WORKBENCH, "test", + "select * where { ?s ?p ?o } group by ?a"); + try { + resetGlobalBreaker(); + System.setProperty(BREAKER_ENABLED, "false"); + try (QueryExecutionContext.Activation ignored = QueryExecutionContext.activate(handle); + GroupIterator groupIterator = new GroupIterator(EVALUATOR, group, EmptyBindingSet.getInstance(), + CONTEXT)) { + heavyOperatorExecutionEnabled.setBoolean(null, false); + assertThatExceptionOfType(QueryInterruptedException.class).isThrownBy(groupIterator::next); + } + assertThat(handle.isCancelRequested()).isTrue(); + } finally { + heavyOperatorExecutionEnabled.setBoolean(null, previousHeavyOperatorExecutionEnabled); + restoreProperty(BREAKER_ENABLED, previousEnabled); + breaker.complete(handle); + resetGlobalBreaker(); + } + } + + private void resetGlobalBreaker() throws Exception { + QueryCircuitBreaker breaker = QueryCircuitBreaker.getInstance(); + Field activeHandlesField = QueryCircuitBreaker.class.getDeclaredField("activeHandles"); + activeHandlesField.setAccessible(true); + ((Map) activeHandlesField.get(breaker)).clear(); + resetCounter(breaker, "handleSequence"); + resetCounter(breaker, "rejectCount"); + resetCounter(breaker, "cancelCount"); + + Field currentStateField = QueryCircuitBreaker.class.getDeclaredField("currentState"); + currentStateField.setAccessible(true); + currentStateField.set(breaker, QueryPressureState.NORMAL); + + Field lastCancelAtField = QueryCircuitBreaker.class.getDeclaredField("lastCancelAt"); + lastCancelAtField.setAccessible(true); + lastCancelAtField.setLong(breaker, Long.MIN_VALUE); + + Class transitionClass = Class.forName("org.eclipse.rdf4j.http.client.QueryCircuitBreaker$Transition"); + Method initialMethod = transitionClass.getDeclaredMethod("initial"); + initialMethod.setAccessible(true); + Object initialTransition = initialMethod.invoke(null); + + Field lastTransitionField = QueryCircuitBreaker.class.getDeclaredField("lastTransition"); + lastTransitionField.setAccessible(true); + lastTransitionField.set(breaker, initialTransition); + } + + private void resetCounter(QueryCircuitBreaker breaker, String fieldName) throws Exception { + Field field = QueryCircuitBreaker.class.getDeclaredField(fieldName); + field.setAccessible(true); + ((AtomicLong) field.get(breaker)).set(0); + } + + private void restoreProperty(String propertyName, String value) { + if (value == null) { + System.clearProperty(propertyName); + } else { + System.setProperty(propertyName, value); + } + } + + @FunctionalInterface + private interface ThrowingRunnable { + void run() throws Exception; + } + private static final class FakeAggregateFunctionFactory implements AggregateFunctionFactory { @Override public String getIri() { diff --git a/core/queryalgebra/evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/OrderIteratorTest.java b/core/queryalgebra/evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/OrderIteratorTest.java index 0e34c2ec43b..e30524a5a54 100644 --- a/core/queryalgebra/evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/OrderIteratorTest.java +++ b/core/queryalgebra/evaluation/src/test/java/org/eclipse/rdf4j/query/algebra/evaluation/iterator/OrderIteratorTest.java @@ -8,26 +8,37 @@ * * SPDX-License-Identifier: BSD-3-Clause *******************************************************************************/ +// Some portions generated by Codex package org.eclipse.rdf4j.query.algebra.evaluation.iterator; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.fail; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; import org.eclipse.rdf4j.common.iteration.AbstractCloseableIteration; +import org.eclipse.rdf4j.http.client.QueryCircuitBreaker; +import org.eclipse.rdf4j.http.client.QueryCircuitBreakerHandle; +import org.eclipse.rdf4j.http.client.QueryExecutionContext; +import org.eclipse.rdf4j.http.client.QueryPressureState; import org.eclipse.rdf4j.model.Value; import org.eclipse.rdf4j.query.Binding; import org.eclipse.rdf4j.query.BindingSet; import org.eclipse.rdf4j.query.QueryEvaluationException; +import org.eclipse.rdf4j.query.QueryInterruptedException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -157,6 +168,11 @@ public String toString() { private SizeComparator cmp; + private static final String BREAKER_ENABLED = "rdf4j.query.breaker.enabled"; + private static final String BREAKER_WARN_FREE_MB = "rdf4j.query.breaker.warn.free.mb"; + private static final String BREAKER_HIGH_FREE_MB = "rdf4j.query.breaker.high.free.mb"; + private static final String BREAKER_CRITICAL_FREE_MB = "rdf4j.query.breaker.critical.free.mb"; + @Test public void testFirstHasNext() { order.hasNext(); @@ -212,6 +228,21 @@ public void testSorting() { assertFalse(order.hasNext()); } + @Test + public void testCircuitBreakerInterruptsOrderIterator() throws Exception { + QueryCircuitBreaker breaker = QueryCircuitBreaker.getInstance(); + withCriticalBreakerProperties(() -> { + QueryCircuitBreakerHandle handle = breaker.register(QueryCircuitBreakerHandle.Source.WORKBENCH, "test", + "select * where { ?s ?p ?o }"); + OrderIterator breakerAwareOrder = new OrderIterator(new IterationStub(list.iterator()), cmp); + try (QueryExecutionContext.Activation ignored = QueryExecutionContext.activate(handle)) { + assertThrows(QueryInterruptedException.class, breakerAwareOrder::hasNext); + } finally { + breaker.complete(handle); + } + }); + } + @BeforeEach protected void setUp() { list = Arrays.asList(b3, b5, b2, b1, b4, b2); @@ -220,4 +251,71 @@ protected void setUp() { order = new OrderIterator(iteration, cmp); } + private void withCriticalBreakerProperties(ThrowingRunnable action) throws Exception { + String previousEnabled = System.getProperty(BREAKER_ENABLED); + String previousWarnFreeMb = System.getProperty(BREAKER_WARN_FREE_MB); + String previousHighFreeMb = System.getProperty(BREAKER_HIGH_FREE_MB); + String previousCriticalFreeMb = System.getProperty(BREAKER_CRITICAL_FREE_MB); + try { + resetGlobalBreaker(); + System.setProperty(BREAKER_ENABLED, "true"); + System.setProperty(BREAKER_WARN_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + System.setProperty(BREAKER_HIGH_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + System.setProperty(BREAKER_CRITICAL_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + action.run(); + } finally { + restoreProperty(BREAKER_ENABLED, previousEnabled); + restoreProperty(BREAKER_WARN_FREE_MB, previousWarnFreeMb); + restoreProperty(BREAKER_HIGH_FREE_MB, previousHighFreeMb); + restoreProperty(BREAKER_CRITICAL_FREE_MB, previousCriticalFreeMb); + resetGlobalBreaker(); + } + } + + private void resetGlobalBreaker() throws Exception { + QueryCircuitBreaker breaker = QueryCircuitBreaker.getInstance(); + Field activeHandlesField = QueryCircuitBreaker.class.getDeclaredField("activeHandles"); + activeHandlesField.setAccessible(true); + ((Map) activeHandlesField.get(breaker)).clear(); + resetCounter(breaker, "handleSequence"); + resetCounter(breaker, "rejectCount"); + resetCounter(breaker, "cancelCount"); + + Field currentStateField = QueryCircuitBreaker.class.getDeclaredField("currentState"); + currentStateField.setAccessible(true); + currentStateField.set(breaker, QueryPressureState.NORMAL); + + Field lastCancelAtField = QueryCircuitBreaker.class.getDeclaredField("lastCancelAt"); + lastCancelAtField.setAccessible(true); + lastCancelAtField.setLong(breaker, Long.MIN_VALUE); + + Class transitionClass = Class.forName("org.eclipse.rdf4j.http.client.QueryCircuitBreaker$Transition"); + Method initialMethod = transitionClass.getDeclaredMethod("initial"); + initialMethod.setAccessible(true); + Object initialTransition = initialMethod.invoke(null); + + Field lastTransitionField = QueryCircuitBreaker.class.getDeclaredField("lastTransition"); + lastTransitionField.setAccessible(true); + lastTransitionField.set(breaker, initialTransition); + } + + private void resetCounter(QueryCircuitBreaker breaker, String fieldName) throws Exception { + Field field = QueryCircuitBreaker.class.getDeclaredField(fieldName); + field.setAccessible(true); + ((AtomicLong) field.get(breaker)).set(0); + } + + private void restoreProperty(String propertyName, String value) { + if (value == null) { + System.clearProperty(propertyName); + } else { + System.setProperty(propertyName, value); + } + } + + @FunctionalInterface + private interface ThrowingRunnable { + void run() throws Exception; + } + } diff --git a/core/sail/lmdb/pom.xml b/core/sail/lmdb/pom.xml index 65ef8f3b116..ab7ef75a6ff 100644 --- a/core/sail/lmdb/pom.xml +++ b/core/sail/lmdb/pom.xml @@ -219,6 +219,18 @@ + + maven-compiler-plugin + + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmhVersion} + + + + maven-assembly-plugin diff --git a/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/benchmark/ParallelMixedQueryPressureBenchmark.java b/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/benchmark/ParallelMixedQueryPressureBenchmark.java new file mode 100644 index 00000000000..21713d63413 --- /dev/null +++ b/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/benchmark/ParallelMixedQueryPressureBenchmark.java @@ -0,0 +1,237 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex + +package org.eclipse.rdf4j.sail.lmdb.benchmark; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.assertj.core.util.Files; +import org.eclipse.rdf4j.benchmark.common.BenchmarkResources; +import org.eclipse.rdf4j.common.transaction.IsolationLevels; +import org.eclipse.rdf4j.query.BindingSet; +import org.eclipse.rdf4j.query.TupleQueryResult; +import org.eclipse.rdf4j.repository.sail.SailRepository; +import org.eclipse.rdf4j.repository.sail.SailRepositoryConnection; +import org.eclipse.rdf4j.rio.RDFFormat; +import org.eclipse.rdf4j.sail.lmdb.LmdbStore; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@State(Scope.Benchmark) +@Warmup(iterations = 1, time = 5, timeUnit = TimeUnit.SECONDS) +@BenchmarkMode({ Mode.AverageTime }) +@Fork(value = 1) +@Threads(1) +@Measurement(iterations = 3, time = 10, timeUnit = TimeUnit.SECONDS) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +public class ParallelMixedQueryPressureBenchmark { + + private static final QuerySpec GROUP_BY_COUNT = query("group_by_count", "benchmarkFiles/query1.qr"); + private static final QuerySpec DISTINCT_PREDICATES = query("distinct_predicates", + "benchmarkFiles/query-distinct-predicates.qr"); + private static final QuerySpec ORDERED_UNION_LIMIT = query("ordered_union_limit", + "benchmarkFiles/ordered-union-limit.qr"); + private static final QuerySpec GROUP_BY_ORDER_LIMIT = query("group_by_order_limit", + "benchmarkFiles/query-persons-count-friends-sorted.qr"); + + private static final QuerySpec WIDE_JOIN = query("wide_join", "benchmarkFiles/query4.qr"); + private static final QuerySpec OPTIONAL_CHAIN = query("optional_chain", "benchmarkFiles/lots-of-optional.qr"); + private static final QuerySpec LONG_CHAIN = query("long_chain", "benchmarkFiles/long-chain.qr"); + private static final QuerySpec OPTIONAL_FILTER = query("optional_filter", + "benchmarkFiles/optional-rhs-filter.qr"); + + private static final QuerySpec[] BALANCED_WORKLOAD = { + GROUP_BY_COUNT, + WIDE_JOIN, + ORDERED_UNION_LIMIT, + OPTIONAL_CHAIN, + GROUP_BY_ORDER_LIMIT, + LONG_CHAIN, + DISTINCT_PREDICATES, + OPTIONAL_FILTER + }; + + private static final QuerySpec[] TERMINAL_HEAVY_WORKLOAD = { + GROUP_BY_COUNT, + ORDERED_UNION_LIMIT, + GROUP_BY_ORDER_LIMIT, + WIDE_JOIN, + DISTINCT_PREDICATES, + ORDERED_UNION_LIMIT, + GROUP_BY_ORDER_LIMIT, + OPTIONAL_CHAIN + }; + + @Param({ "8", "16", "32" }) + public int queryThreads; + + @Param({ "4", "8" }) + public int queriesPerThread; + + private SailRepository repository; + private ExecutorService executorService; + private File dataDir; + + public static void main(String[] args) throws RunnerException { + var options = new OptionsBuilder() + .include(ParallelMixedQueryPressureBenchmark.class.getSimpleName()) + .forks(0) + .build(); + new Runner(options).run(); + } + + @Setup(Level.Trial) + public void setup() throws IOException { + dataDir = Files.newTemporaryFolder(); + repository = new SailRepository(new LmdbStore(dataDir, ConfigUtil.createConfig())); + executorService = Executors.newFixedThreadPool(queryThreads); + + try (SailRepositoryConnection connection = repository.getConnection()) { + connection.begin(IsolationLevels.NONE); + try (InputStream resourceAsStream = getResourceAsStream("benchmarkFiles/datagovbe-valid.ttl.gz")) { + connection.add(resourceAsStream, RDFFormat.TURTLE); + } + connection.commit(); + } + } + + @TearDown(Level.Trial) + public void tearDown() throws IOException { + if (executorService != null) { + executorService.shutdownNow(); + executorService = null; + } + if (repository != null) { + repository.shutDown(); + repository = null; + } + if (dataDir != null) { + FileUtils.deleteDirectory(dataDir); + dataDir = null; + } + } + + @Benchmark + public long balancedMixedQueries() throws InterruptedException, ExecutionException { + return runWorkload(BALANCED_WORKLOAD); + } + + @Benchmark + public long terminalHeavyMixedQueries() throws InterruptedException, ExecutionException { + return runWorkload(TERMINAL_HEAVY_WORKLOAD); + } + + private long runWorkload(QuerySpec[] workload) throws InterruptedException, ExecutionException { + CountDownLatch startSignal = new CountDownLatch(1); + List> futures = new ArrayList<>(queryThreads); + + for (int workerIndex = 0; workerIndex < queryThreads; workerIndex++) { + final int localWorkerIndex = workerIndex; + futures.add(executorService.submit(() -> executeWorker(workload, startSignal, localWorkerIndex))); + } + + startSignal.countDown(); + + long checksum = 0; + for (Future future : futures) { + checksum += future.get(); + } + + return checksum; + } + + private long executeWorker(QuerySpec[] workload, CountDownLatch startSignal, int workerIndex) + throws InterruptedException { + startSignal.await(); + + long checksum = 0; + try (SailRepositoryConnection connection = repository.getConnection()) { + for (int i = 0; i < queriesPerThread; i++) { + QuerySpec query = workload[(workerIndex + i) % workload.length]; + checksum += executeQuery(connection, query); + } + } + + return checksum; + } + + private long executeQuery(SailRepositoryConnection connection, QuerySpec query) + throws InterruptedException { + try (TupleQueryResult result = connection.prepareTupleQuery(query.query).evaluate()) { + long rows = 0; + long checksum = query.name.hashCode(); + while (result.hasNext()) { + BindingSet bindingSet = result.next(); + rows++; + checksum = (checksum * 31) + bindingSet.hashCode(); + } + return checksum + rows; + } + } + + private static QuerySpec query(String name, String resourcePath) { + try (InputStream inputStream = getResourceAsStream(resourcePath)) { + if (inputStream == null) { + throw new IllegalStateException("Missing benchmark query resource " + resourcePath); + } + return new QuerySpec(name, IOUtils.toString(inputStream, StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new IllegalStateException("Unable to load benchmark query " + resourcePath, e); + } + } + + private static InputStream getResourceAsStream(String name) { + if (name.endsWith(".gz")) { + return BenchmarkResources.openDecompressedStream(name); + } + return ParallelMixedQueryPressureBenchmark.class.getClassLoader().getResourceAsStream(name); + } + + private static final class QuerySpec { + + private final String name; + private final String query; + + private QuerySpec(String name, String query) { + this.name = name; + this.query = query; + } + } +} diff --git a/docker/Dockerfile-jetty b/docker/Dockerfile-jetty index f1f052dbf85..a7406153116 100644 --- a/docker/Dockerfile-jetty +++ b/docker/Dockerfile-jetty @@ -16,7 +16,7 @@ LABEL org.opencontainers.image.authors="Bart Hanssens (bart.hanssens@bosa.fgov.b USER root -ENV JAVA_OPTIONS="-Xmx2g -Dorg.eclipse.rdf4j.appdata.basedir=/var/rdf4j -Dorg.eclipse.rdf4j.rio.jsonld_secure_mode=false" +ENV JAVA_OPTIONS="-Xmx2g -Dorg.eclipse.rdf4j.appdata.basedir=/var/rdf4j -Dorg.eclipse.rdf4j.rio.jsonld_secure_mode=false -Drdf4j.query.breaker.enabled=true" ENV JETTY_MODULES="server,bytebufferpool,threadpool,security,servlet,webapp,ext,plus,deploy,annotations,http,jsp,jstl" COPY --from=temp /tmp/eclipse-rdf4j*/war/*.war /var/lib/jetty/webapps/ diff --git a/docker/Dockerfile-tomcat b/docker/Dockerfile-tomcat index 940fd96d30a..f5390769daf 100644 --- a/docker/Dockerfile-tomcat +++ b/docker/Dockerfile-tomcat @@ -17,7 +17,7 @@ MAINTAINER Bart Hanssens (bart.hanssens@bosa.fgov.be) RUN apt-get clean && apt-get update && apt-get upgrade -y && apt-get clean ENV JAVA_OPTS="-Xmx2g" -ENV CATALINA_OPTS="-Dorg.eclipse.rdf4j.appdata.basedir=/var/rdf4j -Dorg.eclipse.rdf4j.rio.jsonld_secure_mode=false" +ENV CATALINA_OPTS="-Dorg.eclipse.rdf4j.appdata.basedir=/var/rdf4j -Dorg.eclipse.rdf4j.rio.jsonld_secure_mode=false -Drdf4j.query.breaker.enabled=true" RUN adduser --system tomcat diff --git a/tools/server-spring/pom.xml b/tools/server-spring/pom.xml index be9f901afa0..bc1ae8b4590 100644 --- a/tools/server-spring/pom.xml +++ b/tools/server-spring/pom.xml @@ -27,6 +27,11 @@ rdf4j-config ${project.version} + + ${project.groupId} + rdf4j-http-client + ${project.version} + ${project.groupId} rdf4j-rio-rdfjson diff --git a/tools/server-spring/src/main/java/org/eclipse/rdf4j/common/webapp/system/QueryCircuitBreakerStatusController.java b/tools/server-spring/src/main/java/org/eclipse/rdf4j/common/webapp/system/QueryCircuitBreakerStatusController.java new file mode 100644 index 00000000000..5678ddb0c07 --- /dev/null +++ b/tools/server-spring/src/main/java/org/eclipse/rdf4j/common/webapp/system/QueryCircuitBreakerStatusController.java @@ -0,0 +1,169 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex +package org.eclipse.rdf4j.common.webapp.system; + +import java.util.Objects; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.eclipse.rdf4j.http.client.QueryCircuitBreaker; +import org.springframework.web.servlet.ModelAndView; +import org.springframework.web.servlet.mvc.Controller; + +public class QueryCircuitBreakerStatusController implements Controller { + + private QueryCircuitBreaker circuitBreaker = QueryCircuitBreaker.getInstance(); + + @Override + public ModelAndView handleRequest(HttpServletRequest request, HttpServletResponse response) throws Exception { + QueryCircuitBreaker.StatusSnapshot snapshot = circuitBreaker.snapshotStatus(); + + response.setStatus(HttpServletResponse.SC_OK); + response.setCharacterEncoding("UTF-8"); + response.setContentType("application/json"); + response.getWriter().write(toJson(snapshot)); + return null; + } + + public void setCircuitBreaker(QueryCircuitBreaker circuitBreaker) { + this.circuitBreaker = Objects.requireNonNull(circuitBreaker, "Circuit breaker was null"); + } + + private String toJson(QueryCircuitBreaker.StatusSnapshot snapshot) { + StringBuilder builder = new StringBuilder(1024); + builder.append('{'); + appendBooleanField(builder, "enabled", snapshot.isEnabled()); + appendStringField(builder, "state", snapshot.getState()); + appendNumberField(builder, "freeMemoryMb", snapshot.getFreeMemoryMb()); + appendNumberField(builder, "rollingGcMs", snapshot.getRollingGcMs()); + appendNumberField(builder, "activeQueryCount", snapshot.getActiveQueryCount()); + appendNumberField(builder, "rejectCount", snapshot.getRejectCount()); + appendNumberField(builder, "cancelCount", snapshot.getCancelCount()); + appendNumberField(builder, "lastTransitionTimestamp", snapshot.getLastTransitionTimestamp()); + appendStringField(builder, "lastTransitionReason", snapshot.getLastTransitionReason()); + appendFieldName(builder, "configuration"); + appendConfiguration(builder, snapshot.getConfiguration()); + builder.append(','); + appendFieldName(builder, "activeQueries"); + appendActiveQueries(builder, snapshot); + builder.append('}'); + return builder.toString(); + } + + private void appendConfiguration(StringBuilder builder, QueryCircuitBreaker.Configuration configuration) { + builder.append('{'); + appendNumberField(builder, "warnGcMs", configuration.getWarnGcMs()); + appendNumberField(builder, "highGcMs", configuration.getHighGcMs()); + appendNumberField(builder, "criticalGcMs", configuration.getCriticalGcMs()); + appendNumberField(builder, "warnFreeMb", configuration.getWarnFreeMb()); + appendNumberField(builder, "highFreeMb", configuration.getHighFreeMb()); + appendNumberField(builder, "criticalFreeMb", configuration.getCriticalFreeMb()); + appendNumberField(builder, "warnAdmissionDelayMs", configuration.getWarnAdmissionDelayMs()); + appendNumberField(builder, "checkpointDelayMs", configuration.getCheckpointDelayMs()); + appendNumberField(builder, "cancelCooldownMs", configuration.getCancelCooldownMs()); + appendNumberField(builder, "retryAfterSeconds", configuration.getRetryAfterSeconds()); + trimTrailingComma(builder); + builder.append('}'); + } + + private void appendActiveQueries(StringBuilder builder, QueryCircuitBreaker.StatusSnapshot snapshot) { + builder.append('['); + boolean first = true; + for (QueryCircuitBreaker.ActiveQueryStatus activeQuery : snapshot.getActiveQueries()) { + if (!first) { + builder.append(','); + } + first = false; + builder.append('{'); + appendStringField(builder, "executionId", activeQuery.getExecutionId()); + appendStringField(builder, "source", activeQuery.getSource()); + appendStringField(builder, "repositoryId", activeQuery.getRepositoryId()); + appendStringField(builder, "queryHash", activeQuery.getQueryHash()); + appendNumberField(builder, "startTimeMillis", activeQuery.getStartTimeMillis()); + appendNumberField(builder, "lastHeavyCheckpointMillis", activeQuery.getLastHeavyCheckpointMillis()); + appendStringField(builder, "lastHeavyOperator", activeQuery.getLastHeavyOperator()); + appendBooleanField(builder, "cancelRequested", activeQuery.isCancelRequested()); + trimTrailingComma(builder); + builder.append('}'); + } + builder.append(']'); + } + + private void appendFieldName(StringBuilder builder, String name) { + builder.append('"').append(escape(name)).append('"').append(':'); + } + + private void appendStringField(StringBuilder builder, String name, String value) { + appendFieldName(builder, name); + if (value == null) { + builder.append("null"); + } else { + builder.append('"').append(escape(value)).append('"'); + } + builder.append(','); + } + + private void appendBooleanField(StringBuilder builder, String name, boolean value) { + appendFieldName(builder, name); + builder.append(value).append(','); + } + + private void appendNumberField(StringBuilder builder, String name, long value) { + appendFieldName(builder, name); + builder.append(value).append(','); + } + + private void trimTrailingComma(StringBuilder builder) { + int lastIndex = builder.length() - 1; + if (lastIndex >= 0 && builder.charAt(lastIndex) == ',') { + builder.deleteCharAt(lastIndex); + } + } + + private String escape(String value) { + StringBuilder escaped = new StringBuilder(value.length()); + for (int i = 0; i < value.length(); i++) { + char ch = value.charAt(i); + switch (ch) { + case '\\': + escaped.append("\\\\"); + break; + case '"': + escaped.append("\\\""); + break; + case '\b': + escaped.append("\\b"); + break; + case '\f': + escaped.append("\\f"); + break; + case '\n': + escaped.append("\\n"); + break; + case '\r': + escaped.append("\\r"); + break; + case '\t': + escaped.append("\\t"); + break; + default: + if (ch <= 0x1F) { + escaped.append(String.format("\\u%04x", (int) ch)); + } else { + escaped.append(ch); + } + } + } + return escaped.toString(); + } +} diff --git a/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/GraphQueryResultView.java b/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/GraphQueryResultView.java index 20426069a62..650062467c0 100644 --- a/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/GraphQueryResultView.java +++ b/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/GraphQueryResultView.java @@ -8,11 +8,11 @@ * * SPDX-License-Identifier: BSD-3-Clause *******************************************************************************/ +// Some portions generated by Codex package org.eclipse.rdf4j.http.server.repository; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_OK; -import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE; import java.io.IOException; import java.io.OutputStream; @@ -79,7 +79,7 @@ protected void renderInternal(Map model, HttpServletRequest request, HttpServlet QueryResults.report(graphQueryResult, rdfWriter); } catch (QueryInterruptedException e) { logger.error("Query interrupted", e); - response.sendError(SC_SERVICE_UNAVAILABLE, "Query evaluation took too long"); + sendServiceUnavailable(response, e, "Query evaluation took too long"); } catch (QueryEvaluationException e) { logger.error("Query evaluation error", e); response.sendError(SC_INTERNAL_SERVER_ERROR, "Query evaluation error: " + e.getMessage()); diff --git a/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/QueryResultView.java b/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/QueryResultView.java index 1040bcf8745..333b012cfcf 100644 --- a/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/QueryResultView.java +++ b/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/QueryResultView.java @@ -8,6 +8,7 @@ * * SPDX-License-Identifier: BSD-3-Clause *******************************************************************************/ +// Some portions generated by Codex package org.eclipse.rdf4j.http.server.repository; import static org.eclipse.rdf4j.http.protocol.Protocol.QUERY_PARAM_NAME; @@ -20,6 +21,10 @@ import javax.servlet.http.HttpServletResponse; import org.eclipse.rdf4j.common.lang.FileFormat; +import org.eclipse.rdf4j.http.client.QueryCircuitBreaker; +import org.eclipse.rdf4j.http.client.QueryCircuitBreakerHandle; +import org.eclipse.rdf4j.http.client.QueryExecutionContext; +import org.eclipse.rdf4j.query.QueryInterruptedException; import org.eclipse.rdf4j.repository.RepositoryConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,15 +69,35 @@ public abstract class QueryResultView implements View { public static final String HEADERS_ONLY = "headersOnly"; + public static final String BREAKER_HANDLE_KEY = "breakerHandle"; + @SuppressWarnings("rawtypes") @Override public final void render(Map model, HttpServletRequest request, HttpServletResponse response) throws IOException { + RepositoryConnection conn = (RepositoryConnection) model.get(CONNECTION_KEY); + QueryCircuitBreakerHandle breakerHandle = (QueryCircuitBreakerHandle) model.get(BREAKER_HANDLE_KEY); + QueryExecutionContext.Activation activation = null; try { + if (breakerHandle != null) { + breakerHandle.attachCurrentThread(conn); + activation = QueryExecutionContext.activate(breakerHandle); + } renderInternal(model, request, response); } finally { - RepositoryConnection conn = (RepositoryConnection) model.get(CONNECTION_KEY); - if (conn != null) { - conn.close(); + try { + if (activation != null) { + activation.close(); + } + } finally { + try { + if (conn != null) { + conn.close(); + } + } finally { + if (breakerHandle != null) { + QueryCircuitBreaker.getInstance().complete(breakerHandle); + } + } } } } @@ -115,4 +140,18 @@ protected void logEndOfRequest(HttpServletRequest request) { } } + protected void sendServiceUnavailable(HttpServletResponse response, QueryInterruptedException exception, + String fallbackMessage) throws IOException { + QueryCircuitBreaker.CircuitBreakerException breakerException = QueryCircuitBreaker + .asCircuitBreakerException(exception); + if (breakerException != null) { + if (breakerException.getRetryAfterSeconds() > 0) { + response.setHeader("Retry-After", String.valueOf(breakerException.getRetryAfterSeconds())); + } + response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, breakerException.getMessage()); + return; + } + response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, fallbackMessage); + } + } diff --git a/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/TupleQueryResultView.java b/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/TupleQueryResultView.java index 815e34a89e1..e5f84fd533f 100644 --- a/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/TupleQueryResultView.java +++ b/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/TupleQueryResultView.java @@ -8,11 +8,11 @@ * * SPDX-License-Identifier: BSD-3-Clause *******************************************************************************/ +// Some portions generated by Codex package org.eclipse.rdf4j.http.server.repository; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_OK; -import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE; import java.io.IOException; import java.io.OutputStream; @@ -104,7 +104,7 @@ protected void renderInternal(Map model, HttpServletRequest request, HttpServlet QueryResults.report(tupleQueryResult, qrWriter); } catch (QueryInterruptedException e) { logger.error("Query interrupted", e); - response.sendError(SC_SERVICE_UNAVAILABLE, "Query evaluation took too long"); + sendServiceUnavailable(response, e, "Query evaluation took too long"); } catch (QueryEvaluationException e) { logger.error("Query evaluation error", e); response.sendError(SC_INTERNAL_SERVER_ERROR, "Query evaluation error: " + e.getMessage()); diff --git a/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/handler/AbstractQueryRequestHandler.java b/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/handler/AbstractQueryRequestHandler.java index c9450c68f1d..9942fc10764 100644 --- a/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/handler/AbstractQueryRequestHandler.java +++ b/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/handler/AbstractQueryRequestHandler.java @@ -8,7 +8,7 @@ * * SPDX-License-Identifier: BSD-3-Clause *******************************************************************************/ - +// Some portions generated by Codex package org.eclipse.rdf4j.http.server.repository.handler; import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE; @@ -24,6 +24,9 @@ import org.eclipse.rdf4j.common.lang.FileFormat; import org.eclipse.rdf4j.common.lang.service.FileFormatServiceRegistry; import org.eclipse.rdf4j.http.client.AsyncExplainCoordinator; +import org.eclipse.rdf4j.http.client.QueryCircuitBreaker; +import org.eclipse.rdf4j.http.client.QueryCircuitBreakerHandle; +import org.eclipse.rdf4j.http.client.QueryExecutionContext; import org.eclipse.rdf4j.http.client.QueryExplanationRequestContext; import org.eclipse.rdf4j.http.protocol.Protocol; import org.eclipse.rdf4j.http.server.ClientHTTPException; @@ -57,6 +60,8 @@ public abstract class AbstractQueryRequestHandler implements QueryRequestHandler private final AsyncExplainCoordinator asyncExplainCoordinator = new AsyncExplainCoordinator(); + private final QueryCircuitBreaker queryCircuitBreaker = QueryCircuitBreaker.getInstance(); + public AbstractQueryRequestHandler(RepositoryResolver repositoryResolver) { this.repositoryResolver = repositoryResolver; } @@ -88,6 +93,7 @@ public ModelAndView handleQueryRequest( RepositoryConnection repositoryCon = null; Object queryResponse = null; + QueryCircuitBreakerHandle breakerHandle = null; try { Repository repository = repositoryResolver.getRepository(request); @@ -96,6 +102,15 @@ public ModelAndView handleQueryRequest( String queryString = getQueryString(request, requestMethod); logQuery(requestMethod, queryString); + breakerHandle = queryCircuitBreaker.register(QueryCircuitBreakerHandle.Source.SERVER, + repositoryResolver.getRepositoryID(request), queryString); + breakerHandle.attachCurrentThread(repositoryCon); + try { + queryCircuitBreaker.beforeExecution(breakerHandle); + } catch (QueryInterruptedException e) { + logger.info("Query interrupted", e); + throw toServiceUnavailable(response, e, "Query evaluation took too long"); + } Query query = getQuery(request, repositoryCon, queryString); @@ -109,7 +124,7 @@ public ModelAndView handleQueryRequest( if (!headersOnly && explainLevel.isPresent() && explainRequestId.isPresent()) { return handleRegisteredExplainRequest(request, response, repository, repositoryCon, query, explainLevel.get(), - explainRequestId.get()); + explainRequestId.get(), breakerHandle); } try { @@ -117,15 +132,21 @@ public ModelAndView handleQueryRequest( // explain param is present, return the query explanation if (explainLevel.isPresent()) { try { - Explanation explanation = explainQuery(query, explainLevel.get()); + Explanation explanation = queryCircuitBreaker.execute(breakerHandle, repositoryCon, + () -> explainQuery(query, explainLevel.get())); + queryCircuitBreaker.complete(breakerHandle); + breakerHandle = null; return getExplainQueryResponse(request, response, explanation); } finally { // explanation is fully evaluated at this point, so we can safely close the connection // before returning the response repositoryCon.close(); + repositoryCon = null; } } - queryResponse = evaluateQuery(query, limit, offset, distinct); + try (QueryExecutionContext.Activation ignored = QueryExecutionContext.activate(breakerHandle)) { + queryResponse = evaluateQuery(query, limit, offset, distinct); + } } FileFormatServiceRegistry registry = getResultWriterFor(query); @@ -140,11 +161,15 @@ public ModelAndView handleQueryRequest( "Unknown view for query of type: " + query.getClass().getName()); } - return getModelAndView(request, response, headersOnly, repositoryCon, view, queryResponse, registry); + ModelAndView modelAndView = getModelAndView(request, response, headersOnly, repositoryCon, view, + queryResponse, registry, breakerHandle); + breakerHandle = null; + repositoryCon = null; + return modelAndView; } catch (QueryInterruptedException e) { logger.info("Query interrupted", e); - throw new ServerHTTPException(SC_SERVICE_UNAVAILABLE, "Query evaluation took too long"); + throw toServiceUnavailable(response, e, "Query evaluation took too long"); } catch (QueryEvaluationException e) { logger.info("Query evaluation error", e); @@ -173,6 +198,8 @@ public ModelAndView handleQueryRequest( } } catch (Exception qre) { logger.warn("Connection closing error", qre); + } finally { + queryCircuitBreaker.complete(breakerHandle); } } throw e; @@ -187,7 +214,7 @@ protected Explanation explainQuery(final Query query, final Explanation.Level ex private ModelAndView handleRegisteredExplainRequest(HttpServletRequest request, HttpServletResponse response, Repository repository, RepositoryConnection repositoryCon, Query query, Explanation.Level explainLevel, - String explainRequestId) + String explainRequestId, QueryCircuitBreakerHandle breakerHandle) throws IOException, HTTPException { final AsyncExplainCoordinator.Handle handle; try { @@ -195,14 +222,23 @@ private ModelAndView handleRegisteredExplainRequest(HttpServletRequest request, explainRequestId)); } catch (IllegalStateException e) { closeConnection(repositoryCon); + queryCircuitBreaker.complete(breakerHandle); response.sendError(HttpServletResponse.SC_BAD_REQUEST, e.getMessage()); return null; } try { Explanation explanation = asyncExplainCoordinator.execute(handle, repositoryCon, - currentHandle -> QueryExplanationRequestContext - .activate(currentHandle.getExplainRequestId())::close, + currentHandle -> { + QueryExplanationRequestContext.Activation explainActivation = QueryExplanationRequestContext + .activate(currentHandle.getExplainRequestId()); + QueryExecutionContext.Activation breakerActivation = QueryExecutionContext + .activate(breakerHandle); + return () -> { + breakerActivation.close(); + explainActivation.close(); + }; + }, () -> explainQuery(query, explainLevel)); if (explanation == null) { return null; @@ -214,7 +250,7 @@ private ModelAndView handleRegisteredExplainRequest(HttpServletRequest request, if (!handle.isActive()) { return null; } - throw new ServerHTTPException(SC_SERVICE_UNAVAILABLE, "Query evaluation took too long"); + throw toServiceUnavailable(response, e, "Query evaluation took too long"); } catch (QueryEvaluationException e) { logger.info("Query evaluation error", e); if (!handle.isActive()) { @@ -234,6 +270,7 @@ private ModelAndView handleRegisteredExplainRequest(HttpServletRequest request, closeConnection(repositoryCon); } finally { asyncExplainCoordinator.complete(handle); + queryCircuitBreaker.complete(breakerHandle); } } } @@ -292,6 +329,33 @@ protected ModelAndView getModelAndView( return new ModelAndView(view, model); } + protected ModelAndView getModelAndView( + HttpServletRequest request, HttpServletResponse response, + boolean headersOnly, RepositoryConnection repositoryCon, View view, Object queryResult, + FileFormatServiceRegistry registry, + QueryCircuitBreakerHandle breakerHandle + ) throws ClientHTTPException { + ModelAndView modelAndView = getModelAndView(request, response, headersOnly, repositoryCon, view, queryResult, + registry); + if (breakerHandle != null) { + modelAndView.addObject(QueryResultView.BREAKER_HANDLE_KEY, breakerHandle); + } + return modelAndView; + } + + private ServerHTTPException toServiceUnavailable(HttpServletResponse response, QueryInterruptedException exception, + String fallbackMessage) { + QueryCircuitBreaker.CircuitBreakerException breakerException = QueryCircuitBreaker + .asCircuitBreakerException(exception); + if (breakerException != null) { + if (breakerException.getRetryAfterSeconds() > 0) { + response.setHeader("Retry-After", String.valueOf(breakerException.getRetryAfterSeconds())); + } + return new ServerHTTPException(SC_SERVICE_UNAVAILABLE, breakerException.getMessage(), exception); + } + return new ServerHTTPException(SC_SERVICE_UNAVAILABLE, fallbackMessage, exception); + } + protected boolean isDistinct(HttpServletRequest request) throws ClientHTTPException { return getParam(request, Protocol.DISTINCT_PARAM_NAME, false, Boolean.TYPE); } diff --git a/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/transaction/Transaction.java b/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/transaction/Transaction.java index a664ee0c6bd..f348fb0d4c2 100644 --- a/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/transaction/Transaction.java +++ b/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/transaction/Transaction.java @@ -8,6 +8,7 @@ * * SPDX-License-Identifier: BSD-3-Clause *******************************************************************************/ +// Some portions generated by Codex package org.eclipse.rdf4j.http.server.repository.transaction; import java.io.IOException; @@ -24,6 +25,8 @@ import org.eclipse.rdf4j.common.transaction.IsolationLevel; import org.eclipse.rdf4j.common.transaction.TransactionSetting; +import org.eclipse.rdf4j.http.client.QueryCircuitBreakerHandle; +import org.eclipse.rdf4j.http.client.QueryExecutionContext; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Resource; import org.eclipse.rdf4j.model.Statement; @@ -203,6 +206,12 @@ TupleQueryResult evaluate(TupleQuery tQuery) throws InterruptedException, Execut return getFromFuture(result); } + TupleQueryResult evaluate(QueryCircuitBreakerHandle handle, TupleQuery tQuery) + throws InterruptedException, ExecutionException { + Future result = submit(() -> executeWithBreaker(handle, tQuery::evaluate)); + return getFromFuture(result); + } + /** * Evaluate a {@link GraphQuery} in this transaction and return the result. * @@ -216,6 +225,12 @@ GraphQueryResult evaluate(GraphQuery gQuery) throws InterruptedException, Execut return getFromFuture(result); } + GraphQueryResult evaluate(QueryCircuitBreakerHandle handle, GraphQuery gQuery) + throws InterruptedException, ExecutionException { + Future result = submit(() -> executeWithBreaker(handle, gQuery::evaluate)); + return getFromFuture(result); + } + /** * Evaluate a {@link BooleanQuery} in this transaction and return the result. * @@ -229,11 +244,23 @@ boolean evaluate(BooleanQuery bQuery) throws InterruptedException, ExecutionExce return getFromFuture(result); } + boolean evaluate(QueryCircuitBreakerHandle handle, BooleanQuery bQuery) + throws InterruptedException, ExecutionException { + Future result = submit(() -> executeWithBreaker(handle, bQuery::evaluate)); + return getFromFuture(result); + } + Explanation explain(Query query, Explanation.Level level) throws InterruptedException, ExecutionException { Future result = submit(() -> query.explain(level)); return getFromFuture(result); } + Explanation explain(QueryCircuitBreakerHandle handle, Query query, Explanation.Level level) + throws InterruptedException, ExecutionException { + Future result = submit(() -> executeWithBreaker(handle, () -> query.explain(level))); + return getFromFuture(result); + } + /** * @param subj * @param pred @@ -468,6 +495,21 @@ private T getFromFuture(Future result) throws InterruptedException, Execu } } + private T executeWithBreaker(QueryCircuitBreakerHandle handle, Operation operation) throws Exception { + if (handle == null) { + return operation.execute(); + } + handle.attachCurrentThread(txnConnection); + try (QueryExecutionContext.Activation ignored = QueryExecutionContext.activate(handle)) { + return operation.execute(); + } + } + + @FunctionalInterface + private interface Operation { + T execute() throws Exception; + } + private static class WildcardRDFRemover extends AbstractRDFHandler { private static final Resource[] ALL_CONTEXT = {}; diff --git a/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/transaction/TransactionController.java b/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/transaction/TransactionController.java index 3231d17b8b3..40e98040f2f 100644 --- a/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/transaction/TransactionController.java +++ b/tools/server-spring/src/main/java/org/eclipse/rdf4j/http/server/repository/transaction/TransactionController.java @@ -50,6 +50,8 @@ import org.eclipse.rdf4j.common.lang.service.FileFormatServiceRegistry; import org.eclipse.rdf4j.common.webapp.views.EmptySuccessView; import org.eclipse.rdf4j.common.webapp.views.SimpleResponseView; +import org.eclipse.rdf4j.http.client.QueryCircuitBreaker; +import org.eclipse.rdf4j.http.client.QueryCircuitBreakerHandle; import org.eclipse.rdf4j.http.protocol.Protocol; import org.eclipse.rdf4j.http.protocol.Protocol.Action; import org.eclipse.rdf4j.http.protocol.error.ErrorInfo; @@ -105,6 +107,7 @@ public class TransactionController extends AbstractController implements DisposableBean { private final Logger logger = LoggerFactory.getLogger(this.getClass()); + private final QueryCircuitBreaker queryCircuitBreaker = QueryCircuitBreaker.getInstance(); public TransactionController() throws ApplicationContextException { setSupportedMethods(METHOD_POST, "PUT", "DELETE"); @@ -367,11 +370,16 @@ private ModelAndView processQuery(Transaction txn, HttpServletRequest request, H "Canceling query explanations is not supported for transaction requests."); } - View view; - Object queryResult; - FileFormatServiceRegistry registry; + View view = null; + Object queryResult = null; + FileFormatServiceRegistry registry = null; + QueryCircuitBreakerHandle breakerHandle = null; + boolean handedOffToView = false; try { + breakerHandle = queryCircuitBreaker.register(QueryCircuitBreakerHandle.Source.TX, + RepositoryInterceptor.getRepositoryID(request), queryStr); + queryCircuitBreaker.beforeExecution(breakerHandle); Query query = getQuery(txn, queryStr, request, response); Optional explainLevel = getExplain(request); @@ -381,7 +389,9 @@ private ModelAndView processQuery(Transaction txn, HttpServletRequest request, H "Tracked query explanations are only supported through the workbench UI."); } try { - Explanation explanation = txn.explain(query, explainLevel.get()); + Explanation explanation = txn.explain(breakerHandle, query, explainLevel.get()); + queryCircuitBreaker.complete(breakerHandle); + breakerHandle = null; return explanation == null ? null : getExplainQueryResponse(explanation); } catch (ExecutionException e) { handleExplainExecutionException(query, e); @@ -391,25 +401,42 @@ private ModelAndView processQuery(Transaction txn, HttpServletRequest request, H if (query instanceof TupleQuery) { TupleQuery tQuery = (TupleQuery) query; - queryResult = txn.evaluate(tQuery); + queryResult = txn.evaluate(breakerHandle, tQuery); registry = TupleQueryResultWriterRegistry.getInstance(); view = TupleQueryResultView.getInstance(); } else if (query instanceof GraphQuery) { GraphQuery gQuery = (GraphQuery) query; - queryResult = txn.evaluate(gQuery); + queryResult = txn.evaluate(breakerHandle, gQuery); registry = RDFWriterRegistry.getInstance(); view = GraphQueryResultView.getInstance(); } else if (query instanceof BooleanQuery) { BooleanQuery bQuery = (BooleanQuery) query; - queryResult = txn.evaluate(bQuery); + queryResult = txn.evaluate(breakerHandle, bQuery); registry = BooleanQueryResultWriterRegistry.getInstance(); view = BooleanQueryResultView.getInstance(); } else { throw new ClientHTTPException(SC_BAD_REQUEST, "Unsupported query type: " + query.getClass().getName()); } + + Object factory = ProtocolUtil.getAcceptableService(request, response, registry); + + Map model = new HashMap<>(); + model.put(QueryResultView.FILENAME_HINT_KEY, "query-result"); + model.put(QueryResultView.QUERY_RESULT_KEY, queryResult); + model.put(QueryResultView.FACTORY_KEY, factory); + model.put(QueryResultView.HEADERS_ONLY, false); // TODO needed for HEAD + // requests. + model.put(QueryResultView.BREAKER_HANDLE_KEY, breakerHandle); + handedOffToView = true; + return new ModelAndView(view, model); } catch (QueryInterruptedException | InterruptedException | ExecutionException e) { + QueryCircuitBreaker.CircuitBreakerException breakerException = findCircuitBreakerException(e); + if (breakerException != null) { + applyRetryAfter(response, breakerException); + throw new ServerHTTPException(SC_SERVICE_UNAVAILABLE, breakerException.getMessage(), e); + } if (e.getCause() != null && e.getCause() instanceof MalformedQueryException) { ErrorInfo errInfo = new ErrorInfo(ErrorType.MALFORMED_QUERY, e.getCause().getMessage()); throw new ClientHTTPException(SC_BAD_REQUEST, errInfo.toString()); @@ -426,16 +453,18 @@ private ModelAndView processQuery(Transaction txn, HttpServletRequest request, H } else { throw new ServerHTTPException("Query evaluation error: " + e.getMessage()); } + } finally { + if (!handedOffToView) { + if (queryResult instanceof AutoCloseable) { + try { + ((AutoCloseable) queryResult).close(); + } catch (Exception e) { + logger.warn("Query result closing error", e); + } + } + queryCircuitBreaker.complete(breakerHandle); + } } - Object factory = ProtocolUtil.getAcceptableService(request, response, registry); - - Map model = new HashMap<>(); - model.put(QueryResultView.FILENAME_HINT_KEY, "query-result"); - model.put(QueryResultView.QUERY_RESULT_KEY, queryResult); - model.put(QueryResultView.FACTORY_KEY, factory); - model.put(QueryResultView.HEADERS_ONLY, false); // TODO needed for HEAD - // requests. - return new ModelAndView(view, model); } private Optional getExplain(HttpServletRequest request) throws ClientHTTPException { @@ -482,6 +511,26 @@ private boolean containsCause(Throwable throwable, Class ex return false; } + private QueryCircuitBreaker.CircuitBreakerException findCircuitBreakerException(Throwable throwable) { + Throwable current = throwable; + while (current != null) { + QueryCircuitBreaker.CircuitBreakerException breakerException = QueryCircuitBreaker + .asCircuitBreakerException(current); + if (breakerException != null) { + return breakerException; + } + current = current.getCause(); + } + return null; + } + + private void applyRetryAfter(HttpServletResponse response, + QueryCircuitBreaker.CircuitBreakerException breakerException) { + if (breakerException.getRetryAfterSeconds() > 0) { + response.setHeader("Retry-After", String.valueOf(breakerException.getRetryAfterSeconds())); + } + } + private static Charset getCharset(HttpServletRequest request) { return request.getCharacterEncoding() != null ? Charset.forName(request.getCharacterEncoding()) : StandardCharsets.UTF_8; diff --git a/tools/server-spring/src/test/java/org/eclipse/rdf4j/common/webapp/system/QueryCircuitBreakerStatusControllerTest.java b/tools/server-spring/src/test/java/org/eclipse/rdf4j/common/webapp/system/QueryCircuitBreakerStatusControllerTest.java new file mode 100644 index 00000000000..3a1993d7d2a --- /dev/null +++ b/tools/server-spring/src/test/java/org/eclipse/rdf4j/common/webapp/system/QueryCircuitBreakerStatusControllerTest.java @@ -0,0 +1,80 @@ +/******************************************************************************* + * Copyright (c) 2026 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +// Some portions generated by Codex +package org.eclipse.rdf4j.common.webapp.system; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; + +class QueryCircuitBreakerStatusControllerTest { + + @Test + void shouldMapSystemBreakerStatusEndpoint() throws Exception { + String servletConfig = Files.readString( + Path.of("..", "server", "src", "main", "webapp", "WEB-INF", "common-webapp-system-servlet.xml")); + + assertThat(servletConfig).contains("/system/breaker/status"); + } + + @Test + void shouldExposeBreakerStatusAsJson() throws Exception { + String previousEnabled = System.getProperty("rdf4j.query.breaker.enabled"); + String previousWarnFreeMb = System.getProperty("rdf4j.query.breaker.warn.free.mb"); + String previousHighFreeMb = System.getProperty("rdf4j.query.breaker.high.free.mb"); + String previousCriticalFreeMb = System.getProperty("rdf4j.query.breaker.critical.free.mb"); + try { + System.setProperty("rdf4j.query.breaker.enabled", "true"); + System.setProperty("rdf4j.query.breaker.warn.free.mb", Integer.toString(Integer.MAX_VALUE)); + System.setProperty("rdf4j.query.breaker.high.free.mb", Integer.toString(Integer.MAX_VALUE)); + System.setProperty("rdf4j.query.breaker.critical.free.mb", "0"); + + Class controllerType = Class + .forName("org.eclipse.rdf4j.common.webapp.system.QueryCircuitBreakerStatusController"); + Object controller = controllerType.getConstructor().newInstance(); + Method handleRequest = controllerType.getMethod("handleRequest", HttpServletRequest.class, + HttpServletResponse.class); + + MockHttpServletRequest request = new MockHttpServletRequest(); + MockHttpServletResponse response = new MockHttpServletResponse(); + + Object modelAndView = handleRequest.invoke(controller, request, response); + + assertThat(modelAndView).isNull(); + assertThat(response.getContentType()).startsWith("application/json"); + assertThat(response.getContentAsString()).contains("\"state\":\"HIGH\""); + assertThat(response.getContentAsString()).contains("\"activeQueryCount\""); + assertThat(response.getContentAsString()).contains("\"configuration\""); + } finally { + restoreProperty("rdf4j.query.breaker.enabled", previousEnabled); + restoreProperty("rdf4j.query.breaker.warn.free.mb", previousWarnFreeMb); + restoreProperty("rdf4j.query.breaker.high.free.mb", previousHighFreeMb); + restoreProperty("rdf4j.query.breaker.critical.free.mb", previousCriticalFreeMb); + } + } + + private void restoreProperty(String propertyName, String value) { + if (value == null) { + System.clearProperty(propertyName); + } else { + System.setProperty(propertyName, value); + } + } +} diff --git a/tools/server-spring/src/test/java/org/eclipse/rdf4j/http/server/repository/TupleQueryResultViewTest.java b/tools/server-spring/src/test/java/org/eclipse/rdf4j/http/server/repository/TupleQueryResultViewTest.java index 28f2670450e..3d9537fa6a7 100644 --- a/tools/server-spring/src/test/java/org/eclipse/rdf4j/http/server/repository/TupleQueryResultViewTest.java +++ b/tools/server-spring/src/test/java/org/eclipse/rdf4j/http/server/repository/TupleQueryResultViewTest.java @@ -11,15 +11,27 @@ package org.eclipse.rdf4j.http.server.repository; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import org.eclipse.rdf4j.http.client.QueryCircuitBreaker; +import org.eclipse.rdf4j.http.client.QueryCircuitBreakerHandle; +import org.eclipse.rdf4j.http.client.QueryPressureState; import org.eclipse.rdf4j.query.QueryEvaluationException; import org.eclipse.rdf4j.query.TupleQueryResult; import org.eclipse.rdf4j.query.resultio.sparqljson.SPARQLResultsJSONWriterFactory; +import org.eclipse.rdf4j.repository.RepositoryConnection; +import org.eclipse.rdf4j.repository.RepositoryException; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; @@ -28,6 +40,11 @@ public class TupleQueryResultViewTest { private static TupleQueryResultView view = TupleQueryResultView.getInstance(); + @AfterEach + public void tearDown() throws Exception { + resetGlobalBreaker(); + } + @Test public void testRender_QueryEvaluationError1() throws Exception { var request = new MockHttpServletRequest(); @@ -44,4 +61,71 @@ public void testRender_QueryEvaluationError1() throws Exception { assertThat(response.getStatus()).isEqualTo(500); } + + @Test + public void testRender_ClosesBreakerHandleWhenConnectionCloseFails() throws Exception { + resetGlobalBreaker(); + + var request = new MockHttpServletRequest(); + var response = new MockHttpServletResponse(); + + TupleQueryResult queryResult = mock(TupleQueryResult.class); + when(queryResult.hasNext()).thenReturn(false); + + RepositoryConnection connection = mock(RepositoryConnection.class); + doThrow(new RepositoryException("close failed")).when(connection).close(); + + QueryCircuitBreaker breaker = QueryCircuitBreaker.getInstance(); + QueryCircuitBreakerHandle handle = breaker.register(QueryCircuitBreakerHandle.Source.SERVER, "repo", + "select * where { ?s ?p ?o }"); + + Map model = new HashMap<>(); + model.put(TupleQueryResultView.FACTORY_KEY, new SPARQLResultsJSONWriterFactory()); + model.put(TupleQueryResultView.QUERY_RESULT_KEY, queryResult); + model.put(TupleQueryResultView.CONNECTION_KEY, connection); + model.put(TupleQueryResultView.BREAKER_HANDLE_KEY, handle); + + try { + assertThatThrownBy(() -> view.render(model, request, response)) + .isInstanceOf(RepositoryException.class) + .hasMessage("close failed"); + assertThat(breaker.snapshotStatus().getActiveQueryCount()).isZero(); + verify(connection).close(); + } finally { + breaker.complete(handle); + } + } + + private void resetGlobalBreaker() throws Exception { + QueryCircuitBreaker breaker = QueryCircuitBreaker.getInstance(); + Field activeHandlesField = QueryCircuitBreaker.class.getDeclaredField("activeHandles"); + activeHandlesField.setAccessible(true); + ((Map) activeHandlesField.get(breaker)).clear(); + resetCounter(breaker, "handleSequence"); + resetCounter(breaker, "rejectCount"); + resetCounter(breaker, "cancelCount"); + + Field currentStateField = QueryCircuitBreaker.class.getDeclaredField("currentState"); + currentStateField.setAccessible(true); + currentStateField.set(breaker, QueryPressureState.NORMAL); + + Field lastCancelAtField = QueryCircuitBreaker.class.getDeclaredField("lastCancelAt"); + lastCancelAtField.setAccessible(true); + lastCancelAtField.setLong(breaker, Long.MIN_VALUE); + + Class transitionClass = Class.forName("org.eclipse.rdf4j.http.client.QueryCircuitBreaker$Transition"); + Method initialMethod = transitionClass.getDeclaredMethod("initial"); + initialMethod.setAccessible(true); + Object initialTransition = initialMethod.invoke(null); + + Field lastTransitionField = QueryCircuitBreaker.class.getDeclaredField("lastTransition"); + lastTransitionField.setAccessible(true); + lastTransitionField.set(breaker, initialTransition); + } + + private void resetCounter(QueryCircuitBreaker breaker, String fieldName) throws Exception { + Field field = QueryCircuitBreaker.class.getDeclaredField(fieldName); + field.setAccessible(true); + ((AtomicLong) field.get(breaker)).set(0); + } } diff --git a/tools/server-spring/src/test/java/org/eclipse/rdf4j/http/server/repository/handler/DefaultQueryRequestHandlerTest.java b/tools/server-spring/src/test/java/org/eclipse/rdf4j/http/server/repository/handler/DefaultQueryRequestHandlerTest.java index 3fd8ead8dc7..91401dd87e1 100644 --- a/tools/server-spring/src/test/java/org/eclipse/rdf4j/http/server/repository/handler/DefaultQueryRequestHandlerTest.java +++ b/tools/server-spring/src/test/java/org/eclipse/rdf4j/http/server/repository/handler/DefaultQueryRequestHandlerTest.java @@ -12,6 +12,7 @@ package org.eclipse.rdf4j.http.server.repository.handler; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -32,6 +33,7 @@ import org.eclipse.rdf4j.query.QueryInterruptedException; import org.eclipse.rdf4j.query.QueryLanguage; import org.eclipse.rdf4j.query.TupleQuery; +import org.eclipse.rdf4j.query.TupleQueryResult; import org.eclipse.rdf4j.query.explanation.Explanation; import org.eclipse.rdf4j.repository.Repository; import org.eclipse.rdf4j.repository.RepositoryConnection; @@ -45,6 +47,11 @@ class DefaultQueryRequestHandlerTest { private static final String SELECT_ALL_QUERY = "select * where { ?s ?p ?o }"; + private static final String BREAKER_ENABLED = "rdf4j.query.breaker.enabled"; + private static final String BREAKER_WARN_FREE_MB = "rdf4j.query.breaker.warn.free.mb"; + private static final String BREAKER_HIGH_FREE_MB = "rdf4j.query.breaker.high.free.mb"; + private static final String BREAKER_CRITICAL_FREE_MB = "rdf4j.query.breaker.critical.free.mb"; + private static final String BREAKER_RETRY_AFTER_SECONDS = "rdf4j.query.breaker.retry.after.seconds"; @Test void explainRequestIdShouldRunInlineAndCloseConnectionOnCancel() throws Exception { @@ -173,6 +180,39 @@ void duplicateAsyncExplainRequestShouldCloseRejectedConnection() throws Exceptio } } + @Test + void shouldRejectNewQueriesWhenCircuitBreakerIsHighPressure() throws Exception { + RepositoryResolver repositoryResolver = mock(RepositoryResolver.class); + Repository repository = mock(Repository.class); + RepositoryConnection connection = mock(RepositoryConnection.class); + TupleQuery tupleQuery = mock(TupleQuery.class); + TupleQueryResult tupleQueryResult = mock(TupleQueryResult.class); + DefaultQueryRequestHandler handler = new DefaultQueryRequestHandler(repositoryResolver); + + MockHttpServletRequest request = new MockHttpServletRequest(); + request.setMethod(RequestMethod.POST.name()); + request.setContentType(Protocol.FORM_MIME_TYPE); + request.setParameter(Protocol.QUERY_PARAM_NAME, SELECT_ALL_QUERY); + request.addHeader("Accept", "application/sparql-results+json"); + MockHttpServletResponse response = new MockHttpServletResponse(); + + when(repositoryResolver.getRepository(request)).thenReturn(repository); + when(repositoryResolver.getRepositoryConnection(request, repository)).thenReturn(connection); + when(connection.prepareQuery(QueryLanguage.SPARQL, SELECT_ALL_QUERY, null)).thenReturn(tupleQuery); + when(tupleQuery.evaluate()).thenReturn(tupleQueryResult); + + withBreakerProperties(() -> { + assertThatThrownBy(() -> handler.handleQueryRequest(request, RequestMethod.POST, response)) + .isInstanceOfSatisfying(org.eclipse.rdf4j.http.server.ServerHTTPException.class, exception -> { + assertThat(exception.getStatusCode()).isEqualTo(MockHttpServletResponse.SC_SERVICE_UNAVAILABLE); + assertThat(exception.getMessage()).contains("circuit breaker"); + }); + assertThat(response.getHeader("Retry-After")).isEqualTo("17"); + }); + + verify(connection).close(); + } + @Test void handleCancelExplainShouldRejectBlankRequestId() throws Exception { DefaultQueryRequestHandler handler = new DefaultQueryRequestHandler(mock(RepositoryResolver.class)); @@ -289,4 +329,39 @@ private static MockHttpServletRequest newCancelExplainRequest(String explainRequ request.setParameter(Protocol.EXPLAIN_REQUEST_ID_PARAM_NAME, explainRequestId); return request; } + + private void withBreakerProperties(ThrowingRunnable action) throws Exception { + String previousEnabled = System.getProperty(BREAKER_ENABLED); + String previousWarnFreeMb = System.getProperty(BREAKER_WARN_FREE_MB); + String previousHighFreeMb = System.getProperty(BREAKER_HIGH_FREE_MB); + String previousCriticalFreeMb = System.getProperty(BREAKER_CRITICAL_FREE_MB); + String previousRetryAfterSeconds = System.getProperty(BREAKER_RETRY_AFTER_SECONDS); + try { + System.setProperty(BREAKER_ENABLED, "true"); + System.setProperty(BREAKER_WARN_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + System.setProperty(BREAKER_HIGH_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + System.setProperty(BREAKER_CRITICAL_FREE_MB, "0"); + System.setProperty(BREAKER_RETRY_AFTER_SECONDS, "17"); + action.run(); + } finally { + restoreProperty(BREAKER_ENABLED, previousEnabled); + restoreProperty(BREAKER_WARN_FREE_MB, previousWarnFreeMb); + restoreProperty(BREAKER_HIGH_FREE_MB, previousHighFreeMb); + restoreProperty(BREAKER_CRITICAL_FREE_MB, previousCriticalFreeMb); + restoreProperty(BREAKER_RETRY_AFTER_SECONDS, previousRetryAfterSeconds); + } + } + + private void restoreProperty(String propertyName, String value) { + if (value == null) { + System.clearProperty(propertyName); + } else { + System.setProperty(propertyName, value); + } + } + + @FunctionalInterface + private interface ThrowingRunnable { + void run() throws Exception; + } } diff --git a/tools/server-spring/src/test/java/org/eclipse/rdf4j/http/server/repository/transaction/TestTransactionControllerExplain.java b/tools/server-spring/src/test/java/org/eclipse/rdf4j/http/server/repository/transaction/TestTransactionControllerExplain.java index a12d0b996f3..d5ac1dbe011 100644 --- a/tools/server-spring/src/test/java/org/eclipse/rdf4j/http/server/repository/transaction/TestTransactionControllerExplain.java +++ b/tools/server-spring/src/test/java/org/eclipse/rdf4j/http/server/repository/transaction/TestTransactionControllerExplain.java @@ -21,6 +21,7 @@ import org.eclipse.rdf4j.http.protocol.Protocol; import org.eclipse.rdf4j.http.server.ClientHTTPException; +import org.eclipse.rdf4j.http.server.ServerHTTPException; import org.eclipse.rdf4j.http.server.repository.ExplainQueryResultView; import org.eclipse.rdf4j.http.server.repository.QueryResultView; import org.eclipse.rdf4j.model.Value; @@ -47,6 +48,11 @@ class TestTransactionControllerExplain { private static final String SELECT_ALL_QUERY = "select * where { ?s ?p ?o }"; + private static final String BREAKER_ENABLED = "rdf4j.query.breaker.enabled"; + private static final String BREAKER_WARN_FREE_MB = "rdf4j.query.breaker.warn.free.mb"; + private static final String BREAKER_HIGH_FREE_MB = "rdf4j.query.breaker.high.free.mb"; + private static final String BREAKER_CRITICAL_FREE_MB = "rdf4j.query.breaker.critical.free.mb"; + private static final String BREAKER_RETRY_AFTER_SECONDS = "rdf4j.query.breaker.retry.after.seconds"; private final String repositoryID = "test-repo"; @@ -213,6 +219,39 @@ void shouldRejectCancelExplainRequestsOnTransactionEndpoint() throws Exception { } } + @Test + void shouldRejectTransactionQueriesWhenCircuitBreakerIsHighPressure() throws Exception { + Transaction txn = new Transaction(repository); + ActiveTransactionRegistry.INSTANCE.register(txn); + + try { + MockHttpServletRequest queryRequest = new MockHttpServletRequest(); + queryRequest.setRequestURI("/repositories/" + repositoryID + "/transactions/" + txn.getID()); + queryRequest.setPathInfo(repositoryID + "/transactions/" + txn.getID()); + queryRequest.setMethod(HttpMethod.PUT.name()); + queryRequest.setCharacterEncoding(StandardCharsets.UTF_8.name()); + queryRequest.setContentType("application/sparql-query; charset=utf-8"); + queryRequest.setContent(SELECT_ALL_QUERY.getBytes(StandardCharsets.UTF_8)); + queryRequest.setParameter(Protocol.ACTION_PARAM_NAME, Protocol.Action.QUERY.toString()); + queryRequest.addHeader("Accept", "application/sparql-results+json"); + + TransactionController transactionController = new TransactionController(); + + withBreakerProperties( + () -> assertThatThrownBy(() -> transactionController.handleRequestInternal(queryRequest, + response)) + .isInstanceOfSatisfying(ServerHTTPException.class, error -> { + assertThat(error.getStatusCode()).isEqualTo(503); + assertThat(error.getMessage()).contains("circuit breaker"); + })); + + assertThat(response.getHeader("Retry-After")).isEqualTo("17"); + } finally { + closeQuietly(txn); + deregisterQuietly(txn); + } + } + private void configureExplainRequest(UUID transactionId) { MockHttpServletRequest explainRequest = newExplainRequest(transactionId, null); request.setRequestURI(explainRequest.getRequestURI()); @@ -268,6 +307,41 @@ private static void deregisterQuietly(Transaction txn) { } } + private void withBreakerProperties(ThrowingRunnable action) throws Exception { + String previousEnabled = System.getProperty(BREAKER_ENABLED); + String previousWarnFreeMb = System.getProperty(BREAKER_WARN_FREE_MB); + String previousHighFreeMb = System.getProperty(BREAKER_HIGH_FREE_MB); + String previousCriticalFreeMb = System.getProperty(BREAKER_CRITICAL_FREE_MB); + String previousRetryAfter = System.getProperty(BREAKER_RETRY_AFTER_SECONDS); + try { + System.setProperty(BREAKER_ENABLED, "true"); + System.setProperty(BREAKER_WARN_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + System.setProperty(BREAKER_HIGH_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + System.setProperty(BREAKER_CRITICAL_FREE_MB, "0"); + System.setProperty(BREAKER_RETRY_AFTER_SECONDS, "17"); + action.run(); + } finally { + restoreProperty(BREAKER_ENABLED, previousEnabled); + restoreProperty(BREAKER_WARN_FREE_MB, previousWarnFreeMb); + restoreProperty(BREAKER_HIGH_FREE_MB, previousHighFreeMb); + restoreProperty(BREAKER_CRITICAL_FREE_MB, previousCriticalFreeMb); + restoreProperty(BREAKER_RETRY_AFTER_SECONDS, previousRetryAfter); + } + } + + private void restoreProperty(String propertyName, String value) { + if (value == null) { + System.clearProperty(propertyName); + } else { + System.setProperty(propertyName, value); + } + } + + @FunctionalInterface + private interface ThrowingRunnable { + void run() throws Exception; + } + private static final class UnsupportedExplainQuery implements Query { private Dataset dataset; diff --git a/tools/server/src/main/webapp/WEB-INF/common-webapp-system-servlet.xml b/tools/server/src/main/webapp/WEB-INF/common-webapp-system-servlet.xml index b116ec96669..b663d93d390 100644 --- a/tools/server/src/main/webapp/WEB-INF/common-webapp-system-servlet.xml +++ b/tools/server/src/main/webapp/WEB-INF/common-webapp-system-servlet.xml @@ -32,6 +32,7 @@ commonWebappSystemOverviewController commonWebappSystemInfoController filenameViewController + commonWebappQueryCircuitBreakerStatusController commonWebappLoggingOverviewController @@ -55,4 +56,5 @@ + diff --git a/tools/workbench/pom.xml b/tools/workbench/pom.xml index 0874cc76e15..34a2dd63c5a 100644 --- a/tools/workbench/pom.xml +++ b/tools/workbench/pom.xml @@ -28,6 +28,11 @@ rdf4j-config ${project.version} + + ${project.groupId} + rdf4j-http-client + ${project.version} + diff --git a/tools/workbench/src/main/java/org/eclipse/rdf4j/workbench/commands/QueryServlet.java b/tools/workbench/src/main/java/org/eclipse/rdf4j/workbench/commands/QueryServlet.java index f7c825a61e2..ff94ba1b11d 100644 --- a/tools/workbench/src/main/java/org/eclipse/rdf4j/workbench/commands/QueryServlet.java +++ b/tools/workbench/src/main/java/org/eclipse/rdf4j/workbench/commands/QueryServlet.java @@ -34,6 +34,7 @@ import org.eclipse.rdf4j.common.exception.RDF4JException; import org.eclipse.rdf4j.common.iteration.Iterations; import org.eclipse.rdf4j.http.client.AsyncExplainCoordinator; +import org.eclipse.rdf4j.http.client.QueryCircuitBreaker; import org.eclipse.rdf4j.http.client.QueryExplanationRequestContext; import org.eclipse.rdf4j.model.IRI; import org.eclipse.rdf4j.model.Literal; @@ -241,7 +242,7 @@ private void writeExplainResponse(final WorkbenchRequest req, final HttpServletR } catch (MalformedQueryException e) { writeExplainErrorResponse(resp, HttpServletResponse.SC_BAD_REQUEST, e.getMessage()); } catch (QueryInterruptedException e) { - writeExplainTimeoutResponse(resp); + writeExplainInterruptedResponse(resp, e); } catch (HTTPQueryEvaluationException e) { if (isExplainTimeout(e)) { writeExplainTimeoutResponse(resp); @@ -311,7 +312,7 @@ private void writeAsyncExplainResponse(final WorkbenchRequest req, final HttpSer } catch (MalformedQueryException e) { writeTrackedExplainError(resp, handle, HttpServletResponse.SC_BAD_REQUEST, e.getMessage()); } catch (QueryInterruptedException e) { - writeTrackedExplainTimeout(resp, handle); + writeTrackedExplainInterrupted(resp, handle, e); } catch (HTTPQueryEvaluationException e) { if (isExplainTimeout(e)) { writeTrackedExplainTimeout(resp, handle); @@ -355,6 +356,31 @@ private void writeTrackedExplainTimeout(HttpServletResponse response, AsyncExpla writeTrackedExplainError(response, handle, HttpServletResponse.SC_SERVICE_UNAVAILABLE, EXPLAIN_TIMEOUT_MESSAGE); } + private void writeExplainInterruptedResponse(HttpServletResponse response, QueryInterruptedException exception) + throws IOException { + QueryCircuitBreaker.CircuitBreakerException breakerException = QueryCircuitBreaker + .asCircuitBreakerException(exception); + if (breakerException != null) { + applyRetryAfter(response, breakerException); + writeExplainErrorResponse(response, HttpServletResponse.SC_SERVICE_UNAVAILABLE, + breakerException.getMessage()); + return; + } + writeExplainTimeoutResponse(response); + } + + private void writeTrackedExplainInterrupted(HttpServletResponse response, AsyncExplainCoordinator.Handle handle, + QueryInterruptedException exception) { + if (!handle.isActive()) { + return; + } + try { + writeExplainInterruptedResponse(response, exception); + } catch (IOException e) { + LOGGER.debug("Explain error response write failed for request {}", handle.getExplainRequestId(), e); + } + } + private void closeConnection(RepositoryConnection con, Throwable failure) { if (failure == null) { con.close(); @@ -437,12 +463,19 @@ private void handleStandardBrowserRequest(WorkbenchRequest req, HttpServletRespo service(req, resp, out, xslPath); } catch (BadRequestException | HTTPQueryEvaluationException exc) { LOGGER.warn(exc.toString(), exc); - TupleResultBuilder builder = getTupleResultBuilder(req, resp, out); - builder.transform(xslPath, "query.xsl"); - builder.start("error-message"); - builder.link(Arrays.asList(INFO, "namespaces")); - builder.result(exc.getMessage()); - builder.end(); + writeBrowserErrorResponse(req, resp, out, xslPath, exc.getMessage()); + } catch (QueryInterruptedException exc) { + LOGGER.warn(exc.toString(), exc); + QueryCircuitBreaker.CircuitBreakerException breakerException = QueryCircuitBreaker + .asCircuitBreakerException(exc); + if (breakerException != null) { + applyRetryAfter(resp, breakerException); + resp.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + writeBrowserErrorResponse(req, resp, out, xslPath, breakerException.getMessage()); + } else { + resp.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + writeBrowserErrorResponse(req, resp, out, xslPath, "Query evaluation took too long"); + } } finally { flushResponseOutputStream(out); } @@ -729,7 +762,7 @@ private void service(final WorkbenchRequest req, final HttpServletResponse resp, } else { try { EVAL.extractQueryAndEvaluate(builder, resp, out, xslPath, con, query, req, this.cookies, - getResponseQueryText(req, query)); + getResponseQueryText(req, query), getRepositoryReference()); } catch (MalformedQueryException exc) { throw new BadRequestException(exc.getMessage(), exc); } catch (HTTPQueryEvaluationException exc) { @@ -754,6 +787,23 @@ private void cacheLongQueryReferenceIfNeeded(WorkbenchRequest req, HttpServletRe } } + private void writeBrowserErrorResponse(WorkbenchRequest req, HttpServletResponse resp, OutputStream out, + String xslPath, String message) throws IOException, QueryResultHandlerException { + TupleResultBuilder builder = getTupleResultBuilder(req, resp, out); + builder.transform(xslPath, "query.xsl"); + builder.start("error-message"); + builder.link(Arrays.asList(INFO, "namespaces")); + builder.result(message); + builder.end(); + } + + private void applyRetryAfter(HttpServletResponse response, + QueryCircuitBreaker.CircuitBreakerException breakerException) { + if (breakerException.getRetryAfterSeconds() > 0) { + response.setHeader("Retry-After", String.valueOf(breakerException.getRetryAfterSeconds())); + } + } + private String getResponseQueryText(WorkbenchRequest req, String queryText) { if (req.isParameterPresent(ACCEPT)) { return null; diff --git a/tools/workbench/src/main/java/org/eclipse/rdf4j/workbench/util/QueryEvaluator.java b/tools/workbench/src/main/java/org/eclipse/rdf4j/workbench/util/QueryEvaluator.java index d79de823b5f..8ad1e2540d1 100644 --- a/tools/workbench/src/main/java/org/eclipse/rdf4j/workbench/util/QueryEvaluator.java +++ b/tools/workbench/src/main/java/org/eclipse/rdf4j/workbench/util/QueryEvaluator.java @@ -19,7 +19,9 @@ import javax.servlet.http.HttpServletResponse; import org.eclipse.rdf4j.common.exception.RDF4JException; -import org.eclipse.rdf4j.common.iteration.Iterations; +import org.eclipse.rdf4j.http.client.QueryCircuitBreaker; +import org.eclipse.rdf4j.http.client.QueryCircuitBreakerHandle; +import org.eclipse.rdf4j.http.client.QueryExecutionContext; import org.eclipse.rdf4j.model.Statement; import org.eclipse.rdf4j.query.BindingSet; import org.eclipse.rdf4j.query.BooleanQuery; @@ -67,6 +69,10 @@ public final class QueryEvaluator { private static final String METADATA_QUERY_TIMEOUT = "query-timeout"; + private static final int MATERIALIZATION_CHECKPOINT_INTERVAL = 128; + + private static final QueryCircuitBreaker QUERY_CIRCUIT_BREAKER = QueryCircuitBreaker.getInstance(); + private QueryEvaluator() { // do nothing } @@ -160,6 +166,32 @@ public void extractQueryAndEvaluate(final TupleResultBuilder builder, final Http final OutputStream out, final String xslPath, final RepositoryConnection con, String queryText, final WorkbenchRequest req, final CookieHandler cookies, final String responseQueryText) throws BadRequestException, RDF4JException { + extractQueryAndEvaluate(builder, resp, out, xslPath, con, queryText, req, cookies, responseQueryText, + extractRepositoryId(con)); + } + + public void extractQueryAndEvaluate(final TupleResultBuilder builder, final HttpServletResponse resp, + final OutputStream out, final String xslPath, final RepositoryConnection con, String queryText, + final WorkbenchRequest req, final CookieHandler cookies, final String responseQueryText, + final String repositoryId) + throws BadRequestException, RDF4JException { + QueryCircuitBreakerHandle breakerHandle = QUERY_CIRCUIT_BREAKER.register( + QueryCircuitBreakerHandle.Source.WORKBENCH, repositoryId, queryText); + try { + QUERY_CIRCUIT_BREAKER.execute(breakerHandle, con, () -> { + extractQueryAndEvaluateInternal(builder, resp, out, xslPath, con, queryText, req, cookies, + responseQueryText); + return null; + }); + } finally { + QUERY_CIRCUIT_BREAKER.complete(breakerHandle); + } + } + + private void extractQueryAndEvaluateInternal(final TupleResultBuilder builder, final HttpServletResponse resp, + final OutputStream out, final String xslPath, final RepositoryConnection con, String queryText, + final WorkbenchRequest req, final CookieHandler cookies, final String responseQueryText) + throws BadRequestException, RDF4JException { final QueryLanguage queryLn = QueryLanguage.valueOf(req.getParameter("queryLn")); Query query = prepareQuery(con, queryText, req); if (req.isParameterPresent(EXPLAIN)) { @@ -213,8 +245,16 @@ public ExplainRequest extractExplainRequest(final WorkbenchRequest req) throws B public ExplainQueryResult explain(final RepositoryConnection con, final String queryText, final ExplainRequest explainRequest) throws RDF4JException, BadRequestException { - Query query = prepareQuery(con, queryText, explainRequest); - return explain(query, explainRequest); + QueryCircuitBreakerHandle breakerHandle = QUERY_CIRCUIT_BREAKER.register( + QueryCircuitBreakerHandle.Source.WORKBENCH, extractRepositoryId(con), queryText); + try { + return QUERY_CIRCUIT_BREAKER.execute(breakerHandle, con, () -> { + Query query = prepareQuery(con, queryText, explainRequest); + return explain(query, explainRequest); + }); + } finally { + QUERY_CIRCUIT_BREAKER.complete(breakerHandle); + } } private Query prepareQuery(final RepositoryConnection con, final String queryText, final WorkbenchRequest req) @@ -354,9 +394,12 @@ public void evaluateTupleQuery(final TupleResultBuilder builder, String xslPath, HttpServletResponse resp, CookieHandler cookies, final TupleQuery query, boolean writeCookie, boolean paged, int offset, int limit, String responseQueryText) throws QueryEvaluationException, QueryResultHandlerException { - final TupleQueryResult result = query.evaluate(); - final String[] names = result.getBindingNames().toArray(new String[0]); - List bindings = Iterations.asList(result); + List bindings; + final String[] names; + try (TupleQueryResult result = query.evaluate()) { + names = result.getBindingNames().toArray(new String[0]); + bindings = collectBindingSets(result); + } if (writeCookie) { cookies.addTotalResultCountCookie(req, resp, bindings.size()); } @@ -427,7 +470,7 @@ private void evaluateGraphQuery(final TupleResultBuilder builder, String xslPath HttpServletResponse resp, CookieHandler cookies, final GraphQuery query, boolean writeCookie, boolean paged, int offset, int limit, String responseQueryText) throws QueryEvaluationException, QueryResultHandlerException { - List statements = Iterations.asList(query.evaluate()); + List statements = collectStatements(query); if (writeCookie) { cookies.addTotalResultCountCookie(req, resp, statements.size()); } @@ -504,4 +547,35 @@ private void addWorkbenchMetadata(TupleResultBuilder builder, WorkbenchRequest r queryTimeout == null || queryTimeout.isBlank() ? Integer.valueOf(0) : queryTimeout); } + private List collectBindingSets(TupleQueryResult result) throws QueryEvaluationException { + List bindings = new ArrayList<>(); + while (result.hasNext()) { + if (bindings.size() % MATERIALIZATION_CHECKPOINT_INTERVAL == 0) { + QueryExecutionContext.checkpoint("WORKBENCH_TUPLE_RESULT"); + } + bindings.add(result.next()); + } + return bindings; + } + + private List collectStatements(GraphQuery query) throws QueryEvaluationException { + List statements = new ArrayList<>(); + try (var result = query.evaluate()) { + while (result.hasNext()) { + if (statements.size() % MATERIALIZATION_CHECKPOINT_INTERVAL == 0) { + QueryExecutionContext.checkpoint("WORKBENCH_GRAPH_RESULT"); + } + statements.add(result.next()); + } + } + return statements; + } + + private String extractRepositoryId(RepositoryConnection connection) { + if (connection == null || connection.getRepository() == null) { + return "unknown-workbench-repository"; + } + return connection.getRepository().getClass().getName(); + } + } diff --git a/tools/workbench/src/test/java/org/eclipse/rdf4j/workbench/util/QueryEvaluatorTest.java b/tools/workbench/src/test/java/org/eclipse/rdf4j/workbench/util/QueryEvaluatorTest.java index 39581a29605..520a9ca95c2 100644 --- a/tools/workbench/src/test/java/org/eclipse/rdf4j/workbench/util/QueryEvaluatorTest.java +++ b/tools/workbench/src/test/java/org/eclipse/rdf4j/workbench/util/QueryEvaluatorTest.java @@ -11,6 +11,9 @@ // Some portions generated by Codex package org.eclipse.rdf4j.workbench.util; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -18,12 +21,21 @@ import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import javax.servlet.http.HttpServletResponse; +import org.eclipse.rdf4j.http.client.QueryCircuitBreaker; +import org.eclipse.rdf4j.http.client.QueryPressureState; import org.eclipse.rdf4j.model.impl.SimpleValueFactory; import org.eclipse.rdf4j.query.BindingSet; +import org.eclipse.rdf4j.query.QueryInterruptedException; import org.eclipse.rdf4j.query.QueryLanguage; import org.eclipse.rdf4j.query.TupleQuery; import org.eclipse.rdf4j.query.TupleQueryResult; @@ -35,6 +47,11 @@ class QueryEvaluatorTest { + private static final String BREAKER_ENABLED = "rdf4j.query.breaker.enabled"; + private static final String BREAKER_WARN_FREE_MB = "rdf4j.query.breaker.warn.free.mb"; + private static final String BREAKER_HIGH_FREE_MB = "rdf4j.query.breaker.high.free.mb"; + private static final String BREAKER_CRITICAL_FREE_MB = "rdf4j.query.breaker.critical.free.mb"; + @Test void shouldUseExplainInsteadOfEvaluateWhenExplainParameterIsPresent() throws Exception { String queryText = "select * where { ?s ?p ?o }"; @@ -77,6 +94,7 @@ void shouldUseDotExplanationWhenExplainFormatIsDot() throws Exception { RepositoryConnection con = mock(RepositoryConnection.class); CookieHandler cookies = mock(CookieHandler.class); TupleQuery tupleQuery = mock(TupleQuery.class); + TupleQueryResult tupleQueryResult = mock(TupleQueryResult.class); Explanation explanation = mock(Explanation.class); when(req.getParameter("queryLn")).thenReturn("SPARQL"); @@ -148,6 +166,80 @@ void shouldApplyQueryTimeoutWhenTimeoutParameterIsProvided() throws Exception { verify(tupleQuery).setMaxExecutionTime(12); } + @Test + void shouldRejectTupleQueryWhenCircuitBreakerIsHighPressure() throws Exception { + String queryText = "select * where { ?s ?p ?o }"; + String xslPath = "/xsl"; + TupleResultBuilder builder = mock(TupleResultBuilder.class); + WorkbenchRequest req = mock(WorkbenchRequest.class); + HttpServletResponse resp = mock(HttpServletResponse.class); + RepositoryConnection con = mock(RepositoryConnection.class); + CookieHandler cookies = mock(CookieHandler.class); + TupleQuery tupleQuery = mock(TupleQuery.class); + TupleQueryResult tupleQueryResult = mock(TupleQueryResult.class); + + when(req.getParameter("queryLn")).thenReturn("SPARQL"); + when(req.isParameterPresent("explain")).thenReturn(false); + when(req.isParameterPresent("infer")).thenReturn(false); + when(req.isParameterPresent("Accept")).thenReturn(false); + when(req.getInt("query-timeout")).thenReturn(0); + when(req.getInt("limit_query")).thenReturn(0); + when(req.getInt("offset")).thenReturn(0); + when(req.getInt("know_total")).thenReturn(0); + when(con.prepareQuery(QueryLanguage.SPARQL, queryText)).thenReturn(tupleQuery); + when(tupleQuery.evaluate()).thenReturn(tupleQueryResult); + when(tupleQueryResult.getBindingNames()).thenReturn(List.of("s")); + when(tupleQueryResult.hasNext()).thenReturn(false); + + withBreakerProperties(() -> { + QueryInterruptedException exception = assertThrows(QueryInterruptedException.class, + () -> QueryEvaluator.INSTANCE.extractQueryAndEvaluate(builder, resp, new ByteArrayOutputStream(), + xslPath, con, queryText, req, cookies, null)); + assertThat(exception).hasMessageContaining("circuit breaker"); + }); + + verify(tupleQuery, never()).evaluate(); + } + + @Test + void shouldAbortTupleMaterializationWhenBreakerTurnsCriticalMidStream() throws Exception { + String queryText = "select * where { ?s ?p ?o }"; + String xslPath = "/xsl"; + TupleResultBuilder builder = mock(TupleResultBuilder.class); + WorkbenchRequest req = mock(WorkbenchRequest.class); + HttpServletResponse resp = mock(HttpServletResponse.class); + RepositoryConnection con = mock(RepositoryConnection.class); + CookieHandler cookies = mock(CookieHandler.class); + TupleQuery tupleQuery = mock(TupleQuery.class); + AtomicBoolean closed = new AtomicBoolean(); + TupleQueryResult tupleQueryResult = new SwitchingTupleQueryResult(200, 64, closed, () -> { + System.setProperty(BREAKER_WARN_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + System.setProperty(BREAKER_HIGH_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + System.setProperty(BREAKER_CRITICAL_FREE_MB, Integer.toString(Integer.MAX_VALUE)); + }); + + when(req.getParameter("queryLn")).thenReturn("SPARQL"); + when(req.isParameterPresent("explain")).thenReturn(false); + when(req.isParameterPresent("infer")).thenReturn(false); + when(req.isParameterPresent("Accept")).thenReturn(false); + when(req.getInt("query-timeout")).thenReturn(0); + when(req.getInt("limit_query")).thenReturn(0); + when(req.getInt("offset")).thenReturn(0); + when(req.getInt("know_total")).thenReturn(0); + when(con.prepareQuery(QueryLanguage.SPARQL, queryText)).thenReturn(tupleQuery); + when(tupleQuery.evaluate()).thenReturn(tupleQueryResult); + + withBreakerProperties("0", "0", "0", () -> { + QueryInterruptedException exception = assertThrows(QueryInterruptedException.class, + () -> QueryEvaluator.INSTANCE.extractQueryAndEvaluate(builder, resp, new ByteArrayOutputStream(), + xslPath, con, queryText, req, cookies, null)); + assertThat(exception).hasMessageContaining("circuit breaker"); + }); + + assertTrue(closed.get()); + verify(tupleQuery).evaluate(); + } + @Test void shouldReapplyInferAndTimeoutWhenPagingUsesKnownTotalResultCount() throws Exception { String queryText = "select * where { ?s ?p ?o }"; @@ -224,4 +316,128 @@ private static BindingSet binding(String value) { bindingSet.addBinding("s", SimpleValueFactory.getInstance().createLiteral(value)); return bindingSet; } + + private void withBreakerProperties(ThrowingRunnable action) throws Exception { + withBreakerProperties(Integer.toString(Integer.MAX_VALUE), Integer.toString(Integer.MAX_VALUE), "0", action); + } + + private void withBreakerProperties(String warnFreeMb, String highFreeMb, String criticalFreeMb, + ThrowingRunnable action) + throws Exception { + String previousEnabled = System.getProperty(BREAKER_ENABLED); + String previousWarnFreeMb = System.getProperty(BREAKER_WARN_FREE_MB); + String previousHighFreeMb = System.getProperty(BREAKER_HIGH_FREE_MB); + String previousCriticalFreeMb = System.getProperty(BREAKER_CRITICAL_FREE_MB); + try { + resetGlobalBreaker(); + System.setProperty(BREAKER_ENABLED, "true"); + System.setProperty(BREAKER_WARN_FREE_MB, warnFreeMb); + System.setProperty(BREAKER_HIGH_FREE_MB, highFreeMb); + System.setProperty(BREAKER_CRITICAL_FREE_MB, criticalFreeMb); + action.run(); + } finally { + restoreProperty(BREAKER_ENABLED, previousEnabled); + restoreProperty(BREAKER_WARN_FREE_MB, previousWarnFreeMb); + restoreProperty(BREAKER_HIGH_FREE_MB, previousHighFreeMb); + restoreProperty(BREAKER_CRITICAL_FREE_MB, previousCriticalFreeMb); + resetGlobalBreaker(); + } + } + + private void resetGlobalBreaker() throws Exception { + QueryCircuitBreaker breaker = QueryCircuitBreaker.getInstance(); + Field activeHandlesField = QueryCircuitBreaker.class.getDeclaredField("activeHandles"); + activeHandlesField.setAccessible(true); + ((Map) activeHandlesField.get(breaker)).clear(); + resetCounter(breaker, "handleSequence"); + resetCounter(breaker, "rejectCount"); + resetCounter(breaker, "cancelCount"); + + Field currentStateField = QueryCircuitBreaker.class.getDeclaredField("currentState"); + currentStateField.setAccessible(true); + currentStateField.set(breaker, QueryPressureState.NORMAL); + + Field lastCancelAtField = QueryCircuitBreaker.class.getDeclaredField("lastCancelAt"); + lastCancelAtField.setAccessible(true); + lastCancelAtField.setLong(breaker, Long.MIN_VALUE); + + Class transitionClass = Class.forName("org.eclipse.rdf4j.http.client.QueryCircuitBreaker$Transition"); + Method initialMethod = transitionClass.getDeclaredMethod("initial"); + initialMethod.setAccessible(true); + Object initialTransition = initialMethod.invoke(null); + + Field lastTransitionField = QueryCircuitBreaker.class.getDeclaredField("lastTransition"); + lastTransitionField.setAccessible(true); + lastTransitionField.set(breaker, initialTransition); + } + + private void resetCounter(QueryCircuitBreaker breaker, String fieldName) throws Exception { + Field field = QueryCircuitBreaker.class.getDeclaredField(fieldName); + field.setAccessible(true); + ((AtomicLong) field.get(breaker)).set(0); + } + + private void restoreProperty(String propertyName, String value) { + if (value == null) { + System.clearProperty(propertyName); + } else { + System.setProperty(propertyName, value); + } + } + + @FunctionalInterface + private interface ThrowingRunnable { + void run() throws Exception; + } + + private static final class SwitchingTupleQueryResult implements TupleQueryResult { + private final List bindings; + private final int switchAfter; + private final AtomicBoolean closed; + private final Runnable switchAction; + private int index; + private boolean switched; + + private SwitchingTupleQueryResult(int totalRows, int switchAfter, AtomicBoolean closed, Runnable switchAction) { + this.bindings = new java.util.ArrayList<>(totalRows); + for (int i = 0; i < totalRows; i++) { + bindings.add(binding("row-" + i)); + } + this.switchAfter = switchAfter; + this.closed = closed; + this.switchAction = switchAction; + } + + @Override + public List getBindingNames() { + return List.of("s"); + } + + @Override + public boolean hasNext() { + return index < bindings.size(); + } + + @Override + public BindingSet next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + if (!switched && index >= switchAfter) { + switched = true; + switchAction.run(); + } + return bindings.get(index++); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + closed.set(true); + } + } }