From fbb3970d186782808022254f025d0e965f22b800 Mon Sep 17 00:00:00 2001 From: Tobias Kuhn Date: Mon, 31 Aug 2026 13:14:19 +0200 Subject: [PATCH] fix: disable LMDB value-cache eviction in new repo configs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With eviction enabled (rdf4j's 60s default), unused value IDs are freed and then REUSED; a stale reference afterwards dereferences to a different value. That is the value-ID-remap corruption behind eight events on the fleet between 2026-08-20 and 2026-08-31 (eclipse-rdf4j/rdf4j#5970), where unrelated records shared identifiers, records were absorbed into others, and literals appeared in the wrong slots. Setting valueEvictionInterval to 0 disables the pass and is the workaround recommended upstream on 2026-08-31. It costs disk — dead values are never reclaimed — but not memory: gcIds() returns early when eviction is off, so nothing accumulates on the heap. Our stores are append-only apart from last30d pruning, so the disk cost is small. The eviction pass "is also run after opening the database" (ValueStore.java), which finally explains why every one of those eight events followed a restart of the affected store — something we had recorded repeatedly without a mechanism. This covers newly created shards only. Repositories that already exist keep the config.ttl they were created with and need a one-off sweep with rdf4j stopped. Also plumb NANOPUB_QUERY_FETCHING_SOCKET_TIMEOUT through docker-compose.yml. Utils.getHttpRequestConfig has always read it, but the query service declares an explicit environment list and no env_file, so setting it in .env did nothing. The 10s default is aggressive for a store that is slow rather than broken: the read raises QueryInterruptedException, the loader's retry loops treat that as transient, and the retries add load. The default is unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- docker-compose.yml | 9 +++++++++ .../java/com/knowledgepixels/query/TripleStore.java | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 3f640754..3ade3ca1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,15 @@ services: # concurrent with writes are a suspected corruption trigger upstream # (rdf4j#5960/#4806), so this is a backstop, not a first-line limit. - RDF4J_QUERY_TIMEOUT_SECONDS=${RDF4J_QUERY_TIMEOUT_SECONDS:-60} + # Client-side socket timeout (ms) for this app's own reads from rdf4j + # (Utils.getHttpRequestConfig). 10s is aggressive for a store that is slow + # rather than broken: the read raises QueryInterruptedException, the loader's + # generic retry loops treat that as transient, and the retries add load, which + # makes the next read slower still. That storm saturated the OVH node on + # 2026-08-31 and again after the migration. Raise it where rdf4j shares CPU + # with other services; it is a timeout, so a higher value costs nothing when + # reads are fast. + - NANOPUB_QUERY_FETCHING_SOCKET_TIMEOUT=${NANOPUB_QUERY_FETCHING_SOCKET_TIMEOUT:-10000} # One-shot repair/resync switch: re-streams the whole registry at startup; # per-repo isLoaded checks skip everything already present, so WITHOUT a # store wipe this fills holes (missing nanopubs) idempotently. Set it for diff --git a/src/main/java/com/knowledgepixels/query/TripleStore.java b/src/main/java/com/knowledgepixels/query/TripleStore.java index 5fff9ab4..c2090fe5 100644 --- a/src/main/java/com/knowledgepixels/query/TripleStore.java +++ b/src/main/java/com/knowledgepixels/query/TripleStore.java @@ -423,6 +423,16 @@ private void createRepo(String repoName) { indexTypes = "spoc,posc,ospc"; } + // valueEvictionInterval 0 disables LMDB's value-cache garbage collection. + // Without it, rdf4j frees unused value IDs and REUSES them; a stale reference + // then dereferences to a different value, which is the value-ID-remap + // corruption we hit eight times between 2026-08-20 and 2026-08-31 (upstream + // eclipse-rdf4j/rdf4j#5970). Crucially the eviction pass "is also run after + // opening the database" (ValueStore.java), which explains why every one of + // those events followed a restart of the affected store. Disabling it is the + // workaround recommended upstream on 2026-08-31; it costs disk (dead values + // are never reclaimed) but not memory — gcIds() returns early, so nothing + // accumulates on the heap. Revisit once the upstream race is fixed. String createRegularRepoQueryString = "@prefix rdfs: .\n" + "@prefix rep: .\n" + @@ -441,6 +451,7 @@ private void createRepo(String repoName) { " sail:sailType \"rdf4j:LmdbStore\" ;\n" + " sail:iterationCacheSyncThreshold \"10000\";\n" + " lmdb:tripleIndexes \"" + indexTypes + "\" ;\n" + + " lmdb:valueEvictionInterval 0 ;\n" + " sb:defaultQueryEvaluationMode \"STANDARD\"\n" + " ]\n" + " ].\n"; @@ -468,6 +479,7 @@ private void createRepo(String repoName) { " sail:sailType \"rdf4j:LmdbStore\" ;\n" + " sail:iterationCacheSyncThreshold \"10000\";\n" + " lmdb:tripleIndexes \"" + indexTypes + "\" ;\n" + + " lmdb:valueEvictionInterval 0 ;\n" + " sb:defaultQueryEvaluationMode \"STANDARD\"\n" + " ]\n" + " ]\n" +