diff --git a/.fluree-memory/repo.ttl b/.fluree-memory/repo.ttl index 2c7285fc48..52abb0f253 100644 --- a/.fluree-memory/repo.ttl +++ b/.fluree-memory/repo.ttl @@ -1897,6 +1897,81 @@ mem:fact-01kv6sahj5se39hv5hkmjytznz a mem:Fact ; mem:createdAt "2026-06-15T23:19:32.677756+00:00"^^xsd:dateTime ; mem:rationale "Serial writes (summed S3 round-trips) blew the 15-min Lambda cap on a real 21 GB/74k-artifact DBLP restore; user chose the parallelize-writes fix. Sequential-stream-read ceiling is still unaddressed." . +mem:constraint-01m0sy4avrsbj4xs4tnaxw3xnr a mem:Constraint ; + mem:content "Every new TransactOperation variant MUST get an arm in `stage_under_lock` (tx_builder.rs), not only stage_plan / owned execute / owned stage. stage_under_lock is the path Raft (`build_commit_with_handle`) and every policy-gated, pre-built, SPARQL and Cypher local commit takes, and its JSON-like fallthrough silently stages an unknown op as `TxnType::Insert` into the default graph — the sync verb shipped that way until review." ; + mem:tag "new-verb-checklist" ; + mem:tag "raft" ; + mem:tag "stage-under-lock" ; + mem:tag "transact" ; + mem:scope mem:repo ; + mem:severity "must" ; + mem:artifactRef "fluree-db-api/src/tx_builder.rs" ; + mem:branch "feature/graph-sync-delta" ; + mem:createdAt "2026-08-24T13:07:35.420277+00:00"^^xsd:dateTime ; + mem:rationale "Local no-policy tests pass via the optimistic path, so the gap is invisible until raft or policy is involved; pinned by policy_gated_sync_targets_the_named_graph." . + +mem:constraint-01m0sy4gbwwx44d2yc3kepwv5a a mem:Constraint ; + mem:content "`BodyKind` and `TransactionBody` are postcard-positional in persisted Raft state (`QueueEntry.body_kind` in state snapshots): append new variants at the END only (pinned by body_kind_ordinals_are_append_only). Separately, `RefTransactBuilder::build_commit` returns `Ok(None)` for a zero-flake update/upsert/sync that registers no graph; the raft worker republishes the current head with `install: None` (mirroring revert NoOp). Before this, a no-change write under Raft failed with EmptyTransaction and poisoned the queued request." ; + mem:tag "consensus" ; + mem:tag "no-op" ; + mem:tag "postcard" ; + mem:tag "raft" ; + mem:tag "wire-format" ; + mem:scope mem:repo ; + mem:severity "must" ; + mem:artifactRef "fluree-db-api/src/tx_builder.rs" ; + mem:artifactRef "fluree-db-consensus/src/lib.rs" ; + mem:artifactRef "fluree-db-consensus/src/raft/commit_worker.rs" ; + mem:branch "feature/graph-sync-delta" ; + mem:createdAt "2026-08-24T13:07:41.052490+00:00"^^xsd:dateTime ; + mem:rationale "Mid-enum insertion shifts every later ordinal so existing snapshots/mixed-version nodes decode the wrong operation; the no-op terminal is the only way consensus can express committed=false." . + +mem:decision-01m0svwaqmb1h9dweex82pcxd7 a mem:Decision ; + mem:content "Graph sync (feature/graph-sync-delta) = Insert-typed Txn + `sync_graph: Option` directive: staging adds a wave after the upsert wave pushing every scanned target-graph flake as a retraction; FlakeAccumulator::mixed nets A∩B to zero so the commit is exactly the delta. Chosen over a GraphMgmtOp variant (payload doesn't fit SPARQL-shaped ops) and over the materialize.rs text-diff pipeline (needs a built binary index + bulk-import path drops retractions; the accumulator path works on any live ledger and inherits policy/SHACL/cascade/no-op machinery). Retraction scan follows CLEAR's policy model (not view-filtered, O4). Zero-staged sync joins the Update|Upsert no-op skip via StageResult.sync_graph." ; + mem:tag "delta" ; + mem:tag "graph-sync" ; + mem:tag "named-graph" ; + mem:tag "staging" ; + mem:tag "transact" ; + mem:scope mem:repo ; + mem:artifactRef "docs/transactions/sync.md" ; + mem:artifactRef "fluree-db-api/src/admin.rs" ; + mem:artifactRef "fluree-db-api/tests/it_sync_graph.rs" ; + mem:artifactRef "fluree-db-transact/src/parse/jsonld.rs" ; + mem:artifactRef "fluree-db-transact/src/stage.rs" ; + mem:branch "feature/graph-sync-delta" ; + mem:createdAt "2026-08-24T12:28:15.988437+00:00"^^xsd:dateTime ; + mem:rationale "Whole-graph ops that materialize scans + payload are the CLEAR-class memory profile; chunked staging remains the known follow-up. Bnode stability = deterministic graph-scoped skolem_txn_id (sync+doc_scope(doc_id)) set in stage_sync_transaction_tracked — label-unstable exporters (Protégé genid) still churn; RDFC canonicalization is the designed seam." ; + mem:alternatives "GraphMgmtOp::Sync variant; CLEAR+INSERT relying on accumulator (no explicit no-op detection); offline export/external-sort/diff pipeline (v2 scale path)" . + +mem:decision-01m0t021b2wwasg494g0yxre7w a mem:Decision ; + mem:content "`fluree sync` CLI grammar: top-level verb (not `graph sync`), target graph is the constant (`--graph ` required), the SOURCE of desired contents is pluggable via `commands/graph_sync.rs::SyncSource` (today RdfText; an R2RML-over-Iceberg/CSV/Excel source is one new variant that materializes to the same JSON-LD payload). Turtle converts client-side (endpoint is JSON-LD only). `--dry-run --json` is the scripting pre-flight; `--json` output = the server dry-run report shape on both local and remote paths. NOTE: `commands/sync.rs` already exists (fetch/pull/push/publish replication) — never reuse that name." ; + mem:tag "cli" ; + mem:tag "command-grammar" ; + mem:tag "graph-sync" ; + mem:tag "r2rml" ; + mem:scope mem:repo ; + mem:artifactRef "docs/cli/server-integration.md" ; + mem:artifactRef "docs/cli/sync.md" ; + mem:artifactRef "fluree-db-cli/src/commands/graph_sync.rs" ; + mem:branch "feature/graph-sync-delta" ; + mem:createdAt "2026-08-24T13:41:17.282471+00:00"^^xsd:dateTime ; + mem:rationale "User wants sync to later drive from Iceberg/CSV/Excel via R2RML (\"latest downstream\"); keeping the source a client-side seam means no new server endpoint or command per source, and --remote only moves where materialization happens." . + +mem:fact-01m0svwr4r77f4v4xy9k7w5q6e a mem:Fact ; + mem:content "CommitReceipt now carries assert_count/retract_count (count_ops in commit.rs, populated in finalize_state_with_base from commit_record.flakes). Raft-applied receipts report 0/0 — AppliedReceipt/idempotency records don't persist the split (raft wire-format stability), so sync reports over raft lack the convenience counts. Sync's HTTP dry-run path computes counts locally instead. New verbs need ALL of: TransactionBody variant + operation_tag + body_hash domain tag + BodyKind + BOTH appliers (local.rs and raft/commit_worker.rs body dispatch) + server route family + tx_builder TransactOperation/OpPlan/stage_plan arms + BOTH no-op skip conditions (tx_builder owned execute + commit_and_finalize, and the three tx.rs transact paths)." ; + mem:tag "commit-receipt" ; + mem:tag "consensus" ; + mem:tag "new-verb-checklist" ; + mem:tag "transact" ; + mem:scope mem:repo ; + mem:artifactRef "fluree-db-api/src/tx_builder.rs" ; + mem:artifactRef "fluree-db-consensus/src/lib.rs" ; + mem:artifactRef "fluree-db-transact/src/commit.rs" ; + mem:branch "feature/graph-sync-delta" ; + mem:createdAt "2026-08-24T12:28:29.720563+00:00"^^xsd:dateTime ; + mem:rationale "The verb-seam checklist is easy to miss partially (a missed applier silently breaks the verb under raft only)." . + mem:fact-01kzemcff3b773aq7yckej9dhk a mem:Fact ; mem:content "Local-filesystem Iceberg support (commit 15ca56807, E2E-verified against a real pyiceberg table): Direct table_location accepts file:///, file:/, and bare absolute paths. Adds FileIcebergStorage and an IcebergStorageBackend enum (S3|File) threaded through the api scan surface, session caches, and lazy path; the version-hint fallback lists metadata/ via a new list_files trait method. GOTCHA the E2E test caught: Direct location validation lives in TWO places — fluree_db_iceberg::config and fluree-db-api graph_source/config.rs validate() — and both must accept a new scheme." ; mem:tag "direct-mode" ; @@ -2664,7 +2739,7 @@ mem:fact-01kynrrhdfqt0n6rx1wpcv5843 a mem:Fact ; mem:rationale "Answers \"would batching help existing-subject upserts?\" with measurements — avoids speculative batching work; documents the store-cached p_sid_table invariant (store immutable behind Arc → OnceLock safe)." . mem:fact-01m0srrpntk3mxseprnpcvey08 a mem:Fact ; - mem:content "REVERTED, do not re-land as written: lowering star-block VALUES to FILTER(?v IN ...) at block assembly (convert_star_values_to_membership_filters) did not work. inline_singleton_values_objects rewrites a singleton VALUES into the triple object but RETAINS the VALUES pattern, so its var is no longer produced by any star triple, membership_filter_from_values declines it, and the all-or-nothing gate declines the whole block — which is exactly the two-VALUES repro shape. Measured: 0 firings across the 6 tests that shipped with it, 1 across the whole 349-test SPARQL group. Where it does fire it is SLOWER: Function::In yields no range constraint from extract_range_constraints, so the block carries no object_bounds, has_selective_anchor is false, and it leaves the fused PropertyJoinOperator for the NLJ chain. Push the constraint down as a SEED that keeps the star anchored, not a filter that unanchors it." ; + mem:content "REVERTED, do not re-land as written: lowering star-block VALUES to FILTER(?v IN ...) at block assembly failed two ways. inline_singleton_values_objects rewrites a singleton VALUES into the triple object but RETAINS the VALUES pattern, so membership_filter_from_values declines its var and the all-or-nothing gate declines the whole block — exactly the two-VALUES repro shape; 1 firing across the 349-test SPARQL group. Where it does fire it is SLOWER: Function::In yields no object_bounds, has_selective_anchor goes false, and the block leaves the fused PropertyJoinOperator for the NLJ chain. Push the constraint down as a SEED that keeps the star anchored, not a filter that unanchors it." ; mem:tag "performance" ; mem:tag "planner" ; mem:tag "reverted" ; @@ -2676,7 +2751,7 @@ mem:fact-01m0srrpntk3mxseprnpcvey08 a mem:Fact ; mem:artifactRef "fluree-db-query/src/planner.rs" ; mem:branch "fix/values-object-scan-constraint" ; mem:createdAt "2026-08-24T11:33:51.418875+00:00"^^xsd:dateTime ; - mem:updatedAt "2026-08-25T23:36:20.896572+00:00"^^xsd:dateTime ; + mem:updatedAt "2026-08-26T13:52:55.183934+00:00"^^xsd:dateTime ; mem:rationale "The approach looks obviously right and was already tried and reverted; the singleton-fold interaction and the missing object_bounds are the two things that kill it." . mem:fact-01m0srrz87xpy9dy11z0b084hv a mem:Fact ; diff --git a/Cargo.lock b/Cargo.lock index cab363734d..7fdf95a52f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2610,6 +2610,7 @@ dependencies = [ "fluree-db-query", "fluree-db-r2rml", "fluree-db-server", + "fluree-graph-turtle", "fluree-vocab", "futures", "hex", diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index e6a29d96d1..8aa0e6e82c 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -14,6 +14,7 @@ - [graph](cli/graph.md) - [insert](cli/insert.md) - [upsert](cli/upsert.md) + - [sync](cli/sync.md) - [update](cli/update.md) - [load](cli/load.md) - [query](cli/query.md) @@ -134,6 +135,7 @@ - [Overview](transactions/overview.md) - [Insert](transactions/insert.md) - [Upsert](transactions/upsert.md) + - [Sync (graph synchronization)](transactions/sync.md) - [Update (WHERE/DELETE/INSERT)](transactions/update-where-delete-insert.md) - [Conditional updates (atomic / compare-and-swap)](transactions/conditional-updates.md) - [Retractions](transactions/retractions.md) diff --git a/docs/api/endpoints.md b/docs/api/endpoints.md index db9ae0e093..334b9edf4b 100644 --- a/docs/api/endpoints.md +++ b/docs/api/endpoints.md @@ -324,6 +324,38 @@ Both W3C TriG graph-block forms are accepted: the SPARQL-style keyword form The compact form is what stock RDF tooling — rdflib, Apache Jena, RDF4J — emits by default, so payloads generated by those libraries are ingested as-is. +### POST /sync + +Synchronize a named graph: make its contents exactly the JSON-LD payload, +committing only the delta (`current − payload` retracted, `payload − +current` asserted; unchanged facts produce no flakes). An identical payload +produces no commit. See [Sync](../transactions/sync.md). + +**URL:** +``` +POST /sync?ledger={ledger-id}&graph={graph-iri} +POST /sync/{ledger-id}?graph={graph-iri} +``` + +**Query parameters:** `graph` (required target graph IRI), `dryRun=true` +(stage and report the delta without committing), `allowEmpty=true` (confirm +an explicitly empty payload, which clears the graph). + +**Supported Content Types:** +- `application/json` - JSON-LD (Turtle payloads must be converted client-side for now) + +**Example:** +```bash +curl -X POST "http://localhost:8090/v1/fluree/sync?ledger=mydb:main&graph=http://example.org/graphs/ontology" \ + -H "Content-Type: application/json" \ + -d '{ + "@context": { "ex": "http://example.org/ns/" }, + "@graph": [ + { "@id": "ex:alice", "ex:name": "Alice" } + ] + }' +``` + ### POST /push/*ledger Push precomputed commit v2 blobs to the server. diff --git a/docs/cli/README.md b/docs/cli/README.md index 39fdbccd84..376b64424d 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -58,6 +58,7 @@ fluree query 'SELECT ?name WHERE { ?s ?name }' | [`branch`](branch.md) | Branches: create, list, drop, rebase, merge, diff, revert | | [`insert`](insert.md) | Insert data into a ledger | | [`upsert`](upsert.md) | Upsert data (insert or update existing) | +| [`sync`](sync.md) | Make a named graph's contents exactly the supplied data, committing only the delta | | [`update`](update.md) | Update with WHERE/DELETE/INSERT patterns | | [`load`](load.md) | Stream a CSV into a ledger as batched Cypher/JSON-LD upserts (`LOAD CSV`) | | [`query`](query.md) | Query a ledger | @@ -147,7 +148,7 @@ When you run `fluree init`, a `.fluree/` directory is created with: ## Input Resolution -Commands that accept data input (`insert`, `upsert`, `update`, `query`) use flexible argument resolution: +Commands that accept data input (`insert`, `upsert`, `sync`, `update`, `query`) use flexible argument resolution: | Arguments | Behavior | |-----------|----------| diff --git a/docs/cli/server-integration.md b/docs/cli/server-integration.md index 04ce565db0..353c8722d9 100644 --- a/docs/cli/server-integration.md +++ b/docs/cli/server-integration.md @@ -4,7 +4,7 @@ This document is for implementers building a custom server (for example in `../s The CLI supports two broad categories of remote operations: -- **Data API**: query / update / insert / upsert / info / exists / show / log / history / context / explain, plus admin operations like create / drop / reindex / branch (create / drop / rebase / merge) / publish / export / import. +- **Data API**: query / update / insert / upsert / sync / info / exists / show / log / history / context / explain, plus admin operations like create / drop / reindex / branch (create / drop / rebase / merge) / publish / export / import. - **Replication / sync**: clone / pull / fetch (content-addressed replication by CID, via pack + storage proxy), ledger-archive (`export --format ledger`), and wholesale restore (`create --remote --from .flpack`, via `POST /import`). ## Base URL And Discovery @@ -251,6 +251,7 @@ See [Ledger portability](#ledger-portability-flpack-files) below for the on-disk - `POST {api_base_url}/query/*ledger` - `POST {api_base_url}/insert/*ledger` - `POST {api_base_url}/upsert/*ledger` +- `POST {api_base_url}/sync/*ledger` — see [Sync Contract](#sync-contract). - `POST {api_base_url}/update/*ledger` - `GET {api_base_url}/info/*ledger` - `GET {api_base_url}/exists/*ledger` @@ -330,6 +331,20 @@ MATCH (n:Person {id: 7}) RETURN n - Bearer ledger scope (`can_read`) and `Fluree-Min-T` apply as on the query path. `--at` is rejected for remote Cypher explain (use `--direct`). +### `fluree sync --remote ` (graph synchronization) + +- `POST {api_base_url}/sync/*ledger?graph=[&dryRun=true][&allowEmpty=true]` + +Makes one named graph's contents exactly the JSON-LD payload, committing +only the delta. Data-bearer auth (same bracket as `/insert` / `/upsert`), +not admin. The CLI converts Turtle to JSON-LD client-side, so the endpoint +only ever sees `application/json`. A dry run answers with a delta report and +must commit nothing; a real run answers with the standard transact response. +Designed so the CLI's source of desired contents (today RDF text; later +R2RML-mapped Iceberg / CSV / spreadsheet data) is invisible to the server — +every source arrives as the same payload. See +[Sync Contract](#sync-contract). + ### `fluree load` (CSV → batched upserts), `fluree update --format cypher` `fluree load` streams a local CSV into a ledger as a sequence of batched @@ -1408,6 +1423,83 @@ them all. | Report struct | `fluree_db_api::DropNamedGraphReport` | | Graph registry | `fluree_db_core::graph_registry` (system graph constants and IRI helpers) | +## Sync Contract + +`fluree sync --graph [--dry-run] [--allow-empty] --remote ` +issues: + +``` +POST {api_base_url}/sync/{ledger}?graph=urn%3Aexample%3Aontology[&dryRun=true][&allowEmpty=true] +Content-Type: application/json + +{ "@context": { ... }, "@graph": [ ... desired full contents of the graph ... ] } +``` + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `graph` (query) | Yes | Full **absolute** IRI of the target named graph (same validation rules as `/drop-graph`'s `graph`). The sync scope is exactly this graph — the payload must not address named graphs itself, and the ledger's `txn-meta` / `config` system graphs are rejected. | +| `dryRun` (query) | No | `true` → stage and report the delta; commit nothing. | +| `allowEmpty` (query) | No | `true` → accept an explicitly empty payload (`"@graph": []`), which clears the graph. Without it an empty payload is a `400`. | +| body | Yes | Insert-shaped JSON-LD describing the graph's desired full contents. The CLI always sends JSON-LD (Turtle is converted client-side). Policy headers / `opts` injection follow the [Policy Enforcement Contract](#policy-enforcement-contract). | + +### Auth + +Data-bearer auth, same bracket as `/insert` and `/upsert` (a token scoped to +the ledger with write access). Not admin. + +### Required semantics + +Given the graph's current contents `A` and the payload `B`: + +1. Retract `A − B`, assert `B − A`; facts in `A ∩ B` produce no flakes. +2. Identical payload (`A = B`) → **no commit** and a successful response + whose `t` is the unchanged head. +3. One commit for the whole delta (`t = current + 1`); history preserved. +4. Policy, SHACL, and uniqueness constraints apply exactly as for a normal + transaction. The current-contents scan is an authoritative replacement + (not view-policy filtered) — a row the caller cannot see is still + retracted if absent from the payload; modify-policy is enforced on the + resulting delta. +5. A dry run stages under the **same** policy / option inputs as the real + run, so its counts (and its failures) predict the real run. +6. Blank nodes are skolemized with a deterministic, graph-scoped key so a + payload with stable labels resyncs bnode structures without churn. + +### Response + +Real run (`200 OK`): the standard transact response (`ledger`, `t`, +`tx-id`, commit info) — identical in shape to `/upsert`. + +Dry run (`200 OK`): + +```json +{ + "ledger": "mydb:main", + "graph": "urn:example:ontology", + "asserted": 2, + "retracted": 2, + "committed": false, + "dryRun": true, + "t": 7 +} +``` + +The CLI's `--json` output uses this same shape for both local and remote +runs, so scripts consume either path identically. + +### Error responses + +| Status | When | +|--------|------| +| `400` | missing `graph`; malformed / relative graph IRI; system-graph target; empty payload without `allowEmpty`; payload addressing named graphs; non-JSON body (Turtle/TriG are not accepted here) | +| `401` / `403` | per the policy contract | +| `404` | unknown ledger | + +### Reference implementation + +`fluree-db-server/src/routes/transact.rs` (`sync`, `sync_ledger`, +`sync_local`) and `Fluree::sync_named_graph_with` in `fluree-db-api`. + ## Rebase Contract `fluree branch rebase --remote ` issues: diff --git a/docs/cli/sync.md b/docs/cli/sync.md new file mode 100644 index 0000000000..19eb142577 --- /dev/null +++ b/docs/cli/sync.md @@ -0,0 +1,102 @@ +# fluree sync + +Synchronize a named graph: make its contents exactly the supplied data, +committing only the delta. + +## Usage + +```bash +fluree sync [LEDGER] [DATA] --graph [OPTIONS] +``` + +## Arguments + +| Arguments | Behavior | +|-----------|----------| +| (none) | Active ledger; provide data via `-e`, `-f`, or stdin | +| `` | Auto-detected: if it looks like data (JSON, Turtle), uses it inline with the active ledger; if it's an existing file, reads from it; otherwise treats it as a ledger name | +| ` ` | Specified ledger + inline data | + +## Options + +| Option | Description | +|--------|-------------| +| `-g, --graph ` | **Required.** Target named graph IRI — the sync scope. The payload never widens or narrows it. | +| `-l, --ledger ` | Ledger name (defaults to active ledger) | +| `-e, --expr ` | Inline data expression (Turtle or JSON-LD) | +| `-f, --file ` | Read data from a file | +| `--format ` | Data format: `turtle` or `jsonld` (auto-detected if omitted) | +| `--dry-run` | Compute and report the delta (asserted / retracted counts) without committing | +| `--allow-empty` | Allow an empty payload, which clears the graph (off by default so a truncated export cannot silently wipe it) | +| `--json` | Emit the report as JSON — the same shape as the server's dry-run response — instead of a sentence | +| `--remote ` | Execute against a remote server (by remote name, e.g., `origin`) | +| policy flags | `--as`, `--policy-class`, `--policy`, … — same as `insert` / `upsert` | + +## Description + +Sync is the "full replacement" verb for data whose source of truth lives +outside Fluree — an ontology maintained in an editor, a reference table +regenerated by a pipeline. Given the graph's current contents `A` and the +payload `B`, one commit retracts `A − B` and asserts `B − A`; unchanged facts +produce no flakes, and an identical payload produces **no commit**. History +is preserved. See [Sync (graph synchronization)](../transactions/sync.md) +for the full semantics, safety rails, and blank-node behavior. + +Turtle input is converted to JSON-LD client-side before submission, so a +Turtle export works against any server that implements the `/sync` +endpoint (which is JSON-LD only). + +### Sources + +The target graph is the constant of this command; the **source** of the +desired contents is pluggable. Today the source is RDF text — a file, an +inline expression, or stdin. The same command shape is where mapped sources +will plug in: an R2RML mapping applied to an Iceberg table, CSV, or +spreadsheet resolves to the same payload and flows through the same verb, +so "sync this graph to the latest downstream data" is one flag set away +rather than a new command. Running with `--remote` moves where that +materialization happens, not what is committed. + +### Pipelines + +`--dry-run --json` is the pre-flight for scheduled syncs: run it, inspect +`retracted` (an unexpectedly large count is a cheap tripwire for a truncated +export), then run for real. The real run is idempotent — re-running with the +same export is a no-op. + +## Examples + +```bash +# Sync an ontology from a Turtle export +fluree sync mydb --graph urn:example:ontology -f ontology.ttl + +# Pre-flight: what would change? +fluree sync mydb --graph urn:example:ontology -f ontology.ttl --dry-run + +# Machine-readable, against a remote +cat export.jsonld | fluree sync --graph urn:example:ontology --remote origin --json + +# Clear the graph on purpose +echo '{"@graph": []}' | fluree sync mydb --graph urn:example:ontology --allow-empty +``` + +## Output + +``` +Synced graph in 'mydb:main': +2 asserted, -2 retracted (t=7). +Graph in 'mydb:main' already matches the payload — no commit produced (t=7). +Would sync graph in 'mydb:main': +2 asserted, -2 retracted (dry run; head t=7). +``` + +With `--json`: + +```json +{ "ledger": "mydb:main", "graph": "urn:example:ontology", + "asserted": 2, "retracted": 2, "committed": true, "dryRun": false, "t": 7 } +``` + +## See Also + +- [upsert](upsert.md) - Replace values for supplied predicates only +- [graph](graph.md) - List and drop named graphs +- [Sync (graph synchronization)](../transactions/sync.md) - Semantics and HTTP endpoint diff --git a/docs/operations/configuration.md b/docs/operations/configuration.md index 3108517592..e49593112c 100644 --- a/docs/operations/configuration.md +++ b/docs/operations/configuration.md @@ -156,6 +156,7 @@ A few operational knobs are environment-only (no CLI flag): | `FLUREE_REASONING_MAX_FACTS` | 1,000,000 | Server-wide default OWL2-RL materialization budget (max derived facts). Overridden per ledger by `f:reasoningMaxFacts` and per query by `"reasoningBudget"`; see [Reasoning](../query/reasoning.md#materialization-budget). | | `FLUREE_REASONING_MAX_SECONDS` | 30 | Server-wide default OWL2-RL materialization budget (wall-clock seconds). Same override chain as above. | | `FLUREE_CYPHER_ALLOW_FULL_SCAN` | off | Allow bare Cypher `MATCH (n)` (no label/property/relationship constraint) to run as a whole-graph distinct-subject scan. Off by default — intended for benchmarks and ad-hoc exploration, not production queries. | +| `FLUREE_MAX_GRAPH_SCAN_FLAKES` | 10,000,000 | Memory backstop for whole-graph transactions (graph sync, `CLEAR`, `DROP`, `COPY`, `MOVE`). Staging materializes the target graph's currently-asserted flakes, so peak memory scales with the graph, not the delta; the scan stops and the transaction fails with a clear resource-limit error once it passes this many flakes. `0` disables. Read per operation, not cached. The streaming-diff follow-up that removes the materialization is [#1691](https://github.com/fluree/db/issues/1691). | | `FLUREE_PATH_MAX_VISITED` | 1,000,000 | Visited-node cap for path traversals (variable-length paths, `shortestPath`) — a runaway-closure backstop. Traversals that exceed it fail with a clear resource-limit error; raise for graphs whose legitimate closures are larger (the cap also bounds per-query traversal memory). Read once at startup. | | `FLUREE_CYPHER_AST_CACHE` | 512 | Capacity (entries) of the process-wide Cypher parsed-AST cache, keyed on statement text. Repeated statements (parameterized workloads, benchmark loops) skip re-parsing; parameters are substituted into a per-request clone. `0` disables the cache. Read once at startup. | | `FLUREE_STORAGE_FSYNC` | on | File-storage durability. On, a write is reported complete once its bytes and the directory entry naming them are flushed to the device, so an acknowledged commit survives power loss. Set to `0`/`false`/`off`/`no` to report completion once the bytes reach the OS page cache instead — faster, but a power loss or kernel panic can lose acknowledged commits. Read once per storage construction, and **overrides** a storage node's `durability` property so a one-off run needs no config edit. Applies only to the local file backend; S3 acknowledges after replication and the Raft log flushes independently. Derived content (index nodes, dictionaries, sketches, arenas) is written page-cache in either setting, since it is recomputable from the commit chain. See [Storage durability](storage.md#durability). | diff --git a/docs/transactions/README.md b/docs/transactions/README.md index e238e78ecd..643f89bf70 100644 --- a/docs/transactions/README.md +++ b/docs/transactions/README.md @@ -28,6 +28,13 @@ Idempotent transactions that replace values for supplied predicates: - Idempotent operations - Synchronization patterns +### [Sync (graph synchronization)](sync.md) + +Whole-graph replacement that commits only the delta: +- Retracts what the payload omits, asserts what it adds +- Identical payload → no commit +- Dry run and empty-payload safety rails + ### [Update (WHERE/DELETE/INSERT)](update-where-delete-insert.md) Targeted updates to existing data: @@ -215,6 +222,7 @@ POST /upsert?ledger=mydb:main - **Insert** (`POST /insert`) — add triples (JSON-LD or Turtle) - **Update** (`POST /update`) — WHERE/DELETE/INSERT (JSON-LD) or SPARQL UPDATE - **Upsert** (`POST /upsert`) — replace values for the predicates you supply (JSON-LD, Turtle, TriG) +- **Sync** (`POST /sync`) — make one named graph's contents exactly the payload, committing only the delta (JSON-LD) ## Transaction Validation diff --git a/docs/transactions/overview.md b/docs/transactions/overview.md index e21883bfcc..a5fd4bbd82 100644 --- a/docs/transactions/overview.md +++ b/docs/transactions/overview.md @@ -221,13 +221,14 @@ See [SPARQL UPDATE](../query/sparql.md#sparql-update) for complete documentation ## Transaction Endpoints -Fluree exposes three transaction endpoints (all under `/v1/fluree/`): +Fluree exposes four transaction endpoints (all under `/v1/fluree/`): - `POST /insert` — add triples (JSON-LD or Turtle) - `POST /update` — WHERE/DELETE/INSERT (JSON-LD) and SPARQL UPDATE - `POST /upsert` — replace values for the predicates you supply (JSON-LD, Turtle, TriG) +- `POST /sync` — make one named graph's contents exactly the payload, committing only the delta (JSON-LD) -See [Insert](insert.md), [Update](update-where-delete-insert.md), and [Upsert](upsert.md) for details. +See [Insert](insert.md), [Update](update-where-delete-insert.md), [Upsert](upsert.md), and [Sync](sync.md) for details. ## Transaction Semantics diff --git a/docs/transactions/sync.md b/docs/transactions/sync.md new file mode 100644 index 0000000000..981f43a70b --- /dev/null +++ b/docs/transactions/sync.md @@ -0,0 +1,136 @@ +# Sync (graph synchronization) + +Sync makes a named graph's contents **exactly** the payload you supply, +committing only the delta. It is the "full replacement" verb for data whose +source of truth lives outside Fluree — an ontology maintained in an editor, a +reference table regenerated by a pipeline — where you want the graph to match +the latest export and the commit to show exactly what changed. + +| | Insert | Upsert | **Sync** | +|---|---|---|---| +| Asserts new triples | ✓ | ✓ | ✓ | +| Retracts changed values | — | for supplied `(subject, predicate)` pairs | ✓ | +| Retracts triples absent from the payload | — | — | ✓ | +| Scope | payload | payload's subjects/predicates | **one whole named graph** | + +## Semantics + +Given the graph's current contents `A` and the payload `B`: + +- retract `A − B` +- assert `B − A` +- `A ∩ B` produces **no flakes** — unchanged facts do not appear in the commit +- `A = B` produces **no commit** (`committed: false`) + +Sync is transactional and history-preserving: one normal commit at +`t = current + 1`; queries `as-of` an earlier `t` still see the previous +contents. SHACL validation, modify-policy enforcement, and novelty +backpressure apply exactly as for any other transaction. + +The scope is **exactly one named graph**, named by the caller — never +inferred from the payload. The payload may not address named graphs itself, +and reserved system graphs (and, for now, the default graph) are rejected. + +## HTTP endpoint + +```bash +curl -X POST "http://localhost:8090/v1/fluree/sync?ledger=mydb:main&graph=http://example.org/graphs/ontology" \ + -H "Content-Type: application/json" \ + -d '{ + "@context": { "ex": "http://example.org/" }, + "@graph": [ + { "@id": "ex:alice", "ex:name": "Alice", "ex:role": "engineer" }, + { "@id": "ex:bob", "ex:name": "Bob" } + ] + }' +``` + +Query parameters: + +| Parameter | Meaning | +|---|---| +| `ledger` | Target ledger (`name:branch`) | +| `graph` | **Required.** Target graph IRI — the sync scope | +| `dryRun=true` | Stage and report the delta (`asserted`/`retracted` counts) without committing — under the same policy, header, and inline-constraint inputs the real run uses, so it reports what the real run would do (or fails the way it would) | +| `allowEmpty=true` | Confirm an explicitly empty payload (`"@graph": []`), which clears the graph | + +The payload is JSON-LD (`application/json`). Convert Turtle exports +client-side (e.g. `fluree-graph-turtle`'s `parse_to_json`) for now. + +A dry run responds with the delta report: + +```json +{ "ledger": "mydb:main", "graph": "http://example.org/graphs/ontology", + "asserted": 2, "retracted": 2, "committed": false, "dryRun": true, "t": 7 } +``` + +## Rust API + +```rust +use fluree_db_api::SyncGraphOpts; + +let report = fluree + .sync_named_graph("mydb:main", "http://example.org/graphs/ontology", + &payload, SyncGraphOpts::default()) + .await?; +assert!(report.committed || (report.asserted == 0 && report.retracted == 0)); +``` + +`SyncGraphOpts { dry_run, allow_empty }` mirror the query parameters; +`sync_named_graph_with` additionally takes explicit `TxnOpts` and a +`PolicyContext`. The builder form +`fluree.stage(&handle).sync_graph(graph_iri, &payload)` is also available +(note: the builder does not apply the `allow_empty` gate). Its consensus +terminal `build_commit()` returns `Ok(None)` for a no-change sync. Target +validation (absolute IRI, no system graphs) is enforced at staging, so it +applies on every entry point. + +## Safety rails + +- **Empty payload requires opt-in.** `"@graph": []` means "the graph's + desired contents are empty" — i.e. clear the graph. Without + `allowEmpty`, it is rejected, so a truncated export cannot silently wipe + the graph. +- **Explicit scope.** No graph parameter, no sync. Subjects in the payload + never widen or narrow the scope. +- **Dry run first.** For a periodic pipeline, a `dryRun` call that reports an + unexpectedly large `retracted` count is a cheap tripwire before the real + run. +- **Policy model.** Like `CLEAR`/`COPY`/`MOVE` (and unlike DELETE-WHERE), + the current-contents scan is not view-policy filtered: sync is an + authoritative whole-graph replacement, and a view-filtered scan would + leave rows the caller cannot see in place, breaking "the graph now equals + the payload". Modify-policy is still enforced on the resulting delta. + +## Blank nodes + +Sync skolemizes the payload's blank nodes with a **deterministic, +graph-scoped key**: the same blank-node label in the same target graph mints +the same skolem IRI on every sync. Exporters that keep labels stable +(hand-maintained Turtle, most pipeline generators) therefore resync +bnode-rooted structures (OWL restrictions, RDF lists) with **zero churn**. + +Exporters that regenerate labels on every save (e.g. Protégé's `genid…`) +still churn those structures: the triples are isomorphic but the labels — and +therefore the skolemized identities — differ. The result is correct, just +noisier commits. Structural (RDFC 1.0-style) canonicalization is the designed +follow-up for label-unstable exporters. + +## Scale + +Staging a sync materializes the target graph's current flakes plus the +payload's flakes in memory — the same profile as `CLEAR`/`COPY`/`MOVE` +(chunked staging for whole-graph operations is a known follow-up). A huge +payload with a small delta is fine; the commit only carries the delta. A huge +*delta* is bounded by novelty backpressure (`reindex_max_bytes`): the commit +fails with `NoveltyWouldExceed` rather than overrunning memory, and the graph +is left unchanged. + +A huge *graph* is bounded by `FLUREE_MAX_GRAPH_SCAN_FLAKES` (default +10,000,000; `0` disables): the whole-graph scan stops there and the +transaction fails with a resource-limit error naming the knob, instead of +exhausting memory — an identical resync of a large graph is the worst case, +since it materializes everything and commits nothing. A dry run trips the +cap the same way the real run would, and `CLEAR`/`DROP`/`COPY`/`MOVE` share +it. The follow-up that streams the diff instead of materializing the graph +is [#1691](https://github.com/fluree/db/issues/1691). diff --git a/fluree-db-api/src/admin.rs b/fluree-db-api/src/admin.rs index 2054d72f48..391e19b6ab 100644 --- a/fluree-db-api/src/admin.rs +++ b/fluree-db-api/src/admin.rs @@ -141,6 +141,44 @@ pub struct DropNamedGraphReport { pub t: i64, } +/// Options for [`Fluree::sync_named_graph`]. +#[derive(Debug, Clone, Default)] +pub struct SyncGraphOpts { + /// Compute the delta and report counts without committing anything. + pub dry_run: bool, + /// Allow an explicitly empty payload (`"@graph": []`), which clears the + /// graph. Off by default so a truncated or accidentally-empty export + /// cannot silently wipe the graph. + pub allow_empty: bool, +} + +/// Report of a [`Fluree::sync_named_graph`] call. +/// +/// Graph sync is **transactional and history-preserving**: it produces one +/// normal commit containing exactly the delta between the graph's current +/// contents and the payload (`current − payload` retracted, `payload − +/// current` asserted; unchanged facts produce no flakes). An identical +/// payload produces no commit (`committed = false`). +#[derive(Debug, Clone, Default)] +pub struct SyncGraphReport { + /// Full `ledger:branch` identifier the sync targeted. + pub ledger_id: String, + /// Graph IRI that was synchronized (echoed for clarity). + pub graph_iri: String, + /// Flakes asserted by the delta (`payload − current`). + pub asserted: usize, + /// Flakes retracted by the delta (`current − payload`). + pub retracted: usize, + /// Whether a new commit was created. `false` when the payload matched + /// the graph exactly, and always `false` for a dry run. + pub committed: bool, + /// Whether this was a dry run (staged and counted, nothing committed). + pub dry_run: bool, + /// Current commit `t` for the branch after the call. Equal to the + /// pre-sync `t` when nothing was committed. + pub t: i64, +} + /// Report of a branch drop operation #[derive(Debug, Clone, Default)] pub struct BranchDropReport { @@ -971,6 +1009,173 @@ impl crate::Fluree { }) } + /// Synchronize a named graph's contents with `data`, committing only the + /// delta (see [`SyncGraphReport`]). + /// + /// `data` is an insert-shaped JSON-LD document describing the graph's + /// DESIRED full contents. Staging retracts `current − payload` and + /// asserts `payload − current`; unchanged facts produce no flakes, so an + /// identical payload reports `committed = false` without creating a + /// commit. The scope is exactly the named graph — the payload may not + /// address named graphs itself, and reserved system graphs (and the + /// default graph) are rejected. + /// + /// Like `CLEAR`/`COPY`/`MOVE` (and unlike DELETE-WHERE), the + /// current-contents scan is not view-policy filtered: sync is an + /// authoritative whole-graph replacement, and a view-filtered scan would + /// leave rows the caller cannot see in place. Modify-policy is still + /// enforced on the resulting delta. + pub async fn sync_named_graph( + &self, + ledger_id: &str, + graph_iri: &str, + data: &serde_json::Value, + opts: SyncGraphOpts, + ) -> Result { + self.sync_named_graph_with( + ledger_id, + graph_iri, + data, + opts, + fluree_db_transact::TxnOpts::default(), + None, + ) + .await + } + + /// [`Self::sync_named_graph`] with explicit transaction options and an + /// optional policy context — the form the HTTP layer uses so a dry run + /// stages under exactly the policy / inline-constraint inputs the real + /// (consensus-submitted) run will, and therefore reports the same delta + /// or fails the same way. `policy: None` runs as root. + pub async fn sync_named_graph_with( + &self, + ledger_id: &str, + graph_iri: &str, + data: &serde_json::Value, + opts: SyncGraphOpts, + txn_opts: fluree_db_transact::TxnOpts, + policy: Option, + ) -> Result { + use fluree_db_core::graph_registry::{config_graph_iri, txn_meta_graph_iri}; + + let bad_request = |msg: String| ApiError::Http { + status: 400, + message: msg, + }; + + if graph_iri.is_empty() { + return Err(bad_request( + "graph IRI is required; sync targets exactly one named graph".to_string(), + )); + } + validate_absolute_iri(graph_iri).map_err(bad_request)?; + + let ledger_id = normalize_ledger_id(ledger_id); + + // Reject system graphs by IRI shape (staging re-checks by g_id). + if graph_iri == txn_meta_graph_iri(&ledger_id) { + return Err(bad_request(format!( + "Cannot sync the txn-meta system graph '{graph_iri}'" + ))); + } + if graph_iri == config_graph_iri(&ledger_id) { + return Err(bad_request(format!( + "Cannot sync the config system graph '{graph_iri}'" + ))); + } + + // An explicitly empty payload clears the graph — require the + // explicit opt-in so a truncated export cannot wipe it silently. + let explicitly_empty = data + .get("@graph") + .and_then(serde_json::Value::as_array) + .is_some_and(Vec::is_empty); + if explicitly_empty && !opts.allow_empty { + return Err(bad_request( + "sync payload is empty; this would clear the graph — set allowEmpty to confirm" + .to_string(), + )); + } + + info!(ledger_id = %ledger_id, graph_iri = %graph_iri, dry_run = opts.dry_run, "Syncing named graph"); + + let handle = self.ledger_cached(&ledger_id).await?; + let pre_t = handle.t().await; + + if opts.dry_run { + let snap = handle.snapshot().await; + let ledger_state = snap.to_ledger_state(); + // Report the `t` of the state the delta was actually computed + // against, not a separately-read head. + let staged_against_t = ledger_state.t(); + let stage_result = self + .stage_sync_transaction_tracked( + ledger_state, + graph_iri, + data, + txn_opts, + None, + None, + policy.as_ref(), + ) + .await?; + let flakes = stage_result.view.staged_flakes(); + let asserted = flakes.iter().filter(|f| f.op).count(); + let retracted = flakes.len() - asserted; + return Ok(SyncGraphReport { + ledger_id, + graph_iri: graph_iri.to_string(), + asserted, + retracted, + committed: false, + dry_run: true, + t: staged_against_t, + }); + } + + let mut builder = self + .stage(&handle) + .sync_graph(graph_iri, data) + .txn_opts(txn_opts); + if let Some(policy) = policy { + builder = builder.policy(policy); + } + let result = builder.execute().await?; + // A delta commit always carries flakes. The only zero-flake commit a + // sync can produce is a registration-only one (an explicitly empty + // payload into a never-registered graph), which advances `t` under a + // real commit id; a no-change sync returns either the no-op sentinel + // id (local path) or the unchanged head (consensus path). Keying on + // the flake count first keeps the common cases exact even when an + // unrelated writer advances the head concurrently. + let noop_sentinel = + fluree_db_core::ContentId::new(fluree_db_core::ContentKind::Commit, &[]); + let committed = result.receipt.flake_count > 0 + || (result.receipt.t > pre_t && result.receipt.commit_id != noop_sentinel); + let t = if committed { result.receipt.t } else { pre_t }; + + info!( + ledger_id = %ledger_id, + graph_iri = %graph_iri, + asserted = result.receipt.assert_count, + retracted = result.receipt.retract_count, + committed, + t, + "Named graph synced", + ); + + Ok(SyncGraphReport { + ledger_id, + graph_iri: graph_iri.to_string(), + asserted: result.receipt.assert_count, + retracted: result.receipt.retract_count, + committed, + dry_run: false, + t, + }) + } + /// Cancel indexing, delete storage artifacts, purge nameservice record, /// and disconnect from cache. Returns the parent's new child count. async fn purge_branch( diff --git a/fluree-db-api/src/cypher_seq.rs b/fluree-db-api/src/cypher_seq.rs index ba46385e04..6479190256 100644 --- a/fluree-db-api/src/cypher_seq.rs +++ b/fluree-db-api/src/cypher_seq.rs @@ -411,6 +411,7 @@ impl Fluree { ns_registry, txn_meta, graph_delta, + sync_graph: _, } = outcome.stage_result; let commit_opts = fluree_db_transact::CommitOpts::default() @@ -431,6 +432,8 @@ impl Fluree { ), t: base.t(), flake_count: 0, + assert_count: 0, + retract_count: 0, }, base, ) diff --git a/fluree-db-api/src/cypher_txn.rs b/fluree-db-api/src/cypher_txn.rs index c6347076e0..9e7111dc3e 100644 --- a/fluree-db-api/src/cypher_txn.rs +++ b/fluree-db-api/src/cypher_txn.rs @@ -80,6 +80,10 @@ struct PendingCommit { t: i64, /// Flakes in this pending commit; summed into the commit receipt. flake_count: usize, + /// Asserted flakes in this pending commit. + assert_count: usize, + /// Retracted flakes in this pending commit. + retract_count: usize, } /// Per-statement outcome of a write inside a transaction. @@ -334,6 +338,7 @@ impl Fluree { ns_registry, txn_meta, graph_delta, + sync_graph: _, } = stage_result; if !view.has_staged() { @@ -390,6 +395,8 @@ impl Fluree { bytes, t: receipt.t, flake_count: receipt.flake_count, + assert_count: receipt.assert_count, + retract_count: receipt.retract_count, }); txn.state = next_state; Ok(receipt.flake_count) @@ -431,6 +438,8 @@ impl Fluree { commit_id: ContentId::new(ContentKind::Commit, &[]), t: txn.base_t, flake_count: 0, + assert_count: 0, + retract_count: 0, }, indexing: IndexingStatus { enabled: self.indexing_mode.is_enabled(), @@ -497,6 +506,8 @@ impl Fluree { // Total flakes across every staged statement — the single-commit // accounting the transactor expects for a multi-statement transaction. let flake_count = txn.pending.iter().map(|p| p.flake_count).sum(); + let assert_count = txn.pending.iter().map(|p| p.assert_count).sum(); + let retract_count = txn.pending.iter().map(|p| p.retract_count).sum(); // Capture indexing signals + tally before `txn.state` is moved into // finalize_commit (which triggers background reindex when needed). @@ -511,6 +522,8 @@ impl Fluree { commit_id, t: final_t, flake_count, + assert_count, + retract_count, }, indexing, tally, diff --git a/fluree-db-api/src/lib.rs b/fluree-db-api/src/lib.rs index 53e5922aed..095433b3cf 100644 --- a/fluree-db-api/src/lib.rs +++ b/fluree-db-api/src/lib.rs @@ -121,6 +121,8 @@ pub use admin::{ IndexStatusResult, ReindexOptions, ReindexResult, + SyncGraphOpts, + SyncGraphReport, TriggerIndexOptions, TriggerIndexResult, }; diff --git a/fluree-db-api/src/tx.rs b/fluree-db-api/src/tx.rs index 0a56946f9f..aa2f629548 100644 --- a/fluree-db-api/src/tx.rs +++ b/fluree-db-api/src/tx.rs @@ -240,6 +240,7 @@ impl SequentialStager { ns_registry, txn_meta: self.txn_meta, graph_delta, + sync_graph: None, }) } } @@ -1895,6 +1896,11 @@ pub struct StageResult { pub txn_meta: Vec, /// Named graph IRI to g_id mappings introduced by this transaction pub graph_delta: rustc_hash::FxHashMap, + /// Graph-sync target, when this was a sync transaction (see + /// [`fluree_db_transact::Txn::sync_graph`]). A sync that stages zero + /// flakes is a legitimate no-change outcome, so the commit paths skip + /// the commit for it exactly like a no-op update/upsert. + pub sync_graph: Option, } /// Convert named graph blocks to TripleTemplates with proper graph_id assignments. @@ -2167,10 +2173,93 @@ impl crate::Fluree { txn.graph_delta.extend(named_graph_delta); } + self.stage_built_txn_tracked( + ledger, + txn, + ns_registry, + txn_json, + index_config, + external_tracker, + policy, + ) + .await + } + + /// Stage a graph-sync transaction (see + /// [`fluree_db_transact::Txn::sync_graph`]): the JSON-LD payload is the + /// target graph's desired full contents, parsed with the staging + /// registry (same namespace hand-off as any JSON-LD transaction) and + /// re-homed onto `graph_iri`; staging's sync wave turns it into a + /// delta-only flake set. + #[allow(clippy::too_many_arguments)] + pub async fn stage_sync_transaction_tracked( + &self, + ledger: LedgerState, + graph_iri: &str, + txn_json: &JsonValue, + txn_opts: TxnOpts, + index_config: Option<&IndexConfig>, + external_tracker: Option<&Tracker>, + policy: Option<&crate::PolicyContext>, + ) -> Result { + let mut ns_registry = NamespaceRegistry::from_db(&ledger.snapshot); + // Deterministic, graph-scoped blank-node identity: the payload is + // the graph's authoritative document, so the same source label must + // mint the same skolem IRI on every sync — otherwise every + // bnode-rooted structure (OWL restrictions, RDF lists) would churn + // as retract+assert on each sync even when unchanged. Exporters + // that regenerate labels per save (e.g. Protégé's genid) still + // churn; structural (RDFC-style) canonicalization is the designed + // follow-up for those. A caller-supplied id wins. + let mut txn_opts = txn_opts; + if txn_opts.skolem_txn_id.is_none() { + let scope = fluree_db_core::skolem::doc_scope(fluree_db_core::skolem::doc_id( + "fluree:graph-sync", + graph_iri, + 0, + )); + txn_opts.skolem_txn_id = Some(format!("sync{scope}")); + } + let txn = { + let parse_span = tracing::debug_span!("txn_parse", txn_type = "sync"); + let _guard = parse_span.enter(); + fluree_db_transact::parse_sync_transaction( + txn_json, + graph_iri, + txn_opts, + &mut ns_registry, + )? + }; + self.stage_built_txn_tracked( + ledger, + txn, + ns_registry, + txn_json, + index_config, + external_tracker, + policy, + ) + .await + } + + /// Shared staging tail for a fully-built JSON-LD [`Txn`]: uniqueness / + /// SHACL / reasoning validation and [`StageResult`] assembly. + #[allow(clippy::too_many_arguments)] + async fn stage_built_txn_tracked( + &self, + ledger: LedgerState, + txn: Txn, + ns_registry: NamespaceRegistry, + txn_json: &JsonValue, + index_config: Option<&IndexConfig>, + external_tracker: Option<&Tracker>, + policy: Option<&crate::PolicyContext>, + ) -> Result { // Extract txn_meta, graph_delta, and any inline uniqueness // properties before staging consumes the Txn. let txn_meta = txn.txn_meta.clone(); let graph_delta = txn.graph_delta.clone(); + let sync_graph = txn.sync_graph.clone(); let inline_unique_properties = txn.opts.unique_properties.clone(); // Use external tracker if provided, otherwise fall back to limits-only tracker @@ -2232,6 +2321,7 @@ impl crate::Fluree { ns_registry, txn_meta, graph_delta, + sync_graph, }) } @@ -2249,6 +2339,7 @@ impl crate::Fluree { tracker: Option<&Tracker>, ) -> Result { let ns_registry = NamespaceRegistry::from_db(&ledger.snapshot); + let sync_graph = txn.sync_graph.clone(); let (view, ns_registry, txn_meta, graph_delta) = self .stage_view_once(ledger, txn, ns_registry, index_config, policy, tracker) .await?; @@ -2257,6 +2348,7 @@ impl crate::Fluree { ns_registry, txn_meta, graph_delta, + sync_graph, }) } @@ -2423,6 +2515,7 @@ impl crate::Fluree { ns_registry, txn_meta, graph_delta, + sync_graph: None, }) } @@ -2487,6 +2580,7 @@ impl crate::Fluree { ns_registry, txn_meta: Vec::new(), graph_delta: FxHashMap::default(), + sync_graph: None, }); } @@ -2588,6 +2682,7 @@ impl crate::Fluree { ns_registry, txn_meta, graph_delta, + sync_graph: None, }) } @@ -2623,6 +2718,7 @@ impl crate::Fluree { ns_registry, txn_meta, graph_delta, + sync_graph: _, } = self .stage_transaction_tracked_with_policy(ledger, input, Some(index_config), &tracker) .await?; @@ -2787,6 +2883,7 @@ impl crate::Fluree { ns_registry, txn_meta, graph_delta, + sync_graph, } = self .stage_transaction(ledger, txn_type, txn_json, txn_opts, Some(index_config)) .await?; @@ -2801,25 +2898,31 @@ impl crate::Fluree { // // This allows patterns like "delete if exists, then insert" to execute safely when // there are no matches, and supports conditional updates. - let (receipt, ledger) = - if !view.has_staged() && matches!(txn_type, TxnType::Update | TxnType::Upsert) { - let (base, flakes) = view.into_parts(); - debug_assert!( - flakes.is_empty(), - "no-op transaction path requires zero staged flakes" - ); - ( - CommitReceipt { - commit_id: ContentId::new(ContentKind::Commit, &[]), - t: base.t(), - flake_count: 0, - }, - base, - ) - } else { - self.commit_staged(view, ns_registry, index_config, commit_opts) - .await? - }; + let (receipt, ledger) = if !view.has_staged() + && (matches!(txn_type, TxnType::Update | TxnType::Upsert) + // A sync that stages zero flakes is a no-change outcome + // (payload identical to the graph), not an empty insert. + || sync_graph.is_some()) + { + let (base, flakes) = view.into_parts(); + debug_assert!( + flakes.is_empty(), + "no-op transaction path requires zero staged flakes" + ); + ( + CommitReceipt { + commit_id: ContentId::new(ContentKind::Commit, &[]), + t: base.t(), + flake_count: 0, + assert_count: 0, + retract_count: 0, + }, + base, + ) + } else { + self.commit_staged(view, ns_registry, index_config, commit_opts) + .await? + }; Ok(self .finalize_owned_commit(receipt, ledger, index_config) @@ -2856,6 +2959,7 @@ impl crate::Fluree { ns_registry, txn_meta, graph_delta, + sync_graph, } = self .stage_transaction_with_trig_meta( ledger, @@ -2874,25 +2978,31 @@ impl crate::Fluree { // No-op updates: if WHERE matches nothing (or templates produce no flakes), // return success without committing. - let (receipt, ledger) = - if !view.has_staged() && matches!(txn_type, TxnType::Update | TxnType::Upsert) { - let (base, flakes) = view.into_parts(); - debug_assert!( - flakes.is_empty(), - "no-op transaction path requires zero staged flakes" - ); - ( - CommitReceipt { - commit_id: ContentId::new(ContentKind::Commit, &[]), - t: base.t(), - flake_count: 0, - }, - base, - ) - } else { - self.commit_staged(view, ns_registry, index_config, commit_opts) - .await? - }; + let (receipt, ledger) = if !view.has_staged() + && (matches!(txn_type, TxnType::Update | TxnType::Upsert) + // A sync that stages zero flakes is a no-change outcome + // (payload identical to the graph), not an empty insert. + || sync_graph.is_some()) + { + let (base, flakes) = view.into_parts(); + debug_assert!( + flakes.is_empty(), + "no-op transaction path requires zero staged flakes" + ); + ( + CommitReceipt { + commit_id: ContentId::new(ContentKind::Commit, &[]), + t: base.t(), + flake_count: 0, + assert_count: 0, + retract_count: 0, + }, + base, + ) + } else { + self.commit_staged(view, ns_registry, index_config, commit_opts) + .await? + }; Ok(self .finalize_owned_commit(receipt, ledger, index_config) @@ -2932,6 +3042,7 @@ impl crate::Fluree { ns_registry, txn_meta, graph_delta, + sync_graph, } = self .stage_transaction_with_named_graphs( ledger, @@ -2951,25 +3062,31 @@ impl crate::Fluree { // No-op updates: if WHERE matches nothing (or templates produce no flakes), // return success without committing. - let (receipt, ledger) = - if !view.has_staged() && matches!(txn_type, TxnType::Update | TxnType::Upsert) { - let (base, flakes) = view.into_parts(); - debug_assert!( - flakes.is_empty(), - "no-op transaction path requires zero staged flakes" - ); - ( - CommitReceipt { - commit_id: ContentId::new(ContentKind::Commit, &[]), - t: base.t(), - flake_count: 0, - }, - base, - ) - } else { - self.commit_staged(view, ns_registry, index_config, commit_opts) - .await? - }; + let (receipt, ledger) = if !view.has_staged() + && (matches!(txn_type, TxnType::Update | TxnType::Upsert) + // A sync that stages zero flakes is a no-change outcome + // (payload identical to the graph), not an empty insert. + || sync_graph.is_some()) + { + let (base, flakes) = view.into_parts(); + debug_assert!( + flakes.is_empty(), + "no-op transaction path requires zero staged flakes" + ); + ( + CommitReceipt { + commit_id: ContentId::new(ContentKind::Commit, &[]), + t: base.t(), + flake_count: 0, + assert_count: 0, + retract_count: 0, + }, + base, + ) + } else { + self.commit_staged(view, ns_registry, index_config, commit_opts) + .await? + }; Ok(self .finalize_owned_commit(receipt, ledger, index_config) @@ -3074,6 +3191,7 @@ impl crate::Fluree { ns_registry, txn_meta, graph_delta, + sync_graph: _, } = stage_result; // Add transaction metadata and graph delta (graph_delta typically empty for Turtle) @@ -3189,6 +3307,7 @@ impl crate::Fluree { ns_registry, txn_meta: Vec::new(), graph_delta: rustc_hash::FxHashMap::default(), + sync_graph: None, }) } diff --git a/fluree-db-api/src/tx_builder.rs b/fluree-db-api/src/tx_builder.rs index 0615dfceb2..daeb094ffd 100644 --- a/fluree-db-api/src/tx_builder.rs +++ b/fluree-db-api/src/tx_builder.rs @@ -130,6 +130,12 @@ pub(crate) enum TransactOperation<'a> { UpdateJson(&'a JsonValue), InsertTurtle(&'a str), UpsertTurtle(&'a str), + /// Graph sync: make `graph_iri`'s contents exactly `json`, committing + /// only the delta (see [`fluree_db_transact::Txn::sync_graph`]). + SyncGraph { + graph_iri: &'a str, + json: &'a JsonValue, + }, } /// Result of parsing a transaction operation to JSON. @@ -149,6 +155,9 @@ impl TransactOperation<'_> { TransactOperation::UpdateJson(_) => TxnType::Update, TransactOperation::InsertTurtle(_) => TxnType::Insert, TransactOperation::UpsertTurtle(_) => TxnType::Upsert, + // Sync parses as an insert whose staging adds the whole-graph + // retraction wave. + TransactOperation::SyncGraph { .. } => TxnType::Insert, } } @@ -178,6 +187,11 @@ impl TransactOperation<'_> { trig_meta: None, named_graphs: Vec::new(), }), + TransactOperation::SyncGraph { json, .. } => Ok(ParsedOperation { + json: (*json).clone(), + trig_meta: None, + named_graphs: Vec::new(), + }), TransactOperation::InsertTurtle(ttl) | TransactOperation::UpsertTurtle(ttl) => { // Phase 1: Extract TriG GRAPH block (if present) let phase1 = parse_trig_phase1(ttl)?; @@ -450,6 +464,17 @@ impl<'a> OwnedTransactBuilder<'a> { self } + /// Set the operation to a graph sync: make `graph_iri`'s contents + /// exactly `data`, committing only the delta (see + /// [`fluree_db_transact::Txn::sync_graph`]). + pub fn sync_graph(mut self, graph_iri: &'a str, data: &'a JsonValue) -> Self { + self.core.set_operation(TransactOperation::SyncGraph { + graph_iri, + json: data, + }); + self + } + /// Set a pre-built transaction IR (bypasses JSON/Turtle parsing). /// /// This is used for SPARQL UPDATE where the transaction is already @@ -531,6 +556,7 @@ impl<'a> OwnedTransactBuilder<'a> { ns_registry, txn_meta, graph_delta, + sync_graph, } = if let Some(followup) = self.core.pre_built_txn_followup { // Per-row relationship MERGE … ON MATCH SET: both branches stage // into one commit, or an error returns with nothing committed. @@ -578,7 +604,10 @@ impl<'a> OwnedTransactBuilder<'a> { // No-op updates: return success without committing. let (receipt, ledger) = if !view.has_staged() && !registers_new_graph - && matches!(txn_type, TxnType::Update | TxnType::Upsert) + && (matches!(txn_type, TxnType::Update | TxnType::Upsert) + // A sync that stages zero flakes is a no-change outcome, + // not an empty insert. + || sync_graph.is_some()) { let (base, flakes) = view.into_parts(); debug_assert!( @@ -590,6 +619,8 @@ impl<'a> OwnedTransactBuilder<'a> { commit_id: ContentId::new(ContentKind::Commit, &[]), t: base.t(), flake_count: 0, + assert_count: 0, + retract_count: 0, }, base, ) @@ -626,6 +657,72 @@ impl<'a> OwnedTransactBuilder<'a> { .await; } + // Graph sync: dedicated staging (whole-graph retraction wave) with + // the no-change short-circuit — an identical payload commits nothing. + if let TransactOperation::SyncGraph { graph_iri, json } = op { + let tracker = self + .core + .tracking + .clone() + .map(Tracker::new) + .unwrap_or_default(); + let stage_result = self + .fluree + .stage_sync_transaction_tracked( + self.ledger, + graph_iri, + json, + self.core.txn_opts, + Some(&index_config), + Some(&tracker), + self.core.policy.as_ref(), + ) + .await?; + let StageResult { + view, + ns_registry, + txn_meta, + graph_delta, + sync_graph: _, + } = stage_result; + let registers_new_graph = graph_delta.values().any(|iri| { + view.base() + .snapshot + .graph_registry + .graph_id_for_iri(iri) + .is_none() + }); + let commit_opts = self + .core + .commit_opts + .with_txn_meta(txn_meta) + .with_graph_delta(graph_delta.into_iter().collect()); + let (receipt, ledger) = if !view.has_staged() && !registers_new_graph { + let (base, flakes) = view.into_parts(); + debug_assert!( + flakes.is_empty(), + "no-op sync path requires zero staged flakes" + ); + ( + fluree_db_transact::CommitReceipt { + commit_id: ContentId::new(ContentKind::Commit, &[]), + t: base.t(), + flake_count: 0, + assert_count: 0, + retract_count: 0, + }, + base, + ) + } else { + self.fluree + .commit_staged(view, ns_registry, &index_config, commit_opts) + .await? + }; + return Ok(self + .fluree + .finalize_owned_commit(receipt, ledger, &index_config) + .await); + } let txn_type = op.txn_type(); // Parse transaction, extracting TriG metadata and named graphs for Turtle inputs let parsed = op.to_json_with_trig_meta()?; @@ -741,6 +838,33 @@ impl<'a> OwnedTransactBuilder<'a> { }); } + // Graph sync: dedicated staging (whole-graph retraction wave). + if let TransactOperation::SyncGraph { graph_iri, json } = op { + let tracker = self + .core + .tracking + .clone() + .map(Tracker::new) + .unwrap_or_else(Tracker::disabled); + let tracker_ref = tracker.is_enabled().then_some(&tracker); + let stage_result = self + .fluree + .stage_sync_transaction_tracked( + self.ledger, + graph_iri, + json, + self.core.txn_opts, + Some(&index_config), + tracker_ref, + self.core.policy.as_ref(), + ) + .await?; + return Ok(Staged { + view: stage_result.view, + ns_registry: stage_result.ns_registry, + graph_delta: stage_result.graph_delta, + }); + } let txn_type = op.txn_type(); // Parse transaction, extracting TriG metadata and named graphs for Turtle inputs let parsed = op.to_json_with_trig_meta()?; @@ -867,6 +991,17 @@ impl<'a> RefTransactBuilder<'a> { self } + /// Set the operation to a graph sync: make `graph_iri`'s contents + /// exactly `data`, committing only the delta (see + /// [`fluree_db_transact::Txn::sync_graph`]). + pub fn sync_graph(mut self, graph_iri: &'a str, data: &'a JsonValue) -> Self { + self.core.set_operation(TransactOperation::SyncGraph { + graph_iri, + json: data, + }); + self + } + /// Set a pre-built transaction IR (bypasses JSON/Turtle parsing). /// /// This is used for SPARQL UPDATE where the transaction is already @@ -958,9 +1093,16 @@ impl<'a> RefTransactBuilder<'a> { /// what consensus-coordinated committers /// (e.g. `fluree-db-consensus::RaftCommitter`) use to produce a /// commit they then carry through Raft consensus themselves. + /// + /// Returns `Ok(None)` for a no-change transaction — a zero-flake + /// update/upsert, or a graph sync whose payload already matches the + /// graph — that registers nothing new. Nothing is written and the + /// write guard is released; the caller reports the unchanged head. + /// Without this, a no-change sync under consensus would surface as an + /// `EmptyTransaction` failure instead of `committed: false`. pub async fn build_commit( self, - ) -> Result<(LedgerWriteGuard, fluree_db_transact::StagedCommit)> { + ) -> Result> { self.fluree .build_commit_with_handle(self.handle, self.core) .await @@ -982,6 +1124,11 @@ enum OpPlan<'a> { trig_meta: Option, named_graphs: Vec, }, + /// Graph sync (see [`fluree_db_transact::Txn::sync_graph`]). + Sync { + graph_iri: String, + txn_json: JsonValue, + }, } impl<'a> OpPlan<'a> { @@ -990,6 +1137,10 @@ impl<'a> OpPlan<'a> { fn from_op(op: TransactOperation<'a>) -> Result { match op { TransactOperation::InsertTurtle(turtle) => Ok(OpPlan::InsertTurtle(turtle)), + TransactOperation::SyncGraph { graph_iri, json } => Ok(OpPlan::Sync { + graph_iri: graph_iri.to_string(), + txn_json: json.clone(), + }), _ => { let txn_type = op.txn_type(); let parsed = op.to_json_with_trig_meta()?; @@ -1224,6 +1375,30 @@ impl Fluree { return Ok((stage_result, TxnType::Insert, commit_opts, None)); } + // Graph sync: dedicated staging (whole-graph retraction wave). Must + // be dispatched before the JSON-like fallthrough, which would + // otherwise stage the payload as a plain default-graph insert. + if let TransactOperation::SyncGraph { graph_iri, json } = op { + let stage_result = self + .stage_sync_transaction_tracked( + ledger_state, + graph_iri, + json, + core.txn_opts, + Some(index_config), + tracker_ref, + core.policy.as_ref(), + ) + .await?; + let commit_opts = self.maybe_spawn_txn_upload( + core.commit_opts, + &ledger_id, + json.clone(), + store_raw_txn, + ); + return Ok((stage_result, TxnType::Insert, commit_opts, None)); + } + // JSON-like operation: parse, extracting TriG metadata + named graphs. let txn_type = op.txn_type(); let parsed = op.to_json_with_trig_meta()?; @@ -1313,6 +1488,29 @@ impl Fluree { .await?; Ok((stage_result, *txn_type, commit_opts)) } + OpPlan::Sync { + graph_iri, + txn_json, + } => { + let commit_opts = self.maybe_spawn_txn_upload( + commit_opts_base.clone(), + &ledger_id, + txn_json.clone(), + store_raw_txn, + ); + let stage_result = self + .stage_sync_transaction_tracked( + ledger_state, + graph_iri, + txn_json, + txn_opts, + Some(index_config), + tracker_ref, + None, + ) + .await?; + Ok((stage_result, TxnType::Insert, commit_opts)) + } } } @@ -1337,6 +1535,7 @@ impl Fluree { ns_registry, txn_meta, graph_delta, + sync_graph, } = stage_result; // See the pre_built_txn path: a registration-only commit (new graph // IRI in the delta, zero flakes) must not take the no-op shortcut. @@ -1353,7 +1552,10 @@ impl Fluree { if !view.has_staged() && !registers_new_graph - && matches!(txn_type, TxnType::Update | TxnType::Upsert) + && (matches!(txn_type, TxnType::Update | TxnType::Upsert) + // A sync that stages zero flakes is a no-change outcome, + // not an empty insert. + || sync_graph.is_some()) { let (base, _) = view.into_parts(); return Ok(TransactResultRef { @@ -1361,6 +1563,8 @@ impl Fluree { commit_id: ContentId::new(ContentKind::Commit, &[]), t: base.t(), flake_count: 0, + assert_count: 0, + retract_count: 0, }, indexing: IndexingStatus { enabled: self.indexing_mode.is_enabled(), @@ -1412,7 +1616,7 @@ impl Fluree { &self, ledger: &LedgerHandle, mut core: TransactCore<'_>, - ) -> Result<(LedgerWriteGuard, fluree_db_transact::StagedCommit)> { + ) -> Result> { core.validate().map_err(ApiError::Builder)?; let index_config = core @@ -1430,7 +1634,7 @@ impl Fluree { // The dry-run/consensus path has no response channel for a trailing // Cypher RETURN — callers reject RETURN-carrying sequential // statements before submission. - let (stage_result, _txn_type, commit_opts, _cypher_return) = self + let (stage_result, txn_type, commit_opts, _cypher_return) = self .stage_under_lock( write_guard.clone_state(), core, @@ -1445,7 +1649,24 @@ impl Fluree { ns_registry, txn_meta, graph_delta, + sync_graph, } = stage_result; + // Same no-op rule as `commit_and_finalize`: a registration-only + // transaction must still build a commit; a zero-flake + // update/upsert/sync that registers nothing is a no-change outcome. + let registers_new_graph = graph_delta.values().any(|iri| { + view.base() + .snapshot + .graph_registry + .graph_id_for_iri(iri) + .is_none() + }); + if !view.has_staged() + && !registers_new_graph + && (matches!(txn_type, TxnType::Update | TxnType::Upsert) || sync_graph.is_some()) + { + return Ok(None); + } let mut commit_opts = commit_opts .with_txn_meta(txn_meta) .with_graph_delta(graph_delta.into_iter().collect()); @@ -1525,7 +1746,7 @@ impl Fluree { staged.tally = tracker.tally(); } - Ok((write_guard, staged)) + Ok(Some((write_guard, staged))) } /// Stage and commit a transaction against a cached ledger handle. diff --git a/fluree-db-api/tests/grp_ledger.rs b/fluree-db-api/tests/grp_ledger.rs index 57340704c0..e77adff4b8 100644 --- a/fluree-db-api/tests/grp_ledger.rs +++ b/fluree-db-api/tests/grp_ledger.rs @@ -27,3 +27,5 @@ mod it_revert; mod it_revert_preview; #[path = "it_stable_hashes.rs"] mod it_stable_hashes; +#[path = "it_sync_graph.rs"] +mod it_sync_graph; diff --git a/fluree-db-api/tests/it_sync_graph.rs b/fluree-db-api/tests/it_sync_graph.rs new file mode 100644 index 0000000000..00255f7122 --- /dev/null +++ b/fluree-db-api/tests/it_sync_graph.rs @@ -0,0 +1,706 @@ +//! Graph-sync integration tests. +//! +//! `sync_named_graph` makes a named graph's contents exactly the payload, +//! committing only the delta: `current − payload` retracted, `payload − +//! current` asserted, unchanged facts untouched. An identical payload +//! produces no commit. + +#![cfg(feature = "native")] + +use crate::support::genesis_ledger; +use fluree_db_api::{FlureeBuilder, SyncGraphOpts}; +use serde_json::{json, Value as JsonValue}; + +const ONT_IRI: &str = "http://example.org/graphs/ontology"; +const OTHER_IRI: &str = "http://example.org/graphs/other"; + +fn payload_v1() -> JsonValue { + json!({ + "@context": { "ex": "http://example.org/" }, + "@graph": [ + { "@id": "ex:alice", "ex:name": "Alice", "ex:role": "engineer" }, + { "@id": "ex:bob", "ex:name": "Bob" } + ] + }) +} + +/// v2 = v1 with alice's role changed and bob's name dropped, carol added. +fn payload_v2() -> JsonValue { + json!({ + "@context": { "ex": "http://example.org/" }, + "@graph": [ + { "@id": "ex:alice", "ex:name": "Alice", "ex:role": "manager" }, + { "@id": "ex:carol", "ex:name": "Carol" } + ] + }) +} + +/// Seed a ledger with one default-graph triple and one triple in OTHER_IRI. +async fn seed(fluree: &fluree_db_api::Fluree, ledger_id: &str) -> i64 { + let ledger = genesis_ledger(fluree, ledger_id); + let trig = format!( + r#" + @prefix ex: . + ex:default-subject ex:p "default-graph-value" . + GRAPH <{OTHER_IRI}> {{ + ex:zed ex:name "Zed" . + }} + "#, + ); + let result = fluree + .stage_owned(ledger) + .upsert_turtle(&trig) + .execute() + .await + .expect("seed insert"); + result.receipt.t +} + +async fn rows_in_graph( + fluree: &fluree_db_api::Fluree, + ledger_id: &str, + graph_iri: Option<&str>, +) -> Vec { + let from = match graph_iri { + Some(iri) => format!("{ledger_id}#{iri}"), + None => ledger_id.to_string(), + }; + let q = json!({ + "from": from, + "select": ["?s", "?p", "?o"], + "where": {"@id": "?s", "?p": "?o"} + }); + let result = fluree.query_connection(&q).await.expect("query connection"); + let ledger = fluree.ledger(ledger_id).await.expect("load ledger"); + let rows = result.to_jsonld(&ledger.snapshot).expect("to_jsonld"); + rows.as_array().cloned().unwrap_or_default() +} + +async fn count_in_graph( + fluree: &fluree_db_api::Fluree, + ledger_id: &str, + graph_iri: Option<&str>, +) -> usize { + rows_in_graph(fluree, ledger_id, graph_iri).await.len() +} + +/// `(default, ONT_IRI, OTHER_IRI)` row counts. +async fn graph_counts(fluree: &fluree_db_api::Fluree, ledger_id: &str) -> (usize, usize, usize) { + ( + count_in_graph(fluree, ledger_id, None).await, + count_in_graph(fluree, ledger_id, Some(ONT_IRI)).await, + count_in_graph(fluree, ledger_id, Some(OTHER_IRI)).await, + ) +} + +#[tokio::test] +async fn first_sync_populates_a_new_graph() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/first:main"; + let seed_t = seed(&fluree, ledger_id).await; + + let report = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect("first sync"); + + assert_eq!(report.asserted, 3, "three payload triples asserted"); + assert_eq!(report.retracted, 0, "nothing to retract in a new graph"); + assert!(report.committed); + assert_eq!(report.t, seed_t + 1); + assert_eq!(count_in_graph(&fluree, ledger_id, Some(ONT_IRI)).await, 3); +} + +#[tokio::test] +async fn identical_resync_is_a_noop() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/noop:main"; + seed(&fluree, ledger_id).await; + + let first = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect("first sync"); + assert!(first.committed); + + let second = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect("identical resync"); + assert_eq!(second.asserted, 0, "identical payload asserts nothing"); + assert_eq!(second.retracted, 0, "identical payload retracts nothing"); + assert!( + !second.committed, + "identical payload must not create a commit" + ); + assert_eq!(second.t, first.t, "head t unchanged on a no-op sync"); +} + +#[tokio::test] +async fn delta_sync_commits_only_the_delta() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/delta:main"; + seed(&fluree, ledger_id).await; + + let first = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect("first sync"); + + let second = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v2(), SyncGraphOpts::default()) + .await + .expect("delta sync"); + + // v1 → v2: alice role engineer→manager (1 retract + 1 assert), bob's + // name + node removed (1 retract), carol added (1 assert). Alice's + // unchanged ex:name must NOT appear in the commit. + assert_eq!(second.asserted, 2, "role change + carol"); + assert_eq!(second.retracted, 2, "old role + bob"); + assert!(second.committed); + assert_eq!(second.t, first.t + 1, "one commit for the whole delta"); + + // The graph now equals payload v2 exactly. + assert_eq!(count_in_graph(&fluree, ledger_id, Some(ONT_IRI)).await, 3); +} + +#[tokio::test] +async fn sync_does_not_touch_other_graphs() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/scoped:main"; + seed(&fluree, ledger_id).await; + + fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect("sync"); + fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v2(), SyncGraphOpts::default()) + .await + .expect("delta sync"); + + assert_eq!(count_in_graph(&fluree, ledger_id, Some(OTHER_IRI)).await, 1); + assert_eq!(count_in_graph(&fluree, ledger_id, None).await, 1); +} + +#[tokio::test] +async fn empty_payload_requires_allow_empty() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/empty:main"; + seed(&fluree, ledger_id).await; + fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect("sync"); + + let empty = json!({ "@graph": [] }); + + let err = fluree + .sync_named_graph(ledger_id, ONT_IRI, &empty, SyncGraphOpts::default()) + .await + .expect_err("empty payload without allowEmpty must be rejected"); + assert!( + err.to_string().contains("allowEmpty"), + "error should name the opt-in: {err}" + ); + + let report = fluree + .sync_named_graph( + ledger_id, + ONT_IRI, + &empty, + SyncGraphOpts { + allow_empty: true, + ..Default::default() + }, + ) + .await + .expect("empty sync with allowEmpty"); + assert_eq!(report.asserted, 0); + assert_eq!(report.retracted, 3, "clears the whole graph"); + assert!(report.committed); + assert_eq!(count_in_graph(&fluree, ledger_id, Some(ONT_IRI)).await, 0); +} + +#[tokio::test] +async fn dry_run_reports_the_delta_without_committing() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/dryrun:main"; + seed(&fluree, ledger_id).await; + let first = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect("sync"); + + let dry = fluree + .sync_named_graph( + ledger_id, + ONT_IRI, + &payload_v2(), + SyncGraphOpts { + dry_run: true, + ..Default::default() + }, + ) + .await + .expect("dry run"); + assert!(dry.dry_run); + assert!(!dry.committed); + assert_eq!(dry.asserted, 2); + assert_eq!(dry.retracted, 2); + assert_eq!(dry.t, first.t, "dry run must not advance t"); + + // Nothing changed: the graph still equals payload v1. + assert_eq!(count_in_graph(&fluree, ledger_id, Some(ONT_IRI)).await, 3); + + // The real run matches the dry run's numbers. + let real = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v2(), SyncGraphOpts::default()) + .await + .expect("real run"); + assert_eq!( + (real.asserted, real.retracted), + (dry.asserted, dry.retracted) + ); + assert!(real.committed); +} + +#[tokio::test] +async fn blank_node_payload_resyncs_as_noop() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/bnode:main"; + seed(&fluree, ledger_id).await; + + // An OWL-restriction-shaped payload: bnode-rooted structure under a + // stable label. Sync skolemizes with a deterministic graph-scoped key, + // so an identical resync must be a no-op. + let payload = json!({ + "@context": { "ex": "http://example.org/" }, + "@graph": [ + { + "@id": "ex:Widget", + "ex:restriction": { + "@id": "_:r1", + "ex:onProperty": { "@id": "ex:hasPart" }, + "ex:minCount": 1 + } + } + ] + }); + + let first = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload, SyncGraphOpts::default()) + .await + .expect("first sync"); + assert!(first.committed); + + let second = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload, SyncGraphOpts::default()) + .await + .expect("resync"); + assert!( + !second.committed, + "stable-label bnode payload must not churn: {second:?}" + ); +} + +#[tokio::test] +async fn payload_addressing_named_graphs_is_rejected() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/nested:main"; + seed(&fluree, ledger_id).await; + + let nested = json!({ + "@context": { "ex": "http://example.org/" }, + "@graph": [ + { + "@id": "http://example.org/graphs/inner", + "@graph": [ { "@id": "ex:x", "ex:p": "v" } ] + } + ] + }); + // The insert-shaped JSON-LD parse has no named-graph selector form, so + // the nested-graph document fails parsing; if a future parser learns + // one, `parse_sync_transaction`'s graph_delta guard rejects it with + // "must not address named graphs". Either way: an error, no commit. + fluree + .sync_named_graph(ledger_id, ONT_IRI, &nested, SyncGraphOpts::default()) + .await + .expect_err("payload-internal named graphs must be rejected"); +} + +#[tokio::test] +async fn system_graph_targets_are_rejected() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/system:main"; + seed(&fluree, ledger_id).await; + + let txn_meta_iri = format!("urn:fluree:{ledger_id}#txn-meta"); + let err = fluree + .sync_named_graph( + ledger_id, + &txn_meta_iri, + &payload_v1(), + SyncGraphOpts::default(), + ) + .await + .expect_err("txn-meta graph must be rejected"); + assert!(err.to_string().contains("txn-meta"), "got: {err}"); +} + +/// A policy-gated sync takes the write-locked fast path +/// (`stage_under_lock`), which previously fell through to the JSON-like +/// insert path — staging the payload into the DEFAULT graph and leaving the +/// target untouched. Raft always uses that path. +#[tokio::test] +async fn policy_gated_sync_targets_the_named_graph() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/policy-path:main"; + seed(&fluree, ledger_id).await; + let handle = fluree.ledger_cached(ledger_id).await.expect("handle"); + let root = || fluree_db_api::PolicyContext::new(fluree_db_api::PolicyWrapper::root(), None); + + let v1 = payload_v1(); + let first = fluree + .stage(&handle) + .sync_graph(ONT_IRI, &v1) + .policy(root()) + .execute() + .await + .expect("policy-gated sync"); + assert_eq!(first.receipt.assert_count, 3); + assert_eq!(count_in_graph(&fluree, ledger_id, Some(ONT_IRI)).await, 3); + assert_eq!( + count_in_graph(&fluree, ledger_id, None).await, + 1, + "nothing may leak into the default graph" + ); + + let v2 = payload_v2(); + let second = fluree + .stage(&handle) + .sync_graph(ONT_IRI, &v2) + .policy(root()) + .execute() + .await + .expect("policy-gated delta sync"); + assert_eq!( + (second.receipt.assert_count, second.receipt.retract_count), + (2, 2), + "delta semantics must hold on the locked fast path" + ); +} + +/// The consensus terminal (`build_commit`) must express a no-change sync +/// as `None` rather than failing with `EmptyTransaction` (which would +/// poison the queued request under Raft). +#[tokio::test] +async fn build_commit_reports_a_no_change_sync_as_none() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/build-commit-noop:main"; + seed(&fluree, ledger_id).await; + fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect("first sync"); + let handle = fluree.ledger_cached(ledger_id).await.expect("handle"); + + let v1 = payload_v1(); + let built = fluree + .stage(&handle) + .sync_graph(ONT_IRI, &v1) + .build_commit() + .await + .expect("build_commit must not error on a no-change sync"); + assert!(built.is_none(), "identical payload must build no commit"); + + let v2 = payload_v2(); + let built = fluree + .stage(&handle) + .sync_graph(ONT_IRI, &v2) + .build_commit() + .await + .expect("build_commit"); + let (_guard, staged) = built.expect("a delta builds a commit"); + assert_eq!(staged.commit.flakes.len(), 4, "2 asserts + 2 retracts"); +} + +/// Target validation lives at staging, so every entry point (builder, +/// consensus applier, HTTP) meets it: malformed IRIs cannot be registered +/// as graphs, and the ledger's own system-graph IRIs are refused even when +/// the registry never seeded them. +#[tokio::test] +async fn staging_rejects_malformed_and_system_graph_targets() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/staging-guards:main"; + seed(&fluree, ledger_id).await; + let handle = fluree.ledger_cached(ledger_id).await.expect("handle"); + let v1 = payload_v1(); + + for bad in [ + "graphs/relative", + "", + "http://x.org/has space", + "1abc:scheme", + ] { + let err = fluree + .stage(&handle) + .sync_graph(bad, &v1) + .execute() + .await + .expect_err("malformed target must be rejected at staging"); + assert!( + err.to_string().contains("sync target"), + "unexpected error for {bad:?}: {err}" + ); + } + let config_iri = format!("urn:fluree:{ledger_id}#config"); + let err = fluree + .stage(&handle) + .sync_graph(&config_iri, &v1) + .execute() + .await + .expect_err("system graph target must be rejected at staging"); + assert!(err.to_string().contains("reserved"), "got: {err}"); + assert_eq!(count_in_graph(&fluree, ledger_id, None).await, 1); +} + +/// An identical resync must still be a no-op after the graph has been +/// indexed — the case the memory-backed tests cannot reach. +/// +/// `scan_graph_flakes` reads through the range provider, and +/// `BinaryRangeProvider` materializes flakes with `g: None` +/// (`binary_range.rs:753`, `:1393`). The accumulator buckets on `flake.g`, +/// so without the explicit stamp in the sync wave the retractions land in a +/// different bucket than the payload's assertions, nothing cancels, and an +/// unchanged graph is retracted and re-asserted in full. Novelty-resident +/// flakes carry `g` already, which is why every other test here passes +/// either way. +#[tokio::test] +async fn identical_resync_is_a_noop_against_indexed_data() { + use crate::support::{start_background_indexer_local, trigger_index_and_wait_outcome}; + + let tmp = tempfile::TempDir::new().unwrap(); + let mut fluree = FlureeBuilder::file(tmp.path().to_string_lossy().to_string()) + .build() + .unwrap(); + let (local, handle) = start_background_indexer_local( + fluree.backend().clone(), + fluree.nameservice_mode().publisher_arc().unwrap(), + fluree_db_indexer::IndexerConfig::small(), + ); + fluree.set_indexing_mode(fluree_db_api::tx::IndexingMode::Background(handle.clone())); + + local + .run_until(async move { + let ledger_id = "it/sync-graph/indexed:main"; + fluree + .create_ledger(ledger_id) + .await + .expect("create ledger"); + + let first = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect("first sync"); + assert_eq!(first.asserted, 3); + assert!(first.committed); + + // Push the graph into the persisted index, so the sync wave's + // scan is served by the range provider rather than novelty. + trigger_index_and_wait_outcome(&handle, ledger_id, first.t).await; + let ledger = fluree.ledger(ledger_id).await.expect("reload ledger"); + assert!( + ledger.snapshot.range_provider.is_some(), + "graph must be index-resident for this test to mean anything" + ); + + let second = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect("resync after index"); + assert_eq!(second.retracted, 0, "indexed rows must still cancel"); + assert_eq!(second.asserted, 0, "identical payload asserts nothing"); + assert!( + !second.committed, + "identical resync of an indexed graph must not commit" + ); + assert_eq!(second.t, first.t, "head t unchanged"); + + // And a real delta over indexed data still commits only the delta. + let third = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v2(), SyncGraphOpts::default()) + .await + .expect("delta sync after index"); + assert_eq!(third.retracted, 2, "bob's name and alice's old role"); + assert_eq!(third.asserted, 2, "alice's new role and carol's name"); + assert!(third.committed); + assert_eq!(count_in_graph(&fluree, ledger_id, Some(ONT_IRI)).await, 3); + }) + .await; +} + +/// The whole-graph memory backstop: staging materializes the target +/// graph's current flakes, so a graph past the cap must fail loud (an +/// OOM kill otherwise, with no guard in between — `NoveltyWouldExceed` +/// only ever sees the netted delta). The limit is read per scan, so the +/// env changes below take effect immediately; nextest's process-per-test +/// isolation keeps them from leaking into other tests. +#[tokio::test] +async fn whole_graph_scan_backstop_fails_loud_before_materializing() { + let fluree = FlureeBuilder::memory().build_memory(); + let ledger_id = "it/sync-graph/backstop:main"; + seed(&fluree, ledger_id).await; + + std::env::set_var("FLUREE_MAX_GRAPH_SCAN_FLAKES", "2"); + // First sync scans an EMPTY graph (0 <= 2): the cap bounds the scan, + // not the payload, so populating past the cap succeeds. + let report = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect("first sync scans an empty graph"); + assert_eq!(report.asserted, 3); + + // Now the graph holds 3 > 2: every whole-graph verb refuses. + let err = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect_err("resync over the cap must fail loud"); + assert!( + err.to_string().contains("FLUREE_MAX_GRAPH_SCAN_FLAKES"), + "error names the knob: {err}" + ); + + // Dry run takes the same scan and must fail the same way. + let dry = fluree + .sync_named_graph( + ledger_id, + ONT_IRI, + &payload_v1(), + SyncGraphOpts { + dry_run: true, + ..Default::default() + }, + ) + .await + .expect_err("dry run must fail the way the real run would"); + assert!(dry.to_string().contains("FLUREE_MAX_GRAPH_SCAN_FLAKES")); + + let ledger = fluree.ledger(ledger_id).await.unwrap(); + let clear = fluree + .stage_owned(ledger) + .txn(fluree_db_transact::Txn::clear_graph(ONT_IRI)) + .execute() + .await + .expect_err("CLEAR shares the scan and the backstop"); + assert!(clear.to_string().contains("FLUREE_MAX_GRAPH_SCAN_FLAKES")); + + // 0 disables; the identical resync is a no-op again. + std::env::set_var("FLUREE_MAX_GRAPH_SCAN_FLAKES", "0"); + let report = fluree + .sync_named_graph(ledger_id, ONT_IRI, &payload_v1(), SyncGraphOpts::default()) + .await + .expect("disabled cap syncs normally"); + assert!(!report.committed); + std::env::remove_var("FLUREE_MAX_GRAPH_SCAN_FLAKES"); +} + +/// CLEAR / COPY / MOVE on an index-resident named graph — the same scan sync +/// rides, and broken the same two ways on `main` before this branch: the +/// scan issued `RangeTest::Ge`, which the V3 provider rejects, and once it +/// runs, index-decoded flakes carry `g: None` and route to the default +/// graph. The receipt then reports a commit while the target is untouched +/// (CLEAR), or the destination merges instead of replacing (COPY). +#[tokio::test] +async fn graph_management_verbs_work_on_indexed_data() { + use crate::support::{start_background_indexer_local, trigger_index_and_wait_outcome}; + use fluree_db_transact::Txn; + + let tmp = tempfile::TempDir::new().unwrap(); + let mut fluree = FlureeBuilder::file(tmp.path().to_string_lossy().to_string()) + .build() + .unwrap(); + let (local, handle) = start_background_indexer_local( + fluree.backend().clone(), + fluree.nameservice_mode().publisher_arc().unwrap(), + fluree_db_indexer::IndexerConfig::small(), + ); + fluree.set_indexing_mode(fluree_db_api::tx::IndexingMode::Background(handle.clone())); + + local + .run_until(async move { + let ledger_id = "it/sync-graph/gmgmt-indexed:main"; + fluree + .create_ledger(ledger_id) + .await + .expect("create ledger"); + let ledger = fluree.ledger(ledger_id).await.unwrap(); + fluree + .stage_owned(ledger) + .upsert_turtle(&format!( + r#" + @prefix ex: . + ex:d ex:name "Default" . + GRAPH <{ONT_IRI}> {{ ex:alice ex:name "Alice" . ex:bob ex:name "Bob" . }} + GRAPH <{OTHER_IRI}> {{ ex:zed ex:name "Zed" . }} + "# + )) + .execute() + .await + .expect("seed"); + let ledger = fluree.ledger(ledger_id).await.unwrap(); + trigger_index_and_wait_outcome(&handle, ledger_id, ledger.t()).await; + let ledger = fluree.ledger(ledger_id).await.unwrap(); + assert!( + ledger.snapshot.range_provider.is_some(), + "must be index-resident" + ); + + assert_eq!(graph_counts(&fluree, ledger_id).await, (1, 2, 1)); + + // CLEAR empties exactly the target. + let r = fluree + .stage_owned(ledger) + .txn(Txn::clear_graph(ONT_IRI)) + .execute() + .await + .expect("CLEAR on indexed graph"); + assert_eq!(r.receipt.flake_count, 2); + assert_eq!( + graph_counts(&fluree, ledger_id).await, + (1, 0, 1), + "CLEAR emptied ont only" + ); + + // COPY replaces the destination with the source. + let ledger = fluree.ledger(ledger_id).await.unwrap(); + fluree + .stage_owned(ledger) + .txn(Txn::copy_graph(OTHER_IRI, ONT_IRI)) + .execute() + .await + .expect("COPY on indexed graphs"); + assert_eq!( + graph_counts(&fluree, ledger_id).await, + (1, 1, 1), + "COPY replaced ont with other" + ); + + // Index again so MOVE's source and destination are both index-resident. + let ledger = fluree.ledger(ledger_id).await.unwrap(); + trigger_index_and_wait_outcome(&handle, ledger_id, ledger.t()).await; + let ledger = fluree.ledger(ledger_id).await.unwrap(); + fluree + .stage_owned(ledger) + .txn(Txn::move_graph(OTHER_IRI, ONT_IRI)) + .execute() + .await + .expect("MOVE on indexed graphs"); + assert_eq!( + graph_counts(&fluree, ledger_id).await, + (1, 1, 0), + "MOVE emptied the source" + ); + }) + .await; +} diff --git a/fluree-db-cli/Cargo.toml b/fluree-db-cli/Cargo.toml index 09f290c77c..1fa01de0d7 100644 --- a/fluree-db-cli/Cargo.toml +++ b/fluree-db-cli/Cargo.toml @@ -47,6 +47,7 @@ csv = "1.4" fluree-db-novelty = { path = "../fluree-db-novelty" } fluree-db-nameservice = { path = "../fluree-db-nameservice" } fluree-db-core = { path = "../fluree-db-core" } +fluree-graph-turtle = { path = "../fluree-graph-turtle" } fluree-db-indexer = { path = "../fluree-db-indexer" } fluree-vocab = { path = "../fluree-vocab" } fluree-db-query = { path = "../fluree-db-query" } diff --git a/fluree-db-cli/src/cli.rs b/fluree-db-cli/src/cli.rs index 516eff8640..99e541dc34 100644 --- a/fluree-db-cli/src/cli.rs +++ b/fluree-db-cli/src/cli.rs @@ -585,6 +585,69 @@ pub enum Commands { policy: PolicyArgs, }, + /// Synchronize a named graph: make its contents exactly the supplied + /// data, committing only the delta. + /// + /// The target graph is the constant; the SOURCE of the desired contents + /// is pluggable. Today the source is RDF text (Turtle or JSON-LD) from a + /// file, inline expression, or stdin; the same command shape is where + /// mapped sources (R2RML over Iceberg / CSV / Excel) will plug in. + /// + /// Examples: + /// fluree sync mydb --graph urn:example:ontology -f ontology.ttl + /// fluree sync mydb --graph urn:example:ontology -f ontology.ttl --dry-run + /// cat export.jsonld | fluree sync --graph urn:example:ontology --remote origin + Sync { + /// Optional ledger name and/or inline data (same resolution rules + /// as `upsert`: 0 args = active ledger + -e/-f/stdin; 1 arg = data, + /// file, or ledger; 2 args = ledger + inline data). + #[arg(num_args = 0..=2)] + args: Vec, + + /// Ledger name (defaults to active ledger). + #[arg(short = 'l', long)] + ledger: Option, + + /// Target named graph IRI — the sync scope. Required; the payload + /// never widens or narrows it. + #[arg(short = 'g', long)] + graph: String, + + /// Inline data expression (Turtle or JSON-LD). + #[arg(short = 'e', long = "expr")] + expr: Option, + + /// Read data from a file + #[arg(short = 'f', long = "file")] + file: Option, + + /// Data format (turtle or jsonld); auto-detected if omitted + #[arg(long)] + format: Option, + + /// Compute and report the delta (asserted / retracted counts) + /// without committing. The standard pre-flight for pipelines. + #[arg(long)] + dry_run: bool, + + /// Allow an empty payload, which clears the graph. Off by default so + /// a truncated export cannot silently wipe the graph. + #[arg(long)] + allow_empty: bool, + + /// Emit the report as JSON (machine-readable; same shape as the + /// server's dry-run response) instead of a sentence. + #[arg(long)] + json: bool, + + /// Execute against a remote server (by remote name, e.g., "origin") + #[arg(long)] + remote: Option, + + #[command(flatten)] + policy: PolicyArgs, + }, + /// Bulk-upsert CSV rows into a ledger via a per-row Cypher or JSON-LD /// template (the `LOAD CSV` analog). /// diff --git a/fluree-db-cli/src/commands/graph_sync.rs b/fluree-db-cli/src/commands/graph_sync.rs new file mode 100644 index 0000000000..e23ab9f0d0 --- /dev/null +++ b/fluree-db-cli/src/commands/graph_sync.rs @@ -0,0 +1,222 @@ +//! `fluree sync` — make a named graph's contents exactly the supplied data, +//! committing only the delta. +//! +//! The target graph is the constant of this command; the source of the +//! desired contents is pluggable ([`SyncSource`]). Every source resolves to +//! one JSON-LD payload and flows through the same verb — locally +//! `Fluree::sync_named_graph_with`, remotely `POST /sync` — so adding a +//! mapped source (R2RML over Iceberg / CSV / Excel) is one new variant here, +//! not a new command or endpoint. + +use crate::cli::PolicyArgs; +use crate::commands::insert::{build_policy_ctx, resolve_inputs}; +use crate::context::{self, LedgerMode}; +use crate::detect; +use crate::error::{CliError, CliResult}; +use crate::input; +use fluree_db_api::server_defaults::FlureeDir; +use fluree_db_api::{SyncGraphOpts, SyncGraphReport, TxnOpts}; +use std::path::Path; + +/// Arguments for [`run`]. +pub struct SyncArgs<'a> { + pub args: &'a [String], + pub ledger: Option<&'a str>, + pub graph: &'a str, + pub expr: Option<&'a str>, + pub file: Option<&'a Path>, + pub format: Option<&'a str>, + pub dry_run: bool, + pub allow_empty: bool, + pub json: bool, + pub remote: Option<&'a str>, + pub direct: bool, + pub policy: &'a PolicyArgs, + pub dirs: &'a FlureeDir, +} + +/// Where the graph's desired contents come from. +/// +/// Today: RDF text. Designed as the seam for mapped sources — an R2RML +/// mapping applied to an Iceberg table, CSV, or spreadsheet would be a new +/// variant whose [`SyncSource::into_payload`] materializes the mapping's +/// output (locally, or via a server-side materialization when running +/// `--remote`) into the same JSON-LD payload. +pub enum SyncSource { + /// Turtle or JSON-LD text, already read from a file / expression / stdin. + RdfText { + content: String, + format: detect::DataFormat, + }, +} + +impl SyncSource { + /// Materialize the desired contents as one JSON-LD payload. + /// + /// Turtle is converted client-side: the sync endpoint is JSON-LD only, + /// so a Turtle export (the common ontology-editor case) works against + /// any server that implements it. + pub fn into_payload(self) -> CliResult { + match self { + SyncSource::RdfText { content, format } => match format { + detect::DataFormat::JsonLd => Ok(serde_json::from_str(&content)?), + detect::DataFormat::Turtle => fluree_graph_turtle::parse_to_json(&content) + .map_err(|e| CliError::Usage(format!("failed to parse Turtle: {e}"))), + }, + } + } +} + +pub async fn run(a: SyncArgs<'_>) -> CliResult<()> { + if a.graph.is_empty() { + return Err(CliError::Usage( + "--graph is required: sync targets exactly one named graph".to_string(), + )); + } + + let (explicit_ledger, positional_inline, positional_file) = resolve_inputs(a.ledger, a.args)?; + let source = input::resolve_input( + a.expr, + positional_inline, + a.file, + positional_file.as_deref(), + )?; + let content = input::read_input(&source)?; + let detect_path = a.file.or(positional_file.as_deref()); + let format = detect::detect_data_format(detect_path, &content, a.format)?; + let payload = SyncSource::RdfText { content, format }.into_payload()?; + + // The empty-payload gate is enforced server-side too; checking here + // gives a precise message before any network or staging work. + let explicitly_empty = payload + .get("@graph") + .and_then(serde_json::Value::as_array) + .is_some_and(Vec::is_empty); + if explicitly_empty && !a.allow_empty { + return Err(CliError::Usage( + "payload is empty; syncing it would clear the graph — pass --allow-empty to confirm" + .to_string(), + )); + } + + let mode = if let Some(remote_name) = a.remote { + let alias = context::resolve_ledger(explicit_ledger, a.dirs)?; + context::build_remote_mode(remote_name, &alias, a.dirs).await? + } else { + let mode = context::resolve_ledger_mode(explicit_ledger, a.dirs).await?; + if a.direct { + mode + } else { + context::try_server_route(mode, a.dirs) + } + }; + + match mode { + LedgerMode::Tracked { + client, + remote_alias, + remote_name, + .. + } => { + let client = client.with_policy(a.policy.clone()); + let response = client + .sync_jsonld(&remote_alias, a.graph, &payload, a.dry_run, a.allow_empty) + .await?; + context::persist_refreshed_tokens(&client, &remote_name, a.dirs).await; + if a.json { + println!( + "{}", + serde_json::to_string_pretty(&response) + .unwrap_or_else(|_| response.to_string()) + ); + } else { + print_remote_response(a.graph, &response, a.dry_run); + } + } + LedgerMode::Local { fluree, alias } => { + let policy_ctx = build_policy_ctx(&fluree, &alias, a.policy).await?; + let report = fluree + .sync_named_graph_with( + &alias, + a.graph, + &payload, + SyncGraphOpts { + dry_run: a.dry_run, + allow_empty: a.allow_empty, + }, + TxnOpts::default(), + policy_ctx, + ) + .await?; + if a.json { + println!( + "{}", + serde_json::to_string_pretty(&report_json(&report)).expect("report serializes") + ); + } else { + print_local_report(&report); + } + } + } + Ok(()) +} + +/// The machine-readable report — the same shape the server's dry-run +/// response uses, so scripts consume either path identically. +fn report_json(r: &SyncGraphReport) -> serde_json::Value { + serde_json::json!({ + "ledger": r.ledger_id, + "graph": r.graph_iri, + "asserted": r.asserted, + "retracted": r.retracted, + "committed": r.committed, + "dryRun": r.dry_run, + "t": r.t, + }) +} + +fn print_local_report(r: &SyncGraphReport) { + if r.dry_run { + println!( + "Would sync graph <{}> in '{}': +{} asserted, -{} retracted (dry run; head t={}).", + r.graph_iri, r.ledger_id, r.asserted, r.retracted, r.t + ); + } else if r.committed { + println!( + "Synced graph <{}> in '{}': +{} asserted, -{} retracted (t={}).", + r.graph_iri, r.ledger_id, r.asserted, r.retracted, r.t + ); + } else { + println!( + "Graph <{}> in '{}' already matches the payload — no commit produced (t={}).", + r.graph_iri, r.ledger_id, r.t + ); + } +} + +fn print_remote_response(graph: &str, value: &serde_json::Value, dry_run: bool) { + // Dry runs answer with the report shape; real runs with the standard + // transact response (ledger, t, tx-id, ...). + if dry_run { + let n = |k: &str| { + value + .get(k) + .and_then(serde_json::Value::as_u64) + .unwrap_or(0) + }; + println!( + "Would sync graph <{graph}>: +{} asserted, -{} retracted (dry run; head t={}).", + n("asserted"), + n("retracted"), + value + .get("t") + .and_then(serde_json::Value::as_i64) + .unwrap_or(0) + ); + } else { + println!( + "{}", + serde_json::to_string_pretty(value).unwrap_or_else(|_| value.to_string()) + ); + } +} diff --git a/fluree-db-cli/src/commands/mod.rs b/fluree-db-cli/src/commands/mod.rs index ec2c9cebf3..5506aad329 100644 --- a/fluree-db-cli/src/commands/mod.rs +++ b/fluree-db-cli/src/commands/mod.rs @@ -12,6 +12,7 @@ pub mod docs; pub mod drop; pub mod export; pub mod graph; +pub mod graph_sync; pub mod history; pub mod iceberg; pub mod index; diff --git a/fluree-db-cli/src/lib.rs b/fluree-db-cli/src/lib.rs index ed279d2317..83d2a8af11 100644 --- a/fluree-db-cli/src/lib.rs +++ b/fluree-db-cli/src/lib.rs @@ -304,6 +304,38 @@ pub async fn run(cli: Cli) -> error::CliResult<()> { .await } + Commands::Sync { + args, + ledger, + graph, + expr, + file, + format, + dry_run, + allow_empty, + json, + remote, + policy, + } => { + let fluree_dir = config::require_fluree_dir(config_path)?; + commands::graph_sync::run(commands::graph_sync::SyncArgs { + args: &args, + ledger: ledger.as_deref(), + graph: &graph, + expr: expr.as_deref(), + file: file.as_deref(), + format: format.as_deref(), + dry_run, + allow_empty, + json, + remote: remote.as_deref(), + direct, + policy: &policy, + dirs: &fluree_dir, + }) + .await + } + Commands::Query { args, ledger, diff --git a/fluree-db-cli/src/remote_client.rs b/fluree-db-cli/src/remote_client.rs index 2a59fadb7c..58870315a0 100644 --- a/fluree-db-cli/src/remote_client.rs +++ b/fluree-db-cli/src/remote_client.rs @@ -1236,6 +1236,42 @@ impl RemoteLedgerClient { .await } + // ========================================================================= + // Sync (graph synchronization) + // ========================================================================= + + /// Synchronize a named graph: make its contents exactly `body`, + /// committing only the delta. + /// + /// `POST {base}/sync/{ledger}?graph=[&dryRun=true][&allowEmpty=true]` + /// with a JSON-LD body. A dry run answers with the delta report; a real + /// run with the standard transact response. + pub async fn sync_jsonld( + &self, + ledger: &str, + graph: &str, + body: &serde_json::Value, + dry_run: bool, + allow_empty: bool, + ) -> Result { + let mut url = self.op_url("sync", ledger); + url.push_str("?graph="); + url.push_str(&urlencoding::encode(graph)); + if dry_run { + url.push_str("&dryRun=true"); + } + if allow_empty { + url.push_str("&allowEmpty=true"); + } + self.send_json( + reqwest::Method::POST, + &url, + "application/json", + Some(RequestBody::Json(body)), + ) + .await + } + // ========================================================================= // Update (WHERE/DELETE/INSERT) // ========================================================================= diff --git a/fluree-db-cli/tests/integration.rs b/fluree-db-cli/tests/integration.rs index acdff1125b..2c7e42d3fa 100644 --- a/fluree-db-cli/tests/integration.rs +++ b/fluree-db-cli/tests/integration.rs @@ -1376,6 +1376,89 @@ fn upsert_turtle() { .stdout(predicate::str::contains("Committed t=1")); } +#[test] +fn sync_graph_commits_only_the_delta() { + let tmp = TempDir::new().unwrap(); + fluree_cmd(&tmp).arg("init").assert().success(); + fluree_cmd(&tmp) + .args(["create", "syncdb"]) + .assert() + .success(); + let graph = "urn:example:ontology"; + + let v1 = r#"{"@context": {"ex": "http://example.org/"}, "@graph": [ + {"@id": "ex:alice", "ex:name": "Alice", "ex:role": "engineer"}, + {"@id": "ex:bob", "ex:name": "Bob"}]}"#; + // First sync populates the graph: 3 asserts, nothing to retract. + fluree_cmd(&tmp) + .args(["sync", "syncdb", "--graph", graph, "-e", v1]) + .assert() + .success() + .stdout(predicate::str::contains("+3 asserted, -0 retracted (t=1)")); + + // Identical payload is a no-op: no commit, t unchanged. + fluree_cmd(&tmp) + .args(["sync", "syncdb", "--graph", graph, "-e", v1]) + .assert() + .success() + .stdout(predicate::str::contains("already matches the payload")); + + // Dry run of a delta reports the counts without committing (still t=1). + let v2 = "@prefix ex: .\nex:alice ex:name \"Alice\" ; ex:role \"manager\" .\nex:carol ex:name \"Carol\" ."; + fluree_cmd(&tmp) + .args([ + "sync", + "syncdb", + "--graph", + graph, + "--dry-run", + "--json", + "-e", + v2, + ]) + .assert() + .success() + .stdout(predicate::str::contains("\"asserted\": 2")) + .stdout(predicate::str::contains("\"retracted\": 2")) + .stdout(predicate::str::contains("\"committed\": false")) + .stdout(predicate::str::contains("\"t\": 1")); + + // Real run from Turtle (converted client-side): one commit for the delta. + fluree_cmd(&tmp) + .args(["sync", "syncdb", "--graph", graph, "-e", v2]) + .assert() + .success() + .stdout(predicate::str::contains("+2 asserted, -2 retracted (t=2)")); + + // Empty payload is refused without --allow-empty ... + fluree_cmd(&tmp) + .args([ + "sync", + "syncdb", + "--graph", + graph, + "-e", + r#"{"@graph": []}"#, + ]) + .assert() + .failure() + .stderr(predicate::str::contains("--allow-empty")); + // ... and clears the graph with it. + fluree_cmd(&tmp) + .args([ + "sync", + "syncdb", + "--graph", + graph, + "--allow-empty", + "-e", + r#"{"@graph": []}"#, + ]) + .assert() + .success() + .stdout(predicate::str::contains("+0 asserted, -3 retracted (t=3)")); +} + // ============================================================================ // v1.1 — CSV output tests // ============================================================================ diff --git a/fluree-db-consensus/src/lib.rs b/fluree-db-consensus/src/lib.rs index 3ff5a60f00..2fb9946019 100644 --- a/fluree-db-consensus/src/lib.rs +++ b/fluree-db-consensus/src/lib.rs @@ -194,6 +194,14 @@ pub enum TransactionBody { /// object map, matching `fluree_db_cypher::ParamMap`. params: Option>, }, + /// JSON-LD document staged as a graph sync: `graph_iri`'s contents + /// become exactly the document, committing only the delta (whole-graph + /// retraction wave + accumulator cancellation). + /// + /// Appended last: the queue envelope and its [`BodyKind`] discriminator + /// are postcard-encoded in persisted Raft state, where variant ordinals + /// are positional — never insert a variant mid-enum. + JsonLdGraphSync { graph_iri: String, body: JsonValue }, } impl TransactionBody { @@ -205,6 +213,7 @@ impl TransactionBody { Self::JsonLdInsert(_) | Self::TurtleInsert(_) => "insert", Self::JsonLdUpsert(_) | Self::TurtleUpsert(_) | Self::TrigUpsert(_) => "upsert", Self::JsonLdUpdate(_) => "update", + Self::JsonLdGraphSync { .. } => "graph-sync", Self::Sparql(_) => "sparql-update", Self::Cypher { .. } => "cypher", } @@ -237,6 +246,12 @@ impl TransactionBody { hasher.update(b"jsonld-update"); hasher.update(json.to_string().as_bytes()); } + Self::JsonLdGraphSync { graph_iri, body } => { + hasher.update(b"jsonld-graph-sync"); + hasher.update(graph_iri.as_bytes()); + hasher.update([0u8]); + hasher.update(body.to_string().as_bytes()); + } Self::TurtleInsert(text) => { hasher.update(b"turtle-insert"); hasher.update(text.as_bytes()); @@ -304,6 +319,12 @@ pub enum BodyKind { /// conflict strategy. Worker re-runs `prepare_rebase` and /// advances the branch's head. Rebase, + /// Graph sync (delta-only whole-graph replacement). Appended last: + /// `BodyKind` is postcard-encoded in persisted Raft state snapshots + /// (`QueueEntry.body_kind`), where variant ordinals are positional — + /// inserting mid-enum would make existing snapshots and mixed-version + /// nodes decode every later variant as the wrong operation. + JsonLdGraphSync, } impl From<&TransactionBody> for BodyKind { @@ -312,6 +333,7 @@ impl From<&TransactionBody> for BodyKind { TransactionBody::JsonLdInsert(_) => BodyKind::JsonLdInsert, TransactionBody::JsonLdUpsert(_) => BodyKind::JsonLdUpsert, TransactionBody::JsonLdUpdate(_) => BodyKind::JsonLdUpdate, + TransactionBody::JsonLdGraphSync { .. } => BodyKind::JsonLdGraphSync, TransactionBody::TurtleInsert(_) => BodyKind::TurtleInsert, TransactionBody::TurtleUpsert(_) => BodyKind::TurtleUpsert, TransactionBody::TrigUpsert(_) => BodyKind::TrigUpsert, @@ -909,3 +931,34 @@ mod tests { ); } } + +#[cfg(all(test, feature = "raft"))] +mod body_kind_wire_tests { + use super::BodyKind; + + /// `BodyKind` is postcard-encoded in persisted Raft state snapshots + /// (`QueueEntry.body_kind`), where variant ordinals are positional. + /// This pins every ordinal so a new variant can only ever be appended. + #[test] + fn body_kind_ordinals_are_append_only() { + let expected = [ + (BodyKind::JsonLdInsert, 0u8), + (BodyKind::JsonLdUpsert, 1), + (BodyKind::JsonLdUpdate, 2), + (BodyKind::TurtleInsert, 3), + (BodyKind::TurtleUpsert, 4), + (BodyKind::TrigUpsert, 5), + (BodyKind::Sparql, 6), + (BodyKind::Cypher, 7), + (BodyKind::Pushed, 8), + (BodyKind::Revert, 9), + (BodyKind::Merge, 10), + (BodyKind::Rebase, 11), + (BodyKind::JsonLdGraphSync, 12), + ]; + for (kind, ordinal) in expected { + let bytes = postcard::to_allocvec(&kind).expect("encode"); + assert_eq!(bytes, vec![ordinal], "{kind:?} ordinal moved"); + } + } +} diff --git a/fluree-db-consensus/src/local.rs b/fluree-db-consensus/src/local.rs index c895eb3f76..3f1f1ebdbf 100644 --- a/fluree-db-consensus/src/local.rs +++ b/fluree-db-consensus/src/local.rs @@ -149,6 +149,9 @@ impl Committer for LocalCommitter { TransactionBody::JsonLdInsert(json) => staged.insert(json), TransactionBody::JsonLdUpsert(json) => staged.upsert(json), TransactionBody::JsonLdUpdate(json) => staged.update(json), + TransactionBody::JsonLdGraphSync { graph_iri, body } => { + staged.sync_graph(graph_iri.as_str(), body) + } TransactionBody::TurtleInsert(text) => staged.insert_turtle(text.as_str()), TransactionBody::TurtleUpsert(text) | TransactionBody::TrigUpsert(text) => { staged.upsert_turtle(text.as_str()) diff --git a/fluree-db-consensus/src/raft/commit_worker.rs b/fluree-db-consensus/src/raft/commit_worker.rs index 327fa09288..e2bce47437 100644 --- a/fluree-db-consensus/src/raft/commit_worker.rs +++ b/fluree-db-consensus/src/raft/commit_worker.rs @@ -634,6 +634,9 @@ impl Worker { TransactionBody::JsonLdInsert(json) => staged.insert(json), TransactionBody::JsonLdUpsert(json) => staged.upsert(json), TransactionBody::JsonLdUpdate(json) => staged.update(json), + TransactionBody::JsonLdGraphSync { graph_iri, body } => { + staged.sync_graph(graph_iri.as_str(), body) + } TransactionBody::TurtleInsert(text) => staged.insert_turtle(text.as_str()), TransactionBody::TurtleUpsert(text) | TransactionBody::TrigUpsert(text) => { staged.upsert_turtle(text.as_str()) @@ -672,10 +675,33 @@ impl Worker { builder = builder.policy(policy); } - let (write_guard, staged_commit) = builder + let Some((write_guard, staged_commit)) = builder .build_commit() .await - .map_err(|e| stage_failure(&format!("build_commit failed: {e}")))?; + .map_err(|e| stage_failure(&format!("build_commit failed: {e}")))? + else { + // No-change transaction (e.g. a graph sync whose payload already + // matches the graph): mirror the revert NoOp short-circuit — + // republish the current head with `install: None` so the queue + // entry completes without advancing and the local state is + // untouched. A no-op requires an already-registered graph, hence + // an existing head. + let snap = ledger_handle.snapshot().await; + let head_id = snap.head_commit_id.clone().ok_or_else(|| { + stage(PoisonReason::WorkerPanic { + message: "no-op transaction on a ledger without a head commit".into(), + }) + })?; + return Ok(StagedOutcome { + receipt: AppliedReceipt::Transact(TransactApplied { + commit_id: head_id, + commit_t: snap.t, + flake_count: 0, + tally: None, + }), + install: None, + }); + }; let commit_cid = staged_commit.commit.id.clone().ok_or_else(|| { stage(PoisonReason::WorkerPanic { diff --git a/fluree-db-consensus/src/raft/queued_transactor.rs b/fluree-db-consensus/src/raft/queued_transactor.rs index 9901c7efc1..57ce83d5a7 100644 --- a/fluree-db-consensus/src/raft/queued_transactor.rs +++ b/fluree-db-consensus/src/raft/queued_transactor.rs @@ -489,6 +489,12 @@ impl Committer for QueuedTransactor { commit_id: record.head, t: record.t, flake_count: record.flake_count as usize, + // The raft idempotency record carries only the total — + // the assert/retract split is not persisted in raft + // state (wire-format stability), so a replayed receipt + // reports 0/0. + assert_count: 0, + retract_count: 0, }, tally: record.tally.map(Into::into), cypher_return: None, @@ -853,6 +859,11 @@ fn transaction_receipt_from( commit_id, t: commit_t, flake_count, + // `AppliedReceipt` does not carry the assert/retract split + // (raft wire-format stability), so raft-applied receipts + // report 0/0. + assert_count: 0, + retract_count: 0, }, tally, cypher_return: None, diff --git a/fluree-db-core/src/graph_registry.rs b/fluree-db-core/src/graph_registry.rs index 266fdcabd8..90a1f5e69b 100644 --- a/fluree-db-core/src/graph_registry.rs +++ b/fluree-db-core/src/graph_registry.rs @@ -56,6 +56,53 @@ pub fn txn_meta_graph_iri(ledger_id: &str) -> String { format!("urn:fluree:{ledger_id}#txn-meta") } +/// Validate that `value` is an absolute IRI acceptable as a graph target. +/// +/// A minimal check, not a full RFC 3987 parser: no whitespace / C0 / DEL / +/// RFC 3987-excluded characters (`<`, `>`, `"`, `{`, `}`, `|`, `\\`, `^`, +/// `` ` ``), and a `:` head per RFC 3986 §3.1. Returns the +/// error message a caller surfaces as a bad request. +pub fn validate_absolute_graph_iri(value: &str) -> std::result::Result<(), String> { + if value.is_empty() { + return Err("graph IRI is required and cannot be empty".to_string()); + } + if value.chars().any(|c| { + matches!(c, '<' | '>' | '"' | '{' | '}' | '|' | '\\' | '^' | '`') + || c.is_whitespace() + || c <= '\u{20}' + || c == '\u{7F}' + }) { + return Err(format!( + "Invalid graph IRI '{value}': contains whitespace, a control character, \ + or a character not allowed in an IRI \ + (one of `<`, `>`, `\"`, `{{`, `}}`, `|`, `\\`, `^`, `` ` ``)" + )); + } + let (scheme, rest) = value.split_once(':').ok_or_else(|| { + format!("Invalid graph IRI '{value}': missing scheme (expected an absolute IRI like 'urn:...' or 'http://...')") + })?; + let mut sc = scheme.chars(); + let first = sc + .next() + .ok_or_else(|| format!("Invalid graph IRI '{value}': scheme is empty"))?; + if !first.is_ascii_alphabetic() { + return Err(format!( + "Invalid graph IRI '{value}': scheme must start with an ASCII letter" + )); + } + if !sc.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) { + return Err(format!( + "Invalid graph IRI '{value}': scheme may only contain letters, digits, '+', '-', '.'" + )); + } + if rest.is_empty() { + return Err(format!( + "Invalid graph IRI '{value}': nothing follows the scheme" + )); + } + Ok(()) +} + /// Construct the ledger-scoped config graph IRI from a ledger ID. /// /// Each ledger has its own config graph. The IRI follows the pattern diff --git a/fluree-db-server/src/routes/mod.rs b/fluree-db-server/src/routes/mod.rs index 22cf2ca662..0555f7e31a 100644 --- a/fluree-db-server/src/routes/mod.rs +++ b/fluree-db-server/src/routes/mod.rs @@ -202,7 +202,9 @@ pub fn build_router(state: Arc) -> Router { .route("/insert", post(transact::insert)) .route("/insert/*ledger", post(transact::insert_ledger_tail)) .route("/upsert", post(transact::upsert)) + .route("/sync", post(transact::sync)) .route("/upsert/*ledger", post(transact::upsert_ledger_tail)) + .route("/sync/*ledger", post(transact::sync_ledger)) // Commit-push endpoint (precomputed commits) .route("/push/*ledger", post(push::push_ledger_tail)) // Nameservice ref endpoints (for remote sync) diff --git a/fluree-db-server/src/routes/submissions.rs b/fluree-db-server/src/routes/submissions.rs index f5593f80ba..7aef12ab78 100644 --- a/fluree-db-server/src/routes/submissions.rs +++ b/fluree-db-server/src/routes/submissions.rs @@ -248,6 +248,7 @@ fn body_kind_tag(kind: BodyKind) -> &'static str { BodyKind::JsonLdInsert | BodyKind::JsonLdUpsert | BodyKind::JsonLdUpdate + | BodyKind::JsonLdGraphSync | BodyKind::TurtleInsert | BodyKind::TurtleUpsert | BodyKind::TrigUpsert @@ -369,6 +370,8 @@ mod tests { commit_id, t: 42, flake_count: 3, + assert_count: 3, + retract_count: 0, }, tally: None, cypher_return: None, diff --git a/fluree-db-server/src/routes/transact.rs b/fluree-db-server/src/routes/transact.rs index 7340a3e588..ba70d50021 100644 --- a/fluree-db-server/src/routes/transact.rs +++ b/fluree-db-server/src/routes/transact.rs @@ -51,6 +51,15 @@ use tracing::Instrument; pub struct TransactQueryParams { /// Target ledger (format: name:branch) pub ledger: Option, + /// Sync target graph IRI (`/sync` only). + pub graph: Option, + /// Sync: compute and report the delta without committing (`/sync` only). + #[serde(rename = "dryRun", default)] + pub dry_run: bool, + /// Sync: allow an explicitly empty payload, which clears the graph + /// (`/sync` only). + #[serde(rename = "allowEmpty", default)] + pub allow_empty: bool, } /// Commit information in transaction response @@ -698,6 +707,7 @@ async fn update_local( &state, &ledger_id, TxnType::Update, + None, body_json, &credential, author.as_deref(), @@ -862,6 +872,7 @@ async fn update_ledger_local( &state, &ledger_id, TxnType::Update, + None, body_json, &credential, author.as_deref(), @@ -1008,6 +1019,7 @@ async fn insert_local( &state, &ledger_id, TxnType::Insert, + None, body_json, &credential, author.as_deref(), @@ -1154,6 +1166,182 @@ async fn upsert_local( &state, &ledger_id, TxnType::Upsert, + None, + body_json, + &credential, + author.as_deref(), + &headers, + ) + .await + } + .instrument(span) + .await +} + +/// Synchronize a named graph: make its contents exactly the JSON-LD payload, +/// committing only the delta. +/// +/// POST /sync?ledger=name:branch&graph=[&dryRun=true][&allowEmpty=true] +/// In peer mode, forwards the request to the transaction server. +pub async fn sync( + State(state): State>, + MaybeDataBearer(bearer): MaybeDataBearer, + request: Request, +) -> Response { + if state.config.server_role == ServerRole::Peer { + return forward_write_request(&state, request).await; + } + sync_local(state, bearer, None, request) + .await + .into_response() +} + +/// Synchronize a named graph with ledger in path. +/// +/// POST /:ledger/sync?graph= +pub async fn sync_ledger( + State(state): State>, + Path(ledger): Path, + MaybeDataBearer(bearer): MaybeDataBearer, + request: Request, +) -> Response { + if state.config.server_role == ServerRole::Peer { + return forward_write_request(&state, request).await; + } + sync_local(state, bearer, Some(ledger), request) + .await + .into_response() +} + +/// Local implementation of graph sync. +async fn sync_local( + state: Arc, + bearer: Option, + path_ledger: Option, + request: Request, +) -> Result { + let query_params = extract_query_params(&request); + let headers = FlureeHeaders::from_headers(request.headers())?; + let credential = MaybeCredential::extract(request).await?; + let request_id = extract_request_id(&credential.headers, &state.telemetry_config); + + let span = create_request_span( + "sync", + request_id.as_deref(), + extract_trace_id(&credential.headers).as_deref(), + None, + None, + Some("json-ld"), + ); + async move { + let span = tracing::Span::current(); + tracing::info!(status = "start", "graph sync requested"); + + // v1 is JSON-LD only; Turtle/TriG bodies are not accepted here. + if credential.is_turtle_or_trig() { + set_span_error_code(&span, "error:BadRequest"); + return Err(ServerError::bad_request( + "sync accepts application/json (JSON-LD); convert Turtle payloads client-side", + )); + } + + let Some(graph_iri) = query_params.graph.clone() else { + set_span_error_code(&span, "error:BadRequest"); + return Err(ServerError::bad_request( + "sync requires a `graph` query parameter naming the target graph IRI", + )); + }; + + let body_json = credential.body_json()?; + let ledger_id = match path_ledger { + Some(l) => l, + None => get_ledger_id(None, &query_params, &headers, &body_json)?, + }; + span.record("ledger_id", ledger_id.as_str()); + + enforce_write_access(&state, &ledger_id, bearer.as_ref(), &credential)?; + let author = effective_author(&credential, bearer.as_ref()); + + // Empty-payload gate: an explicitly empty payload clears the graph. + // Enforced here (not only in `sync_named_graph`) because the + // consensus submission below calls the builder directly. + let explicitly_empty = body_json + .get("@graph") + .and_then(serde_json::Value::as_array) + .is_some_and(Vec::is_empty); + if explicitly_empty && !query_params.allow_empty { + set_span_error_code(&span, "error:BadRequest"); + return Err(ServerError::bad_request( + "sync payload is empty; this would clear the graph — pass allowEmpty=true to confirm", + )); + } + + if query_params.dry_run { + // Dry run: stage + count locally, commit nothing. Safe outside + // consensus — it is a read of the delta, not a write. It stages + // under the SAME inputs the real run would (header-injected + // opts, governance-derived policy context built from the ledger + // state, inline shapes / unique properties), so a restricted + // writer sees policy-filtered counts and a run that would fail + // policy / SHACL / uniqueness fails here too. + let prepared = prepare_transaction_body( + &state, + &ledger_id, + body_json, + &headers, + author.as_deref(), + ) + .await; + let txn_opts = txn_opts_from_body(&prepared.body, &span)?; + let handle = state + .fluree + .ledger_cached(&ledger_id) + .await + .map_err(ServerError::from)?; + let snap = handle.snapshot().await; + let policy = fluree_db_api::build_transact_policy_context( + &state.fluree, + &snap.snapshot, + snap.novelty.as_ref(), + Some(snap.novelty.as_ref()), + snap.t, + &prepared.governance, + ) + .await + .map_err(ServerError::from)?; + drop(snap); + let report = state + .fluree + .sync_named_graph_with( + &ledger_id, + &graph_iri, + &prepared.body, + fluree_db_api::SyncGraphOpts { + dry_run: true, + allow_empty: query_params.allow_empty, + }, + txn_opts, + policy, + ) + .await + .map_err(ServerError::from)?; + return Ok(axum::Json(serde_json::json!({ + "ledger": report.ledger_id, + "graph": report.graph_iri, + "asserted": report.asserted, + "retracted": report.retracted, + "committed": report.committed, + "dryRun": report.dry_run, + "t": report.t, + })) + .into_response()); + } + + execute_transaction( + &state, + &ledger_id, + TxnType::Insert, + Some(&graph_iri), body_json, &credential, author.as_deref(), @@ -1301,6 +1489,7 @@ async fn insert_ledger_local( &state, &ledger_id, TxnType::Insert, + None, body_json, &credential, author.as_deref(), @@ -1448,6 +1637,7 @@ async fn upsert_ledger_local( &state, &ledger_id, TxnType::Upsert, + None, body_json, &credential, author.as_deref(), @@ -1477,10 +1667,83 @@ fn is_misrouted_cypher_envelope(body: &JsonValue) -> bool { .all(|k| !obj.contains_key(*k)) } +/// Derive the `TxnOpts` the HTTP layer surfaces from a JSON-LD body's +/// `opts` block (`shapes`, `uniqueProperties`). Shared by the consensus +/// submission path and the sync dry-run path so both stage under the same +/// inline constraints. +fn txn_opts_from_body(body: &JsonValue, span: &tracing::Span) -> Result { + // Pick up `opts.shapes` and `opts.uniqueProperties` from the body + // so inline SHACL shapes and unique-property constraints reach the + // staging path. Other `TxnOpts` fields are not yet surfaced over + // HTTP (branch/context/etc. come from headers or query params); + // add them here if a use case lands. + let mut txn_opts = TxnOpts::default(); + if let Some(shapes) = body.get("opts").and_then(|o| o.get("shapes")) { + // Validate at the boundary: `shapes` must be a JSON-LD + // document (object) or an array of JSON-LD documents. + // Letting scalars / nulls fall through to + // `fluree_graph_json_ld::expand` surfaces as a fuzzy + // internal parse error rather than the precise 400 + // the caller deserves. + match shapes { + JsonValue::Object(_) => {} + JsonValue::Array(arr) => { + for (idx, item) in arr.iter().enumerate() { + if !item.is_object() { + set_span_error_code(span, "error:BadRequest"); + return Err(ServerError::bad_request(format!( + "opts.shapes[{idx}] must be a JSON-LD object; got {item}" + ))); + } + } + } + _ => { + set_span_error_code(span, "error:BadRequest"); + return Err(ServerError::bad_request( + "opts.shapes must be a JSON-LD object or array of objects", + )); + } + } + txn_opts.shapes = Some(shapes.clone()); + } + if let Some(unique_props_raw) = body.get("opts").and_then(|o| o.get("uniqueProperties")) { + // Must be a JSON array. A scalar (or null) is a type + // error, not "empty list". + let Some(arr) = unique_props_raw.as_array() else { + set_span_error_code(span, "error:BadRequest"); + return Err(ServerError::bad_request( + "opts.uniqueProperties must be an array of property IRI strings", + )); + }; + // Every element must be a string. `filter_map` would + // silently drop integers/bools/etc. — that's the silent- + // weakening pattern we deliberately don't want here. + let mut iris: Vec = Vec::with_capacity(arr.len()); + for (idx, v) in arr.iter().enumerate() { + let Some(s) = v.as_str() else { + set_span_error_code(span, "error:BadRequest"); + return Err(ServerError::bad_request(format!( + "opts.uniqueProperties[{idx}] must be a string IRI; got {v}" + ))); + }; + iris.push(s.to_string()); + } + // Empty array is intentionally treated as "no inline + // constraints" rather than an error — operators may build + // the array dynamically and end up with zero entries. + if !iris.is_empty() { + txn_opts.unique_properties = Some(iris); + } + } + Ok(txn_opts) +} + +#[allow(clippy::too_many_arguments)] async fn execute_transaction( state: &AppState, ledger_id: &str, txn_type: TxnType, + sync_graph: Option<&str>, body: JsonValue, credential: &MaybeCredential, author: Option<&str>, @@ -1557,86 +1820,22 @@ async fn execute_transaction( .with_received_at(chrono::Utc::now().to_rfc3339()); } - // Pick up `opts.shapes` and `opts.uniqueProperties` from the body - // so inline SHACL shapes and unique-property constraints reach the - // staging path. Other `TxnOpts` fields are not yet surfaced over - // HTTP (branch/context/etc. come from headers or query params); - // add them here if a use case lands. - let mut txn_opts = TxnOpts::default(); - if let Some(shapes) = prepared_transaction - .body - .get("opts") - .and_then(|o| o.get("shapes")) - { - // Validate at the boundary: `shapes` must be a JSON-LD - // document (object) or an array of JSON-LD documents. - // Letting scalars / nulls fall through to - // `fluree_graph_json_ld::expand` surfaces as a fuzzy - // internal parse error rather than the precise 400 - // the caller deserves. - match shapes { - JsonValue::Object(_) => {} - JsonValue::Array(arr) => { - for (idx, item) in arr.iter().enumerate() { - if !item.is_object() { - set_span_error_code(&span, "error:BadRequest"); - return Err(ServerError::bad_request(format!( - "opts.shapes[{idx}] must be a JSON-LD object; got {item}" - ))); - } - } - } - _ => { - set_span_error_code(&span, "error:BadRequest"); - return Err(ServerError::bad_request( - "opts.shapes must be a JSON-LD object or array of objects", - )); - } - } - txn_opts.shapes = Some(shapes.clone()); - } - if let Some(unique_props_raw) = prepared_transaction - .body - .get("opts") - .and_then(|o| o.get("uniqueProperties")) - { - // Must be a JSON array. A scalar (or null) is a type - // error, not "empty list". - let Some(arr) = unique_props_raw.as_array() else { - set_span_error_code(&span, "error:BadRequest"); - return Err(ServerError::bad_request( - "opts.uniqueProperties must be an array of property IRI strings", - )); - }; - // Every element must be a string. `filter_map` would - // silently drop integers/bools/etc. — that's the silent- - // weakening pattern we deliberately don't want here. - let mut iris: Vec = Vec::with_capacity(arr.len()); - for (idx, v) in arr.iter().enumerate() { - let Some(s) = v.as_str() else { - set_span_error_code(&span, "error:BadRequest"); - return Err(ServerError::bad_request(format!( - "opts.uniqueProperties[{idx}] must be a string IRI; got {v}" - ))); - }; - iris.push(s.to_string()); - } - // Empty array is intentionally treated as "no inline - // constraints" rather than an error — operators may build - // the array dynamically and end up with zero entries. - if !iris.is_empty() { - txn_opts.unique_properties = Some(iris); - } - } + let txn_opts = txn_opts_from_body(&prepared_transaction.body, &span)?; // Every JSON-LD transaction goes through consensus. Policy context, // tracking, and execution are all handled by the submission layer; // policy is built there from the ledger state the transaction // actually stages against. - let body = match txn_type { - TxnType::Insert => TransactionBody::JsonLdInsert(prepared_transaction.body), - TxnType::Upsert => TransactionBody::JsonLdUpsert(prepared_transaction.body), - TxnType::Update => TransactionBody::JsonLdUpdate(prepared_transaction.body), + let body = match sync_graph { + Some(graph_iri) => TransactionBody::JsonLdGraphSync { + graph_iri: graph_iri.to_string(), + body: prepared_transaction.body, + }, + None => match txn_type { + TxnType::Insert => TransactionBody::JsonLdInsert(prepared_transaction.body), + TxnType::Upsert => TransactionBody::JsonLdUpsert(prepared_transaction.body), + TxnType::Update => TransactionBody::JsonLdUpdate(prepared_transaction.body), + }, }; let request = TransactionRequest { idempotency_key, diff --git a/fluree-db-server/tests/integration.rs b/fluree-db-server/tests/integration.rs index 4b9467f651..48186e7e92 100644 --- a/fluree-db-server/tests/integration.rs +++ b/fluree-db-server/tests/integration.rs @@ -3639,3 +3639,170 @@ async fn sparql_graph_pattern_named_graph_without_from_named() { "Expected GRAPH ?g discovery to surface 'urn:probegraph', got: {json}" ); } + +/// `/sync` HTTP contract (what `fluree sync --remote` depends on): delta +/// commit, no-op resync, dry-run report shape, and the 400 guards. +#[tokio::test] +async fn sync_route_contract() { + let (_tmp, state) = test_state().await; + let app = build_router(state.clone()); + + let resp = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/fluree/create") + .header("content-type", "application/json") + .body(Body::from( + serde_json::json!({ "ledger": "sync:test" }).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::CREATED); + + let graph = "urn%3Aexample%3Aontology"; + let post = |uri: String, body: String, ct: &'static str| { + Request::builder() + .method("POST") + .uri(uri) + .header("content-type", ct) + .body(Body::from(body)) + .unwrap() + }; + let v1 = serde_json::json!({ + "@context": { "ex": "http://example.org/" }, + "@graph": [ + { "@id": "ex:alice", "ex:name": "Alice", "ex:role": "engineer" }, + { "@id": "ex:bob", "ex:name": "Bob" } + ] + }) + .to_string(); + let v2 = serde_json::json!({ + "@context": { "ex": "http://example.org/" }, + "@graph": [ + { "@id": "ex:alice", "ex:name": "Alice", "ex:role": "manager" }, + { "@id": "ex:carol", "ex:name": "Carol" } + ] + }) + .to_string(); + + // First sync populates the graph: a real commit at t=1. + let (status, json) = json_body( + app.clone() + .oneshot(post( + format!("/v1/fluree/sync/sync:test?graph={graph}"), + v1.clone(), + "application/json", + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(json.get("t").and_then(serde_json::Value::as_i64), Some(1)); + + // Identical payload: success, no new commit (t unchanged). + let (status, json) = json_body( + app.clone() + .oneshot(post( + format!("/v1/fluree/sync/sync:test?graph={graph}"), + v1.clone(), + "application/json", + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!( + json.get("t").and_then(serde_json::Value::as_i64), + Some(1), + "identical resync must not advance t: {json}" + ); + + // Dry run reports the delta in the report shape and commits nothing. + let (status, json) = json_body( + app.clone() + .oneshot(post( + format!("/v1/fluree/sync/sync:test?graph={graph}&dryRun=true"), + v2.clone(), + "application/json", + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(json["asserted"], 2); + assert_eq!(json["retracted"], 2); + assert_eq!(json["committed"], false); + assert_eq!(json["dryRun"], true); + assert_eq!(json["t"], 1); + + // Real delta run: one commit. + let (status, json) = json_body( + app.clone() + .oneshot(post( + format!("/v1/fluree/sync/sync:test?graph={graph}"), + v2, + "application/json", + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(json.get("t").and_then(serde_json::Value::as_i64), Some(2)); + + // Guards: missing graph, empty payload without allowEmpty, malformed + // graph IRI, and a Turtle body are all 400s. + for (uri, body, ct) in [ + ( + "/v1/fluree/sync/sync:test".to_string(), + v1.clone(), + "application/json", + ), + ( + format!("/v1/fluree/sync/sync:test?graph={graph}"), + serde_json::json!({ "@graph": [] }).to_string(), + "application/json", + ), + ( + "/v1/fluree/sync/sync:test?graph=relative%2Fgraph".to_string(), + v1.clone(), + "application/json", + ), + ( + format!("/v1/fluree/sync/sync:test?graph={graph}"), + "@prefix ex: . ex:a ex:b \"c\" .".to_string(), + "text/turtle", + ), + ] { + let (status, json) = json_body( + app.clone() + .oneshot(post(uri.clone(), body, ct)) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{uri}: {json}"); + } + + // allowEmpty clears the graph (3 retracts at t=3). + let (status, json) = json_body( + app.clone() + .oneshot(post( + format!("/v1/fluree/sync/sync:test?graph={graph}&allowEmpty=true"), + serde_json::json!({ "@graph": [] }).to_string(), + "application/json", + )) + .await + .unwrap(), + ) + .await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(json.get("t").and_then(serde_json::Value::as_i64), Some(3)); +} diff --git a/fluree-db-transact/src/commit.rs b/fluree-db-transact/src/commit.rs index dda9538438..dc28de83fa 100644 --- a/fluree-db-transact/src/commit.rs +++ b/fluree-db-transact/src/commit.rs @@ -46,6 +46,17 @@ pub struct CommitReceipt { pub t: i64, /// Number of flakes in the commit pub flake_count: usize, + /// Asserted (`op = true`) flakes in the commit. + pub assert_count: usize, + /// Retracted (`op = false`) flakes in the commit. + pub retract_count: usize, +} + +/// Count `(asserts, retracts)` in a flake slice — the split every +/// [`CommitReceipt`] carries alongside its total. +pub fn count_ops(flakes: &[Flake]) -> (usize, usize) { + let asserts = flakes.iter().filter(|f| f.op).count(); + (asserts, flakes.len() - asserts) } /// Output of [`build_commit`]. @@ -910,6 +921,7 @@ fn finalize_state_with_base( // temporal metadata current so the next commit's event-time guard and // dual-stamp decision stay in-memory integer checks. let head_temporal = HeadTemporal::from_commit(&commit_record).or(base.head_temporal); + let (assert_count, retract_count) = count_ops(&commit_record.flakes); // 10. Generate commit metadata flakes let commit_metadata_flakes = { @@ -1051,6 +1063,8 @@ fn finalize_state_with_base( commit_id: commit_cid, t: new_t, flake_count, + assert_count, + retract_count, }; Ok((receipt, new_state)) } diff --git a/fluree-db-transact/src/error.rs b/fluree-db-transact/src/error.rs index 58fae02f22..b391ef2904 100644 --- a/fluree-db-transact/src/error.rs +++ b/fluree-db-transact/src/error.rs @@ -112,6 +112,14 @@ pub enum TransactError { max_bytes: usize, }, + /// Whole-graph scan larger than the memory backstop + #[error( + "whole-graph operation would materialize more than {limit} currently-asserted flakes; \ + this is the memory backstop for graph sync / CLEAR / DROP / COPY / MOVE — raise or \ + disable it with FLUREE_MAX_GRAPH_SCAN_FLAKES (0 disables)" + )] + WholeGraphScanTooLarge { limit: usize }, + /// Invalid template term #[error("Invalid template term: {0}")] InvalidTerm(String), diff --git a/fluree-db-transact/src/ir.rs b/fluree-db-transact/src/ir.rs index acfc9e3340..9c0325c054 100644 --- a/fluree-db-transact/src/ir.rs +++ b/fluree-db-transact/src/ir.rs @@ -200,6 +200,24 @@ pub struct Txn { /// /// See [`GraphMgmtOp`] and `stage_graph_mgmt`. pub graph_mgmt: Option, + + /// Graph-synchronization directive: make the named graph's contents + /// exactly this transaction's insert templates, committing only the + /// delta. + /// + /// When `Some(iri)`, staging adds a second wave after assertion + /// generation (like the upsert wave): every currently-asserted flake in + /// the target graph is pushed as a retraction, and the mixed + /// [`FlakeAccumulator`](crate::generate::FlakeAccumulator) nets + /// retract+assert of the same fact to nothing — so the staged set is + /// exactly `A − B` retractions plus `B − A` assertions. An identical + /// payload stages zero flakes (no commit). + /// + /// The transaction's insert templates must all target this graph (the + /// parser re-homes them) and the graph must be a user graph — reserved + /// system graphs and (for now) the default graph are rejected at + /// staging. `None` for every ordinary transaction. + pub sync_graph: Option, } /// A SPARQL graph-management operation, executed by whole-graph scan/re-home @@ -282,6 +300,7 @@ impl Txn { graph_delta: FxHashMap::default(), namespace_delta: std::collections::HashMap::new(), graph_mgmt: None, + sync_graph: None, } } @@ -391,6 +410,12 @@ impl Txn { } } + /// Set the graph-synchronization target (see [`Txn::sync_graph`]). + pub fn with_sync_graph(mut self, iri: impl Into) -> Self { + self.sync_graph = Some(iri.into()); + self + } + /// Add a WHERE pattern pub fn with_where(mut self, pattern: UnresolvedPattern) -> Self { self.where_patterns.push(pattern); diff --git a/fluree-db-transact/src/lib.rs b/fluree-db-transact/src/lib.rs index 795ac46bfe..ca2c2786c4 100644 --- a/fluree-db-transact/src/lib.rs +++ b/fluree-db-transact/src/lib.rs @@ -74,8 +74,8 @@ pub use namespace::{ SharedNamespaceAllocator, BLANK_NODE_ID_PREFIX, BLANK_NODE_PREFIX, }; pub use parse::{ - parse_transaction, parse_trig_phase1, resolve_trig_meta, NamedGraphBlock, RawObject, RawTerm, - RawTrigMeta, RawTriple, TrigPhase1Result, + parse_sync_transaction, parse_transaction, parse_trig_phase1, resolve_trig_meta, + NamedGraphBlock, RawObject, RawTerm, RawTrigMeta, RawTriple, TrigPhase1Result, }; pub use raw_txn_upload::PendingRawTxnUpload; pub use stage::{generate_txn_id, stage, stage_flakes, StageOptions}; diff --git a/fluree-db-transact/src/lower_cypher_update.rs b/fluree-db-transact/src/lower_cypher_update.rs index 663319a543..b04298b107 100644 --- a/fluree-db-transact/src/lower_cypher_update.rs +++ b/fluree-db-transact/src/lower_cypher_update.rs @@ -219,6 +219,7 @@ impl<'a> CypherLowering<'a> { graph_delta: Default::default(), namespace_delta: std::collections::HashMap::new(), graph_mgmt: None, + sync_graph: None, } } diff --git a/fluree-db-transact/src/lower_sparql_update.rs b/fluree-db-transact/src/lower_sparql_update.rs index fe17b3526b..c76f45ae5c 100644 --- a/fluree-db-transact/src/lower_sparql_update.rs +++ b/fluree-db-transact/src/lower_sparql_update.rs @@ -1086,6 +1086,7 @@ fn lower_insert_data( graph_delta: graph_ids.delta(), namespace_delta: std::collections::HashMap::new(), graph_mgmt: None, + sync_graph: None, }) } @@ -1138,6 +1139,7 @@ fn lower_delete_data( graph_delta: graph_ids.delta(), namespace_delta: std::collections::HashMap::new(), graph_mgmt: None, + sync_graph: None, }) } @@ -1239,6 +1241,7 @@ fn lower_delete_where( graph_delta: FxHashMap::default(), namespace_delta: std::collections::HashMap::new(), graph_mgmt: None, + sync_graph: None, }) } @@ -1301,6 +1304,7 @@ fn lower_delete_where_with_graphs( graph_delta: graph_ids.delta(), namespace_delta: std::collections::HashMap::new(), graph_mgmt: None, + sync_graph: None, }) } @@ -1543,6 +1547,7 @@ fn lower_modify( graph_delta: graph_ids.delta(), namespace_delta: std::collections::HashMap::new(), graph_mgmt: None, + sync_graph: None, }) } diff --git a/fluree-db-transact/src/parse/jsonld.rs b/fluree-db-transact/src/parse/jsonld.rs index c675103f44..5218b79dc0 100644 --- a/fluree-db-transact/src/parse/jsonld.rs +++ b/fluree-db-transact/src/parse/jsonld.rs @@ -163,6 +163,56 @@ pub fn parse_transaction( } } +/// Transaction-local graph id assigned to the sync target graph. The +/// payload may not address named graphs itself (rejected below), so the +/// assigner never hands this id to anything else. +const SYNC_GRAPH_LOCAL_ID: u16 = 2; + +/// Parse a graph-sync transaction (see [`Txn::sync_graph`]). +/// +/// The payload is an ordinary insert-shaped JSON-LD document describing the +/// target graph's DESIRED full contents. Parsing is exactly insert parsing +/// (same context handling, annotation lowering, txn-meta extraction), after +/// which every template is re-homed onto `graph_iri` and the sync directive +/// is stamped on the transaction. +/// +/// Differences from insert: +/// - an explicitly empty document (`"@graph": []`) is allowed — it means +/// "the graph's desired contents are empty" (the API layer gates this +/// behind an explicit opt-in before it becomes a whole-graph clear); +/// - a payload that addresses named graphs itself (`@graph` with a graph +/// `@id`) is rejected: the sync scope is exactly one graph, named by the +/// caller, never inferred from the data. +pub fn parse_sync_transaction( + json: &Value, + graph_iri: &str, + opts: TxnOpts, + ns_registry: &mut NamespaceRegistry, +) -> Result { + let explicitly_empty = json + .get("@graph") + .and_then(Value::as_array) + .is_some_and(Vec::is_empty); + let mut txn = if explicitly_empty { + Txn::insert().with_opts(opts) + } else { + parse_transaction(json, TxnType::Insert, opts, ns_registry)? + }; + if !txn.graph_delta.is_empty() { + return Err(TransactError::Parse( + "sync payload must not address named graphs; the target graph is the sync scope" + .to_string(), + )); + } + for t in &mut txn.insert_templates { + t.graph_id = Some(SYNC_GRAPH_LOCAL_ID); + } + txn.graph_delta + .insert(SYNC_GRAPH_LOCAL_ID, graph_iri.to_string()); + txn.sync_graph = Some(graph_iri.to_string()); + Ok(txn) +} + /// Parse an insert transaction fn parse_insert(json: &Value, opts: TxnOpts, ns_registry: &mut NamespaceRegistry) -> Result { let mut vars = VarRegistry::new(); diff --git a/fluree-db-transact/src/parse/mod.rs b/fluree-db-transact/src/parse/mod.rs index ad57bed43b..0dbe8dd884 100644 --- a/fluree-db-transact/src/parse/mod.rs +++ b/fluree-db-transact/src/parse/mod.rs @@ -47,7 +47,7 @@ pub(crate) const RESERVED_TXN_KEYS: &[&str] = &[ /// never stripped. pub(crate) const CLAUSE_KEYS: &[&str] = &["where", "delete", "insert", "upsert", "values"]; -pub use jsonld::parse_transaction; +pub use jsonld::{parse_sync_transaction, parse_transaction}; pub use nquads::nquads_to_trig; pub use trig_meta::{ extract_trig_txn_meta, parse_trig_phase1, resolve_trig_meta, NamedGraphBlock, RawObject, diff --git a/fluree-db-transact/src/stage.rs b/fluree-db-transact/src/stage.rs index 922e9cd9cf..a68851f767 100644 --- a/fluree-db-transact/src/stage.rs +++ b/fluree-db-transact/src/stage.rs @@ -690,6 +690,40 @@ pub async fn stage( } } + // Graph-sync target: resolve the g_id + graph Sid now, before the + // generator takes `ns_registry` mutably. An unregistered target is a + // first population — nothing to retract (`None` scan). Reserved + // system graphs are refused the same way CLEAR refuses them. + let sync_scan: Option<(GraphId, Sid)> = match &txn.sync_graph { + Some(iri) => { + // Guard the target by shape, independent of registration: + // every entry point (builder, consensus applier, HTTP) meets + // this check, so a malformed IRI can't be registered as a + // graph and the ledger's own system-graph IRIs are refused + // even on a ledger whose registry never seeded them. + fluree_db_core::graph_registry::validate_absolute_graph_iri(iri) + .map_err(|msg| TransactError::Parse(format!("sync target: {msg}")))?; + let ledger_id = ledger.snapshot.ledger_id.as_ref(); + if *iri == fluree_db_core::graph_registry::txn_meta_graph_iri(ledger_id) + || *iri == fluree_db_core::graph_registry::config_graph_iri(ledger_id) + { + return Err(TransactError::ReservedGraphTarget { + graph_iri: iri.clone(), + }); + } + match ledger.snapshot.graph_registry.graph_id_for_iri(iri) { + Some(g_id) if g_id < FIRST_USER_GRAPH_ID => { + return Err(TransactError::ReservedGraphTarget { + graph_iri: iri.clone(), + }); + } + Some(g_id) => Some((g_id, ns_registry.sid_for_iri(iri))), + None => None, + } + } + None => None, + }; + let mut generator = FlakeGenerator::new(new_t, &mut ns_registry, txn_id) .with_graph_sids(graph_sids.clone()); @@ -754,6 +788,44 @@ pub async fn stage( acc.push_retractions(upsert_retractions); } + // Graph-sync wave: push every currently-asserted flake of the target + // graph as a retraction (see [`Txn::sync_graph`]). The accumulator + // nets retract+assert of the same fact to nothing, so what survives + // `finalize()` is exactly `current − payload` retractions plus + // `payload − current` assertions — the delta. Scanned flakes carry + // correct `m` from storage, so (like the upsert wave) no hydration + // is needed. + // + // Policy model follows CLEAR (roadmap O4): the scan is not + // view-policy filtered — sync is an authoritative whole-graph + // replacement, and a view-filtered scan would leave rows the caller + // cannot see in place, breaking "the graph now equals the payload". + // Modify-policy is still enforced on the resulting flakes below. + // + // Scale note: like CLEAR/COPY/MOVE, this materializes the whole + // graph's flakes at staging time; backpressure is the pre-check + // above plus `NoveltyWouldExceed` sizing at commit (which sees only + // the surviving delta). Chunked staging for whole-graph ops is the + // same known follow-up flagged on `scan_graph_flakes`. + if let Some((sync_g_id, sync_graph_sid)) = &sync_scan { + // The scan attributes every flake to the graph Sid, matching the + // payload's assertions — both sides must agree on `flake.g` for + // the accumulator's unchanged-fact cancellation to fire. + let mut sync_retractions = + scan_graph_flakes(&ledger, *sync_g_id, Some(sync_graph_sid), options.tracker) + .await?; + for f in &mut sync_retractions { + f.op = false; + f.t = new_t; + } + tracing::debug!( + graph_id = sync_g_id, + scanned = sync_retractions.len(), + "graph-sync retractions generated" + ); + acc.push_retractions(sync_retractions); + } + let retraction_count = stream_stats.retraction_count; let assertion_count = stream_stats.assertion_count; let total_inputs = acc.input_count(); @@ -1052,8 +1124,41 @@ fn flake_content(f: &Flake) -> FlakeContent { ) } +/// Default for [`whole_graph_scan_limit`]: ~2 GB peak at the accumulator's +/// two-copies-per-fact profile. Any graph that worked before the limit +/// existed still works — whole-graph verbs errored outright on +/// index-resident graphs, and novelty-resident graphs are already bounded +/// well below this by `reindex_max_bytes`. +const DEFAULT_MAX_GRAPH_SCAN_FLAKES: usize = 10_000_000; + +/// Memory backstop for whole-graph scans (graph sync, CLEAR, DROP, COPY, +/// MOVE): staging materializes the target graph's currently-asserted +/// flakes, so peak memory scales with the graph, not the delta — an +/// identical resync of a huge graph is the worst case, and no other guard +/// sees it (`NoveltyWouldExceed` measures only the surviving delta, after +/// materialization). `FLUREE_MAX_GRAPH_SCAN_FLAKES` overrides; `0` +/// disables. Read per call — once per graph-management op, never per +/// flake — so tests and embedders can change it at runtime. +fn whole_graph_scan_limit() -> Option { + match std::env::var("FLUREE_MAX_GRAPH_SCAN_FLAKES") { + Ok(v) => match v.trim().parse::() { + Ok(0) => None, + Ok(n) => Some(n), + Err(_) => Some(DEFAULT_MAX_GRAPH_SCAN_FLAKES), + }, + Err(_) => Some(DEFAULT_MAX_GRAPH_SCAN_FLAKES), + } +} + /// Scan every currently-asserted flake in graph `g_id` (merged snapshot + -/// novelty view as of the ledger's current `t`). +/// novelty view as of the ledger's current `t`), attributed to `g_sid`. +/// +/// Every flake comes back with `g = g_sid` (`None` for the default graph). +/// The range provider materializes index-resident rows with `g: None` +/// regardless of graph — only novelty-resident flakes carry it — and every +/// caller here routes by `flake.g` (`resolve_flake_graph_id`, where `None` +/// is the default graph). Without the stamp, retracting an indexed named +/// graph silently retracted phantoms from the default graph instead. /// /// Scale note: a whole-graph operation (`CLEAR ALL`, a large COPY/MOVE) /// materializes every scanned flake into a `Vec` and re-stages it, and @@ -1075,17 +1180,38 @@ fn flake_content(f: &Flake) -> FlakeContent { async fn scan_graph_flakes( ledger: &LedgerState, g_id: GraphId, + g_sid: Option<&Sid>, tracker: Option<&Tracker>, ) -> Result> { let db_ref = match tracker { Some(t) => ledger.as_graph_db_ref(g_id).with_tracker(t), None => ledger.as_graph_db_ref(g_id), }; - // Unbounded SPOT scan (empty match, `>= min`) returns the whole graph. - db_ref - .range(IndexType::Spot, RangeTest::Ge, RangeMatch::new()) + // `Eq` with an empty match is the whole-graph scan on both range paths: + // the V3 provider treats "nothing bound" as a full-index cursor and + // rejects every other `RangeTest`, and the genesis (overlay-only) path + // matches an empty `Eq` against every flake. `Ge` only ever worked on + // the genesis path, where non-`Eq` tests pass through unfiltered. + // `flake_limit` stops the provider's drain loop mid-scan, so the + // backstop bounds what is materialized, not just what is returned. + let limit = whole_graph_scan_limit(); + let opts = fluree_db_core::RangeOptions { + flake_limit: limit.map(|l| l.saturating_add(1)), + ..Default::default() + }; + let mut flakes = db_ref + .range_with_opts(IndexType::Spot, RangeTest::Eq, RangeMatch::new(), opts) .await - .map_err(|e| TransactError::FlakeGeneration(format!("graph scan failed: {e}"))) + .map_err(|e| TransactError::FlakeGeneration(format!("graph scan failed: {e}")))?; + if let Some(l) = limit { + if flakes.len() > l { + return Err(TransactError::WholeGraphScanTooLarge { limit: l }); + } + } + for f in &mut flakes { + f.g = g_sid.cloned(); + } + Ok(flakes) } /// Resolve the ledger `GraphId` and graph `Sid` for a named graph IRI, if it @@ -1198,10 +1324,12 @@ async fn stage_graph_mgmt( } for (g_id, sid) in targets { - if let Some(sid) = sid { - graph_sids.insert(g_id, sid); + if let Some(sid) = &sid { + graph_sids.insert(g_id, sid.clone()); } - for mut f in scan_graph_flakes(&ledger, g_id, options.tracker).await? { + for mut f in + scan_graph_flakes(&ledger, g_id, sid.as_ref(), options.tracker).await? + { f.op = false; f.t = new_t; flakes.push(f); @@ -1238,7 +1366,7 @@ async fn stage_graph_mgmt( // `from == to` is a spec no-op for ADD/COPY/MOVE. if from != to { // Resolve the source (existing only) and destination. - let (src_g_id, _src_sid): (Option, Option) = match from { + let (src_g_id, src_sid): (Option, Option) = match from { GraphSel::Default => (Some(0), None), GraphSel::Graph(iri) => { match resolve_named_graph(&ledger, &mut ns_registry, iri) { @@ -1310,12 +1438,15 @@ async fn stage_graph_mgmt( } let src_flakes = match src_g_id { - Some(g) => scan_graph_flakes(&ledger, g, options.tracker).await?, + Some(g) => { + scan_graph_flakes(&ledger, g, src_sid.as_ref(), options.tracker).await? + } None => Vec::new(), }; let dest_flakes = - scan_graph_flakes(&ledger, dest_g_id, options.tracker).await?; + scan_graph_flakes(&ledger, dest_g_id, dest_sid.as_ref(), options.tracker) + .await?; let dest_contents: HashSet = dest_flakes.iter().map(flake_content).collect();