From 1f6d249212aaa9bb5527a4f12db4a0c3bb459a0e Mon Sep 17 00:00:00 2001 From: Josh Heinrichs Date: Wed, 14 Oct 2026 15:28:17 -0400 Subject: [PATCH 1/2] libexpr: builtins.tecnixQueryFile, and fingerprints that record observations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the three World-specific manifest builtins from #72 (tectonixManifestEntry, tectonixManifestKeys, tectonixManifestIdToPath) and their `.meta/manifest.json#` synthetic keys with one generic builtin that queries any JSON or TOML file in the repository, and with a fingerprint format that records what was asked and what the answer was. Tecnix no longer knows anything about World. builtins.tecnixQueryFile { path = ../../.meta/manifest.json; # format inferred from the extension op = "get"; # get | has | keys | find at = [ "//areas/tools/dev" ]; # object keys, root down # find only: where = "id"; value = "W-123456"; } The model. A closure maps a repo path to a fingerprint, and a fingerprint records what evaluation observed about the path: `git:;mode=…` is the object observed there, `absent` that none was. A target that only queried a data file observed less than the whole object, so it records the answers it got: ".meta/manifest.json": "query:json\tget\t//areas/tools/dev\t{\"id\":\"W-c10e84\"}\njson\thas\t//areas\ttrue" one line per distinct query, sorted. The key stays the repo path, so everything that only knows paths keeps working (tec's affected-targets stays conservatively correct with no change), and the TXDC row format is untouched. A target that also read the file whole records the oid, which subsumes the queries. #72 had moved the observation into the key instead, which is why it needed a key grammar, a `#` special case in validation, and a tec change. How it works. The builtin reads the file unrecorded through the repo accessor, parses it once per evaluation, evaluates the op, and records an access `\0` through the ordinary recordAccess, so labels, tecnixMemoize replay and thread confinement apply unchanged; the line is built in thread-local scratch and interned on first sight like a path. Flattening groups accesses by file into one `query:` fingerprint. Validation (`dependencyFingerprintHolds`, replacing the string equality in candidateMatches) splits a stored `query:` fingerprint into views, re-asks each line of the file as the current tree serves it (parsed once per thread per run, read unrecorded), and memoizes per (path, line) in the existing thread-local memo. Evaluation and validation run the same evaluateDataQuery on the same parsed tree, so they cannot disagree. A line that does not parse, an unknown op or format, or a file that no longer parses does not hold: a miss, never a stale hit. A missing file answers every query as an empty document would, and that answer is recorded. Format: a line is `\t\t\t\t`, the answer last so a variable number of segments is unambiguous. Fields never contain tab, newline or NUL: the builtin rejects string arguments that do, and answers are `absent`, compact JSON when at most 64 bytes (which escapes control characters), or `sha256:` of it. Parsing a fingerprint is splitting, with no unescaping. Outside the repo accessor the builtin is fromJSON (readFile path) plus the selection; a path pure evaluation forbids throws as readFile would (lstat, not maybeLstat, which reports forbidden paths as absent). Removed: the three builtins, getTecnixManifestJson and its EvalData fields, manifestPathFingerprint and the per-thread parsed manifest in eval-cache.cc. The unsafeTectonixInternalManifest* builtins stay; they record the whole file, correctly. Old cache rows (keys `.meta/manifest.json#…`) read as paths that do not exist and miss; no blob version bump is needed. Tests: src/libexpr-tests/tecnix-data-query.cc (27: op semantics, line round-trips, rejected characters, answer hashing, TOML and JSON of one tree agreeing, every hold/miss case including unparseable and unknown lines, grouping and subsumption in dependencyFingerprints); tests/functional/tecnix/data-query.sh (replaces manifest-tracking.sh, keeping its memoize cases): exact fingerprints per target; across ten commits, adding a zone misses only `keys` and the whole-file reader, `has b` survives b's renumbering, a comment-only TOML edit misses only the whole-file reader, a file whose absence was observed misses when it appears, a dirty data file is queried from disk and its observation follows; argument validation; the builtin outside tracking; the older-rev read. The tecnix suite and the Tecnix unit tests pass. Docs: explainer §4.2 and the fingerprint vocabulary; a guardrail on fingerprints as observations. --- plans/tecnix-data-query/plan.md | 408 ++++++++++++++++ plans/tecnix-target-eval-caching/explainer.md | 28 +- .../tecnix-target-eval-caching/guardrails.md | 1 + src/libexpr-tests/meson.build | 2 +- src/libexpr-tests/tecnix-data-query.cc | 439 ++++++++++++++++++ .../tecnix-manifest-fingerprint.cc | 110 ----- src/libexpr/include/nix/expr/meson.build | 1 + .../include/nix/expr/tecnix/data-query.hh | 150 ++++++ .../include/nix/expr/tecnix/eval-cache.hh | 28 +- .../nix/expr/tecnix/source-accessors.hh | 20 +- src/libexpr/meson.build | 1 + src/libexpr/primops/meson.build | 1 + src/libexpr/primops/tecnix-query.cc | 245 ++++++++++ src/libexpr/primops/tectonix.cc | 140 ------ src/libexpr/tecnix/data-query.cc | 350 ++++++++++++++ src/libexpr/tecnix/eval-cache.cc | 189 ++++---- src/libexpr/tecnix/eval-data.hh | 14 +- src/libexpr/tecnix/repo-accessor.cc | 36 +- tests/functional/tecnix/data-query.sh | 391 ++++++++++++++++ tests/functional/tecnix/manifest-tracking.sh | 284 ----------- tests/functional/tecnix/memoize.sh | 2 +- tests/functional/tecnix/meson.build | 2 +- 22 files changed, 2188 insertions(+), 654 deletions(-) create mode 100644 plans/tecnix-data-query/plan.md create mode 100644 src/libexpr-tests/tecnix-data-query.cc delete mode 100644 src/libexpr-tests/tecnix-manifest-fingerprint.cc create mode 100644 src/libexpr/include/nix/expr/tecnix/data-query.hh create mode 100644 src/libexpr/primops/tecnix-query.cc create mode 100644 src/libexpr/tecnix/data-query.cc create mode 100755 tests/functional/tecnix/data-query.sh delete mode 100755 tests/functional/tecnix/manifest-tracking.sh diff --git a/plans/tecnix-data-query/plan.md b/plans/tecnix-data-query/plan.md new file mode 100644 index 000000000..b02b4580d --- /dev/null +++ b/plans/tecnix-data-query/plan.md @@ -0,0 +1,408 @@ +# `builtins.tecnixQueryFile`: fine-grained dependencies on data files + +Status: implemented on branch `tecnix-query-file` (§6 names the code). Replaces the three World-specific manifest builtins from #72 +(`tectonixManifestEntry`, `tectonixManifestKeys`, `tectonixManifestIdToPath`) +and their `.meta/manifest.json#` synthetic keys with one builtin that queries +any JSON or TOML file in the repo, and a fingerprint format that records what +was asked and what the answer was. Tecnix stops knowing anything about World. + +## 1. Why + +#72 made the mechanism: a target that looks up one zone depends on that +lookup's answer, not on the manifest's blob oid, so adding a zone no longer +invalidates every target. Only three facts in it are World's: the file path, +"top-level keys are zone paths", and "`id` is the field to look up by". +Pulling those into arguments gives a builtin that also serves `Cargo.toml`, +`package.json`, generated lock data, and any config file World wants +fine-grained dependencies on. + +The constraint that fixes the design: **validation is a pure function of the +stored row and the tree, with no evaluation** (explainer §4.1, §8). Warm runs +skip the evaluator entirely, and the cache layer cannot see the revision. So a +query is a fixed operation tecnix can recompute from the file's bytes alone. + +## 2. The model: a fingerprint records an observation of a path + +A source closure is `path → fingerprint`. Today's fingerprints already record +what evaluation observed: `git:;mode=` ("the object at this path was +exactly this"), `absent` ("nothing was here"), `;dirty=` ("and the +uncommitted content hashed to this"). Validation asks whether the current tree +still yields each stored observation; for these, that is string equality with +the current fingerprint. + +This proposal adds one more kind of observation, narrower than the oid: + +``` +query:\n… the file, parsed as a data format, answers these + queries this way +``` + +The key stays the repo path. A target that read the file wholesale records the +oid, as today; a target that only queried it records the sorted set of +`query → answer` lines it observed; a target that did both records the oid, +which subsumes the queries. One path, one entry, however many queries. + +This is the "narrower observed property" format the guardrails anticipate +("future formats may represent narrower observed properties when they can +prove the same oracle"), and #72's synthetic keys were the deviation from it: +they moved the observation into the key, which is why they needed a key +grammar, a `#` special case in validation, and a tec change. With the +observation back in the value: + +- keys are repo paths, period; explainer §4.2 and the `.meta/manifest.json#` + rule go away; +- `tec query affected-targets` is correct unchanged: a manifest edit matches + the key `.meta/manifest.json`, so every target that queried it is selected + (conservatively, as it is for every path today). #2123770 becomes + unnecessary; +- the public shape stays `path = fingerprint` (guardrail), and the TXDC row + format is untouched. + +## 3. The builtin + +```nix +builtins.tecnixQueryFile { + path = ../../.meta/manifest.json; # path (or string) naming a file + format = "json"; # "json" | "toml"; default: from the extension + op = "get"; # "get" | "has" | "keys" | "find" + at = [ "//areas/tools/dev" ]; # list of object keys, root-down + # find only: + # where = "id"; # field of each child of `at` to compare + # value = "W-123456"; # string +} +``` + +| op | returns | when `at` does not resolve | +|---|---|---| +| `get` | the value at `at`, as `fromJSON` would produce it | `null` | +| `has` | `true` if `at` resolves (a present `null` counts) | `false` | +| `keys` | sorted list of the object's attribute names | `null` (also when not an object) | +| `find` | sorted list of child keys `k` of the object at `at` whose field `where` is a string equal to `value` | `[ ]` | + +`at` is a list, not a dotted string: zone paths contain `/` and `.`. Segments, +`where` and `value` are strings in v1; array indices, nested `where` paths and +non-string `find` values can be added as new ops or a tagged segment form later +without disturbing the format (§5 explains why). + +A missing file is not an error: every op sees an absent document (`get` → +`null`, `has` → `false`, `keys` → `null`, `find` → `[ ]`). An unparseable file +is an evaluation error, like `fromJSON`. TOML is parsed with the toml11 parser +`fromTOML` uses, into the same tree shape; datetimes become +`{ _type = "timestamp"; value = ""; }` unconditionally. + +Outside the Tecnix repo accessor (a plain `nix eval` in a checkout) the +builtin is `fromJSON (readFile path)` plus the selection, and the read records +whatever a normal read of that path records. The same resolver code works +tracked and untracked. + +One builtin with an attrset: one slot in the fixed-size base environment +(`baseEnvCapacity`, which #72 ran into; this is net −2 builtins), and a new +operation is a new `op` value. `has` stays because it is finer than `get` (a +zone-root check would otherwise be invalidated by an id change); `find` stays +because it is the generic form of the id → path lookup, which otherwise +depends on the whole file. + +## 4. Recording, flattening, validating + +### 4.1 Recording (eval time) + +The builtin resolves `path` (symlinks like `readFile`), checks that it lies +under the Tecnix repo mount, reads the file **unrecorded** +(`TecnixSourceAccessor::readFileUnrecorded`), parses it once per evaluation +(a concurrent map on `TecnixEvalData`, replacing `tecnixManifestJson`), +evaluates the op, and records one observation into the current frame: + +``` +\0 e.g. .meta/manifest.json\0json\tget\t//areas/tools/dev\t{"id":"W-c10e84"} +``` + +through the ordinary `TrackingContext::recordAccess`. The observation carries +the answer the target actually saw, built into a thread-local scratch buffer; +`internAccess` allocates only on first sight, like a path. From here on the +label machinery does everything it does for paths: memoized values carry it, +`tecnixMemoize` replays it, parallel workers keep it thread-confined. + +### 4.2 Flattening (end of a target's evaluation) + +`dependencyFingerprints` receives the flattened observation strings. It +partitions them: plain paths fingerprint as today; `\0` observations are +grouped by file. A file with a plain observation gets the oid and its query +lines are dropped (the oid subsumes them). A file with only query lines gets +`query:` + the lines, deduplicated, sorted, joined with `\n`. Output is a +`DependencyClosure` as before; nothing downstream (rows, `includeDependencies`, +tec) changes. + +### 4.3 Validation (warm lookup) + +`candidateMatches` currently computes the path's current fingerprint and +compares strings. It becomes `fingerprintHolds(path, stored)`: + +- stored does not begin with `query:` → current fingerprint equals stored, as + today, memoized per path; +- stored begins with `query:` → split into lines on `\n` and fields on `\t` + (`string_view`s, no allocation), and for each line: parse the file (once per + run, per file, shared across threads on the per-run + `DependencyFingerprintCache`; read unrecorded), evaluate the op, encode the + answer, compare with the line's answer. Memoized per `(path, line)`, so a + `has //areas` shared by thousands of targets is evaluated once. A line that + does not parse, an unknown op or format, or a file that does not parse → + `false` (a miss; fail closed). + +The hit path still works directly on the stored row's bytes; the only heap +structure is the parsed document, one per file per run, which plays the role +the git object store plays for oids. + +### 4.4 What this does not change + +Fingerprints are computed by one C++ function on both sides +(`evaluateQuery` → `encodeAnswer`), so int-vs-float, duplicate keys (nlohmann: +last wins), sorting and unicode cannot disagree between eval and validation. +Old rows (keys `.meta/manifest.json#…`, fingerprints `manifest-entry:…`) are +plain paths that don't exist → `absent` ≠ stored → a miss. No TXDC version +bump; `formatIsPinned` says so if it's wrong. + +## 5. Formats + +### 5.1 Observation line + +``` + \t \t \t +``` + +- `format` ∈ `json`, `toml`; `op` ∈ `get`, `has`, `keys`, `find`. +- Fixed-arity args come first (`find`'s `where` and `value`), then the `at` + segments, then the answer. The **answer is always the last field**, so a + variable number of `at` segments is unambiguous, including zero + (`json\tkeys\tsha256:…`). + +Per op: + +``` +json\tget\t\t +json\thas\t\ttrue|false +json\tkeys\t\t +json\tfind\t\t\t\t +``` + +`where` is one field name in v1 (it covers the id lookup); a nested `where` +path would be a second variable-length run and needs a count or a new op, +which is why it is left out until something needs it. + +String arguments may not contain `\t`, `\n` or NUL (an evaluation error; no +config key, zone path, crate name or id does). Compact JSON never contains +them raw, so answers are safe without escaping. This is what makes parsing a +`split`, and why strings-only arguments in v1 cost nothing later: a tagged +segment form (`i:0`) or a new op fits the same grammar. + +### 5.2 Answer + +``` +absent get/keys on a path that does not resolve + when ≤ 64 bytes: {"id":"W-c10e84"}, true, ["//areas/tools/dev"] +sha256: otherwise (the 1.5k-entry key list) +``` + +A bare `absent` cannot collide with JSON (no JSON value starts with `a`). The +64-byte bound is a constant; it only trades row bytes for readability. + +### 5.3 Fingerprint + +``` +query:\n… lines deduplicated and sorted bytewise +``` + +Readable in `nix eval --json` output (`\t`, `\n` escapes), opaque to everything +that only compares strings, and `tec` sees the path it already knows. + +## 6. Implementation + +### 6.1 Pure core: `src/libexpr/tecnix/data-query.{hh,cc}` + +No `EvalState`, no accessor, no tracking. The header is the normative +statement of the format. In outline: + +```cpp +enum class DataFormat { Json, Toml }; +enum class DataQueryOp { Get, Has, Keys, Find }; + +struct DataQuery { // one line, minus the answer; views borrow + DataFormat format; DataQueryOp op; + std::vector at; + std::string_view where, value; // Find only +}; + +class ParsedDataFile { // shared per evaluation / per validation run + static std::shared_ptr parse(DataFormat, std::string_view); // throws DataFileParseError + static std::shared_ptr absent(); + std::shared_ptr findIndex(const DataQuery &) const; // (at, where) → value → keys, lazy +}; + +DataQueryAnswer evaluateDataQuery(const ParsedDataFile &, const DataQuery &); // optional; nullopt = absent +void appendDataQueryAnswer(std::string &, const DataQueryAnswer &); // absent | compact JSON ≤ 64 | sha256: +void appendDataObservation(std::string &, const DataQuery &, std::string_view answer); +std::optional parseDataObservation(std::string_view line); + +std::string dataQueryFingerprint(std::vector lines); // "query:" + sorted, deduplicated +std::optional> dataQueryFingerprintLines(std::string_view); +void appendDataAccess(std::string &, std::string_view file, std::string_view line); // \0 +std::optional splitDataAccess(std::string_view); +``` + +TOML goes through toml11 into the same `nlohmann::json` tree shape +`fromTOML` produces (`fromTOML.cc` itself is untouched). + +### 6.2 Flatten and validation: `tecnix/eval-cache.cc` + +- `dependencyFingerprints` partitions observations and groups queries by file + (§4.2). +- `dependencyFingerprintHolds(accessor, path, stored, cache)` replaces the + string equality in `candidateMatches`. Object fingerprints are memoized per + path as before; observations per `(path, line)` in the same thread-local + cache; parsed documents live on the per-run `DependencyFingerprintCache` + (`DependencyDataFiles`, a `Sync` keyed by `\0`, shared + across threads; a null entry is a file that does not parse). +- Validation reads the data file through `readSourceFileUnrecorded` + (`source-accessors.hh`), which uses `TecnixSourceAccessor::readFileUnrecorded` + when it can and a plain read otherwise (unit tests pass a + `MemorySourceAccessor`). Today validation runs before the target's tracking + context exists, so a recorded read would be harmless, but #80 validates + memo rows *inside* a tracked evaluation, where it would put the whole file + back into the enclosing target's closure. +- The manifest thread-locals and `manifestPathFingerprint` are gone. + +### 6.3 The builtin: `primops/tecnix-query.cc` + +Argument validation naming the argument in every error; `realisePath` like +`readFile`; `tecnixRepoRelativePath` (`source-accessors.hh`) decides whether +the file is under the repo accessor's mount; the per-evaluation document cache +is `TecnixEvalData::dataFileCache`; the answer becomes a value through +`parseJSON` of its compact dump; the observation is built in thread-local +scratch buffers and recorded only when tracking is active and the file is in +the repo. Outside the repo the read is an ordinary one (`lstat`, not +`maybeLstat`: a path pure evaluation forbids must throw as `readFile` would, +not read as absent). + +### 6.4 Removals + +`tectonixManifestEntry`, `tectonixManifestKeys`, `tectonixManifestIdToPath` +and docs; `getTecnixManifestJson` and `TecnixEvalData::tecnixManifestJson{,Flag}`. +Nothing on World `main` calls the three builtins; Burke's open stack +feature-detects them and falls back to whole-file reads. The +`unsafeTectonixInternalManifest*` builtins stay: World's non-repo-rooted path +uses them, and they correctly record the whole file. + +### 6.5 Tests + +Unit, `src/libexpr-tests/tecnix-data-query.cc` (replaces +`tecnix-manifest-fingerprint.cc`): + +- line round-trips; rejected characters; answer encoding thresholds +- op semantics including every absent case +- `fingerprintHolds`: a stored `get` holds when other entries change and key + order changes; fails when its entry changes, appears or disappears; `keys` + holds unless the key set changes; `find` holds unless the match set + changes; the same tree as JSON and as TOML; reformat and comments hold; + missing file: stored `absent`/`false` answers hold until the queried entry + appears; unparseable file never holds; unknown op or format never holds +- grouping: oid subsumes query lines; lines dedupe and sort; two files, two + entries +- the parsed-document cache follows the accessor across + `DependencyFingerprintCache` generations (today's + `FollowsTheAccessorAcrossCaches`) +- `find` via the index equals a naive scan on a generated 2k-entry document + +Functional, `tests/functional/tecnix/data-query.sh` (replaces +`manifest-tracking.sh`, keeping its memoize cases): a JSON manifest and a TOML +file; one target per op, one that queries twice (two lines, one entry), one +that reads the whole file (oid only); across commits: adding an entry hits +`get`/`has`/`find` and misses `keys` and the whole-file reader; changing one +entry misses only that entry's `get` and the `find` that pointed at it; a +comment-only TOML edit hits everything but the whole-file reader; a dirty +working-tree edit is seen by the value and by validation; an older rev of a +moved-on checkout reads that rev's file; tracked and untracked evaluations +give identical values; `base-env.sh` still passes. + +Mutation checks: dropping the unrecorded read makes a query target's closure +contain the oid (the test checks it is absent); dropping the subsumption emits both. + +Bench: `tecnix-eval-bench.cc` gains validating 1.5k `get` lines plus 1.5k +`find` lines against a 90 KB document. + +### 6.6 Docs + +Explainer §4: fingerprints record observations; the vocabulary gains `query:`; +§4.2 is deleted. Guardrails: "Public dependency output remains +path-to-fingerprint data" gains "a fingerprint may record a narrower +observation than the object id; validation must prove the observed result". +The builtin's `.doc` is the normative statement of §3 and §5. + +## 7. World and tec + +Everything funnels through the one file Burke's #2121412 introduces, +`manifest-access.nix`: + +```nix +let + q = args: builtins.tecnixQueryFile ({ path = ../../.meta/manifest.json; format = "json"; } // args); +in { + entry = zonePath: q { op = "get"; at = [ zonePath ]; }; # { id = …; } or null + hasZone = zonePath: q { op = "has"; at = [ zonePath ]; }; + zonePaths = q { op = "keys"; at = [ ]; }; + pathsForId = id: q { op = "find"; at = [ ]; where = "id"; value = id; }; +} +``` + +with the existing fallbacks (whole-file `fromJSON (readFile …)` when +`builtins ? tecnixQueryFile` is false; `unsafeTectonixInternalManifest*` when +not repo-rooted). `__invertedManifestById.${id}` becomes `pathsForId id` +(`[ ]` → not found, `[ p ]` → `p`, more → duplicate, as the inverted builtin +throws today). `isZone` in per-crate Rust becomes `hasZone`. The +`repoRooted = pathExists ../../.meta/manifest.json` check still records the +oid; #2117071 moves it, or `q { op = "has"; at = [ ]; }` is an existence +check that depends on no content. + +Landing order: tecnix PR → `tecnix-package.nix` bump → `manifest-access.nix`. +No tec change is required. Later, a precise `affected-targets` is one small +builtin, `builtins.tecnixFingerprintsHold { gitDir; rev; closure; }`, that +runs §4.3's `fingerprintHolds` over a stored closure at a revision; tec never +parses a fingerprint. + +Housekeeping: the in-repo `//system/tecnix` copy is pinned to a September fork +commit without #72 and would have to catch up before anyone pointed the +overlay at it. + +## 8. Cost + +Eval side, per call: force a small attrset, a map lookup, the op, a Value +conversion, one scratch-buffer write, one `internAccess` (allocates on first +sight only). Validation: each file parsed once per run and shared across +threads (stand-in measurement: 0.4 ms for the 91 KB manifest); each distinct +`(path, line)` evaluated once (~3 ms for all 1.5k entries); `find` O(1) after +a one-time index. Rows: a query entry's string is the target's sorted lines, +so two targets share it only when they asked exactly the same questions; +~100–200 bytes per target against today's ~187-entry closures. Not doing now: +a persisted `(file oid, line) → holds` memo; it needs no format change if it +is ever wanted. + +## 9. Interaction with #80 (`tecnixPersistentMemo`) + +Validation reads must be unrecorded (§6.2). On a memo hit, #80 replays the +row's closure into the enclosing frame; for a `query:` entry that means +recording each line as a `\0` observation, so the enclosing target +records the queries, not the oid. Queries and memos stay complementary: +queries make rows depend on less, memos make misses cheaper. + +## 10. Rejected shapes + +- **Synthetic keys** (#72's `.meta/manifest.json#…`, and a generic + `/` or `//#` form). Moves the observation into the key, + so it needs an exact key grammar, a special case in validation that parses + keys, and a tec change to map keys back to files. The observation already + lives in the value for every other fingerprint. +- **A Nix function as the query, with early cutoff.** Validation would + evaluate Nix for every query whose inputs changed, needs an evaluator at the + revision inside the cache layer, and `affected-targets` could no longer + decide without evaluating. Belongs in #80's memo, not in target validation. +- **Auto-tracking `fromJSON` attribute selection.** Hooks in the evaluator's + hottest path, precision depends on how calling code is written, and id + lookups still depend on the whole file. diff --git a/plans/tecnix-target-eval-caching/explainer.md b/plans/tecnix-target-eval-caching/explainer.md index 25218af3a..5268ee0fc 100644 --- a/plans/tecnix-target-eval-caching/explainer.md +++ b/plans/tecnix-target-eval-caching/explainer.md @@ -177,6 +177,7 @@ git:;mode= clean file or directory, at this commit's git:;mode=;dirty= git base, plus a hash of uncommitted content beneath the path absent the path does not currently exist absent;dirty= no git object exists, but uncommitted content does +query:\n… a data file answers these queries this way (§4.2) ``` This path-level fingerprinting is intentionally conservative. There is room to refine the closure model from "this path's full fingerprint" toward "the specific property of this path that evaluation observed." For example, a directory listing depends on the complete child name/type map it returned — which also proves that no other child names were present — not necessarily on the full tree object ID or on every descendant's content. The current tree fingerprint is exact enough to avoid stale cache hits, but it can over-invalidate; a future closure format could record operation-specific observations such as existence, file type, executable bit, symlink target, directory child name/type sets, or file contents separately. @@ -202,23 +203,26 @@ This is the precise sense in which the system works across commits. After a comm The same property makes the system fail-safe. A row is never believed on the strength of its key; it is believed only when proven. Format changes, corruption, and storage defects therefore all degrade into cache misses, never into wrong answers. In practice, the difference between a cold run and a warm run is the difference between roughly a minute of full tracked evaluation and well under a second — several orders of magnitude, with the gap consisting precisely of "all of evaluation" versus "fingerprint checks and output construction." -### 4.2 Synthetic keys: parts of the manifest +### 4.2 Fingerprints are observations; `query:` records answers, not objects -Not every closure key is a file. A target that looks up one zone in `.meta/manifest.json` depends on that zone's entry, not on the whole file, and recording the file would invalidate every such target whenever any zone is added or renumbered. The manifest builtins therefore record *synthetic* keys, each naming one observation of the manifest. This is the complete list: +A fingerprint records what evaluation observed about a path, and a closure holds while the tree still yields the same observations. `git:;mode=` is the object observed at the path; `absent` records that none was. For a *data file* that a target only queried, the object is more than the target observed: a target that looks up one zone in `.meta/manifest.json` depends on that zone's entry, not on the whole file, and recording the file's oid would invalidate every such target whenever any zone is added or renumbered. `builtins.tecnixQueryFile` therefore records the question it asked and the answer it saw, and the file's closure entry becomes a list of such observations: -| key | recorded by | fingerprint changes exactly when | -|---|---|---| -| `.meta/manifest.json#` | `builtins.tectonixManifestEntry zonePath` | the JSON value of that zone's entry (any field, not only `id`) changes, appears, or disappears | -| `.meta/manifest.json#keys` | `builtins.tectonixManifestKeys` | the set of zone paths (manifest keys) changes | -| `.meta/manifest.json#id/` | `builtins.tectonixManifestIdToPath zoneId` | a different zone path, or none, is the first in sorted order whose entry has the id `zoneId` | +``` +".meta/manifest.json": "query:json\tget\t//areas/tools/dev\t{\"id\":\"W-c10e84\"}\njson\thas\t//areas\ttrue" +``` + +The key is still the repo path, so everything that only knows paths keeps working: a change to the file still maps to every target that read or queried it (`tec query affected-targets` is conservative, not wrong), and the row format is untouched. What changes is how validation proves the entry: an object fingerprint must equal the current one; a `query:` fingerprint holds when every line, re-asked of the file as the current tree serves it, gets the recorded answer. A target that both queried a file and read it whole records the oid alone, which implies every answer. + +The format, exactly (`src/libexpr/include/nix/expr/tecnix/data-query.hh` is normative): -The builtins read the manifest through the same accessor as every other tracked read (§7), so a value and the fingerprint of its key always describe the same manifest. The grammar is exact: +- `query:` followed by observation lines separated by `\n`, deduplicated and sorted bytewise. +- A line is `\t\t\t\t`: `json` or `toml`; `get`, `has`, `keys` or `find` (`find` carries `where` and `value` before the segments); the object keys walked from the root; and the answer last, so a variable number of segments is unambiguous. +- An answer is `absent`, the compact JSON of the result when that is at most 64 bytes, or `sha256:` of it otherwise. Compact JSON escapes control characters, and the builtin rejects string arguments containing tab, newline or NUL, so a line is split, never unescaped. +- Validation and evaluation compute answers with the same function on the same parsed document, so they cannot disagree. A line that does not parse, an op or format this evaluator does not know, or a file that no longer parses, does not hold: a miss, never a stale hit. -- A key is synthetic if and only if it begins with `.meta/manifest.json#`. Every other key is a repo-relative path, even one that contains `#`; splitting arbitrary keys at `#` is wrong. -- The fragment after that prefix is `keys`, or `id/` followed by a zone id (everything after `id/`, verbatim), or a zone path, which begins with `//`. `tectonixManifestEntry` given an argument that does not begin with `//` records the whole file, `.meta/manifest.json`, so the three forms never overlap. -- A synthetic key depends on the file `.meta/manifest.json`. A consumer of closure keys (for example, one mapping changed files to affected targets) that cannot interpret a fragment must treat any change to that file as affecting the key; one that can compares the named observation of the manifest before and after the change. +The file is parsed once per evaluation (shared by every query on it) and once per validation run (shared across threads); each distinct `(path, line)` is checked once per run, so the cost scales with unique observations, like path fingerprints. Reads of the queried file record nothing themselves: the query is the observation. A missing file answers every query as an empty document would, and that answer is recorded, so a target that observed a file's absence is invalidated when the file appears with the key it asked about, and not before. -Reading the whole manifest (the `unsafeTectonixInternalManifest*` builtins, or `builtins.readFile` of the file) still records the plain path `.meta/manifest.json`. +This is the first instance of the refinement anticipated above, from "this path's full fingerprint" toward "the property of this path that evaluation observed". Reading the whole manifest (the `unsafeTectonixInternalManifest*` builtins, or `builtins.readFile` of the file) still records the object fingerprint. --- diff --git a/plans/tecnix-target-eval-caching/guardrails.md b/plans/tecnix-target-eval-caching/guardrails.md index 58ab7e273..646f7c494 100644 --- a/plans/tecnix-target-eval-caching/guardrails.md +++ b/plans/tecnix-target-eval-caching/guardrails.md @@ -83,6 +83,7 @@ Use this as a review checklist for source-dependency tracking and target-eval ca - **Public dependency output remains path-to-fingerprint data.** - Internal storage may use IDs/dictionaries, but the public dependency shape remains compact `path = fingerprint` entries. + - A fingerprint records an observation of the path, not necessarily its object id: `query:` fingerprints record the answers a data file gave (explainer §4.2). Validation must prove the observed result by recomputing it from the current tree with the same code evaluation used, and anything it cannot interpret is a miss. ## Public API boundaries diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index f42699d73..1951457a9 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -67,9 +67,9 @@ sources = files( 'nix_api_value_internal.cc', 'primops.cc', 'search-path.cc', + 'tecnix-data-query.cc', 'tecnix-dependency-tracking.cc', 'tecnix-eval-cache-row.cc', - 'tecnix-manifest-fingerprint.cc', 'tecnix-worldtree.cc', 'tectonix.cc', 'trivial.cc', diff --git a/src/libexpr-tests/tecnix-data-query.cc b/src/libexpr-tests/tecnix-data-query.cc new file mode 100644 index 000000000..ee5e4a34c --- /dev/null +++ b/src/libexpr-tests/tecnix-data-query.cc @@ -0,0 +1,439 @@ +#include +#include +#include +#include + +#include + +#include "nix/expr/tecnix/data-query.hh" +#include "nix/expr/tecnix/eval-cache.hh" +#include "nix/util/memory-source-accessor.hh" + +namespace nix { + +namespace { + +constexpr auto manifestPath = ".meta/manifest.json"; + +constexpr auto twoZones = R"({ "//zones/a": { "id": "W-000001" }, "//zones/b": { "id": "W-000002" } })"; +constexpr auto threeZones = + R"({ "//zones/a": { "id": "W-000001" }, "//zones/b": { "id": "W-000002" }, "//zones/c": { "id": "W-000003" } })"; +constexpr auto bRenumbered = R"({ "//zones/a": { "id": "W-000001" }, "//zones/b": { "id": "W-000009" } })"; +constexpr auto reordered = R"({ "//zones/b": { "id": "W-000002" }, "//zones/a": { "id": "W-000001" } })"; +constexpr auto twoZonesToml = R"( +# the same tree as twoZones +["//zones/a"] +id = "W-000001" + +["//zones/b"] +id = "W-000002" +)"; + +/** A source tree with one file. */ +ref treeWith(std::string_view path, std::string content) +{ + auto accessor = make_ref(); + accessor->addFile(CanonPath(path), std::move(content)); + return accessor; +} + +std::vector at(std::initializer_list segments) +{ + return segments; +} + +DataQuery get(std::initializer_list segments, DataFormat format = DataFormat::Json) +{ + return {.format = format, .op = DataQueryOp::Get, .at = at(segments)}; +} + +DataQuery has(std::initializer_list segments) +{ + return {.op = DataQueryOp::Has, .at = at(segments)}; +} + +DataQuery keys(std::initializer_list segments) +{ + return {.op = DataQueryOp::Keys, .at = at(segments)}; +} + +DataQuery find(std::string_view where, std::string_view value, std::initializer_list segments = {}) +{ + return {.op = DataQueryOp::Find, .at = at(segments), .where = where, .value = value}; +} + +ParsedDataFile parsed(std::string_view content, DataFormat format = DataFormat::Json) +{ + return parseDataFile(format, content); +} + +std::string answer(const ParsedDataFile & file, const DataQuery & query) +{ + std::string out; + appendDataQueryAnswer(out, evaluateDataQuery(file, query)); + return out; +} + +std::string line(const ParsedDataFile & file, const DataQuery & query) +{ + std::string out; + appendDataObservation(out, query, answer(file, query)); + return out; +} + +/** The fingerprint a target that asked exactly `queries` of `content` records. */ +std::string fingerprintOf(std::string_view content, const std::vector & queries) +{ + auto file = parsed(content); + std::vector lines; + for (auto & query : queries) + lines.push_back(line(file, query)); + return dataQueryFingerprint({lines.begin(), lines.end()}); +} + +/** Whether `fingerprint`, recorded for `manifestPath`, holds against a tree with `content` there. */ +bool holdsAgainst(std::string_view fingerprint, std::optional content) +{ + DependencyFingerprintCache cache; + auto tree = content ? treeWith(manifestPath, *content) : make_ref(); + return dependencyFingerprintHolds(tree, manifestPath, fingerprint, cache); +} + +} // namespace + +// --------------------------------------------------------------------------- +// Queries +// --------------------------------------------------------------------------- + +TEST(TecnixDataQuery, GetReturnsTheValueOrAbsent) +{ + auto file = parsed(twoZones); + EXPECT_EQ(answer(file, get({"//zones/a"})), R"({"id":"W-000001"})"); + EXPECT_EQ(answer(file, get({"//zones/a", "id"})), R"("W-000001")"); + EXPECT_EQ(answer(file, get({"//zones/c"})), "absent"); + // Descending through a string, not an object + EXPECT_EQ(answer(file, get({"//zones/a", "id", "x"})), "absent"); + // The whole document + EXPECT_EQ(answer(file, get({})), R"({"//zones/a":{"id":"W-000001"},"//zones/b":{"id":"W-000002"}})"); +} + +TEST(TecnixDataQuery, HasIsPresenceNotTruthiness) +{ + auto file = parsed(R"({ "a": null, "b": false })"); + EXPECT_EQ(answer(file, has({"a"})), "true"); + EXPECT_EQ(answer(file, has({"b"})), "true"); + EXPECT_EQ(answer(file, has({"c"})), "false"); + EXPECT_EQ(answer(file, has({})), "true"); +} + +TEST(TecnixDataQuery, KeysAreSortedAndOnlyForObjects) +{ + auto file = parsed(reordered); + EXPECT_EQ(answer(file, keys({})), R"(["//zones/a","//zones/b"])"); + EXPECT_EQ(answer(file, keys({"//zones/a", "id"})), "absent"); + EXPECT_EQ(answer(file, keys({"nope"})), "absent"); +} + +TEST(TecnixDataQuery, FindReturnsEveryMatchingKeySorted) +{ + auto file = parsed( + R"({ "zones": { "//b": { "id": "X" }, "//a": { "id": "X" }, "//c": { "id": "Y" }, "//d": 3, "//e": { "id": 7 } } })"); + EXPECT_EQ(answer(file, find("id", "X", {"zones"})), R"(["//a","//b"])"); + EXPECT_EQ(answer(file, find("id", "Y", {"zones"})), R"(["//c"])"); + EXPECT_EQ(answer(file, find("id", "Z", {"zones"})), "[]"); + // Only string fields compare + EXPECT_EQ(answer(file, find("id", "7", {"zones"})), "[]"); + // No object there: empty, not absent + EXPECT_EQ(answer(file, find("id", "X", {"elsewhere"})), "[]"); +} + +TEST(TecnixDataQuery, FindOnALargeDocumentMatchesAScan) +{ + nlohmann::json doc = nlohmann::json::object(); + for (int i = 0; i < 2000; i++) + doc["//zone/" + std::to_string(i)] = {{"id", "W-" + std::to_string(i % 700)}}; + auto file = parsed(doc.dump()); + for (int i = 0; i < 700; i += 97) { + auto id = "W-" + std::to_string(i); + std::vector expected; + for (auto & [key, value] : doc.items()) + if (value["id"] == id) + expected.push_back(key); + EXPECT_EQ(answer(file, find("id", id)), nlohmann::json(expected).dump()) << id; + } +} + +TEST(TecnixDataQuery, AbsentFileAnswersEveryQuery) +{ + ParsedDataFile file; + EXPECT_EQ(answer(file, get({"x"})), "absent"); + EXPECT_EQ(answer(file, has({"x"})), "false"); + EXPECT_EQ(answer(file, keys({})), "absent"); + EXPECT_EQ(answer(file, find("id", "x")), "[]"); +} + +TEST(TecnixDataQuery, LongAnswersAreHashed) +{ + std::string big = R"({"k":")" + std::string(100, 'x') + R"("})"; + auto file = parsed(big); + auto hashed = answer(file, get({})); + EXPECT_TRUE(hashed.starts_with("sha256:")) << hashed; + EXPECT_EQ(hashed.size(), 7 + 64); + // ...and are still a function of the content. + EXPECT_NE(hashed, answer(parsed(R"({"k":")" + std::string(100, 'y') + R"("})"), get({}))); +} + +TEST(TecnixDataQuery, TomlParsesToTheSameTree) +{ + auto json = parsed(twoZones); + auto toml = parsed(twoZonesToml, DataFormat::Toml); + EXPECT_EQ(answer(json, get({"//zones/a"})), answer(toml, get({"//zones/a"}))); + EXPECT_EQ(answer(json, keys({})), answer(toml, keys({}))); + EXPECT_EQ(answer(json, find("id", "W-000002")), answer(toml, find("id", "W-000002"))); + + auto typed = + parsed("i = 1\nf = 1.5\nb = true\ns = \"s\"\na = [1, 2]\nd = 1979-05-27T07:32:00Z\n", DataFormat::Toml); + EXPECT_EQ(answer(typed, get({"i"})), "1"); + EXPECT_EQ(answer(typed, get({"f"})), "1.5"); + EXPECT_EQ(answer(typed, get({"b"})), "true"); + EXPECT_EQ(answer(typed, get({"s"})), R"("s")"); + EXPECT_EQ(answer(typed, get({"a"})), "[1,2]"); + EXPECT_EQ(answer(typed, get({"d", "_type"})), R"("timestamp")"); +} + +TEST(TecnixDataQuery, UnparseableFilesThrow) +{ + EXPECT_THROW(parsed("{ nope"), DataFileParseError); + EXPECT_THROW(parsed("= nope", DataFormat::Toml), DataFileParseError); +} + +TEST(TecnixDataQuery, FormatFromExtension) +{ + EXPECT_EQ(dataFormatFromExtension(".meta/manifest.json"), DataFormat::Json); + EXPECT_EQ(dataFormatFromExtension("Cargo.toml"), DataFormat::Toml); + EXPECT_EQ(dataFormatFromExtension("dir.json/file"), std::nullopt); + EXPECT_EQ(dataFormatFromExtension("Makefile"), std::nullopt); + EXPECT_EQ(dataFormatFromExtension("x.yaml"), std::nullopt); +} + +// --------------------------------------------------------------------------- +// Observation lines +// --------------------------------------------------------------------------- + +TEST(TecnixDataQuery, LinesRoundTrip) +{ + std::vector> cases = { + {get({"//zones/a"}), R"({"id":"W-000001"})"}, + {get({}), "absent"}, + {get({"a", "b", "c"}, DataFormat::Toml), "1"}, + {has({"x"}), "true"}, + {keys({}), "sha256:0000000000000000000000000000000000000000000000000000000000000000"}, + {find("id", "W-000001"), R"(["//zones/a"])"}, + {find("id", "W-000001", {"zones", "nested"}), "[]"}, + // Empty segments are keys too. + {get({""}), "absent"}, + {get({"", ""}), "absent"}, + }; + for (auto & [query, answer] : cases) { + std::string encoded; + appendDataObservation(encoded, query, answer); + auto parsed = parseDataObservation(encoded); + ASSERT_TRUE(parsed) << encoded; + EXPECT_EQ(parsed->query, query) << encoded; + EXPECT_EQ(parsed->answer, answer) << encoded; + std::string reencoded; + appendDataObservation(reencoded, parsed->query, parsed->answer); + EXPECT_EQ(reencoded, encoded); + } +} + +TEST(TecnixDataQuery, LinesAreTabSeparatedWithTheAnswerLast) +{ + std::string encoded; + appendDataObservation(encoded, find("id", "W-1", {"zones"}), "[]"); + EXPECT_EQ(encoded, "json\tfind\tid\tW-1\tzones\t[]"); + encoded.clear(); + appendDataObservation(encoded, keys({}), "absent"); + EXPECT_EQ(encoded, "json\tkeys\tabsent"); +} + +TEST(TecnixDataQuery, MalformedLinesDoNotParse) +{ + EXPECT_FALSE(parseDataObservation("")); + EXPECT_FALSE(parseDataObservation("json\tget")); + EXPECT_FALSE(parseDataObservation("yaml\tget\tabsent")); + EXPECT_FALSE(parseDataObservation("json\tselect\tx\tabsent")); + EXPECT_FALSE(parseDataObservation("json\tfind\tid\tabsent")); + EXPECT_TRUE(parseDataObservation("json\tfind\tid\tW-1\t[]")); +} + +TEST(TecnixDataQuery, FieldsMayNotContainSeparators) +{ + EXPECT_TRUE(isValidDataQueryField("//zones/a")); + EXPECT_TRUE(isValidDataQueryField("")); + EXPECT_TRUE(isValidDataQueryField("with space and \"quotes\" and ;semicolons")); + EXPECT_FALSE(isValidDataQueryField("a\tb")); + EXPECT_FALSE(isValidDataQueryField("a\nb")); + EXPECT_FALSE(isValidDataQueryField(std::string_view("a\0b", 3))); +} + +// --------------------------------------------------------------------------- +// Fingerprints +// --------------------------------------------------------------------------- + +TEST(TecnixDataQuery, FingerprintLinesAreSortedAndDeduplicated) +{ + auto fp = dataQueryFingerprint({"json\tkeys\tabsent", "json\tget\ta\t1", "json\tkeys\tabsent"}); + EXPECT_EQ(fp, "query:json\tget\ta\t1\njson\tkeys\tabsent"); + auto lines = dataQueryFingerprintLines(fp); + ASSERT_TRUE(lines); + EXPECT_EQ(*lines, (std::vector{"json\tget\ta\t1", "json\tkeys\tabsent"})); +} + +TEST(TecnixDataQuery, ObjectFingerprintsAreNotQueries) +{ + EXPECT_FALSE(dataQueryFingerprintLines("git:0123;mode=100644")); + EXPECT_FALSE(dataQueryFingerprintLines("absent")); + EXPECT_FALSE(dataQueryFingerprintLines("query:")); +} + +TEST(TecnixDataQuery, ObservationsSplitAtTheFirstNul) +{ + std::string observation; + appendDataAccess(observation, "x/y.json", "json\tget\ta\t1"); + auto split = splitDataAccess(observation); + ASSERT_TRUE(split); + EXPECT_EQ(split->file, "x/y.json"); + EXPECT_EQ(split->line, "json\tget\ta\t1"); + EXPECT_FALSE(splitDataAccess("lib/common.nix")); +} + +// --------------------------------------------------------------------------- +// Validation: does a recorded fingerprint still hold against a tree? +// --------------------------------------------------------------------------- + +TEST(TecnixDataQueryHolds, EntryQueryIgnoresOtherEntries) +{ + auto fp = fingerprintOf(twoZones, {get({"//zones/a"})}); + EXPECT_TRUE(holdsAgainst(fp, twoZones)); + EXPECT_TRUE(holdsAgainst(fp, threeZones)); + EXPECT_TRUE(holdsAgainst(fp, bRenumbered)); + EXPECT_TRUE(holdsAgainst(fp, reordered)); + EXPECT_TRUE(holdsAgainst(fp, "{\n \"//zones/a\": {\n \"id\": \"W-000001\"\n }\n}\n")); +} + +TEST(TecnixDataQueryHolds, EntryQueryFollowsItsOwnEntry) +{ + auto fp = fingerprintOf(twoZones, {get({"//zones/b"})}); + EXPECT_TRUE(holdsAgainst(fp, twoZones)); + EXPECT_FALSE(holdsAgainst(fp, bRenumbered)); + EXPECT_FALSE(holdsAgainst(fp, R"({ "//zones/a": { "id": "W-000001" } })")); + EXPECT_FALSE(holdsAgainst(fp, std::nullopt)); +} + +TEST(TecnixDataQueryHolds, AbsentEntryHoldsUntilItAppears) +{ + auto fp = fingerprintOf(twoZones, {get({"//zones/c"}), has({"//zones/c"})}); + EXPECT_TRUE(holdsAgainst(fp, twoZones)); + EXPECT_TRUE(holdsAgainst(fp, bRenumbered)); + // A missing file gives the same answers. + EXPECT_TRUE(holdsAgainst(fp, std::nullopt)); + EXPECT_FALSE(holdsAgainst(fp, threeZones)); +} + +TEST(TecnixDataQueryHolds, KeysFollowTheKeySetOnly) +{ + auto fp = fingerprintOf(twoZones, {keys({})}); + EXPECT_TRUE(holdsAgainst(fp, twoZones)); + EXPECT_TRUE(holdsAgainst(fp, bRenumbered)); + EXPECT_TRUE(holdsAgainst(fp, reordered)); + EXPECT_FALSE(holdsAgainst(fp, threeZones)); +} + +TEST(TecnixDataQueryHolds, FindFollowsTheMatchSet) +{ + auto fp = fingerprintOf(twoZones, {find("id", "W-000002")}); + EXPECT_TRUE(holdsAgainst(fp, twoZones)); + EXPECT_TRUE(holdsAgainst(fp, threeZones)); + EXPECT_FALSE(holdsAgainst(fp, bRenumbered)); + EXPECT_FALSE(holdsAgainst(fp, R"({ "//zones/b": { "id": "W-000002" }, "//zones/z": { "id": "W-000002" } })")); +} + +TEST(TecnixDataQueryHolds, EveryLineMustHold) +{ + auto fp = fingerprintOf(twoZones, {get({"//zones/a"}), get({"//zones/b"})}); + EXPECT_TRUE(holdsAgainst(fp, threeZones)); + EXPECT_FALSE(holdsAgainst(fp, bRenumbered)); +} + +TEST(TecnixDataQueryHolds, UnparseableAndUnknownNeverHold) +{ + auto fp = fingerprintOf(twoZones, {get({"//zones/c"})}); + EXPECT_FALSE(holdsAgainst(fp, "{ nope")); + EXPECT_FALSE(holdsAgainst("query:yaml\tget\tx\tabsent", twoZones)); + EXPECT_FALSE(holdsAgainst("query:json\tselect\tx\tabsent", twoZones)); + EXPECT_FALSE(holdsAgainst("query:", twoZones)); + EXPECT_FALSE(holdsAgainst("query:\n", twoZones)); +} + +TEST(TecnixDataQueryHolds, ObjectFingerprintsCompareAsBefore) +{ + // A MemorySourceAccessor fingerprints every path with its own fingerprint. + DependencyFingerprintCache cache; + auto tree = treeWith(manifestPath, twoZones); + tree->fingerprint = "git:0123;mode=100644"; + auto current = dependencyFingerprint(tree, manifestPath, cache); + ASSERT_TRUE(current); + EXPECT_EQ(*current, "git:0123;mode=100644"); + EXPECT_TRUE(dependencyFingerprintHolds(tree, manifestPath, *current, cache)); + EXPECT_FALSE(dependencyFingerprintHolds(tree, manifestPath, "absent", cache)); + EXPECT_FALSE(dependencyFingerprintHolds(tree, manifestPath, "git:4567;mode=100644", cache)); +} + +TEST(TecnixDataQueryHolds, FollowsTheAccessorAcrossCaches) +{ + // The parsed document must not outlive the run it was read for: another + // evaluation on the same thread validates against its own tree. + auto fp = fingerprintOf(twoZones, {get({"//zones/b"})}); + EXPECT_TRUE(holdsAgainst(fp, twoZones)); + EXPECT_FALSE(holdsAgainst(fp, bRenumbered)); + EXPECT_TRUE(holdsAgainst(fp, twoZones)); +} + +// --------------------------------------------------------------------------- +// Flattening: observations to closure entries +// --------------------------------------------------------------------------- + +TEST(TecnixDataQueryClosure, QueriesGroupByFileAndTheObjectSubsumesThem) +{ + DependencyFingerprintCache cache; + auto tree = make_ref(); + tree->addFile(CanonPath(manifestPath), twoZones); + tree->addFile(CanonPath("config.toml"), "a = 1\n"); + tree->addFile(CanonPath("lib.nix"), "{}"); + tree->fingerprint = "git:0123;mode=100644"; + + auto manifest = parsed(twoZones); + auto config = parsed("a = 1\n", DataFormat::Toml); + std::string obsA, obsB, obsConfig, obsLib; + appendDataAccess(obsA, manifestPath, line(manifest, get({"//zones/a"}))); + appendDataAccess(obsB, manifestPath, line(manifest, get({"//zones/b"}))); + appendDataAccess(obsConfig, "config.toml", line(config, get({"a"}, DataFormat::Toml))); + + auto closure = dependencyFingerprints(tree, {obsB, "lib.nix", obsA, obsConfig, obsA}, cache); + ASSERT_EQ(closure.size(), 3u); + EXPECT_EQ(closure[0].path, manifestPath); + EXPECT_EQ(closure[0].fingerprint, fingerprintOf(twoZones, {get({"//zones/a"}), get({"//zones/b"})})); + EXPECT_EQ(closure[1].path, "config.toml"); + EXPECT_EQ(closure[1].fingerprint, "query:toml\tget\ta\t1"); + EXPECT_EQ(closure[2].path, "lib.nix"); + EXPECT_FALSE(closure[2].fingerprint.starts_with("query:")); + + // Reading the manifest whole as well: its object fingerprint subsumes the queries. + auto whole = dependencyFingerprints(tree, {obsA, std::string(manifestPath)}, cache); + ASSERT_EQ(whole.size(), 1u); + EXPECT_EQ(whole[0].path, manifestPath); + EXPECT_EQ(whole[0].fingerprint, dependencyFingerprint(tree, manifestPath, cache)); +} + +} // namespace nix diff --git a/src/libexpr-tests/tecnix-manifest-fingerprint.cc b/src/libexpr-tests/tecnix-manifest-fingerprint.cc deleted file mode 100644 index c7b6d999f..000000000 --- a/src/libexpr-tests/tecnix-manifest-fingerprint.cc +++ /dev/null @@ -1,110 +0,0 @@ -#include -#include -#include - -#include - -#include "nix/expr/tecnix/eval-cache.hh" -#include "nix/util/memory-source-accessor.hh" - -namespace nix { - -namespace { - -/** A source tree whose only file is `.meta/manifest.json` with the given content. */ -ref treeWithManifest(std::string manifest) -{ - auto accessor = make_ref(); - accessor->addFile(CanonPath(".meta/manifest.json"), std::move(manifest)); - return accessor; -} - -/** Fingerprint one synthetic manifest path, with a fresh cache like a fresh evaluation. */ -std::optional fingerprint(ref accessor, std::string_view syntheticPath) -{ - DependencyFingerprintCache cache; - return dependencyFingerprint(accessor, syntheticPath, cache); -} - -constexpr auto twoZones = R"({ "//zones/a": { "id": "W-000001" }, "//zones/b": { "id": "W-000002" } })"; -constexpr auto threeZones = - R"({ "//zones/a": { "id": "W-000001" }, "//zones/b": { "id": "W-000002" }, "//zones/c": { "id": "W-000003" } })"; -constexpr auto bRenumbered = R"({ "//zones/a": { "id": "W-000001" }, "//zones/b": { "id": "W-000009" } })"; - -} // namespace - -TEST(TecnixManifestFingerprint, EntryFingerprintIgnoresOtherEntries) -{ - auto entryA = ".meta/manifest.json#//zones/a"; - auto before = fingerprint(treeWithManifest(twoZones), entryA); - ASSERT_TRUE(before); - EXPECT_EQ(fingerprint(treeWithManifest(threeZones), entryA), before); - EXPECT_EQ(fingerprint(treeWithManifest(bRenumbered), entryA), before); -} - -TEST(TecnixManifestFingerprint, EntryFingerprintFollowsItsOwnEntry) -{ - auto entryB = ".meta/manifest.json#//zones/b"; - EXPECT_NE(fingerprint(treeWithManifest(twoZones), entryB), fingerprint(treeWithManifest(bRenumbered), entryB)); - EXPECT_NE( - fingerprint(treeWithManifest(twoZones), ".meta/manifest.json#//zones/a"), - fingerprint(treeWithManifest(twoZones), entryB)); -} - -TEST(TecnixManifestFingerprint, AbsentEntryHasItsOwnFingerprint) -{ - auto entryC = ".meta/manifest.json#//zones/c"; - auto absent = fingerprint(treeWithManifest(twoZones), entryC); - ASSERT_TRUE(absent); - EXPECT_NE(fingerprint(treeWithManifest(threeZones), entryC), absent); -} - -TEST(TecnixManifestFingerprint, KeyFingerprintTracksTheKeySetOnly) -{ - auto keys = ".meta/manifest.json#keys"; - auto before = fingerprint(treeWithManifest(twoZones), keys); - ASSERT_TRUE(before); - EXPECT_NE(fingerprint(treeWithManifest(threeZones), keys), before); - // Renumbering a zone changes no key. - EXPECT_EQ(fingerprint(treeWithManifest(bRenumbered), keys), before); -} - -TEST(TecnixManifestFingerprint, KeyFingerprintIgnoresKeyOrder) -{ - auto reordered = R"({ "//zones/b": { "id": "W-000002" }, "//zones/a": { "id": "W-000001" } })"; - EXPECT_EQ( - fingerprint(treeWithManifest(twoZones), ".meta/manifest.json#keys"), - fingerprint(treeWithManifest(reordered), ".meta/manifest.json#keys")); -} - -TEST(TecnixManifestFingerprint, IdFingerprintTracksWhereTheIdPoints) -{ - auto idOfB = ".meta/manifest.json#id/W-000002"; - auto before = fingerprint(treeWithManifest(twoZones), idOfB); - ASSERT_TRUE(before); - // Adding an unrelated zone leaves the lookup alone; renumbering removes the id. - EXPECT_EQ(fingerprint(treeWithManifest(threeZones), idOfB), before); - EXPECT_NE(fingerprint(treeWithManifest(bRenumbered), idOfB), before); -} - -TEST(TecnixManifestFingerprint, FollowsTheAccessorAcrossCaches) -{ - // The parsed manifest must not outlive the cache it was read for: another - // evaluation on the same thread fingerprints against its own commit. - auto entryB = ".meta/manifest.json#//zones/b"; - auto first = fingerprint(treeWithManifest(twoZones), entryB); - auto second = fingerprint(treeWithManifest(bRenumbered), entryB); - EXPECT_NE(first, second); - EXPECT_NE( - fingerprint(treeWithManifest(twoZones), ".meta/manifest.json#keys"), - fingerprint(treeWithManifest(threeZones), ".meta/manifest.json#keys")); -} - -TEST(TecnixManifestFingerprint, NoManifestMeansNoFingerprint) -{ - EXPECT_EQ(fingerprint(make_ref(), ".meta/manifest.json#keys"), std::nullopt); - // ...and that must not leak into the next tree either. - EXPECT_TRUE(fingerprint(treeWithManifest(twoZones), ".meta/manifest.json#keys")); -} - -} // namespace nix diff --git a/src/libexpr/include/nix/expr/meson.build b/src/libexpr/include/nix/expr/meson.build index 2b31e77ef..972574a63 100644 --- a/src/libexpr/include/nix/expr/meson.build +++ b/src/libexpr/include/nix/expr/meson.build @@ -40,6 +40,7 @@ headers = [ config_pub_h ] + files( 'static-string-data.hh', 'symbol-table.hh', 'tecnix/access-set-graph.hh', + 'tecnix/data-query.hh', 'tecnix/eval-cache-row.hh', 'tecnix/eval-cache.hh', 'tecnix/force-value.hh', diff --git a/src/libexpr/include/nix/expr/tecnix/data-query.hh b/src/libexpr/include/nix/expr/tecnix/data-query.hh new file mode 100644 index 000000000..47dc200d8 --- /dev/null +++ b/src/libexpr/include/nix/expr/tecnix/data-query.hh @@ -0,0 +1,150 @@ +#pragma once +/** + * @file + * + * Queries on data files (`builtins.tecnixQueryFile`) and the observations + * they leave in a target's source closure. + * + * A fingerprint records what evaluation observed about a path, and a closure + * holds while the tree still yields the same observations. A target that only + * queried a data file observed the answers it got, not the whole object, so + * its fingerprint for the file is + * + * query:\n… + * + * one sorted line per distinct query, each + * + * \t \t \t \t + * + * with the answer last, so a variable number of segments is unambiguous. + * Fields never contain tab, newline or NUL (the builtin rejects arguments + * that do; answers are a bare word, a hash, or compact JSON, which escapes + * control characters), so parsing is splitting, with no unescaping. + * + * Everything here is pure, so the evaluator (which records an answer while + * a target runs) and the cache validator (which recomputes it against the + * current tree, with no evaluator at hand) share one implementation. + */ + +#include "nix/util/error.hh" + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +enum class DataFormat { Json, Toml }; + +std::string_view dataFormatName(DataFormat format); +std::optional parseDataFormat(std::string_view name); +/** The format a file name's extension implies (`.json`, `.toml`), if any. */ +std::optional dataFormatFromExtension(std::string_view fileName); + +enum class DataQueryOp { + /** The value at `at`, or absent. */ + Get, + /** Whether `at` resolves (a present `null` counts). */ + Has, + /** The sorted keys of the object at `at`, or absent if there is no object there. */ + Keys, + /** The sorted keys of the children of the object at `at` whose string + field `where` equals `value`; empty when there is no object there. */ + Find, +}; + +std::string_view dataQueryOpName(DataQueryOp op); +std::optional parseDataQueryOp(std::string_view name); + +/** + * One question asked of a data file. The views borrow from whatever the + * query was built or parsed from (the builtin's forced arguments, or an + * observation line), so a `DataQuery` does not outlive that. + */ +struct DataQuery +{ + DataFormat format = DataFormat::Json; + DataQueryOp op = DataQueryOp::Get; + /** Object keys, root down. */ + std::vector at; + /** `Find` only: the field of each child to compare. */ + std::string_view where; + /** `Find` only: the string the field must equal. */ + std::string_view value; + + bool operator==(const DataQuery &) const = default; +}; + +/** Whether a string may be a field of an observation line (no tab, newline or NUL). */ +bool isValidDataQueryField(std::string_view s); + +/** The file exists but is not valid ``. */ +MakeError(DataFileParseError, Error); + +/** Parse `content` as `format`; throws `DataFileParseError`. */ +nlohmann::json parseDataFile(DataFormat format, std::string_view content); + +/** + * A data file's parsed document, or nullopt for a file that does not exist, + * which answers every query as an empty document would. + */ +using ParsedDataFile = std::optional; + +/** A query's answer: the resulting JSON value, or nullopt for absent. */ +using DataQueryAnswer = std::optional; + +DataQueryAnswer evaluateDataQuery(const ParsedDataFile & file, const DataQuery & query); + +/** + * Append the encoding of an answer: `absent`; the compact JSON when it is + * short; otherwise `sha256:` of the compact JSON. + */ +void appendDataQueryAnswer(std::string & out, const DataQueryAnswer & answer); + +/** Append the observation line for `query` with an already encoded `answer`. */ +void appendDataObservation(std::string & out, const DataQuery & query, std::string_view answer); + +struct DataObservation +{ + DataQuery query; + std::string_view answer; +}; + +/** Parse one observation line; the views borrow from `line`. Nullopt if malformed. */ +std::optional parseDataObservation(std::string_view line); + +/** Every fingerprint built from observation lines begins with this. */ +constexpr std::string_view dataQueryFingerprintPrefix = "query:"; + +/** The fingerprint recording the observation `lines` (deduplicated and sorted). */ +std::string dataQueryFingerprint(std::vector lines); + +/** + * The observation lines of a `query:` fingerprint, or nullopt if `fingerprint` + * is not one (an object fingerprint, or a `query:` with no lines, which + * nothing ever writes). + */ +std::optional> dataQueryFingerprintLines(std::string_view fingerprint); + +/** + * What the builtin records into a tracking frame: the repo path of the file + * and one observation line, as `\0`. Repo paths contain no NUL, so + * the first NUL is the separator and a plain path observation never has one. + * Flattening a closure groups these by file into one `query:` fingerprint. + */ +void appendDataAccess(std::string & out, std::string_view file, std::string_view line); + +struct DataAccess +{ + std::string_view file; + std::string_view line; +}; + +/** Split an observation; nullopt for a plain path. */ +std::optional splitDataAccess(std::string_view observation); + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/tecnix/eval-cache.hh b/src/libexpr/include/nix/expr/tecnix/eval-cache.hh index ee26b2e24..3a1f23c04 100644 --- a/src/libexpr/include/nix/expr/tecnix/eval-cache.hh +++ b/src/libexpr/include/nix/expr/tecnix/eval-cache.hh @@ -33,8 +33,9 @@ using DependencyClosure = std::vector; /** * Per-run token for the thread-local fingerprint memo: fingerprints are - * memoized per unique path for the lifetime of one of these, so validation - * cost scales with unique paths, not total closure entries. + * memoized per unique path, and data-file observations per unique (path, + * line), for the lifetime of one of these, so validation cost scales with + * unique observations, not total closure entries. */ struct DependencyFingerprintCache { @@ -43,12 +44,31 @@ struct DependencyFingerprintCache DependencyFingerprintCache(); }; +/** The current fingerprint of the object at `path` (`git:;mode=`, `absent`, ...). */ std::optional dependencyFingerprint(ref accessor, std::string_view path, DependencyFingerprintCache & cache); -/** Fingerprint every path, throwing if any path cannot be certified. */ +/** + * Whether the tree `accessor` serves still satisfies a stored fingerprint for + * `path`: an object fingerprint must equal the current one; a `query:` + * fingerprint (see data-query.hh) holds when every observation line still gets + * the answer it records. Anything unreadable or unknown does not hold. + */ +bool dependencyFingerprintHolds( + ref accessor, + std::string_view path, + std::string_view fingerprint, + DependencyFingerprintCache & cache); + +/** + * The closure of a set of observations: plain paths fingerprint as the + * objects at them; data observations (`\0`, see data-query.hh) + * are grouped by file into one `query:` fingerprint each, unless the file + * was also observed whole, in which case its object fingerprint subsumes + * them. Throws if any path cannot be certified. Sorted by path. + */ DependencyClosure dependencyFingerprints( - ref accessor, const std::vector & paths, DependencyFingerprintCache & cache); + ref accessor, const std::vector & observations, DependencyFingerprintCache & cache); /** * Reserved cache key under which target discovery stores its closure and diff --git a/src/libexpr/include/nix/expr/tecnix/source-accessors.hh b/src/libexpr/include/nix/expr/tecnix/source-accessors.hh index 9f9451881..d8c019bab 100644 --- a/src/libexpr/include/nix/expr/tecnix/source-accessors.hh +++ b/src/libexpr/include/nix/expr/tecnix/source-accessors.hh @@ -17,12 +17,24 @@ const std::string & getManifestContent(const EvalState & state); const nlohmann::json & getManifestJson(const EvalState & state); StorePath getLegacyTectonixZoneStorePath(EvalState & state, std::string_view zonePath); ref getTecnixRepoAccessor(EvalState & state); + +/** + * The regular file at `path`, read without recording a Tecnix dependency on + * it, or nullopt if there is no regular file there. For a caller that records + * its own, finer-grained observation of the file instead (`tecnixQueryFile`, + * and the cache validator checking such an observation). Through a Tecnix + * repo accessor the read is genuinely unrecorded; any other accessor records + * whatever its reads record. + */ +std::optional readSourceFileUnrecorded(SourceAccessor & accessor, const CanonPath & path); + /** - * `.meta/manifest.json` as the Tecnix repo accessor serves it (the view that - * also fingerprints Tecnix dependencies), parsed once per evaluation. The - * read records nothing: callers record the part of the manifest they observe. + * Where `path` lies in the Tecnix repo, if it is under the repo accessor's + * mount (`mountTecnixRepoAccessor`): the repo-relative path, which is what + * tracked reads of it record and what fingerprints are keyed by. Nullopt + * for any other path, and when no repo accessor has been mounted. */ -const nlohmann::json & getTecnixManifestJson(EvalState & state); +std::optional tecnixRepoRelativePath(EvalState & state, const SourcePath & path); /** Resolve a checkout's HEAD to a commit SHA (throws if it has none). */ std::string resolveCheckoutHeadRev(const std::string & checkoutPath); diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index f58c7e081..ca2da2fa8 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -188,6 +188,7 @@ sources = files( 'root-value.cc', 'search-path.cc', 'symbol-table.cc', + 'tecnix/data-query.cc', 'tecnix/eval-cache.cc', 'tecnix/repo-accessor.cc', 'tecnix/source-accessors.cc', diff --git a/src/libexpr/primops/meson.build b/src/libexpr/primops/meson.build index 0b6419137..5dcad3513 100644 --- a/src/libexpr/primops/meson.build +++ b/src/libexpr/primops/meson.build @@ -10,6 +10,7 @@ sources += files( 'fetchMercurial.cc', 'fetchTree.cc', 'fromTOML.cc', + 'tecnix-query.cc', 'tecnix.cc', 'tectonix.cc', ) diff --git a/src/libexpr/primops/tecnix-query.cc b/src/libexpr/primops/tecnix-query.cc new file mode 100644 index 000000000..60657ef8b --- /dev/null +++ b/src/libexpr/primops/tecnix-query.cc @@ -0,0 +1,245 @@ +/** + * `builtins.tecnixQueryFile`: query a JSON or TOML file and depend on the + * answer, not on the file. The query engine and the observation format are in + * tecnix/data-query.cc; the cache validator re-asks the recorded questions in + * tecnix/eval-cache.cc. + */ + +#include "tecnix/eval-data.hh" +#include "nix/expr/eval-inline.hh" +#include "nix/expr/json-to-value.hh" +#include "nix/expr/primops.hh" +#include "nix/expr/tecnix/data-query.hh" +#include "nix/expr/tecnix/source-accessors.hh" +#include "nix/expr/tecnix/source-deps.hh" +#include "nix/expr/tecnix/thread-state.hh" + +namespace nix { + +/** + * Where a queried file lives: in the Tecnix repo (read unrecorded, each query + * records its own observation under `repoPath`), or anywhere else (read like + * `readFile`, recording whatever that records). + */ +struct QueriedFile +{ + SourcePath path; + std::optional repoPath; +}; + +static std::optional readQueriedFile(EvalState & state, const QueriedFile & file) +{ + if (file.repoPath) + return readSourceFileUnrecorded(*getTecnixRepoAccessor(state), *file.repoPath); + + // Outside the repo accessor this is an ordinary existence check and read. + // Existence is recorded at the primop layer, like `pathExists`. + if (currentTecnixThreadState.trackingContext && file.path.accessor->tracksEvalAccesses(file.path.path)) + file.path.accessor->recordEvalAccess(file.path.path); + // `maybeLstat` answers "absent" for a path pure evaluation forbids; only + // `lstat` tells a missing file (FileNotFound, so: absent) from a forbidden + // one (RestrictedPathError, which propagates, as it does from `readFile`). + SourceAccessor::Stat st; + try { + st = file.path.lstat(); + } catch (FileNotFound &) { + return std::nullopt; + } + if (st.type != SourceAccessor::tRegular) + return std::nullopt; + return file.path.readFile(); +} + +/** The parsed file, once per evaluation and shared by every query on it. */ +static std::shared_ptr +queriedDataFile(EvalState & state, const PosIdx pos, const QueriedFile & file, DataFormat format) +{ + auto & cache = *state.tecnixEvalData().dataFileCache; + + std::string key = file.path.to_string(); + key += '\0'; + key += dataFormatName(format); + + std::shared_ptr parsed; + if (cache.cvisit(key, [&](const auto & kv) { parsed = kv.second; })) + return parsed; + + try { + auto content = readQueriedFile(state, file); + parsed = std::make_shared( + content ? ParsedDataFile(parseDataFile(format, *content)) : ParsedDataFile()); + } catch (DataFileParseError & e) { + state.error("while parsing '%s' as %s: %s", file.path, dataFormatName(format), e.what()) + .atPos(pos) + .debugThrow(); + } + // Two threads parsing the same content produce interchangeable documents. + cache.try_emplace(std::move(key), parsed); + return parsed; +} + +static std::string_view forceQueryField(EvalState & state, const PosIdx pos, Value & v, std::string_view argument) +{ + auto s = + state.forceStringNoCtx(v, pos, fmt("while evaluating the '%s' argument to builtins.tecnixQueryFile", argument)); + if (!isValidDataQueryField(s)) + state + .error( + "the '%s' argument to builtins.tecnixQueryFile may not contain a tab, newline or NUL", argument) + .atPos(pos) + .debugThrow(); + return s; +} + +static void prim_tecnixQueryFile(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.tecnixQueryFile"); + auto & attrs = *args[0]->attrs(); + + const Attr *pathAttr = nullptr, *formatAttr = nullptr, *opAttr = nullptr, *atAttr = nullptr, *whereAttr = nullptr, + *valueAttr = nullptr; + for (auto & attr : attrs) { + std::string_view name = state.symbols[attr.name]; + if (name == "path") + pathAttr = &attr; + else if (name == "format") + formatAttr = &attr; + else if (name == "op") + opAttr = &attr; + else if (name == "at") + atAttr = &attr; + else if (name == "where") + whereAttr = &attr; + else if (name == "value") + valueAttr = &attr; + else + state.error("unexpected attribute '%s' passed to builtins.tecnixQueryFile", name) + .atPos(attr.pos) + .debugThrow(); + } + for (auto [attr, name] : {std::pair{pathAttr, "path"}, std::pair{opAttr, "op"}, std::pair{atAttr, "at"}}) + if (!attr) + state.error("builtins.tecnixQueryFile: missing attribute '%s'", name).atPos(pos).debugThrow(); + + DataQuery query; + + auto opName = state.forceStringNoCtx( + *opAttr->value, opAttr->pos, "while evaluating the 'op' argument to builtins.tecnixQueryFile"); + if (auto op = parseDataQueryOp(opName)) + query.op = *op; + else + state + .error( + "unknown op '%s' passed to builtins.tecnixQueryFile (expected get, has, keys or find)", opName) + .atPos(opAttr->pos) + .debugThrow(); + + QueriedFile file{.path = state.realisePath(pathAttr->pos, *pathAttr->value)}; + file.repoPath = tecnixRepoRelativePath(state, file.path); + + if (formatAttr) { + auto formatName = state.forceStringNoCtx( + *formatAttr->value, formatAttr->pos, "while evaluating the 'format' argument to builtins.tecnixQueryFile"); + if (auto format = parseDataFormat(formatName)) + query.format = *format; + else + state + .error( + "unknown format '%s' passed to builtins.tecnixQueryFile (expected json or toml)", formatName) + .atPos(formatAttr->pos) + .debugThrow(); + } else if (auto format = dataFormatFromExtension(file.path.path.abs())) { + query.format = *format; + } else { + state + .error( + "cannot infer the format of '%s' from its extension; pass 'format' to builtins.tecnixQueryFile", + file.path) + .atPos(pathAttr->pos) + .debugThrow(); + } + + state.forceList(*atAttr->value, atAttr->pos, "while evaluating the 'at' argument to builtins.tecnixQueryFile"); + query.at.reserve(atAttr->value->listSize()); + for (auto * segment : atAttr->value->listView()) + query.at.push_back(forceQueryField(state, atAttr->pos, *segment, "at")); + + if (query.op == DataQueryOp::Find) { + if (!whereAttr || !valueAttr) + state.error("builtins.tecnixQueryFile requires 'where' and 'value' attributes for op 'find'") + .atPos(pos) + .debugThrow(); + query.where = forceQueryField(state, whereAttr->pos, *whereAttr->value, "where"); + query.value = forceQueryField(state, valueAttr->pos, *valueAttr->value, "value"); + } else if (whereAttr || valueAttr) { + state.error("builtins.tecnixQueryFile accepts 'where' and 'value' only for op 'find'") + .atPos(pos) + .debugThrow(); + } + + auto answer = evaluateDataQuery(*queriedDataFile(state, pos, file, query.format), query); + + // The observation: this file, this question, this answer. Built in scratch + // buffers so a repeated query allocates nothing; interning copies the + // string on first sight only, like a path. + if (auto ctx = currentTecnixThreadState.trackingContext; ctx && file.repoPath) { + static thread_local std::string encodedAnswer, line, observation; + encodedAnswer.clear(); + appendDataQueryAnswer(encodedAnswer, answer); + line.clear(); + appendDataObservation(line, query, encodedAnswer); + observation.clear(); + appendDataAccess(observation, file.repoPath->rel(), line); + ctx->recordAccess(observation); + } + + if (!answer) + v.mkNull(); + else + parseJSON(state, answer->dump(), v); +} + +static RegisterPrimOp primop_tecnixQueryFile({ + .name = "__tecnixQueryFile", + .args = {"query"}, + .doc = R"( + Query a JSON or TOML file, depending on the answer rather than on the file. + + ```nix + builtins.tecnixQueryFile { + path = ../../.meta/manifest.json; # a file, as a path or string + format = "json"; # "json" or "toml"; default: from the extension + op = "get"; # "get", "has", "keys" or "find" + at = [ "//areas/tools/dev" ]; # object keys, root down + # find only: + # where = "id"; # field of each child of `at` + # value = "W-123456"; # the string it must equal + } + ``` + + | `op` | returns | when `at` does not resolve | + |--------|-------------------------------------------------------------------------|----------------------------| + | `get` | the value at `at`, as `builtins.fromJSON` would produce it | `null` | + | `has` | `true` if `at` resolves (a present `null` counts) | `false` | + | `keys` | the sorted attribute names of the object at `at` | `null` (also for a non-object) | + | `find` | the sorted keys of the children of the object at `at` whose string field `where` equals `value` | `[ ]` | + + `at` is a list, not a dotted string, so keys may contain `/` and `.`. + Segments, `where` and `value` are strings and may not contain a tab, + newline or NUL. A missing file is not an error: every op sees an absent + document. A file that does not parse is an error. TOML datetimes become + `{ _type = "timestamp"; value = "..."; }`. + + Under Tecnix source tracking, a query on a file in the repository records + what was asked and what the answer was, not the file: the target's + closure entry for the file becomes a `query:` fingerprint listing one + observation per distinct query, and the entry stays valid while the file + keeps giving those answers, whatever else in it changes. A target that + also reads the file whole depends on the whole file, as before. Outside + the repository the builtin is `fromJSON (readFile path)` plus the + selection. + )", + .impl = prim_tecnixQueryFile, +}); + +} // namespace nix diff --git a/src/libexpr/primops/tectonix.cc b/src/libexpr/primops/tectonix.cc index 00f235813..8925e3437 100644 --- a/src/libexpr/primops/tectonix.cc +++ b/src/libexpr/primops/tectonix.cc @@ -107,146 +107,6 @@ static RegisterPrimOp primop_worldManifestInverted({ .impl = prim_worldManifestInverted, }); -// ============================================================================ -// builtins.tectonixManifestEntry zonePath -// Returns null or { id = "W-xxxxxx" } for a single zone, recording only that -// entry as a tracked dependency (synthetic path .meta/manifest.json#) -// instead of the whole manifest file. The doc below is the one statement of -// the synthetic-key grammar; eval-cache.cc fingerprints the keys. -// ============================================================================ -static void prim_tectonixManifestEntry(EvalState & state, const PosIdx pos, Value ** args, Value & v) -{ - auto zonePath = state.forceStringNoCtx( - *args[0], pos, "while evaluating the 'zonePath' argument to builtins.tectonixManifestEntry"); - if (auto ctx = currentTecnixThreadState.trackingContext; ctx) { - // Manifest keys start with `//`. Anything else could spell one of the - // other key forms (`keys`, `id/...`), so it records the whole file. - if (zonePath.starts_with("//")) - ctx->recordAccess(".meta/manifest.json#" + std::string(zonePath)); - else - ctx->recordAccess(".meta/manifest.json"); - } - auto & manifest = getTecnixManifestJson(state); - auto it = manifest.find(std::string(zonePath)); - if (it == manifest.end() || !it->is_object() || !it->contains("id") || !(*it).at("id").is_string()) { - v.mkNull(); - return; - } - auto zoneAttrs = state.buildBindings(1); - zoneAttrs.alloc("id").mkString((*it).at("id").get(), state.mem); - v.mkAttrs(zoneAttrs); -} - -static RegisterPrimOp primop_tectonixManifestEntry({ - .name = "__tectonixManifestEntry", - .args = {"zonePath"}, - .doc = R"( - Get a single zone's manifest entry as a Nix attrset `{ id = "W-xxxxxx"; }`, - or null if the manifest has no entry for `zonePath` (or its entry has no - string `id`). Reads `.meta/manifest.json` through the Tecnix repository - view (the evaluated rev plus the checkout's uncommitted changes) that - every tracked read and every dependency fingerprint uses. - - Under Tecnix source tracking, the manifest builtins record what they - read of `.meta/manifest.json` as synthetic dependency keys, not the - whole file. These are all of them: - - - `.meta/manifest.json#`, from this builtin: the entry for - `zonePath`. Its fingerprint changes when the entry's JSON value (any - field, not only `id`) changes, appears or disappears. `zonePath` is - recorded verbatim; manifest keys start with `//`, and an argument - that does not records the whole file, `.meta/manifest.json`, instead. - - `.meta/manifest.json#keys`, from `builtins.tectonixManifestKeys`: the - set of zone paths (manifest keys). - - `.meta/manifest.json#id/`, from - `builtins.tectonixManifestIdToPath`: which zone path has the id - `zoneId` (the first in sorted order), or that none has. `` is - everything after `id/`, verbatim. - - Only keys that begin with `.meta/manifest.json#` are synthetic; every - other dependency key is a repo-relative path, even one that contains - `#`. A synthetic key depends on the file `.meta/manifest.json`: a - consumer of dependency keys that cannot interpret the part after the - `#` must treat any change to that file as affecting the key. - - Example: `builtins.tectonixManifestEntry "//areas/tools/dev"` returns `{ id = "W-123456"; }`. - )", - .impl = prim_tectonixManifestEntry, -}); - -// ============================================================================ -// builtins.tectonixManifestKeys -// Returns the sorted list of zone paths (manifest keys), recording only the -// key set as a tracked dependency (synthetic path .meta/manifest.json#keys). -// ============================================================================ -static void prim_tectonixManifestKeys(EvalState & state, const PosIdx pos, Value ** args, Value & v) -{ - if (auto ctx = currentTecnixThreadState.trackingContext; ctx) - ctx->recordAccess(".meta/manifest.json#keys"); - auto & manifest = getTecnixManifestJson(state); - std::vector keys; - keys.reserve(manifest.size()); - for (auto & [path, value] : manifest.items()) - keys.push_back(path); - std::sort(keys.begin(), keys.end()); - auto list = state.buildList(keys.size()); - for (size_t i = 0; i < keys.size(); i++) { - auto * val = state.allocValue(); - val->mkString(keys[i], state.mem); - list[i] = val; - } - v.mkList(list); -} - -static RegisterPrimOp primop_tectonixManifestKeys({ - .name = "__tectonixManifestKeys", - .args = {}, - .doc = R"( - Get the sorted list of zone paths (manifest keys). Records only the key - set as a Tecnix tracked dependency (synthetic path .meta/manifest.json#keys), - not the whole manifest file. Only world-wide folds that enumerate every - zone should use this; per-target resolution should use tectonixManifestEntry. - Reads the manifest through the same Tecnix repository view as - tectonixManifestEntry, whose documentation lists every synthetic key. - )", - .impl = prim_tectonixManifestKeys, -}); - -// ============================================================================ -// builtins.tectonixManifestIdToPath zoneId -// Returns null or the zone path for a zone ID, recording only that id lookup -// as a tracked dependency (synthetic path .meta/manifest.json#id/). -// ============================================================================ -static void prim_tectonixManifestIdToPath(EvalState & state, const PosIdx pos, Value ** args, Value & v) -{ - auto zoneId = state.forceStringNoCtx( - *args[0], pos, "while evaluating the 'zoneId' argument to builtins.tectonixManifestIdToPath"); - if (auto ctx = currentTecnixThreadState.trackingContext; ctx) - ctx->recordAccess(".meta/manifest.json#id/" + std::string(zoneId)); - auto & manifest = getTecnixManifestJson(state); - for (auto & [path, value] : manifest.items()) { - if (value.contains("id") && value.at("id").is_string() && value.at("id").get() == zoneId) { - v.mkString(path, state.mem); - return; - } - } - v.mkNull(); -} - -static RegisterPrimOp primop_tectonixManifestIdToPath({ - .name = "__tectonixManifestIdToPath", - .args = {"zoneId"}, - .doc = R"( - Get the zone path for a zone ID, or null if not found. Records only that - id lookup as a Tecnix tracked dependency (synthetic path - .meta/manifest.json#id/), not the whole manifest file. Reads the - manifest through the same Tecnix repository view as - tectonixManifestEntry, whose documentation lists every synthetic key. - Example: `builtins.tectonixManifestIdToPath "W-123456"` returns `"//areas/tools/dev"`. - )", - .impl = prim_tectonixManifestIdToPath, -}); - static std::string normalizeTrackedRepoPath(std::string_view path); // ============================================================================ diff --git a/src/libexpr/tecnix/data-query.cc b/src/libexpr/tecnix/data-query.cc new file mode 100644 index 000000000..5471230a4 --- /dev/null +++ b/src/libexpr/tecnix/data-query.cc @@ -0,0 +1,350 @@ +#include "nix/expr/tecnix/data-query.hh" + +#include "nix/util/hash.hh" + +#include "expr-config-private.hh" + +#include +#include + +#include + +namespace nix { + +using namespace std::literals; + +void DataFileParseError::anchor() {} + +/** Answers up to this long are stored verbatim; longer ones as a hash. */ +static constexpr size_t maxVerbatimAnswerBytes = 64; + +static constexpr char fieldSeparator = '\t'; +static constexpr char lineSeparator = '\n'; +static constexpr std::string_view absentAnswer = "absent"; + +// ============================================================================ +// Names +// ============================================================================ + +std::string_view dataFormatName(DataFormat format) +{ + switch (format) { + case DataFormat::Json: + return "json"; + case DataFormat::Toml: + return "toml"; + } + unreachable(); +} + +std::optional parseDataFormat(std::string_view name) +{ + if (name == "json") + return DataFormat::Json; + if (name == "toml") + return DataFormat::Toml; + return std::nullopt; +} + +std::optional dataFormatFromExtension(std::string_view fileName) +{ + auto base = fileName.substr(fileName.rfind('/') + 1); + auto dot = base.rfind('.'); + if (dot == std::string_view::npos) + return std::nullopt; + return parseDataFormat(base.substr(dot + 1)); +} + +std::string_view dataQueryOpName(DataQueryOp op) +{ + switch (op) { + case DataQueryOp::Get: + return "get"; + case DataQueryOp::Has: + return "has"; + case DataQueryOp::Keys: + return "keys"; + case DataQueryOp::Find: + return "find"; + } + unreachable(); +} + +std::optional parseDataQueryOp(std::string_view name) +{ + if (name == "get") + return DataQueryOp::Get; + if (name == "has") + return DataQueryOp::Has; + if (name == "keys") + return DataQueryOp::Keys; + if (name == "find") + return DataQueryOp::Find; + return std::nullopt; +} + +bool isValidDataQueryField(std::string_view s) +{ + return s.find_first_of("\t\n\0"sv) == std::string_view::npos; +} + +// ============================================================================ +// Parsing data files +// ============================================================================ + +static nlohmann::json tomlToJson(const toml::value & t) +{ + switch (t.type()) { + case toml::value_t::table: { + auto object = nlohmann::json::object(); + for (auto & [key, value] : toml::get(t)) + object[key] = tomlToJson(value); + return object; + } + case toml::value_t::array: { + auto array = nlohmann::json::array(); + for (auto & value : toml::get>(t)) + array.push_back(tomlToJson(value)); + return array; + } + case toml::value_t::boolean: + return toml::get(t); + case toml::value_t::integer: + return toml::get(t); + case toml::value_t::floating: + return toml::get(t); + case toml::value_t::string: + return std::string(toml::get(t)); + case toml::value_t::local_datetime: + case toml::value_t::offset_datetime: + case toml::value_t::local_date: + case toml::value_t::local_time: { + // The shape `builtins.fromTOML` gives timestamps. The text is + // whatever toml11 prints, which is deterministic for a given input, + // and that is all a fingerprint needs. + std::ostringstream s; + s << t; + return nlohmann::json{{"_type", "timestamp"}, {"value", s.str()}}; + } + case toml::value_t::empty: + return nullptr; + } + unreachable(); +} + +nlohmann::json parseDataFile(DataFormat format, std::string_view content) +{ + switch (format) { + case DataFormat::Json: + try { + return nlohmann::json::parse(content); + } catch (const nlohmann::json::exception & e) { + throw DataFileParseError("%s", e.what()); + } + case DataFormat::Toml: + try { + std::istringstream stream{std::string(content)}; + return tomlToJson( + toml::parse( + stream, + "tecnixQueryFile" +#if HAVE_TOML11_4 + , + toml::spec::v(1, 0, 0) +#endif + )); + } catch (const std::exception & e) { + throw DataFileParseError("%s", e.what()); + } + } + unreachable(); +} + +// ============================================================================ +// Evaluating queries +// ============================================================================ + +/** The node `at` names under `root`, or null if some step is not an object with that key. */ +static const nlohmann::json * resolveAt(const nlohmann::json * node, std::span at) +{ + for (auto segment : at) { + if (!node->is_object()) + return nullptr; + auto it = node->find(segment); + if (it == node->end()) + return nullptr; + node = &*it; + } + return node; +} + +DataQueryAnswer evaluateDataQuery(const ParsedDataFile & file, const DataQuery & query) +{ + auto * node = file ? resolveAt(&*file, query.at) : nullptr; + + switch (query.op) { + case DataQueryOp::Get: + if (!node) + return std::nullopt; + return *node; + + case DataQueryOp::Has: + return nlohmann::json(node != nullptr); + + case DataQueryOp::Keys: { + if (!node || !node->is_object()) + return std::nullopt; + auto keys = nlohmann::json::array(); + // Object iteration is in key order. + for (auto & [key, _] : node->items()) + keys.push_back(key); + return keys; + } + + case DataQueryOp::Find: { + auto keys = nlohmann::json::array(); + if (!node || !node->is_object()) + return keys; + // Object iteration is in key order, so the result is sorted. + for (auto & [key, child] : node->items()) { + if (!child.is_object()) + continue; + auto field = child.find(query.where); + if (field != child.end() && field->is_string() && field->get_ref() == query.value) + keys.push_back(key); + } + return keys; + } + } + unreachable(); +} + +// ============================================================================ +// Observation lines +// ============================================================================ + +void appendDataQueryAnswer(std::string & out, const DataQueryAnswer & answer) +{ + if (!answer) { + out += absentAnswer; + return; + } + // Compact JSON escapes every control character, so the text never + // contains a field or line separator. Invalid UTF-8 is replaced rather + // than thrown on: both sides see the same bytes either way. + auto text = answer->dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); + if (text.size() <= maxVerbatimAnswerBytes) { + out += text; + return; + } + out += "sha256:"; + out += hashString(HashAlgorithm::SHA256, text).to_string(HashFormat::Base16, false); +} + +void appendDataObservation(std::string & out, const DataQuery & query, std::string_view answer) +{ + out += dataFormatName(query.format); + out += fieldSeparator; + out += dataQueryOpName(query.op); + if (query.op == DataQueryOp::Find) { + out += fieldSeparator; + out += query.where; + out += fieldSeparator; + out += query.value; + } + for (auto segment : query.at) { + out += fieldSeparator; + out += segment; + } + out += fieldSeparator; + out += answer; +} + +static std::vector split(std::string_view s, char separator) +{ + std::vector fields; + while (true) { + auto end = s.find(separator); + fields.push_back(s.substr(0, end)); + if (end == std::string_view::npos) + return fields; + s.remove_prefix(end + 1); + } +} + +std::optional parseDataObservation(std::string_view line) +{ + auto fields = split(line, fieldSeparator); + // format, op, answer at least + if (fields.size() < 3) + return std::nullopt; + + DataObservation observation; + auto format = parseDataFormat(fields[0]); + auto op = parseDataQueryOp(fields[1]); + if (!format || !op) + return std::nullopt; + observation.query.format = *format; + observation.query.op = *op; + observation.answer = fields.back(); + + size_t atBegin = 2; + if (*op == DataQueryOp::Find) { + // format, op, where, value, answer at least + if (fields.size() < 5) + return std::nullopt; + observation.query.where = fields[2]; + observation.query.value = fields[3]; + atBegin = 4; + } + observation.query.at.assign(fields.begin() + atBegin, fields.end() - 1); + return observation; +} + +// ============================================================================ +// Fingerprints and observations +// ============================================================================ + +std::string dataQueryFingerprint(std::vector lines) +{ + std::sort(lines.begin(), lines.end()); + lines.erase(std::unique(lines.begin(), lines.end()), lines.end()); + + std::string fingerprint(dataQueryFingerprintPrefix); + for (size_t i = 0; i < lines.size(); i++) { + if (i > 0) + fingerprint += lineSeparator; + fingerprint += lines[i]; + } + return fingerprint; +} + +std::optional> dataQueryFingerprintLines(std::string_view fingerprint) +{ + if (!fingerprint.starts_with(dataQueryFingerprintPrefix)) + return std::nullopt; + auto lines = fingerprint.substr(dataQueryFingerprintPrefix.size()); + if (lines.empty()) + return std::nullopt; + return split(lines, lineSeparator); +} + +void appendDataAccess(std::string & out, std::string_view file, std::string_view line) +{ + out += file; + out += '\0'; + out += line; +} + +std::optional splitDataAccess(std::string_view observation) +{ + auto separator = observation.find('\0'); + if (separator == std::string_view::npos) + return std::nullopt; + return DataAccess{ + .file = observation.substr(0, separator), + .line = observation.substr(separator + 1), + }; +} + +} // namespace nix diff --git a/src/libexpr/tecnix/eval-cache.cc b/src/libexpr/tecnix/eval-cache.cc index 47ce1f60b..d368f8d46 100644 --- a/src/libexpr/tecnix/eval-cache.cc +++ b/src/libexpr/tecnix/eval-cache.cc @@ -7,6 +7,7 @@ #include "nix/expr/tecnix/eval-cache.hh" #include "nix/expr/tecnix/eval-cache-row.hh" +#include "nix/expr/tecnix/data-query.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/tecnix/source-accessors.hh" @@ -26,11 +27,10 @@ #include #include #include +#include #include #include #include -#include -#include "nix/util/hash.hh" #include namespace nix { @@ -47,13 +47,17 @@ DependencyFingerprintCache::DependencyFingerprintCache() { } +/** A data file as this run's tree serves it; null if it exists but does not parse. */ +using DependencyDataFile = std::shared_ptr; + struct DependencyFingerprintThreadLocalCache { uint64_t generation = 0; boost::unordered_flat_map, StringViewHash, std::equal_to<>> fingerprints; - /** Cached parsed manifest JSON for synthetic path fingerprinting (per-batch). */ - std::optional cachedManifest; - bool manifestChecked = false; + /** Whether a data observation still holds, keyed by `\0`. */ + boost::unordered_flat_map> observations; + /** Parsed data files, keyed by `\0`. */ + std::map> dataFiles; DependencyFingerprintThreadLocalCache() { @@ -567,17 +571,6 @@ struct TecnixEvalCache return CurrentFingerprints{.accessor = accessor, .fingerprintCache = &fingerprintCache}; } - std::optional currentFingerprint(uint32_t pathId, CurrentFingerprints & current) const - { - auto p = path(pathId); - if (!p) - return std::nullopt; - auto cached = dependencyFingerprintCached(current.accessor, *p, *current.fingerprintCache); - if (!cached || !*cached) - return std::nullopt; - return std::string_view(**cached); - } - bool candidateMatches(uint32_t candidateIndex, CurrentFingerprints & currentFingerprints) const { auto pairStart = candidatePairStart(candidateIndex); @@ -593,9 +586,11 @@ struct TecnixEvalCache auto fingerprintId = pairFingerprint(pairIndex); if (!pathId || !fingerprintId || *pathId >= pathCount || *fingerprintId >= fingerprintCount) return false; - auto currentFp = currentFingerprint(*pathId, currentFingerprints); + auto p = path(*pathId); auto expectedFp = fingerprint(*fingerprintId); - if (!currentFp || !expectedFp || *currentFp != *expectedFp) + if (!p || !expectedFp + || !dependencyFingerprintHolds( + currentFingerprints.accessor, *p, *expectedFp, *currentFingerprints.fingerprintCache)) return false; } return true; @@ -1356,48 +1351,12 @@ static DependencyFingerprintThreadLocalCache & getDependencyFingerprintThreadCac if (threadCache.generation != cache.generation) { threadCache.generation = cache.generation; threadCache.fingerprints.clear(); - // The parsed manifest belongs to the accessor this generation reads. - threadCache.cachedManifest.reset(); - threadCache.manifestChecked = false; + threadCache.observations.clear(); + threadCache.dataFiles.clear(); } return threadCache; } -static std::optional manifestPathFingerprint(const nlohmann::json & manifest, std::string_view suffix) -{ - constexpr std::string_view keysToken = "keys"; - if (suffix == keysToken) { - // Fingerprint the sorted key list - std::vector keys; - keys.reserve(manifest.size()); - for (auto & [k, v] : manifest.items()) - keys.push_back(k); - std::sort(keys.begin(), keys.end()); - std::string concatenated; - for (auto & k : keys) - concatenated += k + "\n"; - auto hash = hashString(HashAlgorithm::SHA256, concatenated); - return "manifest-keys:" + hash.to_string(HashFormat::Base64, false); - } - constexpr std::string_view idPrefix = "id/"; - if (suffix.starts_with(idPrefix)) { - auto zoneId = suffix.substr(idPrefix.size()); - for (auto & [path, value] : manifest.items()) { - if (value.contains("id") && value.at("id").is_string() && value.at("id").get() == zoneId) - return "manifest-id:" + std::string(zoneId) + "=" + path; - } - return "manifest-id:" + std::string(zoneId) + "=absent"; - } - // suffix is a zone path like "//zone/path" - auto it = manifest.find(std::string(suffix)); - if (it == manifest.end()) - return "manifest-entry:" + std::string(suffix) + "=absent"; - // Entries are { "id": "W-xxxxxx" }; fingerprint from the entry JSON - auto entryStr = it->dump(); - auto hash = hashString(HashAlgorithm::SHA256, entryStr); - return "manifest-entry:" + std::string(suffix) + "=" + hash.to_string(HashFormat::Base64, false); -} - static const std::optional * dependencyFingerprintCached(ref accessor, std::string_view path, DependencyFingerprintCache & cache) { @@ -1407,26 +1366,7 @@ dependencyFingerprintCached(ref accessor, std::string_view path, if (it != threadCache.fingerprints.end()) return &it->second; - std::optional fp; - constexpr std::string_view manifestPrefix = ".meta/manifest.json#"; - if (path.starts_with(manifestPrefix)) { - // Synthetic manifest path: fingerprint from manifest content, not the - // filesystem. This lets per-entry tracking avoid whole-file invalidation. - if (!threadCache.manifestChecked) { - threadCache.manifestChecked = true; - try { - auto content = accessor->readFile(CanonPath(".meta/manifest.json")); - threadCache.cachedManifest = nlohmann::json::parse(content); - } catch (...) { - threadCache.cachedManifest = std::nullopt; - } - } - if (threadCache.cachedManifest) - fp = manifestPathFingerprint(*threadCache.cachedManifest, path.substr(manifestPrefix.size())); - } else { - auto [_, rawFp] = accessor->getFingerprint(CanonPath(path)); - fp = std::move(rawFp); - } + auto [_, fp] = accessor->getFingerprint(CanonPath(path)); auto inserted = threadCache.fingerprints.emplace(std::string(path), std::move(fp)).first; return &inserted->second; } @@ -1440,17 +1380,106 @@ dependencyFingerprint(ref accessor, std::string_view path, Depen return *fp; } +/** + * The data file at `path` as this run's tree serves it, parsed as `format`, + * once per thread per run. The read records nothing: the observations being + * checked are the observation of this file, and recording the whole file + * here would put it back into the closure of whatever evaluation is + * validating (#80's memo rows validate inside a tracked evaluation). + */ +static DependencyDataFile dependencyDataFile( + ref accessor, + std::string_view path, + DataFormat format, + DependencyFingerprintThreadLocalCache & threadCache) +{ + std::string key(path); + key += '\0'; + key += dataFormatName(format); + if (auto it = threadCache.dataFiles.find(key); it != threadCache.dataFiles.end()) + return it->second; + + DependencyDataFile file; + try { + auto content = readSourceFileUnrecorded(*accessor, CanonPath(path)); + file = std::make_shared( + content ? ParsedDataFile(parseDataFile(format, *content)) : ParsedDataFile()); + } catch (DataFileParseError & e) { + debug("tecnix: data file '%s' does not parse as %s: %s", path, dataFormatName(format), e.what()); + } + threadCache.dataFiles.emplace(std::move(key), file); + return file; +} + +/** Whether one observation of the data file at `path` still holds. */ +static bool dataObservationHolds( + ref accessor, std::string_view path, std::string_view line, DependencyFingerprintCache & cache) +{ + auto & threadCache = getDependencyFingerprintThreadCache(cache); + + static thread_local std::string memoKey; + memoKey.clear(); + appendDataAccess(memoKey, path, line); + if (auto it = threadCache.observations.find(memoKey); it != threadCache.observations.end()) + return it->second; + + bool holds = false; + if (auto observation = parseDataObservation(line)) { + if (auto file = dependencyDataFile(accessor, path, observation->query.format, threadCache)) { + static thread_local std::string answer; + answer.clear(); + appendDataQueryAnswer(answer, evaluateDataQuery(*file, observation->query)); + holds = answer == observation->answer; + } + } + threadCache.observations.emplace(memoKey, holds); + return holds; +} + +bool dependencyFingerprintHolds( + ref accessor, + std::string_view path, + std::string_view fingerprint, + DependencyFingerprintCache & cache) +{ + auto lines = dataQueryFingerprintLines(fingerprint); + if (!lines) { + auto current = dependencyFingerprintCached(accessor, path, cache); + return current && *current && **current == fingerprint; + } + for (auto line : *lines) + if (!dataObservationHolds(accessor, path, line, cache)) + return false; + return true; +} + DependencyClosure dependencyFingerprints( - ref accessor, const std::vector & paths, DependencyFingerprintCache & cache) + ref accessor, const std::vector & observations, DependencyFingerprintCache & cache) { + // Data observations by file; a file also observed whole drops out, since + // its object fingerprint implies every answer. + std::map> queried; + std::vector paths; + for (auto & observation : observations) { + if (auto data = splitDataAccess(observation)) + queried[data->file].push_back(data->line); + else + paths.push_back(observation); + } + DependencyClosure result; - result.reserve(paths.size()); - for (auto & path : paths) { + result.reserve(paths.size() + queried.size()); + for (auto path : paths) { auto fp = dependencyFingerprint(accessor, path, cache); if (!fp) throw Error("failed to fingerprint Tecnix dependency path '%s'", path); - result.push_back({path, std::move(*fp)}); + result.push_back({std::string(path), std::move(*fp)}); + queried.erase(path); } + for (auto & [file, lines] : queried) + result.push_back({std::string(file), dataQueryFingerprint(std::move(lines))}); + + std::sort(result.begin(), result.end(), [](auto & a, auto & b) { return a.path < b.path; }); return result; } diff --git a/src/libexpr/tecnix/eval-data.hh b/src/libexpr/tecnix/eval-data.hh index b29c6bc87..0ad158d6a 100644 --- a/src/libexpr/tecnix/eval-data.hh +++ b/src/libexpr/tecnix/eval-data.hh @@ -49,6 +49,9 @@ struct EvalTecnixModuleCacheEntry using EvalTecnixModuleCache = boost::concurrent_flat_map; +/** Parsed data files (`std::optional`, see data-query.hh). */ +using EvalDataFileCache = boost::concurrent_flat_map>>; + /** * A `builtins.tecnixMemoize` table key: the memoized function, identified by * the address of its value, and the key it was called with. `Lookup` is the @@ -176,10 +179,13 @@ struct EvalState::TecnixEvalData mutable std::once_flag tecnixRepoAccessorFlag; mutable std::optional> tecnixRepoAccessor; - /** `.meta/manifest.json` as `tecnixRepoAccessor` serves it, parsed - (thread-safe via once_flag). */ - mutable std::once_flag tecnixManifestJsonFlag; - mutable std::unique_ptr tecnixManifestJson; + /** + * Data files `builtins.tecnixQueryFile` has read, parsed once per + * evaluation and shared by every query on them, keyed by the file's + * `SourcePath` and format (`\0`). Reads record nothing; + * each query records its own observation. + */ + const ref dataFileCache = make_ref(); /** * Virtual store path where the Tecnix repo-wide accessor is lazily mounted. diff --git a/src/libexpr/tecnix/repo-accessor.cc b/src/libexpr/tecnix/repo-accessor.cc index 9559a5c55..74a14f8f0 100644 --- a/src/libexpr/tecnix/repo-accessor.cc +++ b/src/libexpr/tecnix/repo-accessor.cc @@ -778,20 +778,30 @@ ref getTecnixRepoAccessor(EvalState & state) return *tecnixData(state)->tecnixRepoAccessor; } -const nlohmann::json & getTecnixManifestJson(EvalState & state) +std::optional readSourceFileUnrecorded(SourceAccessor & accessor, const CanonPath & path) { - std::call_once(tecnixData(state)->tecnixManifestJsonFlag, [&state]() { - // getTecnixRepoAccessor always builds a TecnixSourceAccessor. - auto & accessor = static_cast(*getTecnixRepoAccessor(state)); - auto content = accessor.readFileUnrecorded(CanonPath(".meta/manifest.json")); - try { - tecnixData(state)->tecnixManifestJson = std::make_unique(nlohmann::json::parse(content)); - } catch (nlohmann::json::parse_error & e) { - throw Error( - "cannot parse '.meta/manifest.json' at commit '%s': %s", requireTectonixGitSha(state), e.what()); - } - }); - return *tecnixData(state)->tecnixManifestJson; + // Stats are untracked by design (see TecnixSourceAccessor::maybeLstat). + auto st = accessor.maybeLstat(path); + if (!st || st->type != SourceAccessor::tRegular) + return std::nullopt; + if (auto * tecnix = dynamic_cast(&accessor)) + return tecnix->readFileUnrecorded(path); + return accessor.readFile(path); +} + +std::optional tecnixRepoRelativePath(EvalState & state, const SourcePath & path) +{ + // The mount is created by `tecnixTargets` before it imports the resolver, + // on the calling thread, so by the time any evaluation can ask, this + // optional is settled. + auto & mount = tecnixData(state)->tecnixRepoMountStorePath; + auto * accessor = &*path.accessor; + if (!mount || (accessor != &*state.rootFS && accessor != &*state.storeFS)) + return std::nullopt; + auto mountPath = CanonPath(state.store->printStorePath(*mount)); + if (!path.path.isWithin(mountPath)) + return std::nullopt; + return path.path.removePrefix(mountPath); } StorePath mountTecnixRepoAccessor(EvalState & state) diff --git a/tests/functional/tecnix/data-query.sh b/tests/functional/tecnix/data-query.sh new file mode 100755 index 000000000..73d1fbf89 --- /dev/null +++ b/tests/functional/tecnix/data-query.sh @@ -0,0 +1,391 @@ +#!/usr/bin/env bash +# `builtins.tecnixQueryFile` and the memoized zone loader. +# +# A target that queries a data file depends on the answers it saw, not on the +# file: its closure entry for the file is a `query:` fingerprint listing one +# observation per distinct query. So adding an unrelated zone to the manifest, or +# reformatting a TOML file, leaves such a target's eval-cache row valid, while +# a target that read the file whole still depends on the whole file. A function +# memoized with `tecnixMemoize` shares one evaluation, and the source +# dependencies recorded during it, among every consumer, but never shares a +# result computed outside tracking with a tracked target (or the other way +# round). + +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +assert_jq() { + local json="$1" + local filter="$2" + local message="$3" + if ! jq -e "$filter" >/dev/null <<< "$json"; then + echo "$json" >&2 + fail "$message" + fi +} + +WORLD="$TEST_ROOT/query-world" +createGitRepo "$WORLD" + +write_manifest() { + # $1: extra manifest lines, already comma-terminated + cat > "$WORLD/.meta/manifest.json" < "$WORLD/config.toml" < "$WORLD/shared.txt" +echo "unrelated one" > "$WORLD/unrelated.txt" +# Two files naming zone a, one memo key read from each by its own target. +printf a > "$WORLD/key1.txt" +printf a > "$WORLD/key2.txt" + +cat > "$WORLD/resolve.nix" <<'EOF' +args: +let + drv = name: suffix: { + drvPath = "/nix/store/00000000000000000000000000000000-${name}-${suffix}.drv"; + outputName = "out"; + }; + manifest = q: builtins.tecnixQueryFile ({ path = ./.meta/manifest.json; } // q); + config = q: builtins.tecnixQueryFile ({ path = ./config.toml; } // q); + entry = name: manifest { op = "get"; at = [ "//zones/${name}" ]; }; + zonePaths = manifest { op = "keys"; at = [ ]; }; + pathsForId = id: manifest { op = "find"; at = [ ]; where = "id"; value = id; }; + # Two consumers of one memoized load: the load reads shared.txt once. + loaded = builtins.tecnixMemoize (_: builtins.readFile ./shared.txt) "shared"; + # Memoized lookups that every consumer calls afresh, so every consumer after + # the first is a memo hit rather than a force of one shared thunk. + zoneId = builtins.tecnixMemoize (name: (entry name).id); + zoneRecord = builtins.tecnixMemoize (name: { id = (entry name).id; }); +in +{ + allTargetNames = [ + "a" "b" "keys" "id" "both" "m1" "m2" "whole" "za1" "za2" "not-a-zone" "missing" + "toml-get" "toml-has" "toml-whole" + "peek" "fill-record" "prefilled" "record-field" "agree" "key1" "key2" + ]; + resolve = name: + if name == "whole" then drv name (toString (builtins.length (builtins.attrNames (builtins.fromJSON (builtins.readFile ./.meta/manifest.json))))) + else if name == "keys" then drv name (toString (builtins.length zonePaths)) + else if name == "id" then drv name ( + let paths = pathsForId "W-000001"; in if paths == [ ] then "absent" else builtins.head paths + ) + # Two queries on one file: one closure entry, two observations. + else if name == "both" then drv name "${(entry "a").id}-${if manifest { op = "has"; at = [ "//zones/b" ]; } then "b" else "no-b"}" + else if name == "m1" || name == "m2" then drv name (builtins.hashString "sha256" loaded) + else if name == "za1" || name == "za2" || name == "prefilled" then drv name (zoneId "a") + # Hands both memoized lookups, unforced, to whoever reads the value. + else if name == "peek" then drv name "peek" // { zoneId = zoneId "a"; recordId = (zoneRecord "a").id; } + # Fills the memoized record without forcing its field. + else if name == "fill-record" then builtins.seq (zoneRecord "a") (drv name "record") + else if name == "record-field" then drv name (zoneRecord "a").id + # What the query builtin reports about zone a, beside what reading + # .meta/manifest.json itself shows. + else if name == "agree" then + let + manifest' = builtins.fromJSON (builtins.readFile ./.meta/manifest.json); + id = manifest'."//zones/a".id; + in drv name "agree" // { + seen = { + builtins = { inherit (entry "a") id; keys = zonePaths; paths = pathsForId id; }; + file = { inherit id; keys = builtins.attrNames manifest'; paths = [ "//zones/a" ]; }; + }; + } + # Any key can be asked about; there is nothing special about "keys". + else if name == "not-a-zone" then drv name (if manifest { op = "get"; at = [ "keys" ]; } == null then "none" else "some") + # A file that does not exist answers like an empty one, and the answer is recorded. + else if name == "missing" then drv name (if builtins.tecnixQueryFile { path = ./optional.json; op = "has"; at = [ ]; } then "present" else "absent") + else if name == "toml-get" then drv name (toString (config { op = "get"; at = [ "build" "jobs" ]; })) + else if name == "toml-has" then drv name (if config { op = "has"; at = [ "build" "cache" ]; } then "cached" else "uncached") + else if name == "toml-whole" then drv name (builtins.hashString "sha256" (builtins.readFile ./config.toml)) + else if name == "key1" || name == "key2" then drv name (zoneId (builtins.readFile ./${name}.txt)) + else drv name (entry name).id; +} +EOF + +commit_world() { + git -C "$WORLD" add -A + git -C "$WORLD" commit -q -m "$1" + git -C "$WORLD" rev-parse HEAD +} + +REV1=$(commit_world "two zones") + +TARGETS='[ "a" "b" "keys" "id" "both" "m1" "m2" "whole" "za1" "za2" "not-a-zone" "missing" "toml-get" "toml-has" "toml-whole" ]' +ALL_TARGETS=(a b keys id both m1 m2 whole za1 za2 not-a-zone missing toml-get toml-has toml-whole) + +# Dependency sets (target -> path -> fingerprint) of every target at $1. +# Extra nix flags follow. With WITH_CHECKOUT set, the checkout's uncommitted +# changes overlay the rev. +deps_at() { + local rev="$1" + shift + local checkout="" + [[ -n "${WITH_CHECKOUT-}" ]] && checkout="checkoutPath = \"$WORLD\";" + nix eval --json -v \ + --extra-experimental-features 'nix-command' \ + --option lazy-trees true \ + --option tectonix-git-dir "$WORLD/.git" \ + --option tectonix-git-sha "$rev" \ + "$@" \ + --expr "builtins.listToAttrs (map (r: { name = r.target; value = r.dependencies; }) (builtins.tecnixTargets { + gitDir = \"$WORLD/.git\"; resolver = \"resolve.nix\"; args = { system = \"test-system\"; }; $checkout + rev = \"$rev\"; targets = $TARGETS; includeDependencies = true; includeTargets = false; }))" +} + +nocache_deps_at() { + deps_at "$1" --option tecnix-eval-cache false +} + +CACHE_HOME="$TEST_ROOT/query-cache-home" +cached_deps_at() { + XDG_CACHE_HOME="$CACHE_HOME" deps_at "$1" --option tecnix-eval-cache true --pure-eval +} + +# `hits ` asserts each target was served from the cache; +# `misses` asserts each was re-evaluated. `hits_except ` +# asserts the given targets missed and every other target hit. +hits() { + local err="$1"; shift + for t in "$@"; do + grepQuiet "dependency cache hit for '$t'" "$err" || fail "$t should be a cache hit ($err)" + done +} +misses() { + local err="$1"; shift + for t in "$@"; do + grepQuiet "dependency cache miss, evaluating '$t'" "$err" || fail "$t should be re-evaluated ($err)" + done +} +hits_except() { + local err="$1"; shift + misses "$err" "$@" + local t m + for t in "${ALL_TARGETS[@]}"; do + for m in "$@"; do + [[ "$t" == "$m" ]] && continue 2 + done + hits "$err" "$t" + done +} + +echo "Testing what each target depends on..." +deps1=$(nocache_deps_at "$REV1") +assert_jq "$deps1" '.a | keys == [".meta/manifest.json", "resolve.nix"]' \ + "a should depend on the manifest (by query) and the resolver" +assert_jq "$deps1" '.a[".meta/manifest.json"] == "query:json\tget\t//zones/a\t{\"id\":\"W-000001\"}"' \ + "a's manifest entry should record its own query and answer" +assert_jq "$deps1" '.b[".meta/manifest.json"] == "query:json\tget\t//zones/b\t{\"id\":\"W-000002\"}"' \ + "b's manifest entry should record its own query and answer" +assert_jq "$deps1" '.keys[".meta/manifest.json"] == "query:json\tkeys\t[\"//zones/a\",\"//zones/b\"]"' \ + "keys should record the key set" +assert_jq "$deps1" '.id[".meta/manifest.json"] == "query:json\tfind\tid\tW-000001\t[\"//zones/a\"]"' \ + "id should record the id lookup" +assert_jq "$deps1" '.both[".meta/manifest.json"] == "query:json\tget\t//zones/a\t{\"id\":\"W-000001\"}\njson\thas\t//zones/b\ttrue"' \ + "two queries on one file should be two sorted observations in one entry" +assert_jq "$deps1" '."not-a-zone"[".meta/manifest.json"] == "query:json\tget\tkeys\tabsent"' \ + "an absent key should be recorded absent" +assert_jq "$deps1" '.missing | keys == ["optional.json", "resolve.nix"] and .["optional.json"] == "query:json\thas\tfalse"' \ + "a query on a missing file should record its answer" +assert_jq "$deps1" '."toml-get"["config.toml"] == "query:toml\tget\tbuild\tjobs\t4"' \ + "a TOML query should record its answer" +assert_jq "$deps1" '."toml-has"["config.toml"] == "query:toml\thas\tbuild\tcache\tfalse"' \ + "a TOML existence query should record its answer" +assert_jq "$deps1" '[.whole[".meta/manifest.json"], ."toml-whole"["config.toml"]] | all(startswith("git:"))' \ + "reading a file whole should still depend on the whole file" +assert_jq "$deps1" '.m1 == .m2 and (.m1 | has("shared.txt"))' \ + "both consumers of a memoized load should inherit the load's dependencies" +assert_jq "$deps1" '[.za1, .za2 | .[".meta/manifest.json"] == "query:json\tget\t//zones/a\t{\"id\":\"W-000001\"}"] | all' \ + "a memo hit should replay the query recorded when its entry was filled" +# How a caller computed a memo key is its own dependency, never the shared +# entry's. Either target may fill the entry first, so check both. +via=$(TARGETS='[ "key1" "key2" ]' nocache_deps_at "$REV1") +assert_jq "$via" '(.key1 | keys) == [".meta/manifest.json", "key1.txt", "resolve.nix"] + and (.key2 | keys) == [".meta/manifest.json", "key2.txt", "resolve.nix"]' \ + "a memo caller should depend on the file it read the key from, and on no other caller's" + +echo "Testing argument validation..." +bad_query() { + # $1: the query attrset; expects an error mentioning $2 + local out + if out=$(nix eval --json --impure --extra-experimental-features nix-command \ + --expr "builtins.tecnixQueryFile ($1)" 2>&1); then + fail "expected $1 to fail" + fi + grepQuiet "$2" <<< "$out" || { echo "$out" >&2; fail "error for $1 should mention $2"; } +} +bad_query "{ path = $WORLD/config.toml; op = \"select\"; at = [ ]; }" "unknown op 'select'" +bad_query "{ path = $WORLD/config.toml; op = \"get\"; }" "missing attribute 'at'" +bad_query "{ path = $WORLD/config.toml; op = \"get\"; at = [ ]; where = \"x\"; }" "only for op 'find'" +bad_query "{ path = $WORLD/config.toml; op = \"find\"; at = [ ]; }" "requires 'where' and 'value'" +bad_query "{ path = $WORLD/config.toml; op = \"get\"; at = [ \"a\tb\" ]; }" "may not contain a tab" +bad_query "{ path = $WORLD/config.toml; op = \"get\"; at = [ ]; format = \"yaml\"; }" "unknown format 'yaml'" +bad_query "{ path = $WORLD/shared.txt; op = \"get\"; at = [ ]; }" "cannot infer the format" +bad_query "{ path = $WORLD/config.toml; op = \"get\"; at = [ ]; extra = 1; }" "unexpected attribute 'extra'" +bad_query "{ path = $WORLD/config.toml; op = \"get\"; at = [ ]; format = \"json\"; }" "while parsing" + +echo "Testing the builtin outside tracking..." +# Outside the repo accessor a query reads like `readFile`: a path pure +# evaluation forbids is an error, not an absent file. +expectStderr 1 nix eval --json --extra-experimental-features nix-command \ + --expr "builtins.tecnixQueryFile { path = $WORLD/config.toml; op = \"has\"; at = [ ]; }" \ + | grepQuiet "forbidden in pure evaluation mode" +plain=$(nix eval --json --impure --extra-experimental-features nix-command --expr "{ + entry = builtins.tecnixQueryFile { path = $WORLD/.meta/manifest.json; op = \"get\"; at = [ \"//zones/a\" ]; }; + keys = builtins.tecnixQueryFile { path = $WORLD/.meta/manifest.json; op = \"keys\"; at = [ ]; }; + found = builtins.tecnixQueryFile { path = $WORLD/.meta/manifest.json; op = \"find\"; at = [ ]; where = \"id\"; value = \"W-000002\"; }; + jobs = builtins.tecnixQueryFile { path = $WORLD/config.toml; op = \"get\"; at = [ \"build\" \"jobs\" ]; }; + table = builtins.tecnixQueryFile { path = \"$WORLD/config.toml\"; op = \"get\"; at = [ \"build\" ]; }; + missing = builtins.tecnixQueryFile { path = $WORLD/optional.json; op = \"get\"; at = [ \"x\" ]; }; +}") +assert_jq "$plain" '. == { entry: { id: "W-000001" }, keys: ["//zones/a", "//zones/b"], found: ["//zones/b"], jobs: 4, table: { jobs: 4 }, missing: null }' \ + "the builtin should answer plain queries like fromJSON (readFile ...) would" + +echo "Testing the eval cache across commits..." +cached_deps_at "$REV1" > "$TEST_ROOT/cold.json" 2> "$TEST_ROOT/cold.err" +misses "$TEST_ROOT/cold.err" "${ALL_TARGETS[@]}" +[[ "$(jq -S . < "$TEST_ROOT/cold.json")" == "$(jq -S . <<< "$deps1")" ]] \ + || fail "the cached evaluation should report the same dependencies" + +# Adding an unrelated zone changes the manifest file and the key set. Only the +# targets that enumerate the keys or read the whole file are invalidated. +write_manifest ' "//zones/c": { "id": "W-000003" },' +REV2=$(commit_world "add an unrelated zone") +cached_deps_at "$REV2" > /dev/null 2> "$TEST_ROOT/add-zone.err" +hits_except "$TEST_ROOT/add-zone.err" keys whole + +# Changing one zone's entry invalidates that zone's target and nothing else: +# `both` only asked whether b exists, and it still does. +sed -i.bak 's/W-000002/W-000009/' "$WORLD/.meta/manifest.json" && rm "$WORLD/.meta/manifest.json.bak" +REV3=$(commit_world "change b's id") +cached_deps_at "$REV3" > "$TEST_ROOT/change-b.json" 2> "$TEST_ROOT/change-b.err" +hits_except "$TEST_ROOT/change-b.err" b whole +assert_jq "$(cat "$TEST_ROOT/change-b.json")" '.b[".meta/manifest.json"] == "query:json\tget\t//zones/b\t{\"id\":\"W-000009\"}"' \ + "b's recorded observation should follow its entry" + +# An id lookup is invalidated when the id moves, and only then. +sed -i.bak 's/W-000001/W-000010/' "$WORLD/.meta/manifest.json" && rm "$WORLD/.meta/manifest.json.bak" +REV4=$(commit_world "renumber a") +cached_deps_at "$REV4" > /dev/null 2> "$TEST_ROOT/renumber-a.err" +hits_except "$TEST_ROOT/renumber-a.err" a id both whole za1 za2 + +# A memoized load's dependencies reach every consumer: changing what the load +# read invalidates both, not just whichever consumer evaluated it first. +echo "shared two" > "$WORLD/shared.txt" +REV5=$(commit_world "change the shared file") +cached_deps_at "$REV5" > /dev/null 2> "$TEST_ROOT/shared.err" +hits_except "$TEST_ROOT/shared.err" m1 m2 + +# A file nothing read leaves every row valid. +echo "unrelated two" > "$WORLD/unrelated.txt" +REV6=$(commit_world "change an unrelated file") +cached_deps_at "$REV6" > /dev/null 2> "$TEST_ROOT/unrelated.err" +hits_except "$TEST_ROOT/unrelated.err" + +# Reformatting a data file changes its object but no answer. +write_config 4 "build settings, now with a longer comment" +REV7=$(commit_world "comment in config.toml") +cached_deps_at "$REV7" > /dev/null 2> "$TEST_ROOT/toml-comment.err" +hits_except "$TEST_ROOT/toml-comment.err" toml-whole + +# Changing a queried value invalidates exactly the query that saw it. +write_config 8 "build settings, now with a longer comment" +REV8=$(commit_world "more jobs") +cached_deps_at "$REV8" > /dev/null 2> "$TEST_ROOT/toml-value.err" +hits_except "$TEST_ROOT/toml-value.err" toml-get toml-whole + +# A file whose absence was observed invalidates its observer when it appears. +echo '{}' > "$WORLD/optional.json" +REV9=$(commit_world "add the optional file") +cached_deps_at "$REV9" > "$TEST_ROOT/optional.json" 2> "$TEST_ROOT/optional.err" +hits_except "$TEST_ROOT/optional.err" missing +assert_jq "$(cat "$TEST_ROOT/optional.json")" '.missing["optional.json"] == "query:json\thas\ttrue"' \ + "the observer should now record the file's presence" + +# A query is answered from the checkout's uncommitted content, and the +# observation follows it, like any dirty read. +write_config 16 "uncommitted" +WITH_CHECKOUT=1 cached_deps_at "$REV9" > "$TEST_ROOT/dirty.json" 2> "$TEST_ROOT/dirty.err" +hits_except "$TEST_ROOT/dirty.err" toml-get toml-whole +assert_jq "$(cat "$TEST_ROOT/dirty.json")" '."toml-get"["config.toml"] == "query:toml\tget\tbuild\tjobs\t16"' \ + "a dirty data file should be queried as it is on disk" +git -C "$WORLD" checkout -q -- config.toml + +# A memoized result computed outside tracking must never reach a tracked +# target without its dependencies, in either direction: an untracked call that +# comes first, and an untracked call that forces part of an entry a tracked +# target filled. `peek` hands both lookups, unforced, to the top-level +# expression, which forces them outside tracking before the tracked targets +# run in the same evaluation. Extra nix flags follow the rev. +prefill_deps_at() { + local rev="$1" + shift + nix eval --json -v \ + --extra-experimental-features 'nix-command' \ + --option lazy-trees true \ + --option tectonix-git-dir "$WORLD/.git" \ + --option tectonix-git-sha "$rev" \ + "$@" \ + --expr "let + call = includeTargets: targets: builtins.tecnixTargets { + gitDir = \"$WORLD/.git\"; resolver = \"resolve.nix\"; args = { system = \"test-system\"; }; + rev = \"$rev\"; inherit targets includeTargets; includeDependencies = true; }; + peek = (builtins.head (call true [ \"peek\" ])).value; + outsideTracking = builtins.seq peek.zoneId (builtins.seq (call false [ \"fill-record\" ]) peek.recordId); + in builtins.seq outsideTracking (builtins.listToAttrs (map (r: { name = r.target; value = r.dependencies; }) + (call false [ \"prefilled\" \"record-field\" ])))" +} + +echo "Testing memoized results computed outside tracking..." +PREFILLED_DEPS='[.prefilled, ."record-field" | .[".meta/manifest.json"] == "query:json\tget\t//zones/a\t{\"id\":\"W-000010\"}"] | all' +# First with tracking from includeDependencies alone, then through the eval +# cache, where a missing dependency turns into a stale warm hit. +assert_jq "$(prefill_deps_at "$REV9" --option tecnix-eval-cache false)" "$PREFILLED_DEPS" \ + "tracked targets should depend on the query behind memoized values first touched outside tracking" + +PREFILL_CACHE_HOME="$TEST_ROOT/query-prefill-cache-home" +XDG_CACHE_HOME="$PREFILL_CACHE_HOME" prefill_deps_at "$REV9" --option tecnix-eval-cache true --pure-eval \ + > "$TEST_ROOT/prefill-cold.json" 2> "$TEST_ROOT/prefill-cold.err" +misses "$TEST_ROOT/prefill-cold.err" prefilled record-field +assert_jq "$(cat "$TEST_ROOT/prefill-cold.json")" "$PREFILLED_DEPS" \ + "cached tracked targets should depend on the query behind memoized values first touched outside tracking" + +# The reviewer's reproduction: renumber zone a, and the warm run must +# re-evaluate instead of serving the old id. +sed -i.bak 's/W-000010/W-000011/' "$WORLD/.meta/manifest.json" && rm "$WORLD/.meta/manifest.json.bak" +REV10=$(commit_world "renumber a again") +XDG_CACHE_HOME="$PREFILL_CACHE_HOME" prefill_deps_at "$REV10" --option tecnix-eval-cache true --pure-eval \ + > /dev/null 2> "$TEST_ROOT/prefill-warm.err" +misses "$TEST_ROOT/prefill-warm.err" prefilled record-field + +# The builtin reads the file through the repo accessor, like every file the +# resolver reads and like the validator re-checking its observations. So evaluating +# an older rev from a checkout whose HEAD has moved on shows it that rev's +# manifest, not the working tree's. +echo "Testing the builtin at an older rev of a checkout..." +seen=$(nix eval --json \ + --extra-experimental-features 'nix-command' \ + --option lazy-trees true \ + --option tecnix-eval-cache false \ + --expr "(builtins.tecnixTargets { gitDir = \"$WORLD/.git\"; resolver = \"resolve.nix\"; args = { system = \"test-system\"; }; + rev = \"$REV1\"; checkoutPath = \"$WORLD\"; targets = [ \"agree\" ]; }).agree.seen") +assert_jq "$seen" '.file.id == "W-000001"' "the resolver should read the evaluated rev's manifest" +assert_jq "$seen" '.builtins == .file' "the builtin should see the manifest the resolver reads" + +echo "All data query tests passed!" diff --git a/tests/functional/tecnix/manifest-tracking.sh b/tests/functional/tecnix/manifest-tracking.sh deleted file mode 100755 index dc307c9c7..000000000 --- a/tests/functional/tecnix/manifest-tracking.sh +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env bash -# Per-entry manifest tracking and the memoized zone loader. -# -# A target that reads one zone's manifest entry (`tectonixManifestEntry`), the -# key set (`tectonixManifestKeys`) or one id (`tectonixManifestIdToPath`) -# depends on exactly that, not on the whole `.meta/manifest.json`. So adding an -# unrelated zone leaves such a target's eval-cache row valid. A function -# memoized with `tecnixMemoize` shares one evaluation, and the source -# dependencies recorded during it, among every consumer, but never shares a -# result computed outside tracking with a tracked target (or the other way -# round). - -source "$(dirname "${BASH_SOURCE[0]}")/common.sh" - -assert_jq() { - local json="$1" - local filter="$2" - local message="$3" - if ! jq -e "$filter" >/dev/null <<< "$json"; then - echo "$json" >&2 - fail "$message" - fi -} - -WORLD="$TEST_ROOT/manifest-world" -createGitRepo "$WORLD" - -write_manifest() { - # $1: extra manifest lines, already comma-terminated - cat > "$WORLD/.meta/manifest.json" < "$WORLD/shared.txt" -echo "unrelated one" > "$WORLD/unrelated.txt" -# Two files naming zone a, one memo key read from each by its own target. -printf a > "$WORLD/key1.txt" -printf a > "$WORLD/key2.txt" - -cat > "$WORLD/resolve.nix" <<'EOF' -args: -let - drv = name: suffix: { - drvPath = "/nix/store/00000000000000000000000000000000-${name}-${suffix}.drv"; - outputName = "out"; - }; - entry = name: builtins.tectonixManifestEntry "//zones/${name}"; - # Two consumers of one memoized load: the load reads shared.txt once. - loaded = builtins.tecnixMemoize (_: builtins.readFile ./shared.txt) "shared"; - # Memoized lookups that every consumer calls afresh, so every consumer after - # the first is a memo hit rather than a force of one shared thunk. - zoneId = builtins.tecnixMemoize (name: (entry name).id); - zoneRecord = builtins.tecnixMemoize (name: { id = (entry name).id; }); -in -{ - allTargetNames = [ "a" "b" "keys" "id" "m1" "m2" "whole" "za1" "za2" "peek" "fill-record" "prefilled" "record-field" "agree" "not-a-zone" "key1" "key2" ]; - resolve = name: - if name == "whole" then drv name (toString (builtins.length (builtins.attrNames builtins.unsafeTectonixInternalManifest))) - else if name == "keys" then drv name (toString (builtins.length (builtins.tectonixManifestKeys))) - else if name == "id" then drv name ( - let path = builtins.tectonixManifestIdToPath "W-000001"; in if path == null then "absent" else path - ) - else if name == "m1" || name == "m2" then drv name (builtins.hashString "sha256" loaded) - else if name == "za1" || name == "za2" || name == "prefilled" then drv name (zoneId "a") - # Hands both memoized lookups, unforced, to whoever reads the value. - else if name == "peek" then drv name "peek" // { zoneId = zoneId "a"; recordId = (zoneRecord "a").id; } - # Fills the memoized record without forcing its field. - else if name == "fill-record" then builtins.seq (zoneRecord "a") (drv name "record") - else if name == "record-field" then drv name (zoneRecord "a").id - # What the manifest builtins report about zone a, beside what reading - # .meta/manifest.json itself shows. - else if name == "agree" then - let - manifest = builtins.fromJSON (builtins.readFile ./.meta/manifest.json); - id = manifest."//zones/a".id; - in drv name "agree" // { - seen = { - builtins = { inherit (entry "a") id; keys = builtins.tectonixManifestKeys; path = builtins.tectonixManifestIdToPath id; }; - file = { inherit id; keys = builtins.attrNames manifest; path = "//zones/a"; }; - }; - } - # Every manifest key starts with `//`; "keys" is not one. - else if name == "not-a-zone" then drv name (if builtins.tectonixManifestEntry "keys" == null then "none" else "some") - else if name == "key1" || name == "key2" then drv name (zoneId (builtins.readFile ./${name}.txt)) - else drv name (entry name).id; -} -EOF - -commit_world() { - git -C "$WORLD" add -A - git -C "$WORLD" commit -q -m "$1" - git -C "$WORLD" rev-parse HEAD -} - -REV1=$(commit_world "two zones") - -TARGETS='[ "a" "b" "keys" "id" "m1" "m2" "whole" "za1" "za2" ]' - -# Dependency sets (target -> path -> fingerprint) of every target at $1. -# Extra nix flags follow. -deps_at() { - local rev="$1" - shift - nix eval --json -v \ - --extra-experimental-features 'nix-command' \ - --option lazy-trees true \ - --option tectonix-git-dir "$WORLD/.git" \ - --option tectonix-git-sha "$rev" \ - "$@" \ - --expr "builtins.listToAttrs (map (r: { name = r.target; value = r.dependencies; }) (builtins.tecnixTargets { - gitDir = \"$WORLD/.git\"; resolver = \"resolve.nix\"; args = { system = \"test-system\"; }; - rev = \"$rev\"; targets = $TARGETS; includeDependencies = true; includeTargets = false; }))" -} - -nocache_deps_at() { - deps_at "$1" --option tecnix-eval-cache false -} - -CACHE_HOME="$TEST_ROOT/manifest-cache-home" -cached_deps_at() { - XDG_CACHE_HOME="$CACHE_HOME" deps_at "$1" --option tecnix-eval-cache true --pure-eval -} - -# `hits ` asserts each target was served from the cache; -# `misses` asserts each was re-evaluated. -hits() { - local err="$1"; shift - for t in "$@"; do - grepQuiet "dependency cache hit for '$t'" "$err" || fail "$t should be a cache hit ($err)" - done -} -misses() { - local err="$1"; shift - for t in "$@"; do - grepQuiet "dependency cache miss, evaluating '$t'" "$err" || fail "$t should be re-evaluated ($err)" - done -} - -echo "Testing what each target depends on..." -deps1=$(nocache_deps_at "$REV1") -assert_jq "$deps1" '.a | keys == [".meta/manifest.json#//zones/a", "resolve.nix"]' \ - "a should depend on its own manifest entry, not the whole manifest" -assert_jq "$deps1" '.b | keys == [".meta/manifest.json#//zones/b", "resolve.nix"]' \ - "b should depend on its own manifest entry, not the whole manifest" -assert_jq "$deps1" '.keys | keys == [".meta/manifest.json#keys", "resolve.nix"]' \ - "keys should depend on the key set only" -assert_jq "$deps1" '.id | keys == [".meta/manifest.json#id/W-000001", "resolve.nix"]' \ - "id should depend on one id lookup only" -assert_jq "$deps1" '[.a, .b, .keys, .id, .m1, .m2, .za1, .za2 | has(".meta/manifest.json")] | any | not' \ - "the per-entry targets should not depend on the whole manifest file" -assert_jq "$deps1" '.whole | has(".meta/manifest.json")' \ - "reading the whole manifest (builtins.unsafeTectonixInternalManifest) should still depend on the file" -assert_jq "$deps1" '.a[".meta/manifest.json#//zones/a"] != .b[".meta/manifest.json#//zones/b"]' \ - "entry fingerprints should differ between zones" -assert_jq "$deps1" '.m1 == .m2 and (.m1 | has("shared.txt"))' \ - "both consumers of a memoized load should inherit the load's dependencies" -assert_jq "$deps1" '[.za1, .za2 | keys == [".meta/manifest.json#//zones/a", "resolve.nix"]] | all' \ - "a memo hit should replay the dependencies recorded when its entry was filled" -# An argument that is not a zone path could spell another key form (here -# `#keys`, the key set), so it records the whole manifest instead. -not_a_zone=$(TARGETS='[ "not-a-zone" ]' nocache_deps_at "$REV1") -assert_jq "$not_a_zone" '."not-a-zone" | keys == [".meta/manifest.json", "resolve.nix"]' \ - "looking up a non-zone path should depend on the whole manifest" -# How a caller computed a memo key is its own dependency, never the shared -# entry's. Either target may fill the entry first, so check both. -via=$(TARGETS='[ "key1" "key2" ]' nocache_deps_at "$REV1") -assert_jq "$via" '(.key1 | keys) == [".meta/manifest.json#//zones/a", "key1.txt", "resolve.nix"] - and (.key2 | keys) == [".meta/manifest.json#//zones/a", "key2.txt", "resolve.nix"]' \ - "a memo caller should depend on the file it read the key from, and on no other caller's" - -echo "Testing the eval cache across commits..." -cached_deps_at "$REV1" > "$TEST_ROOT/cold.json" 2> "$TEST_ROOT/cold.err" -misses "$TEST_ROOT/cold.err" a b keys id m1 m2 whole za1 za2 -[[ "$(jq -S . < "$TEST_ROOT/cold.json")" == "$(jq -S . <<< "$deps1")" ]] \ - || fail "the cached evaluation should report the same dependencies" - -# Adding an unrelated zone changes the manifest file and the key set. Only the -# targets that enumerate the keys or read the whole file are invalidated. -write_manifest ' "//zones/c": { "id": "W-000003" },' -REV2=$(commit_world "add an unrelated zone") -cached_deps_at "$REV2" > /dev/null 2> "$TEST_ROOT/add-zone.err" -hits "$TEST_ROOT/add-zone.err" a b id m1 m2 za1 za2 -misses "$TEST_ROOT/add-zone.err" keys whole - -# Changing one zone's entry invalidates that zone's target and nothing else. -sed -i.bak 's/W-000002/W-000009/' "$WORLD/.meta/manifest.json" && rm "$WORLD/.meta/manifest.json.bak" -REV3=$(commit_world "change b's id") -cached_deps_at "$REV3" > "$TEST_ROOT/change-b.json" 2> "$TEST_ROOT/change-b.err" -hits "$TEST_ROOT/change-b.err" a id keys m1 m2 za1 za2 -misses "$TEST_ROOT/change-b.err" b whole -assert_jq "$(cat "$TEST_ROOT/change-b.json")" \ - ".b[\".meta/manifest.json#//zones/b\"] != $(jq '.b[".meta/manifest.json#//zones/b"]' <<< "$deps1")" \ - "b's recorded entry fingerprint should follow its entry" - -# An id lookup is invalidated when the id moves, and only then. -sed -i.bak 's/W-000001/W-000010/' "$WORLD/.meta/manifest.json" && rm "$WORLD/.meta/manifest.json.bak" -REV4=$(commit_world "renumber a") -cached_deps_at "$REV4" > /dev/null 2> "$TEST_ROOT/renumber-a.err" -hits "$TEST_ROOT/renumber-a.err" b keys m1 m2 -misses "$TEST_ROOT/renumber-a.err" a id whole za1 za2 - -# A memoized load's dependencies reach every consumer: changing what the load -# read invalidates both, not just whichever consumer evaluated it first. -echo "shared two" > "$WORLD/shared.txt" -REV5=$(commit_world "change the shared file") -cached_deps_at "$REV5" > /dev/null 2> "$TEST_ROOT/shared.err" -hits "$TEST_ROOT/shared.err" a b id keys whole za1 za2 -misses "$TEST_ROOT/shared.err" m1 m2 - -# A file nothing read leaves every row valid. -echo "unrelated two" > "$WORLD/unrelated.txt" -REV6=$(commit_world "change an unrelated file") -cached_deps_at "$REV6" > /dev/null 2> "$TEST_ROOT/unrelated.err" -hits "$TEST_ROOT/unrelated.err" a b keys id m1 m2 whole za1 za2 - -# A memoized result computed outside tracking must never reach a tracked -# target without its dependencies, in either direction: an untracked call that -# comes first, and an untracked call that forces part of an entry a tracked -# target filled. `peek` hands both lookups, unforced, to the top-level -# expression, which forces them outside tracking before the tracked targets -# run in the same evaluation. Extra nix flags follow the rev. -prefill_deps_at() { - local rev="$1" - shift - nix eval --json -v \ - --extra-experimental-features 'nix-command' \ - --option lazy-trees true \ - --option tectonix-git-dir "$WORLD/.git" \ - --option tectonix-git-sha "$rev" \ - "$@" \ - --expr "let - call = includeTargets: targets: builtins.tecnixTargets { - gitDir = \"$WORLD/.git\"; resolver = \"resolve.nix\"; args = { system = \"test-system\"; }; - rev = \"$rev\"; inherit targets includeTargets; includeDependencies = true; }; - peek = (builtins.head (call true [ \"peek\" ])).value; - outsideTracking = builtins.seq peek.zoneId (builtins.seq (call false [ \"fill-record\" ]) peek.recordId); - in builtins.seq outsideTracking (builtins.listToAttrs (map (r: { name = r.target; value = r.dependencies; }) - (call false [ \"prefilled\" \"record-field\" ])))" -} - -echo "Testing memoized results computed outside tracking..." -PREFILLED_DEPS='[.prefilled, ."record-field" | keys == [".meta/manifest.json#//zones/a", "resolve.nix"]] | all' -# First with tracking from includeDependencies alone, then through the eval -# cache, where a missing dependency turns into a stale warm hit. -assert_jq "$(prefill_deps_at "$REV6" --option tecnix-eval-cache false)" "$PREFILLED_DEPS" \ - "tracked targets should depend on the entry behind memoized values first touched outside tracking" - -PREFILL_CACHE_HOME="$TEST_ROOT/manifest-prefill-cache-home" -XDG_CACHE_HOME="$PREFILL_CACHE_HOME" prefill_deps_at "$REV6" --option tecnix-eval-cache true --pure-eval \ - > "$TEST_ROOT/prefill-cold.json" 2> "$TEST_ROOT/prefill-cold.err" -misses "$TEST_ROOT/prefill-cold.err" prefilled record-field -assert_jq "$(cat "$TEST_ROOT/prefill-cold.json")" "$PREFILLED_DEPS" \ - "cached tracked targets should depend on the entry behind memoized values first touched outside tracking" - -# The reviewer's reproduction: renumber zone a, and the warm run must -# re-evaluate instead of serving the old id. -sed -i.bak 's/W-000010/W-000011/' "$WORLD/.meta/manifest.json" && rm "$WORLD/.meta/manifest.json.bak" -REV7=$(commit_world "renumber a again") -XDG_CACHE_HOME="$PREFILL_CACHE_HOME" prefill_deps_at "$REV7" --option tecnix-eval-cache true --pure-eval \ - > /dev/null 2> "$TEST_ROOT/prefill-warm.err" -misses "$TEST_ROOT/prefill-warm.err" prefilled record-field - -# The manifest builtins read .meta/manifest.json through the repo accessor, -# like every file the resolver reads and like the fingerprints of their -# dependency keys. So evaluating an older rev from a checkout whose HEAD has -# moved on shows them that rev's manifest, not the working tree's. -echo "Testing the manifest builtins at an older rev of a checkout..." -seen=$(nix eval --json \ - --extra-experimental-features 'nix-command' \ - --option lazy-trees true \ - --option tecnix-eval-cache false \ - --expr "(builtins.tecnixTargets { gitDir = \"$WORLD/.git\"; resolver = \"resolve.nix\"; args = { system = \"test-system\"; }; - rev = \"$REV1\"; checkoutPath = \"$WORLD\"; targets = [ \"agree\" ]; }).agree.seen") -assert_jq "$seen" '.file.id == "W-000001"' "the resolver should read the evaluated rev's manifest" -assert_jq "$seen" '.builtins == .file' "the manifest builtins should see the manifest the resolver reads" - -echo "All manifest tracking tests passed!" diff --git a/tests/functional/tecnix/memoize.sh b/tests/functional/tecnix/memoize.sh index a90525e21..77e730292 100755 --- a/tests/functional/tecnix/memoize.sh +++ b/tests/functional/tecnix/memoize.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # builtins.tecnixMemoize: `g = builtins.tecnixMemoize f` is `f`, with each # `f k` evaluated once per evaluation and shared. What tracked calls record, and -# that tracked and untracked results stay apart, is in manifest-tracking.sh. +# that tracked and untracked results stay apart, is in data-query.sh. # shellcheck disable=SC2016 source "$(dirname "${BASH_SOURCE[0]}")/common.sh" diff --git a/tests/functional/tecnix/meson.build b/tests/functional/tecnix/meson.build index 540675a5b..c811b2055 100644 --- a/tests/functional/tecnix/meson.build +++ b/tests/functional/tecnix/meson.build @@ -5,7 +5,7 @@ suites += { 'base-env.sh', 'builtins.sh', 'gc.sh', - 'manifest-tracking.sh', + 'data-query.sh', 'ingestion-stats.sh', 'memoize.sh', ], From e51b218bdceaa55e9388a7615e59490607e1656a Mon Sep 17 00:00:00 2001 From: Josh Heinrichs Date: Thu, 15 Oct 2026 16:08:10 -0400 Subject: [PATCH 2/2] libexpr: an unreadable channels directory is not on the default search path EvalSettings::getDefaultNixPath adds ~/.nix-defexpr/channels and root's channels to the search path when they exist, asking with the throwing std::filesystem::exists. A stat that fails with anything but ENOENT or ENOTDIR therefore aborts the construction of every impure EvalState: filesystem error: in posix_stat: failed to determine attributes for the specified path: Operation not permitted ["/nix/var/nix/profiles/per-user/root/channels/nixpkgs"] That is what a strict Darwin sandbox answers, since /nix/var/nix/profiles is not among the paths it exposes, so a `nix` run inside a sandboxed build on macOS cannot evaluate impurely at all. It is how Tecnix's own test suite fails when built with `sandbox = true` on a Mac (macOS defaults the sandbox off, which is why nobody saw it): every libexpr test fixture, the C API flake tests, and the inner `nix build` of tests/functional/recursive.sh. On Linux the directory is simply absent inside the chroot and the ENOENT is swallowed, as intended. Ask with the non-throwing overload. A directory the filesystem will not even stat is not a default search path, which is the answer for one that does not exist, and no reason to refuse to evaluate. The test makes root's channels directory unreadable through a parent without search permission and checks that getDefaultNixPath neither throws nor lists it. It is skipped as root, who is never refused a stat. --- src/libexpr-tests/eval.cc | 31 +++++++++++++++++++++++++++++++ src/libexpr/eval-settings.cc | 6 +++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/libexpr-tests/eval.cc b/src/libexpr-tests/eval.cc index 4ddd1b302..123382c36 100644 --- a/src/libexpr-tests/eval.cc +++ b/src/libexpr-tests/eval.cc @@ -7,8 +7,10 @@ #include "nix/store/async-path-writer.hh" #include "nix/store/derivations.hh" #include "nix/store/local-store.hh" +#include "nix/store/globals.hh" #include "nix/util/file-system.hh" #include "nix/util/finally.hh" +#include "nix/util/users.hh" namespace nix { @@ -153,6 +155,35 @@ TEST(nix_isAllowedURI, non_scheme_colon) ASSERT_FALSE(isAllowedURI("https://foo/bar:baz", allowed)); } +/* The default search path lists the channel directories that exist. One the + filesystem refuses to stat (here EACCES, through a parent without search + permission; EPERM from a sandbox is the same to exists()) is left out, and + must not make every impure evaluation fail. */ +TEST(EvalSettings, getDefaultNixPathSkipsUnreadableChannels) +{ + if (isRootUser()) + GTEST_SKIP() << "root is never refused a stat"; + + auto tmp = createTempDir(); + auto root = tmp / "state" / "profiles" / "per-user" / "root"; + createDirs(root); + std::filesystem::permissions(root, std::filesystem::perms::none); + Finally cleanup([&] { + std::filesystem::permissions(root, std::filesystem::perms::owner_all); + deletePath(tmp); + }); + + auto savedStateDir = settings.nixStateDir; + Finally restoreStateDir([&] { settings.nixStateDir = savedStateDir; }); + settings.nixStateDir = tmp / "state"; + + auto channels = (root / "channels").string(); + Strings res; + ASSERT_NO_THROW(res = EvalSettings::getDefaultNixPath()); + for (auto & entry : res) + ASSERT_THAT(entry, testing::Not(testing::HasSubstr(channels))); +} + class EvalStateTest : public LibExprTest {}; diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 630f7b670..1424333a1 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -84,7 +84,11 @@ Strings EvalSettings::getDefaultNixPath() { Strings res; auto add = [&](const std::filesystem::path & p, const std::string & s = std::string()) { - if (std::filesystem::exists(p)) { + /* A directory the filesystem will not even stat (EACCES, or EPERM + from a sandbox that excludes /nix/var/nix/profiles) is not a + default search path; it is not a reason to refuse to evaluate. */ + std::error_code ec; + if (std::filesystem::exists(p, ec)) { if (s.empty()) { res.push_back(p.string()); } else {