From 87baf29d6aebcda00eb2c46275afca7d3ce958b5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Nov 2025 15:30:38 +0100 Subject: [PATCH 001/927] Git fetcher: Don't compute revCount if it's already specified We don't care if the user (or more likely the lock file) specifies an incorrect value for revCount, since it doesn't matter for security (unlikely content hashes like narHash). --- src/libfetchers/fetchers.cc | 5 ----- src/libfetchers/git.cc | 9 +++++++-- tests/nixos/tarball-flakes.nix | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 7e091ef107..0d8bd4514f 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -282,11 +282,6 @@ void Input::checkLocks(Input specified, Input & result) if (result.getRev() != prevRev) throw Error("'rev' attribute mismatch in input '%s', expected %s", result.to_string(), prevRev->gitRev()); } - - if (auto prevRevCount = specified.getRevCount()) { - if (result.getRevCount() != prevRevCount) - throw Error("'revCount' attribute mismatch in input '%s', expected %d", result.to_string(), *prevRevCount); - } } std::pair, Input> Input::getAccessor(const Settings & settings, Store & store) const diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 75e3f12148..f4d492308d 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -891,8 +891,13 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev)); - if (!getShallowAttr(input)) - input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); + if (!getShallowAttr(input)) { + /* Skip revCount computation if it's already supplied by the caller. + We don't care if they specify an incorrect value; it doesn't + matter for security, unlike narHash. */ + if (!input.attrs.contains("revCount")) + input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); + } printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix index 26c20cb1ae..b0402412d8 100644 --- a/tests/nixos/tarball-flakes.nix +++ b/tests/nixos/tarball-flakes.nix @@ -99,7 +99,6 @@ in # Check that fetching fails if we provide incorrect attributes. machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?rev=493300eb13ae6fb387fbd47bf54a85915acc31c0") - machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?revCount=789") machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?narHash=sha256-tbudgBSg+bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw=") ''; From 8f32f28ebdaf3272d386756724d345883eec760c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Nov 2025 15:39:19 +0100 Subject: [PATCH 002/927] Git fetcher: Don't compute lastModified if it's already specified Same as revCount. --- src/libfetchers/fetchers.cc | 9 --------- src/libfetchers/git.cc | 10 ++++++---- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 0d8bd4514f..48a23ad369 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -269,15 +269,6 @@ void Input::checkLocks(Input specified, Input & result) } } - if (auto prevLastModified = specified.getLastModified()) { - if (result.getLastModified() != prevLastModified) - throw Error( - "'lastModified' attribute mismatch in input '%s', expected %d, got %d", - result.to_string(), - *prevLastModified, - result.getLastModified().value_or(-1)); - } - if (auto prevRev = specified.getRev()) { if (result.getRev() != prevRev) throw Error("'rev' attribute mismatch in input '%s', expected %s", result.to_string(), prevRev->gitRev()); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index f4d492308d..96008d45bd 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -889,12 +889,14 @@ struct GitInputScheme : InputScheme auto rev = *input.getRev(); - input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev)); + /* Skip lastModified computation if it's already supplied by the caller. + We don't care if they specify an incorrect value; it doesn't + matter for security, unlike narHash. */ + if (!input.attrs.contains("lastModified")) + input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev)); if (!getShallowAttr(input)) { - /* Skip revCount computation if it's already supplied by the caller. - We don't care if they specify an incorrect value; it doesn't - matter for security, unlike narHash. */ + /* Like lastModified, skip revCount if supplied by the caller. */ if (!input.attrs.contains("revCount")) input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); } From 4ecc09c43f5f42808134ec8144ac2afdcce0fed2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Nov 2025 19:58:08 +0100 Subject: [PATCH 003/927] Make content-encoding test more reliable --- tests/nixos/content-encoding.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/nixos/content-encoding.nix b/tests/nixos/content-encoding.nix index debee377bd..1e188cb060 100644 --- a/tests/nixos/content-encoding.nix +++ b/tests/nixos/content-encoding.nix @@ -131,6 +131,7 @@ in start_all() machine.wait_for_unit("nginx.service") + machine.wait_for_open_port(80) # Original test: zstd archive with gzip content-encoding # Make sure that the file is properly compressed as the test would be meaningless otherwise From 0255d696c6658f1cc1bd0aad57e9df495f177dda Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 28 Jan 2026 03:31:31 -0500 Subject: [PATCH 004/927] Kill build hook with SIGTERM instead of SIGKILL This gives custom build hooks the chance to perform any needed cleanup, which is impossible when killed with SIGKILL. After 500ms, if the hook still hasn't exited, it will be forcibly killed with SIGKILL. Resolves #14760 Signed-off-by: Lisanna Dettwyler --- src/libstore/unix/build/hook-instance.cc | 4 ++++ src/libutil/include/nix/util/processes.hh | 4 ++++ src/libutil/unix/processes.cc | 29 +++++++++++++++++++++-- src/nix/build-remote/build-remote.cc | 17 +++++++++++++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index af92494870..65f528aa18 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -72,6 +72,10 @@ HookInstance::HookInstance() throw SysError("executing %s", PathFmt(buildHook)); }); + /* Give custom build hooks the chance to cleanup. */ + pid.setKillSignal(SIGTERM); + pid.setKillTimeout(500ms); + pid.setSeparatePG(true); fromHook.writeSide = -1; toHook.readSide = -1; diff --git a/src/libutil/include/nix/util/processes.hh b/src/libutil/include/nix/util/processes.hh index 81e6a2b83c..cc33caebfa 100644 --- a/src/libutil/include/nix/util/processes.hh +++ b/src/libutil/include/nix/util/processes.hh @@ -18,6 +18,7 @@ #include #include #include +#include namespace nix { @@ -30,6 +31,8 @@ class Pid pid_t pid = -1; bool separatePG = false; int killSignal = SIGKILL; + std::chrono::milliseconds killTimeout; + std::thread killThread; #else AutoCloseFD pid = INVALID_DESCRIPTOR; #endif @@ -55,6 +58,7 @@ public: #ifndef _WIN32 void setSeparatePG(bool separatePG); void setKillSignal(int signal); + void setKillTimeout(std::chrono::milliseconds duration); pid_t release(); #endif diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 2393da55d4..930020c23e 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -12,7 +12,8 @@ #include #include #include -#include +#include +using namespace std::chrono_literals; #include #include @@ -75,6 +76,20 @@ int Pid::kill(bool allowInterrupts) debug("killing process %1%", pid); + std::atomic killed = false; + + if (killTimeout > 0ms && killSignal != SIGKILL) + killThread = std::thread([&]() { + auto elapsed = 0ms; + while (elapsed < killTimeout) { + std::this_thread::sleep_for(25ms); + elapsed += 25ms; + if (killed) + return; + } + ::kill(separatePG ? -pid : pid, SIGKILL); + }); + /* Send the requested signal to the child. If it has its own process group, send the signal to every process in the child process group (which hopefully includes *all* its children). */ @@ -88,7 +103,12 @@ int Pid::kill(bool allowInterrupts) logError(SysError("killing process %d", pid).info()); } - return wait(allowInterrupts); + int ret = wait(allowInterrupts); + if (killThread.joinable()) { + killed = true; + killThread.join(); + } + return ret; } int Pid::wait(bool allowInterrupts) @@ -118,6 +138,11 @@ void Pid::setKillSignal(int signal) this->killSignal = signal; } +void Pid::setKillTimeout(std::chrono::milliseconds duration) +{ + this->killTimeout = duration; +} + pid_t Pid::release() { pid_t p = pid; diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index eec0a0f284..3333678026 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -53,6 +53,23 @@ static bool allSupportedLocally(Store & store, const StringSet & requiredFeature static int main_build_remote(int argc, char ** argv) { { + /* Upon exiting, Nix will attempt to terminate this process with + SIGTERM. initNix will block or handle SIGTERM, so we need to unblock + and unhandle it here. + */ + struct sigaction act; + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + act.sa_handler = SIG_DFL; + if (sigaction(SIGTERM, &act, 0)) + throw SysError("resetting SIGTERM"); + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGTERM); + if (pthread_sigmask(SIG_UNBLOCK, &set, nullptr)) + throw SysError("unblocking SIGTERM"); + logger = makeJSONLogger(getStandardError()); /* Ensure we don't get any SSH passphrase or host key popups. */ From f25ec8344bddbeeea4a59e45e3522bd57e1bcd22 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Fri, 30 Jan 2026 11:52:41 -0500 Subject: [PATCH 005/927] Display shutdown message on interrupt Signed-off-by: Lisanna Dettwyler --- src/libmain/progress-bar.cc | 7 +++++++ src/libstore/unix/build/hook-instance.cc | 2 ++ src/libutil/unix/signals.cc | 17 +++++++++++++---- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index a973102f95..71c7789de2 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,6 +1,7 @@ #include "nix/main/progress-bar.hh" #include "nix/util/terminal.hh" #include "nix/util/sync.hh" +#include "nix/util/signals.hh" #include "nix/store/store-api.hh" #include "nix/store/names.hh" @@ -94,10 +95,16 @@ class ProgressBar : public Logger bool printBuildLogs = false; bool isTTY; + std::unique_ptr interruptCallback; + public: ProgressBar(bool isTTY) : isTTY(isTTY) + , interruptCallback(createInterruptCallback([&]() { + pause(); + redraw("\rshutting down\e[K"); + })) { state_.lock()->active = isTTY; updateThread = std::thread([&]() { diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index 65f528aa18..364c1c4ff3 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -6,6 +6,8 @@ #include "nix/util/strings.hh" #include "nix/util/executable-path.hh" +using namespace std::chrono_literals; + namespace nix { HookInstance::HookInstance() diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index deff402764..d073c9e46c 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -42,7 +42,16 @@ struct InterruptCallbacks std::map> callbacks; }; -static Sync _interruptCallbacks; +/* Required to avoid static initialization order fiasco. This allows global + objects to safely register callbacks. */ +static Sync & getInterruptCallbacks() +{ + /* Intentionally leak, according to the Construct On First Use Idiom. + An alternative is to use the Nifty Counter Idiom, but + InterruptCallbacks' destructor is not very important. */ + static Sync * _interruptCallbacks = new Sync(); + return *_interruptCallbacks; +} static void signalHandlerThread(sigset_t set) { @@ -68,7 +77,7 @@ void unix::triggerInterrupt() while (true) { std::function callback; { - auto interruptCallbacks(_interruptCallbacks.lock()); + auto interruptCallbacks(getInterruptCallbacks().lock()); auto lb = interruptCallbacks->callbacks.lower_bound(i); if (lb == interruptCallbacks->callbacks.end()) break; @@ -153,14 +162,14 @@ struct InterruptCallbackImpl : InterruptCallback ~InterruptCallbackImpl() override { - auto interruptCallbacks(_interruptCallbacks.lock()); + auto interruptCallbacks(getInterruptCallbacks().lock()); interruptCallbacks->callbacks.erase(token); } }; std::unique_ptr createInterruptCallback(std::function callback) { - auto interruptCallbacks(_interruptCallbacks.lock()); + auto interruptCallbacks(getInterruptCallbacks().lock()); auto token = interruptCallbacks->nextToken++; interruptCallbacks->callbacks.emplace(token, callback); return std::make_unique(token); From 9868310d6f2d6a2225eaf9d48c94b88b67cc9ac5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 6 Feb 2026 14:58:27 +0100 Subject: [PATCH 006/927] Add test for builtins.getFlake --- tests/functional/flakes/common.sh | 2 ++ tests/functional/flakes/get-flake.sh | 20 ++++++++++++++++++++ tests/functional/flakes/meson.build | 1 + 3 files changed, 23 insertions(+) create mode 100644 tests/functional/flakes/get-flake.sh diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 2dcf2e0fd5..6fef789255 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -32,6 +32,8 @@ writeSimpleFlake() { baseName = builtins.baseNameOf ./.; root = ./.; + + number = 123; }; } EOF diff --git a/tests/functional/flakes/get-flake.sh b/tests/functional/flakes/get-flake.sh new file mode 100644 index 0000000000..85ef8c23d1 --- /dev/null +++ b/tests/functional/flakes/get-flake.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +source ./common.sh + +createFlake1 + +mkdir -p "$flake1Dir/subflake" +cat > "$flake1Dir/subflake/flake.nix" < Date: Fri, 6 Feb 2026 15:48:47 +0100 Subject: [PATCH 007/927] builtins.getFlake: Support path values This allows doing `builtins.getFlake ./subflake` instead of ugly hacks. --- src/libflake/flake-primops.cc | 50 +++++++++++++------------ src/libflake/flake.cc | 24 +++++++++--- src/libflake/include/nix/flake/flake.hh | 7 ++++ tests/functional/flakes/get-flake.sh | 6 +++ 4 files changed, 58 insertions(+), 29 deletions(-) diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index a43777574f..6ff66a964e 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -35,35 +35,39 @@ namespace nix::flake::primops { PrimOp getFlake(const Settings & settings) { auto prim_getFlake = [&settings](EvalState & state, const PosIdx pos, Value ** args, Value & v) { - std::string flakeRefS( - state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); - auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); - if (state.settings.pureEval && !flakeRef.input.isLocked(state.fetchSettings)) - throw Error( - "cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", - flakeRefS, - state.positions[pos]); - - callFlake( - state, - lockFlake( - settings, - state, - flakeRef, - LockFlags{ - .updateLockFile = false, - .writeLockFile = false, - .useRegistries = !state.settings.pureEval && settings.useRegistries, - .allowUnlocked = !state.settings.pureEval, - }), - v); + state.forceValue(*args[0], pos); + + LockFlags lockFlags{ + .updateLockFile = false, + .writeLockFile = false, + .useRegistries = !state.settings.pureEval && settings.useRegistries, + .allowUnlocked = !state.settings.pureEval, + }; + + if (args[0]->type() == nPath) { + auto path = state.realisePath(pos, *args[0]); + callFlake(state, lockFlake(settings, state, path, lockFlags), v); + } else { + NixStringContext context; + std::string flakeRefS( + state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); + + auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); + if (state.settings.pureEval && !flakeRef.input.isLocked(state.fetchSettings)) + throw Error( + "cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", + flakeRefS, + state.positions[pos]); + + callFlake(state, lockFlake(settings, state, flakeRef, lockFlags), v); + } }; return PrimOp{ .name = "__getFlake", .args = {"args"}, .doc = R"( - Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: + Fetch a flake from a flake reference or a path, and return its output attributes and some metadata. For example: ```nix (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 9dab20c5c1..13999a5c30 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -416,10 +416,8 @@ static LockFile readLockFile(const fetchers::Settings & fetchSettings, const Sou : LockFile(); } -/* Compute an in-memory lock file for the specified top-level flake, - and optionally write it to file, if the flake is writable. */ -LockedFlake -lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) +LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags, Flake flake) { experimentalFeatureSettings.require(Xp::Flakes); @@ -427,8 +425,6 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; - auto flake = getFlake(state, topRef, useRegistriesTop, {}); - if (lockFlags.applyNixConfig) { flake.config.apply(settings); state.store->setOptions(); @@ -908,6 +904,22 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, } } +LockedFlake +lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) +{ + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); + auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; + return lockFlake(settings, state, topRef, lockFlags, getFlake(state, topRef, useRegistriesTop, {})); +} + +LockedFlake +lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags) +{ + /* We need a fake flakeref to put in the `Flake` struct, but it's not used for anything. */ + auto fakeRef = parseFlakeRef(state.fetchSettings, "flake:get-flake"); + return lockFlake(settings, state, fakeRef, lockFlags, readFlake(state, fakeRef, fakeRef, fakeRef, flakeDir, {})); +} + static ref makeInternalFS() { auto internalFS = make_ref(MemorySourceAccessor{}); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 3bee6556f6..fd52dbebac 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -214,9 +214,16 @@ struct LockFlags std::set inputUpdates; }; +/* + * Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is + * writable. + */ LockedFlake lockFlake(const Settings & settings, EvalState & state, const FlakeRef & flakeRef, const LockFlags & lockFlags); +LockedFlake +lockFlake(const Settings & settings, EvalState & state, const SourcePath & flakeDir, const LockFlags & lockFlags); + void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & v); /** diff --git a/tests/functional/flakes/get-flake.sh b/tests/functional/flakes/get-flake.sh index 85ef8c23d1..e462607db8 100644 --- a/tests/functional/flakes/get-flake.sh +++ b/tests/functional/flakes/get-flake.sh @@ -9,12 +9,18 @@ cat > "$flake1Dir/subflake/flake.nix" < Date: Wed, 18 Feb 2026 22:50:37 +0100 Subject: [PATCH 008/927] Add release note --- doc/manual/rl-next/getflake-path.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/manual/rl-next/getflake-path.md diff --git a/doc/manual/rl-next/getflake-path.md b/doc/manual/rl-next/getflake-path.md new file mode 100644 index 0000000000..2360fe7693 --- /dev/null +++ b/doc/manual/rl-next/getflake-path.md @@ -0,0 +1,6 @@ +--- +synopsis: "`builtins.getFlake` now supports path values" +prs: [15290] +--- + +`builtins.getFlake` now accepts path values in addition to flakerefs, allowing you to write `builtins.getFlake ./subflake` instead of having to use ugly workarounds to construct a pure flakeref. From 6cae299bd972892549ddaee722dc210514d481bc Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Tue, 24 Feb 2026 16:45:55 -0500 Subject: [PATCH 009/927] Error on invalid URL param in github fetcher Resolves #15304 Signed-off-by: Lisanna Dettwyler --- .../github-fetcher-param-validation.md | 7 ++++ src/libfetchers-tests/input.cc | 18 ++++++++++ src/libfetchers/github.cc | 36 +++++++------------ 3 files changed, 38 insertions(+), 23 deletions(-) create mode 100644 doc/manual/rl-next/github-fetcher-param-validation.md diff --git a/doc/manual/rl-next/github-fetcher-param-validation.md b/doc/manual/rl-next/github-fetcher-param-validation.md new file mode 100644 index 0000000000..2fb430ae4b --- /dev/null +++ b/doc/manual/rl-next/github-fetcher-param-validation.md @@ -0,0 +1,7 @@ +--- +synopsis: GitHub fetcher now validates URL parameters +prs: [15331] +issues: [15304] +--- + +The `github:` fetcher now validates URL parameters, and will error if an invalid parameter like `tag` is provided. diff --git a/src/libfetchers-tests/input.cc b/src/libfetchers-tests/input.cc index faff55f2c2..170ff4485c 100644 --- a/src/libfetchers-tests/input.cc +++ b/src/libfetchers-tests/input.cc @@ -1,8 +1,12 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/attrs.hh" #include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/fetch-settings.hh" +#include "nix/util/tests/gmock-matchers.hh" +#include "nix/util/url.hh" #include +#include #include @@ -58,4 +62,18 @@ INSTANTIATE_TEST_SUITE_P( }), [](const ::testing::TestParamInfo & info) { return info.param.description; }); +namespace fetchers { + +class GitHubInputTest : public ::testing::Test +{}; + +TEST_F(GitHubInputTest, throwOnInvalidURLParam) +{ + EXPECT_THAT( + []() { Input::fromURL(fetchers::Settings{}, "github:a/b?tag=foo"); }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher("tag"))); +} + +} // namespace fetchers + } // namespace nix diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index b86fa926a6..ac8c02418d 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -41,16 +41,14 @@ struct GitArchiveInputScheme : InputScheme /* This ignores empty path segments for back-compat. Older versions used a tokenizeString here. */ auto path = url.pathSegments(/*skipEmpty=*/true) | std::ranges::to>(); - std::optional rev; - std::optional ref; - std::optional host_url; + Attrs attrs; auto size = path.size(); if (size == 3) { if (std::regex_match(path[2], revRegex)) - rev = path[2]; + attrs.insert_or_assign("rev", path[2]); else - ref = path[2]; + attrs.insert_or_assign("ref", path[2]); } else if (size > 3) { std::string rs; for (auto i = std::next(path.begin(), 2); i != path.end(); i++) { @@ -59,38 +57,30 @@ struct GitArchiveInputScheme : InputScheme rs += "/"; } } - ref = rs; + attrs.insert_or_assign("ref", rs); } else if (size < 2) throw BadURL("URL '%s' is invalid", url); for (auto & [name, value] : url.query) { if (name == "rev") { - if (rev) + if (attrs.contains(name)) throw BadURL("URL '%s' contains multiple commit hashes", url); - rev = value; + attrs.insert_or_assign("rev", value); } else if (name == "ref") { - if (ref) + if (attrs.contains(name)) throw BadURL("URL '%s' contains multiple branch/tag names", url); - ref = value; + attrs.insert_or_assign("ref", value); } else if (name == "host") - host_url = value; - // FIXME: barf on unsupported attributes + attrs.insert_or_assign("host", value); + else if (name == "narHash") + attrs.insert_or_assign("narHash", value); + else + throw BadURL("URL '%s' contains unknown parameter '%s'", url, name); } - Attrs attrs; attrs.insert_or_assign("type", std::string{schemeName()}); attrs.insert_or_assign("owner", path[0]); attrs.insert_or_assign("repo", path[1]); - if (rev) - attrs.insert_or_assign("rev", *rev); - if (ref) - attrs.insert_or_assign("ref", *ref); - if (host_url) - attrs.insert_or_assign("host", *host_url); - - auto narHash = url.query.find("narHash"); - if (narHash != url.query.end()) - attrs.insert_or_assign("narHash", narHash->second); return inputFromAttrs(settings, attrs); } From 26c1c8fb4a34e2669027c02cb9e480b2c0f53c80 Mon Sep 17 00:00:00 2001 From: eveeifyeve <88671402+Eveeifyeve@users.noreply.github.com> Date: Sun, 1 Mar 2026 05:52:48 +1100 Subject: [PATCH 010/927] feat(nix): add shortname to bundle bundler arg --- src/nix/bundle.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 5bca8e0cb3..a9266a11bd 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -18,6 +18,7 @@ struct CmdBundle : InstallableValueCommand { addFlag({ .longName = "bundler", + .shortName = 'B', .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, From 30b6bba0feb1aa5b37ec0937c647a794a271538a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 26 Feb 2026 14:37:27 -0500 Subject: [PATCH 011/927] Convert `parseURLRelative` tests to parameters test suite This will allow extending it further in the next commit --- src/libutil-tests/url.cc | 490 +++++++++++++++++++++++---------------- 1 file changed, 291 insertions(+), 199 deletions(-) diff --git a/src/libutil-tests/url.cc b/src/libutil-tests/url.cc index efc322feb8..1197adaa8c 100644 --- a/src/libutil-tests/url.cc +++ b/src/libutil-tests/url.cc @@ -168,7 +168,11 @@ INSTANTIATE_TEST_SUITE_P( .expected = ParsedURL{ .scheme = "http", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, .path = {"", "file.tar.gz"}, .query = (StringMap) {}, .fragment = "", @@ -179,7 +183,11 @@ INSTANTIATE_TEST_SUITE_P( .expected = ParsedURL{ .scheme = "https", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, .path = {"", "file.tar.gz"}, .query = (StringMap) {}, .fragment = "", @@ -190,7 +198,11 @@ INSTANTIATE_TEST_SUITE_P( .expected = ParsedURL{ .scheme = "https", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, .path = {"", "file.tar.gz"}, .query = (StringMap) {{"download", "fast"}, {"when", "now"}}, .fragment = "hello", @@ -201,7 +213,11 @@ INSTANTIATE_TEST_SUITE_P( .expected = ParsedURL{ .scheme = "file+https", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, .path = {"", "video.mp4"}, .query = (StringMap) {}, .fragment = "", @@ -212,7 +228,12 @@ INSTANTIATE_TEST_SUITE_P( .expected = ParsedURL{ .scheme = "http", - .authority = Authority{.hostType = HostType::IPv4, .host = "127.0.0.1", .port = 8080}, + .authority = + Authority{ + .hostType = HostType::IPv4, + .host = "127.0.0.1", + .port = 8080, + }, .path = {"", "file.tar.gz"}, .query = (StringMap) {{"download", "fast"}, {"when", "now"}}, .fragment = "hello", @@ -225,7 +246,10 @@ INSTANTIATE_TEST_SUITE_P( .scheme = "http", .authority = Authority{ - .hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c\%enp0s25", .port = 8080}, + .hostType = HostType::IPv6, + .host = "fe80::818c:da4d:8975:415c\%enp0s25", + .port = 8080, + }, .path = {""}, .query = (StringMap) {}, .fragment = "", @@ -277,7 +301,11 @@ TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment) ParsedURL expected{ .scheme = "http", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, .path = {"", "file.tar.gz"}, .query = (StringMap) {{"field", "value"}}, .fragment = "?foo=bar#", @@ -484,7 +512,11 @@ TEST(parseURL, parsesHttpUrlWithEmptyPort) ParsedURL expected{ .scheme = "http", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, .path = {"", "file.tar.gz"}, .query = (StringMap) {{"foo", "bar"}}, .fragment = "", @@ -498,203 +530,263 @@ TEST(parseURL, parsesHttpUrlWithEmptyPort) * parseURLRelative * --------------------------------------------------------------------------*/ -TEST(parseURLRelative, resolvesRelativePath) +struct ParseURLRelativeParam { - ParsedURL base = parseURL("http://example.org/dir/page.html"); - auto parsed = parseURLRelative("subdir/file.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"}, - .path = {"", "dir", "subdir", "file.txt"}, - .query = {}, - .fragment = "", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, baseUrlIpv6AddressWithoutZoneId) -{ - ParsedURL base = parseURL("http://[fe80::818c:da4d:8975:415c]/dir/page.html"); - auto parsed = parseURLRelative("subdir/file.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = ParsedURL::Authority{.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c"}, - .path = {"", "dir", "subdir", "file.txt"}, - .query = {}, - .fragment = "", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, resolvesRelativePathIpv6AddressWithZoneId) -{ - ParsedURL base = parseURL("http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080/dir/page.html"); - auto parsed = parseURLRelative("subdir/file2.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = Authority{.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c\%enp0s25", .port = 8080}, - .path = {"", "dir", "subdir", "file2.txt"}, - .query = {}, - .fragment = "", - }; - - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, resolvesRelativePathWithDot) -{ - ParsedURL base = parseURL("http://example.org/dir/page.html"); - auto parsed = parseURLRelative("./subdir/file.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"}, - .path = {"", "dir", "subdir", "file.txt"}, - .query = {}, - .fragment = "", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, resolvesParentDirectory) -{ - ParsedURL base = parseURL("http://example.org:234/dir/page.html"); - auto parsed = parseURLRelative("../up.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org", .port = 234}, - .path = {"", "up.txt"}, - .query = {}, - .fragment = "", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, resolvesParentDirectoryNotTrickedByEscapedSlash) -{ - ParsedURL base = parseURL("http://example.org:234/dir\%2Ffirst-trick/another-dir\%2Fsecond-trick/page.html"); - auto parsed = parseURLRelative("../up.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org", .port = 234}, - .path = {"", "dir/first-trick", "up.txt"}, - .query = {}, - .fragment = "", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, replacesPathWithAbsoluteRelative) -{ - ParsedURL base = parseURL("http://example.org/dir/page.html"); - auto parsed = parseURLRelative("/rooted.txt", base); - ParsedURL expected{ - .scheme = "http", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"}, - .path = {"", "rooted.txt"}, - .query = {}, - .fragment = "", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, keepsQueryAndFragmentFromRelative) -{ - // But discard query params on base URL - ParsedURL base = parseURL("https://www.example.org/path/index.html?z=3"); - auto parsed = parseURLRelative("other.html?x=1&y=2#frag", base); - ParsedURL expected{ - .scheme = "https", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = {"", "path", "other.html"}, - .query = {{"x", "1"}, {"y", "2"}}, - .fragment = "frag", - }; - ASSERT_EQ(parsed, expected); -} - -TEST(parseURLRelative, absOverride) -{ - ParsedURL base = parseURL("http://example.org/path/page.html"); - std::string_view abs = "https://127.0.0.1.org/secure"; - auto parsed = parseURLRelative(abs, base); - auto parsedAbs = parseURL(abs); - ASSERT_EQ(parsed, parsedAbs); -} - -TEST(parseURLRelative, absOverrideWithZoneId) -{ - ParsedURL base = parseURL("http://example.org/path/page.html"); - std::string_view abs = "https://[fe80::818c:da4d:8975:415c\%25enp0s25]/secure?foo=bar"; - auto parsed = parseURLRelative(abs, base); - auto parsedAbs = parseURL(abs); - ASSERT_EQ(parsed, parsedAbs); -} - -TEST(parseURLRelative, bothWithoutAuthority) -{ - ParsedURL base = parseURL("mailto:mail-base@bar.baz?bcc=alice@asdf.com"); - std::string_view over = "mailto:mail-override@foo.bar?subject=url-testing"; - auto parsed = parseURLRelative(over, base); - auto parsedOverride = parseURL(over); - ASSERT_EQ(parsed, parsedOverride); -} - -TEST(parseURLRelative, emptyRelative) -{ - ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag"); - auto parsed = parseURLRelative("", base); - ParsedURL expected{ - .scheme = "https", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = {"", "path", "index.html"}, - .query = {{"a b", "5 6"}, {"x y", "34"}}, - .fragment = "", - }; - EXPECT_EQ(base.fragment, "frag"); - EXPECT_EQ(parsed, expected); -} + std::string_view base; + std::string_view relative; + ParsedURL expected; + std::string description; +}; -TEST(parseURLRelative, fragmentRelative) -{ - ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag"); - auto parsed = parseURLRelative("#frag2", base); - ParsedURL expected{ - .scheme = "https", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = {"", "path", "index.html"}, - .query = {{"a b", "5 6"}, {"x y", "34"}}, - .fragment = "frag2", - }; - EXPECT_EQ(parsed, expected); -} +class ParseURLRelativeTestSuite : public ::testing::TestWithParam +{}; -TEST(parseURLRelative, queryRelative) +TEST_P(ParseURLRelativeTestSuite, resolve) { - ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag"); - auto parsed = parseURLRelative("?asdf\%20qwer=1\%202\%203", base); - ParsedURL expected{ - .scheme = "https", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = {"", "path", "index.html"}, - .query = {{"asdf qwer", "1 2 3"}}, - .fragment = "", - }; - EXPECT_EQ(parsed, expected); + auto & p = GetParam(); + auto base = parseURL(p.base); + auto parsed = parseURLRelative(p.relative, base); + EXPECT_EQ(parsed, p.expected); } -TEST(parseURLRelative, queryFragmentRelative) -{ - ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag"); - auto parsed = parseURLRelative("?asdf\%20qwer=1\%202\%203#frag2", base); - ParsedURL expected{ - .scheme = "https", - .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = {"", "path", "index.html"}, - .query = {{"asdf qwer", "1 2 3"}}, - .fragment = "frag2", - }; - EXPECT_EQ(parsed, expected); -} +INSTANTIATE_TEST_SUITE_P( + parseURLRelative, + ParseURLRelativeTestSuite, + ::testing::Values( + ParseURLRelativeParam{ + .base = "http://example.org/dir/page.html", + .relative = "subdir/file.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "example.org", + }, + .path = {"", "dir", "subdir", "file.txt"}, + }, + .description = "resolvesRelativePath", + }, + ParseURLRelativeParam{ + .base = "http://[fe80::818c:da4d:8975:415c]/dir/page.html", + .relative = "subdir/file.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::IPv6, + .host = "fe80::818c:da4d:8975:415c", + }, + .path = {"", "dir", "subdir", "file.txt"}, + }, + .description = "baseUrlIpv6AddressWithoutZoneId", + }, + ParseURLRelativeParam{ + .base = "http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080/dir/page.html", + .relative = "subdir/file2.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::IPv6, + .host = "fe80::818c:da4d:8975:415c%enp0s25", + .port = 8080, + }, + .path = {"", "dir", "subdir", "file2.txt"}, + }, + .description = "resolvesRelativePathIpv6AddressWithZoneId", + }, + ParseURLRelativeParam{ + .base = "http://example.org/dir/page.html", + .relative = "./subdir/file.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "example.org", + }, + .path = {"", "dir", "subdir", "file.txt"}, + }, + .description = "resolvesRelativePathWithDot", + }, + ParseURLRelativeParam{ + .base = "http://example.org:234/dir/page.html", + .relative = "../up.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "example.org", + .port = 234, + }, + .path = {"", "up.txt"}, + }, + .description = "resolvesParentDirectory", + }, + ParseURLRelativeParam{ + .base = "http://example.org:234/dir\%2Ffirst-trick/another-dir\%2Fsecond-trick/page.html", + .relative = "../up.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "example.org", + .port = 234, + }, + .path = {"", "dir/first-trick", "up.txt"}, + }, + .description = "resolvesParentDirectoryNotTrickedByEscapedSlash", + }, + ParseURLRelativeParam{ + .base = "http://example.org/dir/page.html", + .relative = "/rooted.txt", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "example.org", + }, + .path = {"", "rooted.txt"}, + }, + .description = "replacesPathWithAbsoluteRelative", + }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?z=3", + .relative = "other.html?x=1&y=2#frag", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "other.html"}, + .query = {{"x", "1"}, {"y", "2"}}, + .fragment = "frag", + }, + .description = "keepsQueryAndFragmentFromRelative", + }, + ParseURLRelativeParam{ + .base = "http://example.org/path/page.html", + .relative = "https://127.0.0.1.org/secure", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "127.0.0.1.org", + }, + .path = {"", "secure"}, + }, + .description = "absOverride", + }, + ParseURLRelativeParam{ + .base = "http://example.org/path/page.html", + .relative = "https://[fe80::818c:da4d:8975:415c\%25enp0s25]/secure?foo=bar", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::IPv6, + .host = "fe80::818c:da4d:8975:415c%enp0s25", + }, + .path = {"", "secure"}, + .query = {{"foo", "bar"}}, + }, + .description = "absOverrideWithZoneId", + }, + ParseURLRelativeParam{ + .base = "mailto:mail-base@bar.baz?bcc=alice@asdf.com", + .relative = "mailto:mail-override@foo.bar?subject=url-testing", + .expected = + ParsedURL{ + .scheme = "mailto", + .path = {"mail-override@foo.bar"}, + .query = {{"subject", "url-testing"}}, + }, + .description = "bothWithoutAuthority", + }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", + .relative = "", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "index.html"}, + .query = {{"a b", "5 6"}, {"x y", "34"}}, + }, + .description = "emptyRelative", + }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", + .relative = "#frag2", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "index.html"}, + .query = {{"a b", "5 6"}, {"x y", "34"}}, + .fragment = "frag2", + }, + .description = "fragmentRelative", + }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", + .relative = "?asdf\%20qwer=1\%202\%203", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "index.html"}, + .query = {{"asdf qwer", "1 2 3"}}, + }, + .description = "queryRelative", + }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", + .relative = "?asdf\%20qwer=1\%202\%203#frag2", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "index.html"}, + .query = {{"asdf qwer", "1 2 3"}}, + .fragment = "frag2", + }, + .description = "queryFragmentRelative", + }), + [](const auto & info) { return info.param.description; }); /* ---------------------------------------------------------------------------- * decodeQuery From e19c0a5a14d8eecd5ca2529fe1028406a9b4838d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 26 Feb 2026 14:45:18 -0500 Subject: [PATCH 012/927] Add test cases for relative `?` and `#` empty explicit params vs none at all Unlike with absolute (complete) URLs, with relative URLs, this does make a difference -- explicit empty overrides, implicit empty does not. --- src/libutil-tests/url.cc | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/libutil-tests/url.cc b/src/libutil-tests/url.cc index 1197adaa8c..a6d464507c 100644 --- a/src/libutil-tests/url.cc +++ b/src/libutil-tests/url.cc @@ -753,6 +753,39 @@ INSTANTIATE_TEST_SUITE_P( }, .description = "fragmentRelative", }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", + .relative = "#", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "index.html"}, + .query = {{"a b", "5 6"}, {"x y", "34"}}, + .fragment = "", + }, + .description = "emptyFragmentRelative", + }, + ParseURLRelativeParam{ + .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", + .relative = "?", + .expected = + ParsedURL{ + .scheme = "https", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + }, + .path = {"", "path", "index.html"}, + .query = {}, + }, + .description = "emptyQueryRelative", + }, ParseURLRelativeParam{ .base = "https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag", .relative = "?asdf\%20qwer=1\%202\%203", From 62d275d7c0709817c7d7f21dfffbe528f9828bef Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 27 Feb 2026 13:38:45 -0500 Subject: [PATCH 013/927] tests: use pathToUrlPath for file:// URL construction in git test --- src/libfetchers-tests/git.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libfetchers-tests/git.cc b/src/libfetchers-tests/git.cc index abc3dd74c5..9fb9a2ce1c 100644 --- a/src/libfetchers-tests/git.cc +++ b/src/libfetchers-tests/git.cc @@ -4,6 +4,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/git-utils.hh" +#include "nix/util/url.hh" #include #include @@ -190,7 +191,7 @@ TEST_F(GitTest, submodulePeriodSupport) auto input = fetchers::Input::fromAttrs( settings, { - {"url", "file://" + repoPath.string()}, + {"url", "file://" + encodeUrlPath(pathToUrlPath(repoPath))}, {"submodules", Explicit{true}}, {"type", "git"}, {"ref", "main"}, From b26f2ca3e68bfc670ab3ffddc7e81ab41dd1ea62 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 27 Feb 2026 14:50:00 -0500 Subject: [PATCH 014/927] tests: clear NIX_STORE env in fetchers, expr, and flake test environments --- src/libexpr-tests/meson.build | 2 ++ src/libfetchers-tests/meson.build | 2 ++ src/libflake-tests/meson.build | 1 + 3 files changed, 5 insertions(+) diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index c5b72851da..0b0a01c206 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -84,6 +84,8 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + 'HOME' : meson.current_build_dir() / 'test-home', + 'NIX_STORE' : '', }, protocol : 'gtest', ) diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index 6bccdb05c9..ba9774e956 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -66,6 +66,8 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + 'HOME' : meson.current_build_dir() / 'test-home', + 'NIX_STORE' : '', }, protocol : 'gtest', ) diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 59094abe86..3512be10bc 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -62,6 +62,7 @@ test( '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', 'NIX_CONFIG' : 'extra-experimental-features = flakes', 'HOME' : meson.current_build_dir() / 'test-home', + 'NIX_STORE' : '', }, protocol : 'gtest', ) From f3792cdad5c6d8e415b5d125c456acb64c026562 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 27 Feb 2026 14:50:00 -0500 Subject: [PATCH 015/927] flakeref: use portable root detection in directory walk loops --- src/libflake/flakeref.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index c35040adb5..2960a158e3 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -146,7 +146,7 @@ std::pair parsePathFlakeRefWithFragment( // Save device to detect filesystem boundary dev_t device = lstat(path).st_dev; bool found = false; - while (path != "/") { + while (path.parent_path() != path) { if (pathExists(path / "flake.nix")) { found = true; break; @@ -171,7 +171,7 @@ std::pair parsePathFlakeRefWithFragment( auto flakeRoot = path; std::string subdir; - while (flakeRoot != "/") { + while (flakeRoot.parent_path() != flakeRoot) { if (pathExists(flakeRoot / ".git")) { auto parsedURL = ParsedURL{ .scheme = "git+file", From 9c59f628905ae4c3c6c3d919f1aa15cd66a6e4d2 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 27 Feb 2026 14:50:00 -0500 Subject: [PATCH 016/927] url: handle Windows drive letters in fixGitURL --- src/libutil/url.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 184eba263f..abf0acf008 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -408,15 +408,22 @@ ParsedURL fixGitURL(std::string url) url = std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"); if (!hasPrefix(url, "file:") && !hasPrefix(url, "git+file:") && url.find("://") == std::string::npos) { auto path = splitString>(url, "/"); +#ifdef _WIN32 + // Windows drive letters (e.g., "C:") look like SCP hosts but are local paths + bool hasDriveLetter = !path.empty() && path[0].size() == 2 + && std::isalpha(static_cast(path[0][0])) && path[0][1] == ':'; +#else + constexpr bool hasDriveLetter = false; +#endif // Reject SCP-like URLs without user (e.g., "github.com:path") - colon in first component - if (!path.empty() && path[0].find(':') != std::string::npos) + if (!path.empty() && path[0].find(':') != std::string::npos && !hasDriveLetter) throw BadURL("SCP-like URL '%s' is not supported; use SSH URL syntax instead (ssh://...)", url); // Absolute paths get an empty authority (file:///path), relative paths get none (file:path) - if (hasPrefix(url, "/")) + if (hasPrefix(url, "/") || hasDriveLetter) return ParsedURL{ .scheme = "file", .authority = ParsedURL::Authority{}, - .path = path, + .path = hasDriveLetter ? pathToUrlPath(std::filesystem::path(url)) : path, }; else return ParsedURL{ From 0d8ca7a8886c3ff0f8ea2f4eff56e33e2f947d3a Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 27 Feb 2026 14:50:00 -0500 Subject: [PATCH 017/927] fetchers: use pathToUrlPath in PathInputScheme::toURL --- src/libfetchers/path.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index cf7f6aa920..9d846c6753 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -95,7 +95,7 @@ struct PathInputScheme : InputScheme query.erase("__final"); return ParsedURL{ .scheme = "path", - .path = splitString>(getStrAttr(input.attrs, "path"), "/"), + .path = pathToUrlPath(std::filesystem::path{getStrAttr(input.attrs, "path")}), .query = query, }; } From 450c1850c9cf3ab509fdc71608cdea7e1f8d6014 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 2 Mar 2026 11:13:03 -0500 Subject: [PATCH 018/927] libutil: Add `finalSymlink` parameter to openDirectory Add a type-safe boolean enum `FinalSymlink` to control whether `openDirectory` follows symlinks on the final path component. - `FinalSymlink::Follow` (default): follow symlinks (current behavior) - `FinalSymlink::DontFollow`: fail if the path is a symlink On Unix, this uses `O_NOFOLLOW`. On Windows, this uses `FILE_FLAG_OPEN_REPARSE_POINT`. Update all call sites with explicit `FinalSymlink` values. --- src/libstore/gc.cc | 3 ++- src/libstore/local-store.cc | 2 +- src/libstore/unix/build/derivation-builder.cc | 2 +- src/libutil-tests/file-system-at.cc | 10 +++++----- src/libutil-tests/source-accessor.cc | 2 +- src/libutil-tests/unix/file-system-at.cc | 8 ++++---- src/libutil/file-system.cc | 4 ++-- src/libutil/include/nix/util/file-system.hh | 9 ++++++++- src/libutil/unix/file-system.cc | 5 +++-- src/libutil/windows/file-system.cc | 4 ++-- 10 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 16b81abf28..a88fe6a64a 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -548,7 +548,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) by another process. We need to be sure that we can acquire an exclusive lock before deleting them. */ if (baseName.find("tmp-", 0) == 0) { - auto tmpDirFd = openDirectory(realPath); + /* TODO Reconsider whether Follow is the right choice, here */ + auto tmpDirFd = openDirectory(realPath, FinalSymlink::Follow); if (!tmpDirFd || !lockFile(tmpDirFd.get(), ltWrite, false)) { debug("skipping locked tempdir %s", PathFmt(realPath)); return; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8631df2184..209ee2fc18 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1285,7 +1285,7 @@ std::pair LocalStore::createTempDirInStore() the GC between createTempDir() and when we acquire a lock on it. We'll repeat until 'tmpDir' exists and we've locked it. */ tmpDirFn = createTempDir(std::filesystem::path{config->realStoreDir.get()}, "tmp"); - tmpDirFd = openDirectory(tmpDirFn); + tmpDirFd = openDirectory(tmpDirFn, FinalSymlink::DontFollow); if (!tmpDirFd) { continue; } diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index da225dc0d1..4996affbeb 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -764,7 +764,7 @@ std::optional DerivationBuilderImpl::startBuild() /* The TOCTOU between the previous mkdir call and this open call is unavoidable due to POSIX semantics.*/ - tmpDirFd = AutoCloseFD{open(tmpDir.c_str(), O_RDONLY | O_NOFOLLOW | O_DIRECTORY)}; + tmpDirFd = openDirectory(tmpDir, FinalSymlink::DontFollow); if (!tmpDirFd) throw SysError("failed to open the build temporary directory descriptor %1%", PathFmt(tmpDir)); diff --git a/src/libutil-tests/file-system-at.cc b/src/libutil-tests/file-system-at.cc index 5c6f93c354..8c75ec14e3 100644 --- a/src/libutil-tests/file-system-at.cc +++ b/src/libutil-tests/file-system-at.cc @@ -32,7 +32,7 @@ TEST(readLinkAt, works) { RestoreSink sink(/*startFsync=*/false); sink.dstPath = tmpDir; - sink.dirFd = openDirectory(tmpDir); + sink.dirFd = openDirectory(tmpDir, FinalSymlink::Follow); sink.createSymlink(CanonPath("link"), "target"); sink.createSymlink(CanonPath("relative"), "../relative/path"); sink.createSymlink(CanonPath("absolute"), "/absolute/path"); @@ -45,7 +45,7 @@ TEST(readLinkAt, works) sink.createDirectory(CanonPath("dir")); } - auto dirFd = openDirectory(tmpDir); + auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("link")), OS_STR("target")); EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("relative")), OS_STR("../relative/path")); @@ -54,7 +54,7 @@ TEST(readLinkAt, works) EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("long")), string_to_os_string(longTarget)); EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("a/b/link")), OS_STR("nested_target")); - auto subDirFd = openDirectory(tmpDir / "a"); + auto subDirFd = openDirectory(tmpDir / "a", FinalSymlink::Follow); EXPECT_EQ(readLinkAt(subDirFd.get(), CanonPath("b/link")), OS_STR("nested_target")); // Test error cases - expect SystemError on both platforms @@ -78,7 +78,7 @@ TEST(openFileEnsureBeneathNoSymlinks, works) { RestoreSink sink(/*startFsync=*/false); sink.dstPath = tmpDir; - sink.dirFd = openDirectory(tmpDir); + sink.dirFd = openDirectory(tmpDir, FinalSymlink::Follow); sink.createDirectory(CanonPath("a")); sink.createDirectory(CanonPath("c")); sink.createDirectory(CanonPath("c/d")); @@ -107,7 +107,7 @@ TEST(openFileEnsureBeneathNoSymlinks, works) SymlinkNotAllowed); } - auto dirFd = openDirectory(tmpDir); + auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); // Helper to open files with platform-specific arguments auto openRead = [&](std::string_view path) -> AutoCloseFD { diff --git a/src/libutil-tests/source-accessor.cc b/src/libutil-tests/source-accessor.cc index b9da508659..fd4e2429e1 100644 --- a/src/libutil-tests/source-accessor.cc +++ b/src/libutil-tests/source-accessor.cc @@ -95,7 +95,7 @@ TEST_F(FSSourceAccessorTest, works) RestoreSink sink(false); sink.dstPath = tmpDir; #ifndef _WIN32 - sink.dirFd = openDirectory(tmpDir); + sink.dirFd = openDirectory(tmpDir, FinalSymlink::Follow); #endif sink.createDirectory(CanonPath("subdir")); sink.createRegularFile(CanonPath("file1"), [](CreateRegularFileSink & crf) { crf("content1"); }); diff --git a/src/libutil-tests/unix/file-system-at.cc b/src/libutil-tests/unix/file-system-at.cc index 609c4e2aa1..6ae2dcce2c 100644 --- a/src/libutil-tests/unix/file-system-at.cc +++ b/src/libutil-tests/unix/file-system-at.cc @@ -29,7 +29,7 @@ TEST(fchmodatTryNoFollow, works) { RestoreSink sink(/*startFsync=*/false); sink.dstPath = tmpDir; - sink.dirFd = openDirectory(tmpDir); + sink.dirFd = openDirectory(tmpDir, FinalSymlink::Follow); sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink & crf) {}); sink.createDirectory(CanonPath("dir")); sink.createSymlink(CanonPath("filelink"), "file"); @@ -39,7 +39,7 @@ TEST(fchmodatTryNoFollow, works) ASSERT_NO_THROW(chmod(tmpDir / "file", 0644)); ASSERT_NO_THROW(chmod(tmpDir / "dir", 0755)); - auto dirFd = openDirectory(tmpDir); + auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); ASSERT_TRUE(dirFd); struct ::stat st; @@ -86,7 +86,7 @@ TEST(fchmodatTryNoFollow, fallbackWithoutProc) { RestoreSink sink(/*startFsync=*/false); sink.dstPath = tmpDir; - sink.dirFd = openDirectory(tmpDir); + sink.dirFd = openDirectory(tmpDir, FinalSymlink::Follow); sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink & crf) {}); sink.createSymlink(CanonPath("link"), "file"); } @@ -104,7 +104,7 @@ TEST(fchmodatTryNoFollow, fallbackWithoutProc) if (mount("tmpfs", "/proc", "tmpfs", 0, 0) == -1) _exit(1); - auto dirFd = openDirectory(tmpDir); + auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); if (!dirFd) exit(1); diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index e1efca0ddf..5dec2decb3 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -363,7 +363,7 @@ void writeFile(const std::filesystem::path & path, Source & source, mode_t mode, void syncParent(const std::filesystem::path & path) { assert(path.has_parent_path()); - AutoCloseFD fd = openDirectory(path.parent_path()); + AutoCloseFD fd = openDirectory(path.parent_path(), FinalSymlink::Follow); if (!fd) throw NativeSysError("opening file %s", PathFmt(path)); /* TODO: Fix on windows, FlushFileBuffers requires GENERIC_WRITE. */ @@ -408,7 +408,7 @@ void recursiveSync(const std::filesystem::path & path) /* Fsync all the directories. */ for (auto dir = dirsToFsync.rbegin(); dir != dirsToFsync.rend(); ++dir) { - AutoCloseFD fd = openDirectory(*dir); /* TODO: O_NOFOLLOW? */ + AutoCloseFD fd = openDirectory(*dir, FinalSymlink::DontFollow); if (!fd) throw NativeSysError("opening directory %1%", PathFmt(*dir)); fd.fsync(); diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 067240812f..58aecade75 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -166,10 +166,17 @@ std::filesystem::path readLink(const std::filesystem::path & path); */ std::filesystem::path descriptorToPath(Descriptor fd); +/** + * Type-safe boolean for whether to follow the final symlink component. + */ +enum struct FinalSymlink : bool { DontFollow = false, Follow = true }; + /** * Open a `Descriptor` with read-only access to the given directory. + * + * @param finalSymlink If `DontFollow`, fail if the path is a symlink. */ -AutoCloseFD openDirectory(const std::filesystem::path & path); +AutoCloseFD openDirectory(const std::filesystem::path & path, FinalSymlink finalSymlink = FinalSymlink::Follow); /** * Open a `Descriptor` with read-only access to the given file. diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index 55fd7584fb..9423b76a1e 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -23,9 +23,10 @@ namespace nix { -AutoCloseFD openDirectory(const std::filesystem::path & path) +AutoCloseFD openDirectory(const std::filesystem::path & path, FinalSymlink finalSymlink) { - return AutoCloseFD{open(path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC)}; + return AutoCloseFD{open( + path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC | (finalSymlink == FinalSymlink::Follow ? 0 : O_NOFOLLOW))}; } AutoCloseFD openFileReadonly(const std::filesystem::path & path) diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index a707343ece..c739ed0413 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -23,7 +23,7 @@ void setWriteTime( warn("Changing file times is not yet implemented on Windows, path is %s", PathFmt(path)); } -AutoCloseFD openDirectory(const std::filesystem::path & path) +AutoCloseFD openDirectory(const std::filesystem::path & path, FinalSymlink finalSymlink) { return AutoCloseFD{CreateFileW( path.c_str(), @@ -31,7 +31,7 @@ AutoCloseFD openDirectory(const std::filesystem::path & path) FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_BACKUP_SEMANTICS | (finalSymlink == FinalSymlink::Follow ? 0 : FILE_FLAG_OPEN_REPARSE_POINT), /*hTemplateFile=*/nullptr)}; } From faca7db6335f64f7b11c5ee5441805cc84f1f6d2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 19 Feb 2026 12:09:03 -0500 Subject: [PATCH 019/927] Revert "Reapply "Use the hash modulo in the derivation outputs"" This reverts commit 100e7cc3376c0f921493e035b7dc9233cd13151b. Unlike the last version, this fixes `--print-out-paths` for old clients When the client lacks `featureRealisationWithPath` (worker) or version < 2.8 (serve), the daemon was sending an empty `StringMap` for `builtOutputs`, causing `--print-out-paths` to print nothing. This commit constructs the old wire format instead, using a dummy `sha256:` hash since the derivation hash no longer exists. Old clients only need the output name and path from the JSON. --- .../json/schema/build-result-v1.yaml | 2 +- .../json/schema/build-trace-entry-v2.yaml | 57 ++-- src/json-schema-checks/meson.build | 3 - src/libcmd/built-path.cc | 18 +- src/libstore-tests/build-result.cc | 16 +- src/libstore-tests/common-protocol.cc | 63 ---- .../data/build-result/success.json | 4 - .../data/dummy-store/one-realisation.json | 3 +- .../data/realisation/simple.json | 12 +- .../with-dependent-realisations.json | 8 - .../data/realisation/with-signature.json | 16 +- .../build-result-2.7-compat.bin | Bin 0 -> 664 bytes .../build-result-2.7-compat.json | 37 +++ .../data/serve-protocol/build-result-2.8.bin | Bin 0 -> 360 bytes .../data/serve-protocol/build-result-2.8.json | 37 +++ .../data/serve-protocol/drv-output-2.8.bin | Bin 0 -> 160 bytes .../data/serve-protocol/drv-output-2.8.json | 10 + .../data/serve-protocol/realisation-2.8.bin | Bin 0 -> 352 bytes .../data/serve-protocol/realisation-2.8.json | 13 + .../unkeyed-realisation-2.8.bin | Bin 0 -> 272 bytes .../unkeyed-realisation-2.8.json | 7 + .../build-result-1.29-compat.bin | Bin 0 -> 664 bytes .../build-result-1.29-compat.json | 37 +++ ...-result-realisation-with-path-not-hash.bin | Bin 0 -> 424 bytes ...result-realisation-with-path-not-hash.json | 39 +++ ...-output-realisation-with-path-not-hash.bin | Bin 0 -> 160 bytes ...output-realisation-with-path-not-hash.json | 10 + ...isation-realisation-with-path-not-hash.bin | Bin 0 -> 352 bytes ...sation-realisation-with-path-not-hash.json | 13 + ...isation-realisation-with-path-not-hash.bin | Bin 0 -> 272 bytes ...sation-realisation-with-path-not-hash.json | 7 + .../ca-drv/store-after.json | 3 +- .../ca-drv/substituter.json | 3 +- .../issue-11928/store-after.json | 3 +- .../issue-11928/substituter.json | 6 +- src/libstore-tests/dummy-store.cc | 16 +- src/libstore-tests/realisation.cc | 17 +- src/libstore-tests/serve-protocol.cc | 226 +++++++++---- src/libstore-tests/worker-protocol.cc | 310 +++++++++++------- src/libstore-tests/worker-substitution.cc | 28 +- src/libstore/binary-cache-store.cc | 12 +- .../build/derivation-building-goal.cc | 7 +- src/libstore/build/derivation-goal.cc | 58 +--- .../build/drv-output-substitution-goal.cc | 7 +- src/libstore/ca-specific-schema.sql | 40 +-- src/libstore/common-protocol.cc | 28 -- src/libstore/daemon.cc | 28 +- src/libstore/derivations.cc | 256 ++++++++------- src/libstore/dummy-store.cc | 12 +- .../include/nix/store/binary-cache-store.hh | 14 +- .../store/build/derivation-building-misc.hh | 1 - .../nix/store/build/derivation-goal.hh | 2 - .../include/nix/store/common-protocol-impl.hh | 5 +- .../include/nix/store/common-protocol.hh | 5 +- src/libstore/include/nix/store/derivations.hh | 59 ++-- .../include/nix/store/dummy-store-impl.hh | 2 +- .../store/length-prefixed-protocol-helper.hh | 20 +- src/libstore/include/nix/store/realisation.hh | 74 +++-- .../include/nix/store/serve-protocol-impl.hh | 6 +- .../include/nix/store/serve-protocol.hh | 18 +- .../include/nix/store/worker-protocol-impl.hh | 6 +- .../include/nix/store/worker-protocol.hh | 27 +- src/libstore/local-store.cc | 22 +- src/libstore/misc.cc | 9 +- src/libstore/nar-info-disk-cache.cc | 53 +-- src/libstore/realisation.cc | 74 +++-- src/libstore/remote-store.cc | 55 +--- src/libstore/restricted-store.cc | 15 +- src/libstore/serve-protocol.cc | 121 ++++++- src/libstore/store-api.cc | 7 +- src/libstore/unix/build/derivation-builder.cc | 5 +- src/libstore/worker-protocol.cc | 153 ++++++++- src/nix/build-remote/build-remote.cc | 6 +- src/perl/lib/Nix/Store.xs | 7 +- tests/functional/ca/common.sh | 3 + tests/functional/ca/substitute.sh | 6 +- 76 files changed, 1368 insertions(+), 879 deletions(-) delete mode 100644 src/libstore-tests/data/realisation/with-dependent-realisations.json create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.7-compat.bin create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/build-result-2.8.json create mode 100644 src/libstore-tests/data/serve-protocol/drv-output-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/drv-output-2.8.json create mode 100644 src/libstore-tests/data/serve-protocol/realisation-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/realisation-2.8.json create mode 100644 src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin create mode 100644 src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json create mode 100644 src/libstore-tests/data/worker-protocol/build-result-1.29-compat.bin create mode 100644 src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json create mode 100644 src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin create mode 100644 src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json create mode 100644 src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin create mode 100644 src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.json create mode 100644 src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.bin create mode 100644 src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json create mode 100644 src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin create mode 100644 src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json diff --git a/doc/manual/source/protocols/json/schema/build-result-v1.yaml b/doc/manual/source/protocols/json/schema/build-result-v1.yaml index 55e55d3e5c..7730f3d939 100644 --- a/doc/manual/source/protocols/json/schema/build-result-v1.yaml +++ b/doc/manual/source/protocols/json/schema/build-result-v1.yaml @@ -83,7 +83,7 @@ properties: description: | A mapping from output names to their build trace entries. additionalProperties: - "$ref": "build-trace-entry-v2.yaml" + "$ref": "build-trace-entry-v2.yaml#/$defs/value" failure: type: object diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml index 4340f82388..2e8cd07da2 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml @@ -16,24 +16,21 @@ description: | - Version 1: Original format - - Version 2: Remove `dependentRealisations` + - Version 2: + - Use `drvPath` not `drvHash` to refer to derivation in a more conventional way. + - Remove `dependentRealisations` + - Separate into `key` and `value` type: object required: - - id - - outPath - - signatures -allOf: - - "$ref": "#/$defs/key" - - "$ref": "#/$defs/value" + - key + - value properties: - id: {} - outPath: {} - signatures: {} -additionalProperties: - dependentRealisations: - description: deprecated field - type: object + key: + "$ref": "#/$defs/key" + value: + "$ref": "#/$defs/value" +additionalProperties: false "$defs": key: @@ -43,23 +40,20 @@ additionalProperties: This is the "key" part, refering to a derivation and output. type: object required: - - id + - drvPath + - outputName properties: - id: + drvPath: + "$ref": "store-path-v1.yaml" + title: Derivation Path + description: | + The store path of the derivation that was built. + outputName: type: string - title: Derivation Output ID - pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$" + title: Output Name description: | - Unique identifier for the derivation output that was built. - - Format: `{hash-quotient-drv}!{output-name}` - - - **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv). - Begins with `sha256:`. - - - **output-name**: Name of the specific output (e.g., "out", "dev", "doc") - - Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"` + Name of the specific output (e.g., "out", "dev", "doc") + additionalProperties: false value: title: Build Trace Value @@ -77,13 +71,6 @@ additionalProperties: description: | The path to the store object that resulted from building this derivation for the given output name. - patternProperties: - "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$": - "$ref": "store-path-v1.yaml" - title: Dependent Store Path - description: Store path that this dependency resolved to during the build - additionalProperties: false - signatures: type: array title: Build Signatures diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index a1b525bc51..a4d9461609 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -65,9 +65,6 @@ schemas = [ 'schema' : schema_dir / 'build-trace-entry-v2.yaml', 'files' : [ 'simple.json', - # The field is no longer supported, but we want to show that we - # ignore it during parsing. - 'with-dependent-realisations.json', 'with-signature.json', ], }, diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index fc7f184938..f60e4499c3 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -108,20 +108,16 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const overloaded{ [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { - auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto & [outputName, outputPath] : p.outputs) { if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - auto drvOutput = get(drvHashes, outputName); - if (!drvOutput) - throw Error( - "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath->outPath()), - outputName); - DrvOutput key{*drvOutput, outputName}; + DrvOutput key{ + .drvPath = p.drvPath->outPath(), + .outputName = outputName, + }; auto thisRealisation = store.queryRealisation(key); - assert(thisRealisation); // We’ve built it, so we must - // have the realisation - res.insert(Realisation{*thisRealisation, std::move(key)}); + // We’ve built it, so we must have the realisation. + assert(thisRealisation); + res.insert(Realisation{*thisRealisation, key}); } else { res.insert(outputPath); } diff --git a/src/libstore-tests/build-result.cc b/src/libstore-tests/build-result.cc index b7e8f83f9e..a1d8ddee64 100644 --- a/src/libstore-tests/build-result.cc +++ b/src/libstore-tests/build-result.cc @@ -75,25 +75,13 @@ INSTANTIATE_TEST_SUITE_P( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index d63cbeeeed..9146af219c 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -108,69 +108,6 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( - drvOutput, - "drv-output", - (std::tuple{ - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "quux", - }, - })) - -CHARACTERIZATION_TEST( - realisation, - "realisation", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - -READ_CHARACTERIZATION_TEST( - realisation_with_deps, - "realisation-with-deps", - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - })) - CHARACTERIZATION_TEST( vector, "vector", diff --git a/src/libstore-tests/data/build-result/success.json b/src/libstore-tests/data/build-result/success.json index 4baadb5477..ec3479d556 100644 --- a/src/libstore-tests/data/build-result/success.json +++ b/src/libstore-tests/data/build-result/success.json @@ -1,14 +1,10 @@ { "builtOutputs": { "bar": { - "dependentRealisations": {}, - "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "signatures": [] }, "foo": { - "dependentRealisations": {}, - "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [] } diff --git a/src/libstore-tests/data/dummy-store/one-realisation.json b/src/libstore-tests/data/dummy-store/one-realisation.json index b5c8b8c562..576de08381 100644 --- a/src/libstore-tests/data/dummy-store/one-realisation.json +++ b/src/libstore-tests/data/dummy-store/one-realisation.json @@ -1,8 +1,7 @@ { "buildTrace": { - "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv": { "out": { - "dependentRealisations": {}, "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [] } diff --git a/src/libstore-tests/data/realisation/simple.json b/src/libstore-tests/data/realisation/simple.json index 2ccb1e7211..1e4760b568 100644 --- a/src/libstore-tests/data/realisation/simple.json +++ b/src/libstore-tests/data/realisation/simple.json @@ -1,6 +1,10 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } } diff --git a/src/libstore-tests/data/realisation/with-dependent-realisations.json b/src/libstore-tests/data/realisation/with-dependent-realisations.json deleted file mode 100644 index a58e0d7fe1..0000000000 --- a/src/libstore-tests/data/realisation/with-dependent-realisations.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "dependentRealisations": { - "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" - }, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [] -} diff --git a/src/libstore-tests/data/realisation/with-signature.json b/src/libstore-tests/data/realisation/with-signature.json index 3270f1cdeb..cca29ef3e4 100644 --- a/src/libstore-tests/data/realisation/with-signature.json +++ b/src/libstore-tests/data/realisation/with-signature.json @@ -1,8 +1,12 @@ { - "dependentRealisations": {}, - "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", - "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ] + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] + } } diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.bin b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.bin new file mode 100644 index 0000000000000000000000000000000000000000..fd745215c0330ffb0fb337617da5b09e71e728f1 GIT binary patch literal 664 zcmZQ&fBEet;BU3Xg0}6p+QeqL<#8#-e)k>KuN>)l#G+s$ZDZjKNAh9F^Xl}Y;xp_vVaUm{R d7w8_PTA*8~?a#FQd|LVwXgq2D1Ze>I696<2ZXf^v literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json new file mode 100644 index 0000000000..9c97678dad --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/build-result-2.7-compat.json @@ -0,0 +1,37 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.bin b/src/libstore-tests/data/serve-protocol/build-result-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..fa072599578a351e0f289c1c6bbdc5c1ff90107c GIT binary patch literal 360 zcmZQ&fBv3WB>s7x-9ts literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/build-result-2.8.json b/src/libstore-tests/data/serve-protocol/build-result-2.8.json new file mode 100644 index 0000000000..9c97678dad --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/build-result-2.8.json @@ -0,0 +1,37 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin b/src/libstore-tests/data/serve-protocol/drv-output-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..5be0b15a34567aed217e56dbf2543f2f60a200be GIT binary patch literal 160 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQsQgQeqXDWenw$YaRN>LeDBQsQgQeqXDr4QwkXdVL- eR9`HVPApDIvvQ;fu(bu+0kfyDJhh0H_5c784ONx^ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.json b/src/libstore-tests/data/serve-protocol/realisation-2.8.json new file mode 100644 index 0000000000..1977d8485b --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/realisation-2.8.json @@ -0,0 +1,13 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "baz" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] + } +} diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.bin new file mode 100644 index 0000000000000000000000000000000000000000..fce3a25dd8811573e84e81985b1cbf3d972bbffa GIT binary patch literal 272 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7EtQ6I5R;luj&8NwadK39z*V+5xkt LuspShmi7Pu*K$Gw literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json new file mode 100644 index 0000000000..2c2d264f9a --- /dev/null +++ b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json @@ -0,0 +1,7 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] +} diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.bin b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.bin new file mode 100644 index 0000000000000000000000000000000000000000..fd745215c0330ffb0fb337617da5b09e71e728f1 GIT binary patch literal 664 zcmZQ&fBEet;BU3Xg0}6p+QeqL<#8#-e)k>KuN>)l#G+s$ZDZjKNAh9F^Xl}Y;xp_vVaUm{R d7w8_PTA*8~?a#FQd|LVwXgq2D1Ze>I696<2ZXf^v literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json new file mode 100644 index 0000000000..9c97678dad --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-result-1.29-compat.json @@ -0,0 +1,37 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.bin new file mode 100644 index 0000000000000000000000000000000000000000..11bec3e6e3b7a82f7b2d792d687a807644ff9d05 GIT binary patch literal 424 zcmZQ&fB``9-Pv>4xRz8I{I`xM*FNUS#vq N^7F|y52hDn001VXF1-K% literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json new file mode 100644 index 0000000000..1d566b13bb --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/build-result-realisation-with-path-not-hash.json @@ -0,0 +1,39 @@ +[ + { + "errorMsg": "no idea why", + "isNonDeterministic": false, + "startTime": 0, + "status": "OutputRejected", + "stopTime": 0, + "success": false, + "timesBuilt": 0 + }, + { + "errorMsg": "no idea why", + "isNonDeterministic": true, + "startTime": 30, + "status": "NotDeterministic", + "stopTime": 50, + "success": false, + "timesBuilt": 3 + }, + { + "builtOutputs": { + "bar": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "signatures": [] + }, + "foo": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] + } + }, + "cpuSystem": 604000000, + "cpuUser": 500000000, + "startTime": 30, + "status": "Built", + "stopTime": 50, + "success": true, + "timesBuilt": 1 + } +] diff --git a/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/drv-output-realisation-with-path-not-hash.bin new file mode 100644 index 0000000000000000000000000000000000000000..5be0b15a34567aed217e56dbf2543f2f60a200be GIT binary patch literal 160 zcmXqJfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7JG>N>LeDBQsQgQeqXDWenw$YaRN>LeDBQsQgQeqXDr4QwkXdVL- eR9`HVPApDIvvQ;fu(bu+0kfyDJhh0H_5c784ONx^ literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json new file mode 100644 index 0000000000..1977d8485b --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json @@ -0,0 +1,13 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputName": "baz" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] + } +} diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.bin new file mode 100644 index 0000000000000000000000000000000000000000..fce3a25dd8811573e84e81985b1cbf3d972bbffa GIT binary patch literal 272 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7EtQ6I5R;luj&8NwadK39z*V+5xkt LuspShmi7Pu*K$Gw literal 0 HcmV?d00001 diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json new file mode 100644 index 0000000000..2c2d264f9a --- /dev/null +++ b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json @@ -0,0 +1,7 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] +} diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json index 7f0f62f876..3271cb9859 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json @@ -1,8 +1,7 @@ { "buildTrace": { - "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", "signatures": [] } diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json index 93f4fb22a6..f10653a3a7 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json @@ -1,8 +1,7 @@ { "buildTrace": { - "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out", "signatures": [] } diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json index 617984a27e..c4d3901236 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json @@ -1,8 +1,7 @@ { "buildTrace": { - "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", "signatures": [] } diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json index a63d6243fa..ecd2282146 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json @@ -1,15 +1,13 @@ { "buildTrace": { - "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": { + "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out", "signatures": [] } }, - "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": { + "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": { "out": { - "dependentRealisations": {}, "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out", "signatures": [] } diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index e3422fe585..cdb92cd1ea 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -37,20 +37,19 @@ TEST(DummyStore, realisation_read) return cfg->openDummyStore(); }(); - auto drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", HashAlgorithm::SHA256, HashFormat::Base16); + StorePath drvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}; auto outputName = "foo"; - EXPECT_EQ(store->queryRealisation({drvHash, outputName}), nullptr); + EXPECT_EQ(store->queryRealisation({drvPath, outputName}), nullptr); UnkeyedRealisation value{ - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }; - store->buildTrace.insert({drvHash, {{outputName, value}}}); + store->buildTrace.insert({drvPath, {{outputName, value}}}); - auto value2 = store->queryRealisation({drvHash, outputName}); + auto value2 = store->queryRealisation({drvPath, outputName}); ASSERT_TRUE(value2); EXPECT_EQ(*value2, value); @@ -131,10 +130,7 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] { [&] { auto store = writeCfg->openDummyStore(); store->buildTrace.insert_or_assign( - Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, std::map{ { "out", diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index d2d7df80f5..9c0ebbe8ab 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -46,13 +46,10 @@ TEST_P(RealisationJsonTest, to_json) Realisation simple{ { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, { - .drvHash = Hash::parseExplicitFormatUnprefixed( - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", - HashAlgorithm::SHA256, - HashFormat::Base16), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, .outputName = "foo", }, }; @@ -77,14 +74,4 @@ INSTANTIATE_TEST_SUITE_P( }(), })); -/** - * We no longer have a notion of "dependent realisations", but we still - * want to parse old realisation files. So make this just be a read test - * (no write direction), accordingly. - */ -TEST_F(RealisationTest, dependent_realisations_from_json) -{ - readJsonTest("with-dependent-realisations", simple); -} - } // namespace nix diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 159abeb608..3eeecd70f4 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -78,16 +78,19 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - drvOutput, - "drv-output", - defaultVersion, + drvOutput_2_8, + "drv-output-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) @@ -96,54 +99,37 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, + unkeyedRealisation_2_8, + "unkeyed-realisation-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, + realisation_2_8, + "realisation-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -209,7 +195,10 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully write + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( ServeProtoTest, buildResult_2_6, "build-result-2.6", @@ -242,27 +231,72 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_8, + "build-result-2.8", + (ServeProto::Version{ + .major = 2, + .minor = 8, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -578,4 +612,62 @@ TEST_F(ServeProtoTest, handshake_client_corrupted_throws) }); } +/** + * The old-protocol fallback writes `builtOutputs` as a `StringMap` + * with a dummy hash so that old clients can still extract output + * paths. This round-trips because the read side only uses the + * `outputName` (from the key) and `outPath` (from the JSON value). + */ +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_7_compat, + "build-result-2.7-compat", + (ServeProto::Version{ + .major = 2, + .minor = 7, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + } // namespace nix diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index aac5822d01..044fbcc920 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -18,17 +18,17 @@ namespace nix { TEST(WorkerProtoVersionNumber, ordering) { using Number = WorkerProto::Version::Number; - EXPECT_LT((Number{1, 10}), (Number{1, 20})); - EXPECT_GT((Number{1, 30}), (Number{1, 20})); - EXPECT_EQ((Number{1, 10}), (Number{1, 10})); - EXPECT_LT((Number{0, 255}), (Number{1, 0})); + EXPECT_LT((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 20})); + EXPECT_GT((Number{.major = 1, .minor = 30}), (Number{.major = 1, .minor = 20})); + EXPECT_EQ((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 10})); + EXPECT_LT((Number{.major = 0, .minor = 255}), (Number{.major = 1, .minor = 0})); } TEST(WorkerProtoVersion, partialOrderingSameFeatures) { using V = WorkerProto::Version; - V v1{.number = {1, 20}, .features = {"a", "b"}}; - V v2{.number = {1, 30}, .features = {"a", "b"}}; + V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; + V v2{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}}; EXPECT_TRUE(v1 < v2); EXPECT_TRUE(v2 > v1); @@ -40,8 +40,8 @@ TEST(WorkerProtoVersion, partialOrderingSameFeatures) TEST(WorkerProtoVersion, partialOrderingSubsetFeatures) { using V = WorkerProto::Version; - V fewer{.number = {1, 30}, .features = {"a"}}; - V more{.number = {1, 30}, .features = {"a", "b"}}; + V fewer{.number = {.major = 1, .minor = 30}, .features = {"a"}}; + V more{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}}; // fewer <= more: JUST the features are a subset EXPECT_TRUE(fewer < more); @@ -54,8 +54,8 @@ TEST(WorkerProtoVersion, partialOrderingUnordered) { using V = WorkerProto::Version; // Same number but incomparable features - V v1{.number = {1, 20}, .features = {"a", "c"}}; - V v2{.number = {1, 20}, .features = {"a", "b"}}; + V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "c"}}; + V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; EXPECT_FALSE(v1 < v2); EXPECT_FALSE(v1 > v2); @@ -69,8 +69,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures) { using V = WorkerProto::Version; // Higher number but fewer features — unordered - V v1{.number = {1, 30}, .features = {"a"}}; - V v2{.number = {1, 20}, .features = {"a", "b"}}; + V v1{.number = {.major = 1, .minor = 30}, .features = {"a"}}; + V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}}; EXPECT_FALSE(v1 < v2); EXPECT_FALSE(v1 > v2); @@ -80,8 +80,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures) TEST(WorkerProtoVersion, partialOrderingEmptyFeatures) { using V = WorkerProto::Version; - V empty{.number = {1, 20}, .features = {}}; - V some{.number = {1, 30}, .features = {"a"}}; + V empty{.number = {.major = 1, .minor = 20}, .features = {}}; + V some{.number = {.major = 1, .minor = 30}, .features = {"a"}}; // empty features is a subset of everything EXPECT_TRUE(empty < some); @@ -223,69 +223,67 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, drvOutput, - "drv-output", - defaultVersion, + "drv-output-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 38, + }, + .features = {"realisation-with-path-not-hash"}, + }), (std::tuple{ { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "baz", }, DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, .outputName = "quux", }, })) VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation, - "realisation", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", - }, - }, - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, + unkeyedRealisation_realisation_with_path, + "unkeyed-realisation-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", + .major = 1, + .minor = 38, }, - }, + .features = {"realisation-with-path-not-hash"}, + }), + (UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, })) -VERSIONED_READ_CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - realisation_with_deps, - "realisation-with-deps", - defaultVersion, - (std::tuple{ - Realisation{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - Signature{.keyName = "qwer", .sig = std::string(64, '\0')}, - }, - }, + realisation_realisation_with_path, + "realisation-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = { - .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - .outputName = "baz", + .major = 1, + .minor = 38, }, + .features = {"realisation-with-path-not-hash"}, + }), + (Realisation{ + UnkeyedRealisation{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + .signatures = + {Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + Signature{.keyName = "qwer", .sig = std::string(64, '\0')}}, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .outputName = "baz", }, })) @@ -318,7 +316,10 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +/* We now do a lossy read which does not allow us to faithfully write + back, since we changed the data type. We still however want to test + that this read works, and so for that we have a one-way test. */ +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_28, "build-result-1.28", @@ -347,25 +348,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -374,7 +363,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_29, "build-result-1.29", @@ -410,27 +400,13 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -443,7 +419,8 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) -VERSIONED_CHARACTERIZATION_TEST( +// See above note +VERSIONED_READ_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_37, "build-result-1.37", @@ -479,27 +456,71 @@ VERSIONED_CHARACTERIZATION_TEST( { "foo", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, }, }, { "bar", { - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, - }, - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_realisation_with_path, + "build-result-realisation-with-path-not-hash", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 38, + }, + .features = {"realisation-with-path-not-hash"}, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, }, }, }, @@ -920,7 +941,11 @@ TEST_F(WorkerProtoTest, handshake_features) out, in, WorkerProto::Version{ - .number = {.major = 1, .minor = 123}, + .number = + { + .major = 1, + .minor = 123, + }, .features = {"bar", "aap", "mies", "xyzzy"}, }); }); @@ -931,7 +956,11 @@ TEST_F(WorkerProtoTest, handshake_features) out, in, WorkerProto::Version{ - .number = {.major = 1, .minor = 200}, + .number = + { + .major = 1, + .minor = 200, + }, .features = {"foo", "bar", "xyzzy"}, }); @@ -1013,4 +1042,65 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws) }); } +/** + * The old-protocol fallback writes `builtOutputs` as a `StringMap` + * with a dummy hash so that old clients can still extract output + * paths. This round-trips because the read side only uses the + * `outputName` (from the key) and `outPath` (from the JSON value). + */ +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_29_compat, + "build-result-1.29-compat", + (WorkerProto::Version{ + .number = + { + .major = 1, + .minor = 29, + }, + }), + ({ + using namespace std::literals::chrono_literals; + std::tuple t{ + BuildResult{.inner{BuildResult::Failure{{ + .status = BuildResult::Failure::OutputRejected, + .msg = HintFmt("no idea why"), + }}}}, + BuildResult{ + .inner{BuildResult::Failure{{ + .status = BuildResult::Failure::NotDeterministic, + .msg = HintFmt("no idea why"), + .isNonDeterministic = true, + }}}, + .timesBuilt = 3, + .startTime = 30, + .stopTime = 50, + }, + BuildResult{ + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = + { + { + "foo", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, + }, + { + "bar", + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, + }, + }, + }}, + .timesBuilt = 1, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + } // namespace nix diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc index 3a049b7e72..9cc7b56dcb 100644 --- a/src/libstore-tests/worker-substitution.cc +++ b/src/libstore-tests/worker-substitution.cc @@ -198,12 +198,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Snapshot the destination store before checkpointJson("ca-drv/store-before", dummyStore); - // Compute the hash modulo of the derivation - // For CA floating derivations, the kind is Deferred since outputs aren't known until build - auto hashModulo = hashDerivationModulo(*dummyStore, drv, true); - ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred); - auto drvHash = hashModulo.hashes.at("out"); - // Create the output store object auto outputPath = substituter->addToStore( "test-ca-drv-out", @@ -222,7 +216,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Add the realisation (build trace) to the substituter substituter->buildTrace.insert_or_assign( - drvHash, + drvPath, std::map{ { "out", @@ -236,7 +230,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) checkpointJson("ca-drv/substituter", substituter); // The realisation should not exist in the destination store yet - DrvOutput drvOutput{drvHash, "out"}; + DrvOutput drvOutput{drvPath, "out"}; ASSERT_FALSE(dummyStore->queryRealisation(drvOutput)); // Create a worker with our custom substituter @@ -299,11 +293,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Write the dependency derivation to the destination store auto depDrvPath = dummyStore->writeDerivation(depDrv); - // Compute the hash modulo for the dependency derivation - auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true); - ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred); - auto depDrvHash = depHashModulo.hashes.at("out"); - // Create the output store object for the dependency in the substituter auto depOutputPath = substituter->addToStore( "dep-drv-out", @@ -322,7 +311,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Add the realisation for the dependency to the substituter substituter->buildTrace.insert_or_assign( - depDrvHash, + depDrvPath, std::map{ { "out", @@ -353,11 +342,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Snapshot the destination store before checkpointJson("issue-11928/store-before", dummyStore); - // Compute the hash modulo for the root derivation - auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true); - ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred); - auto rootDrvHash = rootHashModulo.hashes.at("out"); - // Create the output store object for the root derivation // Note: it does NOT reference the dependency's output auto rootOutputPath = substituter->addToStore( @@ -378,12 +362,12 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) HashAlgorithm::SHA256); // The DrvOutputs for both derivations - DrvOutput depDrvOutput{depDrvHash, "out"}; - DrvOutput rootDrvOutput{rootDrvHash, "out"}; + DrvOutput depDrvOutput{depDrvPath, "out"}; + DrvOutput rootDrvOutput{rootDrvPath, "out"}; // Add the realisation for the root derivation to the substituter substituter->buildTrace.insert_or_assign( - rootDrvHash, + rootDrvPath, std::map{ { "out", diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 31fe2c6173..34a3209620 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -514,7 +514,7 @@ StorePath BinaryCacheStore::addToStore( std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id) { - return realisationsPrefix + "/" + id.to_string() + ".doi"; + return realisationsPrefix + "/" + id.drvPath.to_string() + "/" + id.outputName + ".doi"; } void BinaryCacheStore::queryRealisationUncached( @@ -535,7 +535,10 @@ void BinaryCacheStore::queryRealisationUncached( realisation = std::make_shared(nlohmann::json::parse(*data)); } catch (Error & e) { e.addTrace( - {}, "while parsing file '%s' as a realisation for key '%s'", outputInfoFilePath, id.to_string()); + {}, + "while parsing file '%s' as a build trace value for key '%s'", + outputInfoFilePath, + id.to_string()); throw; } return (*callbackPtr)(std::move(realisation)); @@ -551,7 +554,10 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info) { if (diskCache) diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info); - upsertFile(makeRealisationPath(info.id), static_cast(info).dump(), "application/json"); + upsertFile( + makeRealisationPath(info.id), + static_cast(static_cast(info)).dump(), + "application/json"); } ref BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 60ca5492c1..0f577a2007 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -304,9 +304,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths) given this information by the downstream goal, that cannot happen anymore if the downstream goal only cares about one output, but we care about all outputs. */ - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - for (auto & [outputName, outputHash] : outputHashes) { - InitialOutput v{.outputHash = outputHash}; + for (auto & [outputName, _] : drv->outputs) { + InitialOutput v; /* TODO we might want to also allow randomizing the paths for regular CA derivations, e.g. for sake of checking @@ -1276,7 +1275,7 @@ DerivationBuildingGoal::checkPathValidity(std::map & : PathStatus::Corrupt, }; } - auto drvOutput = DrvOutput{info.outputHash, i.first}; + auto drvOutput = DrvOutput{drvPath, i.first}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 4f99928d78..4a726e2950 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -37,12 +37,6 @@ DerivationGoal::DerivationGoal( , drvPath(drvPath) , wantedOutput(wantedOutput) , drv{std::make_unique(drv)} - , outputHash{[&] { - auto outputHashes = staticOutputHashes(worker.evalStore, drv); - if (auto * mOutputHash = get(outputHashes, wantedOutput)) - return *mOutputHash; - throw Error("derivation '%s' does not have output '%s'", worker.store.printStorePath(drvPath), wantedOutput); - }()} , buildMode(buildMode) { @@ -102,7 +96,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) them. */ if (worker.settings.useSubstitutes && drvOptions.substitutesAllowed(worker.settings)) { if (!checkResult) { - DrvOutput id{outputHash, wantedOutput}; + DrvOutput id{drvPath, wantedOutput}; auto g = worker.makeDrvOutputSubstitutionGoal(id); waitees.insert(g); co_await await(std::move(waitees)); @@ -186,12 +180,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) // No `std::visit` for coroutines yet if (auto * successP = resolvedResult.tryGetSuccess()) { auto & success = *successP; - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - auto resolvedHashes = staticOutputHashes(worker.store, drvResolved); - - auto outputHash = get(outputHashes, wantedOutput); - auto resolvedHash = get(resolvedHashes, wantedOutput); - if ((!outputHash) || (!resolvedHash)) + if (!drv->outputs.contains(wantedOutput)) throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", worker.store.printStorePath(drvPath), @@ -200,7 +189,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) auto realisation = [&] { auto take1 = get(success.builtOutputs, wantedOutput); if (take1) - return static_cast(*take1); + return *take1; /* The above `get` should work. But stateful tracking of outputs in resolvedResult, this can get out of sync with the @@ -208,7 +197,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) check the store directly if it fails. */ auto take2 = worker.evalStore.queryRealisation( DrvOutput{ - .drvHash = *resolvedHash, + .drvPath = pathResolved, .outputName = wantedOutput, }); if (take2) @@ -224,7 +213,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) Realisation newRealisation{ realisation, { - .drvHash = *outputHash, + .drvPath = drvPath, .outputName = wantedOutput, }}; newRealisation.signatures.clear(); @@ -270,16 +259,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* In checking mode, the builder will not register any outputs. So we want to make sure the ones that we wanted to check are properly there. */ - success.builtOutputs = {{ - wantedOutput, - { - assertPathValidity(), - { - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}; + success.builtOutputs = {{wantedOutput, assertPathValidity()}}; } else { /* Otherwise the builder will give us info for out output, but also for other outputs. Filter down to just our output so as @@ -298,16 +278,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) if (success.builtOutputs.count(wantedOutput) == 0) { debug( "BUG! wanted output '%s' not in builtOutputs, working around by adding it manually", wantedOutput); - success.builtOutputs = {{ - wantedOutput, - { - assertPathValidity(), - { - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}; + success.builtOutputs = {{wantedOutput, assertPathValidity()}}; } } } @@ -404,7 +375,7 @@ std::optional> DerivationGoal::checkPa if (drv->type().isImpure()) return std::nullopt; - auto drvOutput = DrvOutput{outputHash, wantedOutput}; + auto drvOutput = DrvOutput{drvPath, wantedOutput}; std::optional mRealisation; @@ -444,7 +415,7 @@ std::optional> DerivationGoal::checkPa Realisation{ *mRealisation, { - .drvHash = outputHash, + .drvPath = drvPath, .outputName = wantedOutput, }, }); @@ -475,16 +446,7 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke return Goal::doneSuccess( BuildResult::Success{ .status = status, - .builtOutputs = {{ - wantedOutput, - { - std::move(builtOutput), - DrvOutput{ - .drvHash = outputHash, - .outputName = wantedOutput, - }, - }, - }}, + .builtOutputs = {{wantedOutput, std::move(builtOutput)}}, }); } diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 71f0ae1e7d..5a5fe06d5e 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -10,7 +10,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput & id, Worke : Goal(worker, init()) , id(id) { - name = fmt("substitution of '%s'", id.to_string()); + name = fmt("substitution of '%s'", id.render(worker.store)); trace("created"); } @@ -93,7 +93,8 @@ Goal::Co DrvOutputSubstitutionGoal::init() /* None left. Terminate this goal and let someone else deal with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); + debug( + "derivation output '%s' is required, but there is no substituter that can provide it", id.render(worker.store)); if (substituterFailed) { worker.failedSubstitutions++; @@ -108,7 +109,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() std::string DrvOutputSubstitutionGoal::key() { - return "a$" + std::string(id.to_string()); + return "a$" + std::string(id.render(worker.store)); } } // namespace nix diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index c5e4e38979..5daeef8c85 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -2,7 +2,16 @@ -- Won't be loaded unless the experimental feature `ca-derivations` -- is enabled -create table if not exists Realisations ( +-- Why the `*V` tables +-- +-- We are trying to keep different versions of the experiment to have +-- completely independent extra schemas from one another. This will +-- enable people to switch between versions of the experiment (including +-- newer to older) without migrating between them, but at the cost +-- of having many abandoned tables lying around. Closer to the end of +-- the experiment, we'll provide guidance on how to clean this up. + +create table if not exists BuildTraceV2 ( id integer primary key autoincrement not null, drvPath text not null, outputName text not null, -- symbolic output id, usually "out" @@ -11,31 +20,4 @@ create table if not exists Realisations ( foreign key (outputPath) references ValidPaths(id) on delete cascade ); -create index if not exists IndexRealisations on Realisations(drvPath, outputName); - --- We can end-up in a weird edge-case where a path depends on itself because --- it’s an output of a CA derivation, that happens to be the same as one of its --- dependencies. --- In that case we have a dependency loop (path -> realisation1 -> realisation2 --- -> path) that we need to break by removing the dependencies between the --- realisations -create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths - begin - delete from RealisationsRefs where realisationReference in ( - select id from Realisations where outputPath = old.id - ); - end; - -create table if not exists RealisationsRefs ( - referrer integer not null, - realisationReference integer, - foreign key (referrer) references Realisations(id) on delete cascade, - foreign key (realisationReference) references Realisations(id) on delete restrict -); --- used by deletion trigger -create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); - --- used by QueryRealisationReferences -create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); --- used by cascade deletion when ValidPaths is deleted -create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); +create index if not exists IndexBuildTraceV2 on BuildTraceV2(drvPath, outputName); diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index a0afe948d6..fc88ee8890 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -47,34 +47,6 @@ void CommonProto::Serialise::write( conn.to << renderContentAddress(ca); } -Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - try { - return nlohmann::json::parse(rawInput); - } catch (Error & e) { - e.addTrace({}, "while parsing a realisation object in the remote protocol"); - throw; - } -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << static_cast(realisation).dump(); -} - -DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void CommonProto::Serialise::write( - const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); -} - std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 3e5eac5170..a7be521a3d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -955,14 +955,8 @@ static void performOp( case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) { - auto outputId = WorkerProto::Serialise::read(*store, rconn); - auto outputPath = StorePath(readString(conn.from)); - store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId}); - } else { - auto realisation = WorkerProto::Serialise::read(*store, rconn); - store->registerDrvOutput(realisation); - } + auto realisation = WorkerProto::Serialise::read(*store, rconn); + store->registerDrvOutput(realisation); logger->stopWork(); break; } @@ -970,19 +964,13 @@ static void performOp( case WorkerProto::Op::QueryRealisation: { logger->startWork(); auto outputId = WorkerProto::Serialise::read(*store, rconn); - auto info = store->queryRealisation(outputId); + std::optional info = *store->queryRealisation(outputId); logger->stopWork(); - if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) { - std::set outPaths; - if (info) - outPaths.insert(info->outPath); - WorkerProto::write(*store, wconn, outPaths); - } else { - std::set realisations; - if (info) - realisations.insert({*info, outputId}); - WorkerProto::write(*store, wconn, realisations); - } + /* Only return the new format because if we got past + `DrvOutput` serialization, we know that is what we're using. + */ + assert(conn.protoVersion.features.contains(WorkerProto::featureRealisationWithPath)); + WorkerProto::write(*store, wconn, info); break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index ab969a089f..b20972df7b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -861,13 +861,14 @@ DrvHashes drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) +static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) { - std::optional hash; + std::optional hash; if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) { return *hash; } auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false); + // Cache it drvHashes.insert_or_assign(drvPath, h); return h; @@ -890,12 +891,10 @@ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPa don't leak the provenance of fixed outputs, reducing pointless cache misses as the build itself won't know this. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { - auto type = drv.type(); - /* Return a fixed hash for fixed-output derivations. */ - if (type.isFixed()) { + if (drv.type().isFixed()) { std::map outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw); @@ -905,54 +904,66 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } - return DrvHash{ - .hashes = outputHashes, - .kind = DrvHash::Kind::Regular, - }; + return outputHashes; } - auto kind = std::visit( - overloaded{ - [](const DerivationType::InputAddressed & ia) { - /* This might be a "pesimistically" deferred output, so we don't - "taint" the kind yet. */ - return DrvHash::Kind::Regular; - }, - [](const DerivationType::ContentAddressed & ca) { - return ca.fixed ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred; - }, - [](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }}, - drv.type().raw); + if (std::visit( + overloaded{ + [](const DerivationType::InputAddressed & ia) { + /* This might be a "pesimistically" deferred output, so we don't + "taint" the kind yet. */ + return false; + }, + [](const DerivationType::ContentAddressed & ca) { + // Already covered + assert(!ca.fixed); + return true; + }, + [](const DerivationType::Impure &) { return true; }}, + drv.type().raw)) { + return DrvHashModulo::DeferredDrv{}; + } + /* For other derivations, replace the inputs paths with recursive + calls to this function. */ DerivedPathMap::ChildNode::Map inputs2; for (auto & [drvPath, node] : drv.inputDrvs.map) { - const auto & res = pathDerivationModulo(store, drvPath); - if (res.kind == DrvHash::Kind::Deferred) - kind = DrvHash::Kind::Deferred; - for (auto & outputName : node.value) { - const auto h = get(res.hashes, outputName); - if (!h) - throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); + /* Need to build and resolve dynamic derivations first */ + if (!node.childMap.empty()) { + return DrvHashModulo::DeferredDrv{}; } - } - auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); - - std::map outputHashes; - for (const auto & [outputName, _] : drv.outputs) { - outputHashes.insert_or_assign(outputName, hash); + const auto & res = pathDerivationModulo(store, drvPath); + if (std::visit( + overloaded{ + [&](const DrvHashModulo::DeferredDrv &) { return true; }, + // Regular non-CA derivation, replace derivation + [&](const DrvHashModulo::DrvHash & drvHash) { + inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node); + return false; + }, + // CA derivation's output hashes + [&](const DrvHashModulo::CaOutputHashes & outputHashes) { + for (auto & outputName : node.value) { + /* Put each one in with a single "out" output.. */ + const auto h = get(outputHashes, outputName); + if (!h) + throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); + inputs2.insert_or_assign( + h->to_string(HashFormat::Base16, false), + DerivedPathMap::ChildNode{ + .value = {"out"}, + }); + } + return false; + }, + }, + res.raw)) { + return DrvHashModulo::DeferredDrv{}; + } } - return DrvHash{ - .hashes = outputHashes, - .kind = kind, - }; -} - -std::map staticOutputHashes(Store & store, const Derivation & drv) -{ - return hashDerivationModulo(store, drv, true).hashes; + return hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); } static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) @@ -1102,26 +1113,6 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites) } } -static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) -{ - drv.applyRewrites(rewrites); - - auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); - for (auto & [outputName, output] : drv.outputs) { - if (std::holds_alternative(output.raw)) { - auto h = get(hashModulo.hashes, outputName); - if (!h) - throw Error( - "derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", drv.name, outputName); - auto outPath = store.makeOutputPath(outputName, *h, drv.name); - drv.env[outputName] = store.printStorePath(outPath); - output = DerivationOutput::InputAddressed{ - .path = std::move(outPath), - }; - } - } -} - bool Derivation::shouldResolve() const { /* No input drvs means nothing to resolve. */ @@ -1233,9 +1224,13 @@ std::optional Derivation::tryResolve( queryResolutionChain)) return std::nullopt; - rewriteDerivation(store, resolved, inputRewrites); + resolved.applyRewrites(inputRewrites); + + Derivation resolved2{std::move(resolved)}; + + resolved2.fillInOutputPaths(store); - return resolved; + return resolved2; } /** @@ -1262,7 +1257,15 @@ std::optional Derivation::tryResolve( template static void processDerivationOutputPaths(Store & store, auto && drv, std::string_view drvName) { - std::optional hashesModulo; + std::optional hashModulo_; + + auto hashModulo = [&]() -> const auto & { + if (!hashModulo_) { + // somewhat expensive so we do lazily + hashModulo_ = hashDerivationModulo(store, drv, true); + } + return *hashModulo_; + }; for (auto & [outputName, output] : drv.outputs) { auto envHasRightPath = [&](const StorePath & actual, bool isDeferred = false) { @@ -1299,65 +1302,64 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string } }; auto hash = [&](const Output & outputVariant) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(store, drv, true); - } - switch (hashesModulo->kind) { - case DrvHash::Kind::Regular: { - auto h = get(hashesModulo->hashes, outputName); - if (!h) - throw Error("derivation produced no hash for output '%s'", outputName); - auto outPath = store.makeOutputPath(outputName, *h, drvName); - - if constexpr (std::is_same_v) { - if (outputVariant.path == outPath) { - return; // Correct case - } - /* Error case, an explicitly wrong path is - always an error. */ - throw Error( - "derivation has incorrect output '%s', should be '%s'", - store.printStorePath(outputVariant.path), - store.printStorePath(outPath)); - } else if constexpr (std::is_same_v) { - if constexpr (fillIn) - /* Fill in output path for Deferred - outputs */ - output = DerivationOutput::InputAddressed{ - .path = outPath, - }; - else - /* Validation mode: deferred outputs - should have been filled in */ - warn( - "derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.", - store.printStorePath(outPath)); - } else { - /* Will never happen, based on where - `hash` is called. */ - static_assert(false); - } - envHasRightPath(outPath); - break; - } - case DrvHash::Kind::Deferred: - if constexpr (std::is_same_v) { - /* Error case, an explicitly wrong path is - always an error. */ - throw Error( - "derivation has incorrect output '%s', should be deferred", - store.printStorePath(outputVariant.path)); - } else if constexpr (std::is_same_v) { - /* Correct: Deferred output with Deferred - hash kind. */ - } else { - /* Will never happen, based on where - `hash` is called. */ - static_assert(false); - } - break; - } + std::visit( + overloaded{ + [&](const DrvHashModulo::DrvHash & drvHash) { + auto outPath = store.makeOutputPath(outputName, drvHash, drvName); + + if constexpr (std::is_same_v) { + if (outputVariant.path == outPath) { + envHasRightPath(outPath); + return; // Correct case + } + /* Error case, an explicitly wrong path is + always an error. */ + throw Error( + "derivation has incorrect output '%s', should be '%s'", + store.printStorePath(outputVariant.path), + store.printStorePath(outPath)); + } else if constexpr (std::is_same_v) { + if constexpr (fillIn) { + /* Fill in output path for Deferred outputs */ + output = DerivationOutput::InputAddressed{ + .path = outPath, + }; + envHasRightPath(outPath); + } else { + /* Validation mode: deferred outputs + should have been filled in */ + warn( + "derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.", + store.printStorePath(outPath)); + } + } else { + /* Will never happen, based on where + `hash` is called. */ + static_assert(false); + } + }, + [&](const DrvHashModulo::CaOutputHashes &) { + /* Shouldn't happen as the original output is + input-addressed (or deferred waiting to be). */ + assert(false); + }, + [&](const DrvHashModulo::DeferredDrv &) { + if constexpr (std::is_same_v) { + /* Error case, an explicitly wrong path is + always an error. */ + throw Error( + "derivation has incorrect output '%s', should be deferred", + store.printStorePath(outputVariant.path)); + } else if constexpr (std::is_same_v) { + /* Correct: Deferred output with Deferred hash kind. */ + } else { + /* Will never happen, based on where + `hash` is called. */ + static_assert(false); + } + }, + }, + hashModulo().raw); }; std::visit( overloaded{ diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 943873119c..052ec9b128 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -330,7 +330,7 @@ struct DummyStoreImpl : DummyStore void registerDrvOutput(const Realisation & output) override { - buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, output}}}, [&](auto & kv) { + buildTrace.insert_or_visit({output.id.drvPath, {{output.id.outputName, output}}}, [&](auto & kv) { kv.second.insert_or_assign(output.id.outputName, output); }); } @@ -339,7 +339,7 @@ struct DummyStoreImpl : DummyStore const DrvOutput & drvOutput, Callback> callback) noexcept override { bool visited = false; - buildTrace.cvisit(drvOutput.drvHash, [&](const auto & kv) { + buildTrace.cvisit(drvOutput.drvPath, [&](const auto & kv) { if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) { visited = true; callback(std::make_shared(it->second)); @@ -436,11 +436,7 @@ ref adl_serializer>::from_json(const json & json) for (auto & [k1, v2] : getObject(v)) { UnkeyedRealisation realisation = v2; res->buildTrace.insert_or_visit( - { - Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64), - {{k1, realisation}}, - }, - [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); }); + {StorePath{k0}, {{k1, realisation}}}, [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); }); } } return res; @@ -473,7 +469,7 @@ void adl_serializer::to_json(json & json, const DummyStore & val) auto obj = json::object(); val.buildTrace.cvisit_all([&](const auto & kv) { auto & [k, v] = kv; - auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object(); + auto & obj2 = obj[k.to_string()] = json::object(); for (auto & [k2, v2] : kv.second) obj2[k2] = v2; }); diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 2c3fbdc3a0..bc499130b0 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -91,8 +91,18 @@ protected: /** * The prefix under which realisation infos will be stored + * + * @note The previous (still experimental, though) hash-keyed + * realisations were under "realisations". "build trace" is a better + * name anyways (issue #11895). This is call "v2" accordingly. + * + * While we're experimenting, we'll freely increase this version + * number. Old build traces will just be "abandoned" at the old URL. + * When we are done experimenting, we'll try lean more on versioning + * the build trace entries themselves than the entire directory, for + * a smoother migration path. */ - constexpr const static std::string realisationsPrefix = "realisations"; + constexpr const static std::string realisationsPrefix = "build-trace-v2"; constexpr const static std::string cacheInfoFile = "nix-cache-info"; @@ -101,7 +111,7 @@ protected: /** * Compute the path to the given realisation * - * It's `${realisationsPrefix}/${drvOutput}.doi`. + * It's `${realisationsPrefix}/${drvPath}/${outputName}`. */ std::string makeRealisationPath(const DrvOutput & id); diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh index 2b68fa1782..8d6892839c 100644 --- a/src/libstore/include/nix/store/build/derivation-building-misc.hh +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -45,7 +45,6 @@ struct InitialOutputStatus struct InitialOutput { - Hash outputHash; std::optional known; }; diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index aaded75511..c6bd412182 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -66,8 +66,6 @@ private: */ std::unique_ptr drv; - const Hash outputHash; - const BuildMode buildMode; /** diff --git a/src/libstore/include/nix/store/common-protocol-impl.hh b/src/libstore/include/nix/store/common-protocol-impl.hh index cb1020a3c8..d0eebe0bc2 100644 --- a/src/libstore/include/nix/store/common-protocol-impl.hh +++ b/src/libstore/include/nix/store/common-protocol-impl.hh @@ -26,12 +26,13 @@ namespace nix { LengthPrefixedProtoHelper::write(store, conn, t); \ } -#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +#define COMMA_ , COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) -COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::map) +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, std::map) #undef COMMA_ /* protocol-specific templates */ diff --git a/src/libstore/include/nix/store/common-protocol.hh b/src/libstore/include/nix/store/common-protocol.hh index 341d87b416..2ef0f795b4 100644 --- a/src/libstore/include/nix/store/common-protocol.hh +++ b/src/libstore/include/nix/store/common-protocol.hh @@ -88,8 +88,9 @@ DECLARE_COMMON_SERIALISER(std::set); template DECLARE_COMMON_SERIALISER(std::tuple); -template -DECLARE_COMMON_SERIALISER(std::map); +template +DECLARE_COMMON_SERIALISER(std::map); +#undef COMMA_ /** * These use the empty string for the null case, relying on the fact diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 0d137eaf36..b74fb420e6 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -507,34 +507,39 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName); * derivations (fixed-output or not) will have a different hash for each * output. */ -struct DrvHash +struct DrvHashModulo { /** - * Map from output names to hashes + * Single hash for the derivation + * + * This is for an input-addressed derivation that doesn't + * transitively depend on any floating-CA derivations. */ - std::map hashes; - - enum struct Kind : bool { - /** - * Statically determined derivations. - * This hash will be directly used to compute the output paths - */ - Regular, + using DrvHash = Hash; - /** - * Floating-output derivations (and their reverse dependencies). - */ - Deferred, - }; + /** + * Known CA drv's output hashes, for fixed-output derivations whose + * output hashes are always known since they are fixed up-front. + */ + using CaOutputHashes = std::map; /** - * The kind of derivation this is, simplified for just "derivation hash - * modulo" purposes. + * This derivation doesn't yet have known output hashes. + * + * Either because itself is floating CA, or it (transtively) depends + * on a floating CA derivation. */ - Kind kind; -}; + using DeferredDrv = std::monostate; + + using Raw = std::variant; -void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; + Raw raw; + + bool operator==(const DrvHashModulo &) const = default; + // auto operator <=> (const DrvHashModulo &) const = default; + + MAKE_WRAPPER_CONSTRUCTOR(DrvHashModulo); +}; /** * Returns hashes with the details of fixed-output subderivations @@ -560,15 +565,17 @@ void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; * ATerm, after subderivations have been likewise expunged from that * derivation. */ -DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /** - * Return a map associating each output to a hash that uniquely identifies its - * derivation (modulo the self-references). + * If a derivation is input addressed and doesn't yet have its input + * addressed (is deferred) try using `hashDerivationModulo`. * - * \todo What is the Hash in this map? + * Does nothing if not deferred input-addressed, or + * `hashDerivationModulo` indicates it is missing inputs' output paths + * and is not yet ready (and must stay deferred). */ -std::map staticOutputHashes(Store & store, const Derivation & drv); +void resolveInputAddressed(Store & store, Derivation & drv); struct DrvHashFct { @@ -583,7 +590,7 @@ struct DrvHashFct /** * Memoisation of hashDerivationModulo(). */ -typedef boost::concurrent_flat_map DrvHashes; +typedef boost::concurrent_flat_map DrvHashes; // FIXME: global, though at least thread-safe. extern DrvHashes drvHashes; diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index ac7ab9c680..8fdeeb3625 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -49,7 +49,7 @@ struct DummyStore : virtual Store * outer map for the derivation, and inner maps for the outputs of a * given derivation. */ - boost::concurrent_flat_map> buildTrace; + boost::concurrent_flat_map> buildTrace; DummyStore(ref config) : Store{*config} diff --git a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh index 035019340f..e1a80e8dc5 100644 --- a/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh +++ b/src/libstore/include/nix/store/length-prefixed-protocol-helper.hh @@ -56,14 +56,14 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); #define COMMA_ , template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); -#undef COMMA_ template LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); -template -#define LENGTH_PREFIXED_PROTO_HELPER_X std::map +template +#define LENGTH_PREFIXED_PROTO_HELPER_X std::map LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X); +#undef COMMA_ template std::vector @@ -109,11 +109,11 @@ void LengthPrefixedProtoHelper>::write( } } -template -std::map -LengthPrefixedProtoHelper>::read(const StoreDirConfig & store, typename Inner::ReadConn conn) +template +std::map LengthPrefixedProtoHelper>::read( + const StoreDirConfig & store, typename Inner::ReadConn conn) { - std::map resMap; + std::map resMap; auto size = readNum(conn.from); while (size--) { auto k = S::read(store, conn); @@ -123,9 +123,9 @@ LengthPrefixedProtoHelper>::read(const StoreDirConfig & st return resMap; } -template -void LengthPrefixedProtoHelper>::write( - const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) +template +void LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) { conn.to << resMap.size(); for (auto & i : resMap) { diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index 670f8ce499..9866b1245b 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -18,33 +18,40 @@ struct OutputsSpec; /** * A general `Realisation` key. * - * This is similar to a `DerivedPath::Opaque`, but the derivation is - * identified by its "hash modulo" instead of by its store path. + * This is similar to a `DerivedPath::Built`, except it is only a single + * step: `drvPath` is a `StorePath` rather than a `DerivedPath`. */ struct DrvOutput { /** - * The hash modulo of the derivation. - * - * Computed from the derivation itself for most types of - * derivations, but computed from the (fixed) content address of the - * output for fixed-output derivations. + * The store path to the derivation */ - Hash drvHash; + StorePath drvPath; /** * The name of the output. */ OutputName outputName; + /** + * Skips the store dir on the `drvPath` + */ std::string to_string() const; - std::string strHash() const - { - return drvHash.to_string(HashFormat::Base16, true); - } + /** + * Skips the store dir on the `drvPath` + */ + static DrvOutput from_string(std::string_view); - static DrvOutput parse(const std::string &); + /** + * Includes the store dir on `drvPath` + */ + std::string render(const StoreDirConfig & store) const; + + /** + * Includes the store dir on `drvPath` + */ + static DrvOutput parse(const StoreDirConfig & store, std::string_view); bool operator==(const DrvOutput &) const = default; auto operator<=>(const DrvOutput &) const = default; @@ -64,6 +71,16 @@ struct UnkeyedRealisation size_t checkSignatures(const DrvOutput & key, const PublicKeys & publicKeys) const; + /** + * Just check the `outPath`. Signatures don't matter for this. + * Callers must ensure that the corresponding key is the same for + * most use-cases. + */ + bool isCompatibleWith(const UnkeyedRealisation & other) const + { + return outPath == other.outPath; + } + const StorePath & getPath() const { return outPath; @@ -77,8 +94,6 @@ struct Realisation : UnkeyedRealisation { DrvOutput id; - bool isCompatibleWith(const UnkeyedRealisation & other) const; - bool operator==(const Realisation &) const = default; auto operator<=>(const Realisation &) const = default; }; @@ -89,16 +104,7 @@ struct Realisation : UnkeyedRealisation * Since these are the outputs of a single derivation, we know the * output names are unique so we can use them as the map key. */ -typedef std::map SingleDrvOutputs; - -/** - * Collection type for multiple derivations' outputs' `Realisation`s. - * - * `DrvOutput` is used because in general the derivations are not all - * the same, so we need to identify firstly which derivation, and - * secondly which output of that derivation. - */ -typedef std::map DrvOutputs; +typedef std::map SingleDrvOutputs; struct OpaquePath { @@ -149,19 +155,17 @@ struct RealisedPath class MissingRealisation final : public CloneableError { public: - MissingRealisation(DrvOutput & outputId) - : MissingRealisation(outputId.outputName, outputId.strHash()) + MissingRealisation(const StoreDirConfig & store, DrvOutput & outputId) + : MissingRealisation(store, outputId.drvPath, outputId.outputName) { } - MissingRealisation(std::string_view drv, OutputName outputName) - : CloneableError( - "cannot operate on output '%s' of the " - "unbuilt derivation '%s'", - outputName, - drv) - { - } + MissingRealisation(const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName); + MissingRealisation( + const StoreDirConfig & store, + const SingleDerivedPath & drvPath, + const StorePath & drvPathResolved, + const OutputName & outputName); }; } // namespace nix diff --git a/src/libstore/include/nix/store/serve-protocol-impl.hh b/src/libstore/include/nix/store/serve-protocol-impl.hh index a9617165a7..24fc3c9abd 100644 --- a/src/libstore/include/nix/store/serve-protocol-impl.hh +++ b/src/libstore/include/nix/store/serve-protocol-impl.hh @@ -34,8 +34,10 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA , SERVE_USE_LENGTH_PREFIX_SERIALISER( - template, - std::map) + template + , + std::map) /** * Use `CommonProto` where possible. diff --git a/src/libstore/include/nix/store/serve-protocol.hh b/src/libstore/include/nix/store/serve-protocol.hh index dba05a3454..94a250b2e3 100644 --- a/src/libstore/include/nix/store/serve-protocol.hh +++ b/src/libstore/include/nix/store/serve-protocol.hh @@ -8,12 +8,18 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb +#define SERVE_PROTOCOL_VERSION (2 << 8 | 8) +#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) +#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) struct StoreDirConfig; struct Source; // items being serialised struct BuildResult; struct UnkeyedValidPathInfo; +struct DrvOutput; +struct UnkeyedRealisation; +struct Realisation; /** * The "serve protocol", used by ssh:// stores. @@ -63,7 +69,7 @@ struct ServeProto static constexpr Version latest = { .major = 2, - .minor = 7, + .minor = 8, }; /** @@ -205,6 +211,12 @@ inline std::ostream & operator<<(std::ostream & s, ServeProto::Command op) template<> DECLARE_SERVE_SERIALISER(BuildResult); template<> +DECLARE_SERVE_SERIALISER(DrvOutput); +template<> +DECLARE_SERVE_SERIALISER(UnkeyedRealisation); +template<> +DECLARE_SERVE_SERIALISER(Realisation); +template<> DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); template<> DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); @@ -217,8 +229,8 @@ DECLARE_SERVE_SERIALISER(std::set); template DECLARE_SERVE_SERIALISER(std::tuple); -template -DECLARE_SERVE_SERIALISER(std::map); +template +DECLARE_SERVE_SERIALISER(std::map); #undef COMMA_ } // namespace nix diff --git a/src/libstore/include/nix/store/worker-protocol-impl.hh b/src/libstore/include/nix/store/worker-protocol-impl.hh index 26f6b9d44e..605663d2ee 100644 --- a/src/libstore/include/nix/store/worker-protocol-impl.hh +++ b/src/libstore/include/nix/store/worker-protocol-impl.hh @@ -34,8 +34,10 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) #define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA , WORKER_USE_LENGTH_PREFIX_SERIALISER( - template, - std::map) + template + , + std::map) /** * Use `CommonProto` where possible. diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 4098e8fd91..8e20d3c992 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -12,6 +12,12 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f +/* Note: you generally shouldn't change the protocol version. Define a + new `WorkerProto::Feature` instead. */ +#define PROTOCOL_VERSION (1 << 8 | 39) +#define MINIMUM_PROTOCOL_VERSION (1 << 8 | 18) +#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) +#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define STDERR_NEXT 0x6f6c6d67 #define STDERR_READ 0x64617461 // data needed from source #define STDERR_WRITE 0x64617416 // data for sink @@ -30,6 +36,9 @@ struct BuildResult; struct KeyedBuildResult; struct ValidPathInfo; struct UnkeyedValidPathInfo; +struct DrvOutput; +struct UnkeyedRealisation; +struct Realisation; enum BuildMode : uint8_t; enum TrustedFlag : bool; enum class GCAction; @@ -109,6 +118,12 @@ struct WorkerProto static const Version minimum; + /** + * Feature for transmitting `UnkeyedRealisation` and `DrvOutput` + * using drvPath (store path) instead of the old hash-based JSON format. + */ + static constexpr std::string_view featureRealisationWithPath = "realisation-with-path-not-hash"; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. @@ -307,6 +322,14 @@ DECLARE_WORKER_SERIALISER(ValidPathInfo); template<> DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); template<> +DECLARE_WORKER_SERIALISER(DrvOutput); +template<> +DECLARE_WORKER_SERIALISER(UnkeyedRealisation); +template<> +DECLARE_WORKER_SERIALISER(Realisation); +template<> +DECLARE_WORKER_SERIALISER(std::optional); +template<> DECLARE_WORKER_SERIALISER(BuildMode); template<> DECLARE_WORKER_SERIALISER(GCAction); @@ -325,8 +348,8 @@ DECLARE_WORKER_SERIALISER(std::set); template DECLARE_WORKER_SERIALISER(std::tuple); -template -DECLARE_WORKER_SERIALISER(std::map); +template +DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ } // namespace nix diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8631df2184..b8409b0568 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -357,14 +357,14 @@ LocalStore::LocalStore(ref config) state->stmts->RegisterRealisedOutput.create( state->db, R"( - insert into Realisations (drvPath, outputName, outputPath, signatures) + insert into BuildTraceV2 (drvPath, outputName, outputPath, signatures) values (?, ?, (select id from ValidPaths where path = ?), ?) ; )"); state->stmts->UpdateRealisedOutput.create( state->db, R"( - update Realisations + update BuildTraceV2 set signatures = ? where drvPath = ? and @@ -374,16 +374,16 @@ LocalStore::LocalStore(ref config) state->stmts->QueryRealisedOutput.create( state->db, R"( - select Realisations.id, Output.path, Realisations.signatures from Realisations - inner join ValidPaths as Output on Output.id = Realisations.outputPath + select BuildTraceV2.id, Output.path, BuildTraceV2.signatures from BuildTraceV2 + inner join ValidPaths as Output on Output.id = BuildTraceV2.outputPath where drvPath = ? and outputName = ? ; )"); state->stmts->QueryAllRealisedOutputs.create( state->db, R"( - select outputName, Output.path from Realisations - inner join ValidPaths as Output on Output.id = Realisations.outputPath + select outputName, Output.path from BuildTraceV2 + inner join ValidPaths as Output on Output.id = BuildTraceV2.outputPath where drvPath = ? ; )"); @@ -603,7 +603,7 @@ void LocalStore::upgradeDBSchema(State & state) if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) doUpgrade( - "20220326-ca-derivations", + "20251016-ca-derivations", #include "ca-specific-schema.sql.gen.hh" ); } @@ -647,8 +647,8 @@ void LocalStore::registerDrvOutput(const Realisation & info) auto combinedSignatures = oldR->signatures; combinedSignatures.insert(info.signatures.begin(), info.signatures.end()); state->stmts->UpdateRealisedOutput - .use()(concatStringsSep(" ", Signature::toStrings(combinedSignatures)))(info.id.strHash())( - info.id.outputName) + .use()(concatStringsSep(" ", Signature::toStrings(combinedSignatures)))( + info.id.drvPath.to_string())(info.id.outputName) .exec(); } else { throw Error( @@ -662,7 +662,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) } } else { state->stmts->RegisterRealisedOutput - .use()(info.id.strHash())(info.id.outputName)(printStorePath(info.outPath))( + .use()(info.id.drvPath.to_string())(info.id.outputName)(printStorePath(info.outPath))( concatStringsSep(" ", Signature::toStrings(info.signatures))) .exec(); } @@ -1540,7 +1540,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const std::set> LocalStore::queryRealisationCore_(LocalStore::State & state, const DrvOutput & id) { - auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.strHash())(id.outputName)); + auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.drvPath.to_string())(id.outputName)); if (!useQueryRealisedOutput.next()) return std::nullopt; auto realisationDbId = useQueryRealisedOutput.getInt(0); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e55dc424f0..44c9a72e5f 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -243,16 +243,15 @@ MissingPaths Store::queryMissing(const std::vector & targets) // If there are unknown output paths, attempt to find if the // paths are known to substituters through a realisation. - auto outputHashes = staticOutputHashes(*this, *drv); knownOutputPaths = true; - for (auto [outputName, hash] : outputHashes) { + for (auto & [outputName, _] : drv->outputs) { if (!bfd.outputs.contains(outputName)) continue; bool found = false; for (auto & sub : getDefaultSubstituters()) { - auto realisation = sub->queryRealisation({hash, outputName}); + auto realisation = sub->queryRealisation({drvPath, outputName}); if (!realisation) continue; found = true; @@ -369,7 +368,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, OutputPathMap outputs; for (auto & [outputName, outputPathOpt] : outputsOpt) { if (!outputPathOpt) - throw MissingRealisation(bfd.drvPath->to_string(store), outputName); + throw MissingRealisation(store, *bfd.drvPath, drvPath, outputName); auto & outputPath = *outputPathOpt; outputs.insert_or_assign(outputName, outputPath); } @@ -393,7 +392,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store bfd.output); auto & optPath = outputPaths.at(bfd.output); if (!optPath) - throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output); + throw MissingRealisation(store, *bfd.drvPath, drvPath, bfd.output); return *optPath; }, }, diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 3f330753d9..632b31ce09 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -42,12 +42,18 @@ create table if not exists NARs ( foreign key (cache) references BinaryCaches(id) on delete cascade ); -create table if not exists Realisations ( +create table if not exists BuildTrace ( cache integer not null, - outputId text not null, - content blob, -- Json serialisation of the realisation, or null if the realisation is absent + + drvPath text not null, + outputName text not null, + + -- The following are null if the realisation is absent + outputPath text, + sigs text, + timestamp integer not null, - primary key (cache, outputId), + primary key (cache, drvPath, outputName), foreign key (cache) references BinaryCaches(id) on delete cascade ); @@ -84,7 +90,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache NarInfoDiskCacheImpl( const Settings & settings, SQLiteSettings sqliteSettings, - std::filesystem::path dbPath = getCacheDir() / "binary-cache-v7.sqlite") + std::filesystem::path dbPath = getCacheDir() / "binary-cache-v8.sqlite") : NarInfoDiskCache{settings} { auto state(_state.lock()); @@ -120,24 +126,24 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache state->insertRealisation.create( state->db, R"( - insert or replace into Realisations(cache, outputId, content, timestamp) - values (?, ?, ?, ?) + insert or replace into BuildTrace(cache, drvPath, outputName, outputPath, sigs, timestamp) + values (?, ?, ?, ?, ?, ?) )"); state->insertMissingRealisation.create( state->db, R"( - insert or replace into Realisations(cache, outputId, timestamp) - values (?, ?, ?) + insert or replace into BuildTrace(cache, drvPath, outputName, timestamp) + values (?, ?, ?, ?) )"); state->queryRealisation.create( state->db, R"( - select content from Realisations - where cache = ? and outputId = ? and - ((content is null and timestamp > ?) or - (content is not null and timestamp > ?)) + select outputPath, sigs from BuildTrace + where cache = ? and drvPath = ? and outputName = ? and + ((outputPath is null and timestamp > ?) or + (outputPath is not null and timestamp > ?)) )"); /* Periodically purge expired entries from the database. */ @@ -298,22 +304,27 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache auto now = time(nullptr); - auto queryRealisation(state->queryRealisation.use()(cache.id)(id.to_string())( + auto queryRealisation(state->queryRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)( now - settings.ttlNegative)(now - settings.ttlPositive)); if (!queryRealisation.next()) - return {oUnknown, 0}; + return {oUnknown, nullptr}; if (queryRealisation.isNull(0)) - return {oInvalid, 0}; + return {oInvalid, nullptr}; try { return { oValid, - std::make_shared(nlohmann::json::parse(queryRealisation.getStr(0))), + std::make_shared( + UnkeyedRealisation{ + .outPath = StorePath{queryRealisation.getStr(0)}, + .signatures = nlohmann::json::parse(queryRealisation.getStr(1)), + }, + id), }; } catch (Error & e) { - e.addTrace({}, "while parsing the local disk cache"); + e.addTrace({}, "reading build trace key-value from the local disk cache"); throw; } }); @@ -359,7 +370,8 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache auto & cache(getCache(*state, uri)); state->insertRealisation - .use()(cache.id)(realisation.id.to_string())(static_cast(realisation).dump())( + .use()(cache.id)(realisation.id.drvPath.to_string())(realisation.id.outputName)( + realisation.outPath.to_string())(static_cast(realisation.signatures).dump())( time(nullptr)) .exec(); }); @@ -371,7 +383,8 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache auto state(_state.lock()); auto & cache(getCache(*state, uri)); - state->insertMissingRealisation.use()(cache.id)(id.to_string())(time(nullptr)).exec(); + state->insertMissingRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)(time(nullptr)) + .exec(); }); } }; diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 433d67f26b..90a032612d 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -8,28 +8,34 @@ namespace nix { MakeError(InvalidDerivationOutputId, Error); -DrvOutput DrvOutput::parse(const std::string & strRep) +DrvOutput DrvOutput::parse(const StoreDirConfig & store, std::string_view s) { - size_t n = strRep.find("!"); - if (n == strRep.npos) - throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); - + size_t n = s.rfind('^'); + if (n == s.npos) + throw InvalidDerivationOutputId("Invalid derivation output id '%s': missing '^'", s); return DrvOutput{ - .drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)), - .outputName = strRep.substr(n + 1), + .drvPath = store.parseStorePath(s.substr(0, n)), + .outputName = OutputName{s.substr(n + 1)}, }; } +std::string DrvOutput::render(const StoreDirConfig & store) const +{ + return std::string(store.printStorePath(drvPath)) + "^" + outputName; +} + std::string DrvOutput::to_string() const { - return strHash() + "!" + outputName; + return std::string(drvPath.to_string()) + "^" + outputName; } std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const { - nlohmann::json serialized = Realisation{*this, key}; - serialized.erase("signatures"); - return serialized.dump(); + auto serialised = static_cast(Realisation{*this, key}); + auto value = serialised.find("value"); + assert(value != serialised.end()); + value->erase("signatures"); + return serialised.dump(); } void UnkeyedRealisation::sign(const DrvOutput & key, const Signer & signer) @@ -61,9 +67,21 @@ const StorePath & RealisedPath::path() const & return std::visit([](auto & arg) -> auto & { return arg.getPath(); }, raw); } -bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const +MissingRealisation::MissingRealisation( + const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName) + : CloneableError( + "cannot operate on output '%s' of the unbuilt derivation '%s'", outputName, store.printStorePath(drvPath)) +{ +} + +MissingRealisation::MissingRealisation( + const StoreDirConfig & store, + const SingleDerivedPath & drvPath, + const StorePath & drvPathResolved, + const OutputName & outputName) + : MissingRealisation{store, drvPathResolved, outputName} { - return outPath == other.outPath; + addTrace({}, "looking up realisation for derivation '%s'", drvPath.to_string(store)); } } // namespace nix @@ -74,12 +92,20 @@ using namespace nix; DrvOutput adl_serializer::from_json(const json & json) { - return DrvOutput::parse(getString(json)); + auto obj = getObject(json); + + return { + .drvPath = valueAt(obj, "drvPath"), + .outputName = getString(valueAt(obj, "outputName")), + }; } void adl_serializer::to_json(json & json, const DrvOutput & drvOutput) { - json = drvOutput.to_string(); + json = { + {"drvPath", drvOutput.drvPath}, + {"outputName", drvOutput.outputName}, + }; } UnkeyedRealisation adl_serializer::from_json(const json & json0) @@ -101,25 +127,25 @@ void adl_serializer::to_json(json & json, const UnkeyedReali json = { {"outPath", r.outPath}, {"signatures", r.signatures}, - // back-compat - {"dependentRealisations", json::object()}, }; } -Realisation adl_serializer::from_json(const json & json0) +Realisation adl_serializer::from_json(const json & json) { - auto json = getObject(json0); + auto obj = getObject(json); - return Realisation{ - static_cast(json0), - valueAt(json, "id"), + return { + static_cast(valueAt(obj, "value")), + static_cast(valueAt(obj, "key")), }; } void adl_serializer::to_json(json & json, const Realisation & r) { - json = static_cast(r); - json["id"] = r.id; + json = { + {"key", r.id}, + {"value", static_cast(r)}, + }; } } // namespace nlohmann diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 05dd5cf13a..c261e3384a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -503,12 +503,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); conn->to << WorkerProto::Op::RegisterDrvOutput; - if (conn->protoVersion.number < WorkerProto::Version::Number{1, 31}) { - WorkerProto::write(*this, *conn, info.id); - conn->to << std::string(info.outPath.to_string()); - } else { - WorkerProto::write(*this, *conn, info); - } + WorkerProto::write(*this, *conn, info); conn.processStderr(); } @@ -518,8 +513,10 @@ void RemoteStore::queryRealisationUncached( try { auto conn(getConnection()); - if (conn->protoVersion.number < WorkerProto::Version::Number{1, 27}) { - warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4"); + if (!conn->protoVersion.features.contains(WorkerProto::featureRealisationWithPath)) { + warn( + "the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations", + WorkerProto::featureRealisationWithPath); return callback(nullptr); } @@ -527,21 +524,12 @@ void RemoteStore::queryRealisationUncached( WorkerProto::write(*this, *conn, id); conn.processStderr(); - auto real = [&]() -> std::shared_ptr { - if (conn->protoVersion.number < WorkerProto::Version::Number{1, 31}) { - auto outPaths = WorkerProto::Serialise>::read(*this, *conn); - if (outPaths.empty()) - return nullptr; - return std::make_shared(UnkeyedRealisation{.outPath = *outPaths.begin()}); - } else { - auto realisations = WorkerProto::Serialise>::read(*this, *conn); - if (realisations.empty()) - return nullptr; - return std::make_shared(*realisations.begin()); - } - }(); - - callback(std::shared_ptr(real)); + callback([&]() -> std::shared_ptr { + auto realisation = WorkerProto::Serialise>::read(*this, *conn); + if (!realisation) + return nullptr; + return std::make_shared(*realisation); + }()); } catch (...) { return callback.rethrow(); } @@ -623,30 +611,19 @@ std::vector RemoteStore::buildPathsWithResults( OutputPathMap outputs; auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); - auto drv = evalStore->readDerivation(drvPath); - const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { - auto outputHash = get(outputHashes, output); - if (!outputHash) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - printStorePath(drvPath), - output); - auto outputId = DrvOutput{*outputHash, output}; + auto outputId = DrvOutput{drvPath, output}; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = queryRealisation(outputId); if (!realisation) - throw MissingRealisation(outputId); - success.builtOutputs.emplace(output, Realisation{*realisation, outputId}); + throw MissingRealisation(*this, outputId); + success.builtOutputs.emplace(output, *realisation); } else { success.builtOutputs.emplace( output, - Realisation{ - UnkeyedRealisation{ - .outPath = outputPath, - }, - outputId, + UnkeyedRealisation{ + .outPath = outputPath, }); } } diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 5248a313c2..8001d43ec8 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -280,9 +280,18 @@ std::vector RestrictedStore::buildPathsWithResults( for (auto & result : results) { if (auto * successP = result.tryGetSuccess()) { - for (auto & [outputName, output] : successP->builtOutputs) { - newPaths.insert(output.outPath); - newRealisations.insert(output); + if (auto * pathBuilt = std::get_if(&result.path)) { + // TODO ugly extra IO + auto drvPath = resolveDerivedPath(*next, *pathBuilt->drvPath); + for (auto & [outputName, output] : successP->builtOutputs) { + newPaths.insert(output.outPath); + newRealisations.insert( + {output, + { + .drvPath = drvPath, + .outputName = outputName, + }}); + } } } } diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index ec7b85ed88..bddd17da8b 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -7,6 +7,7 @@ #include "nix/store/serve-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/path-info.hh" +#include "nix/util/json-utils.hh" #include @@ -28,10 +29,19 @@ BuildResult ServeProto::Serialise::read(const StoreDirConfig & stor if (conn.version >= ServeProto::Version{2, 3}) conn.from >> res.timesBuilt >> isNonDeterministic >> res.startTime >> res.stopTime; - if (conn.version >= ServeProto::Version{2, 6}) { - auto builtOutputs = ServeProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + + if (conn.version >= ServeProto::Version{2, 8}) { + success.builtOutputs = ServeProto::Serialise>::read(store, conn); + } else if (conn.version >= ServeProto::Version{2, 6}) { + for (auto & [output, realisation] : ServeProto::Serialise::read(store, conn)) { + size_t n = output.find("!"); + if (n == output.npos) + throw Error("Invalid derivation output id %s", output); + success.builtOutputs.insert_or_assign( + output.substr(n + 1), + UnkeyedRealisation{ + StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}}); + } } res.inner = std::visit( @@ -63,15 +73,30 @@ void ServeProto::Serialise::write( default value for the fields that don't exist in that case. */ auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { conn.to << errorMsg; + if (conn.version >= ServeProto::Version{2, 3}) conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; - if (conn.version >= ServeProto::Version{2, 6}) { - DrvOutputs builtOutputsFullKey; - for (auto & [output, realisation] : builtOutputs) - builtOutputsFullKey.insert_or_assign(realisation.id, realisation); - ServeProto::write(store, conn, builtOutputsFullKey); + + if (conn.version >= ServeProto::Version{2, 8}) { + ServeProto::write(store, conn, builtOutputs); + } else if (conn.version >= ServeProto::Version{2, 6}) { + // Old clients read `builtOutputs` as a `StringMap` keyed + // by `sha256:!` with JSON-encoded + // realisations. The derivation hash no longer exists, but + // old clients only extract `outputName` and `outPath`, so + // a dummy hash suffices. + StringMap sm; + for (auto & [outputName, realisation] : builtOutputs) { + auto dummyId = Hash::dummy.to_string(HashFormat::Base16, true) + "!" + outputName; + nlohmann::json j; + j["id"] = dummyId; + j["outPath"] = realisation.outPath.to_string(); + sm[dummyId] = j.dump(); + } + ServeProto::write(store, conn, sm); } }; + std::visit( overloaded{ [&](const BuildResult::Failure & failure) { @@ -158,4 +183,82 @@ void ServeProto::Serialise::write( } } +UnkeyedRealisation ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (conn.version < ServeProto::Version{2, 8}) { + throw Error( + "serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations", + conn.version.major, + conn.version.minor); + } + + auto outPath = ServeProto::Serialise::read(store, conn); + auto signatures = ServeProto::Serialise>::read(store, conn); + + return UnkeyedRealisation{ + .outPath = std::move(outPath), + .signatures = std::move(signatures), + }; +} + +void ServeProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info) +{ + if (conn.version < ServeProto::Version{2, 8}) { + throw Error( + "serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations", + conn.version.major, + conn.version.minor); + } + ServeProto::write(store, conn, info.outPath); + ServeProto::write(store, conn, info.signatures); +} + +DrvOutput ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (conn.version < ServeProto::Version{2, 8}) { + throw Error( + "serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations", + conn.version.major, + conn.version.minor); + } + + auto drvPath = ServeProto::Serialise::read(store, conn); + auto outputName = ServeProto::Serialise::read(store, conn); + + return DrvOutput{ + .drvPath = std::move(drvPath), + .outputName = std::move(outputName), + }; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info) +{ + if (conn.version < ServeProto::Version{2, 8}) { + throw Error( + "serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations", + conn.version.major, + conn.version.minor); + } + ServeProto::write(store, conn, info.drvPath); + ServeProto::write(store, conn, info.outputName); +} + +Realisation ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + auto id = ServeProto::Serialise::read(store, conn); + auto unkeyed = ServeProto::Serialise::read(store, conn); + + return Realisation{ + std::move(unkeyed), + std::move(id), + }; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info) +{ + ServeProto::write(store, conn, info.id); + ServeProto::write(store, conn, static_cast(info)); +} + } // namespace nix diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 845e7bcb37..dd38a36bc8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -403,9 +403,8 @@ Store::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore return outputs; auto drv = evalStore.readInvalidDerivation(path); - auto drvHashes = staticOutputHashes(*this, drv); - for (auto & [outputName, hash] : drvHashes) { - auto realisation = queryRealisation(DrvOutput{hash, outputName}); + for (auto & [outputName, _] : drv.outputs) { + auto realisation = queryRealisation(DrvOutput{path, outputName}); if (realisation) { outputs.insert_or_assign(outputName, realisation->outPath); } else { @@ -425,7 +424,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * ev OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw MissingRealisation(printStorePath(path), outName); + throw MissingRealisation(*this, path, outName); result.insert_or_assign(outName, *optOutPath); } return result; diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index da225dc0d1..911d4b3f81 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1968,7 +1968,10 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() { .outPath = newInfo.path, }, - DrvOutput{oldinfo->outputHash, outputName}, + DrvOutput{ + .drvPath = drvPath, + .outputName = outputName, + }, }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().isImpure()) { store.signRealisation(thisRealisation); diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 49ab4c36b1..3da966a620 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -8,6 +8,7 @@ #include "nix/store/worker-protocol-impl.hh" #include "nix/util/archive.hh" #include "nix/store/path-info.hh" +#include "nix/util/json-utils.hh" #include #include @@ -20,6 +21,12 @@ const WorkerProto::Version WorkerProto::latest = { .major = 1, .minor = 38, }, + .features = + { + std::string{ + WorkerProto::featureRealisationWithPath, + }, + }, }; const WorkerProto::Version WorkerProto::minimum = { @@ -254,10 +261,19 @@ BuildResult WorkerProto::Serialise::read(const StoreDirConfig & sto res.cpuUser = WorkerProto::Serialise>::read(store, conn); res.cpuSystem = WorkerProto::Serialise>::read(store, conn); } - if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { - auto builtOutputs = WorkerProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + + if (conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + success.builtOutputs = WorkerProto::Serialise>::read(store, conn); + } else if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { + for (auto && [output, realisation] : WorkerProto::Serialise::read(store, conn)) { + size_t n = output.find("!"); + if (n == output.npos) + throw Error("Invalid derivation output id %s", output); + success.builtOutputs.insert_or_assign( + output.substr(n + 1), + UnkeyedRealisation{ + StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}}); + } } res.inner = std::visit( @@ -296,13 +312,27 @@ void WorkerProto::Serialise::write( WorkerProto::write(store, conn, res.cpuUser); WorkerProto::write(store, conn, res.cpuSystem); } - if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { - DrvOutputs builtOutputsFullKey; - for (auto & [output, realisation] : builtOutputs) - builtOutputsFullKey.insert_or_assign(realisation.id, realisation); - WorkerProto::write(store, conn, builtOutputsFullKey); + + if (conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + WorkerProto::write(store, conn, builtOutputs); + } else if (conn.version >= WorkerProto::Version{.number = {1, 28}}) { + // Old clients read `builtOutputs` as a `StringMap` keyed + // by `sha256:!` with JSON-encoded + // realisations. The derivation hash no longer exists, but + // old clients only extract `outputName` and `outPath`, so + // a dummy hash suffices. + StringMap sm; + for (auto & [outputName, realisation] : builtOutputs) { + auto dummyId = Hash::dummy.to_string(HashFormat::Base16, true) + "!" + outputName; + nlohmann::json j; + j["id"] = dummyId; + j["outPath"] = realisation.outPath.to_string(); + sm[dummyId] = j.dump(); + } + WorkerProto::write(store, conn, sm); } }; + std::visit( overloaded{ [&](const BuildResult::Failure & failure) { @@ -395,4 +425,109 @@ void WorkerProto::Serialise::write( } } +UnkeyedRealisation WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + throw Error( + "the daemon is missing the '%s' protocol feature, needed to understand build trace", + WorkerProto::featureRealisationWithPath); + } + + auto outPath = WorkerProto::Serialise::read(store, conn); + auto signatures = WorkerProto::Serialise>::read(store, conn); + + return UnkeyedRealisation{ + .outPath = std::move(outPath), + .signatures = std::move(signatures), + }; +} + +void WorkerProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info) +{ + if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + throw Error( + "the daemon is missing the '%s' protocol feature, needed to understand build trace", + WorkerProto::featureRealisationWithPath); + } + WorkerProto::write(store, conn, info.outPath); + WorkerProto::write(store, conn, info.signatures); +} + +std::optional +WorkerProto::Serialise>::read(const StoreDirConfig & store, ReadConn conn) +{ + if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + // Hack to improve compat + (void) WorkerProto::Serialise::read(store, conn); + return std::nullopt; + } else { + auto temp = readNum(conn.from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return WorkerProto::Serialise::read(store, conn); + default: + throw Error("Invalid optional build trace from remote"); + } + } +} + +void WorkerProto::Serialise>::write( + const StoreDirConfig & store, WriteConn conn, const std::optional & info) +{ + if (!info) { + conn.to << uint8_t{0}; + } else { + conn.to << uint8_t{1}; + WorkerProto::write(store, conn, *info); + } +} + +DrvOutput WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + throw Error( + "the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations", + WorkerProto::featureRealisationWithPath); + } + + auto drvPath = WorkerProto::Serialise::read(store, conn); + auto outputName = WorkerProto::Serialise::read(store, conn); + + return DrvOutput{ + .drvPath = std::move(drvPath), + .outputName = std::move(outputName), + }; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info) +{ + if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) { + throw Error( + "the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations", + WorkerProto::featureRealisationWithPath); + } + WorkerProto::write(store, conn, info.drvPath); + WorkerProto::write(store, conn, info.outputName); +} + +Realisation WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + auto id = WorkerProto::Serialise::read(store, conn); + auto unkeyed = WorkerProto::Serialise::read(store, conn); + + return Realisation{ + std::move(unkeyed), + std::move(id), + }; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info) +{ + WorkerProto::write(store, conn, info.id); + WorkerProto::write(store, conn, static_cast(info)); +} + } // namespace nix diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index d46f3011a4..14a0cad760 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -346,13 +346,11 @@ static int main_build_remote(int argc, char ** argv) optResult = std::move(res[0]); } - auto outputHashes = staticOutputHashes(*store, drv); std::set missingRealisations; StorePathSet missingPaths; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { for (auto & outputName : wantedOutputs) { - auto thisOutputHash = outputHashes.at(outputName); - auto thisOutputId = DrvOutput{thisOutputHash, outputName}; + auto thisOutputId = DrvOutput{*drvPath, outputName}; if (!store->queryRealisation(thisOutputId)) { debug("missing output %s", outputName); assert(optResult); @@ -362,7 +360,7 @@ static int main_build_remote(int argc, char ** argv) auto i = success.builtOutputs.find(outputName); assert(i != success.builtOutputs.end()); auto & newRealisation = i->second; - missingRealisations.insert(newRealisation); + missingRealisations.insert({newRealisation, thisOutputId}); missingPaths.insert(newRealisation.outPath); } } diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index 6c4bf2e705..b0f70be3a9 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -163,10 +163,13 @@ StoreWrapper::queryPathInfo(char * path, int base32) } SV * -StoreWrapper::queryRawRealisation(char * outputId) +StoreWrapper::queryRawRealisation(char * drvPath, char * outputName) PPCODE: try { - auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId)); + auto realisation = THIS->store->queryRealisation(DrvOutput{ + .drvPath = THIS->store->parseStorePath(drvPath), + .outputName = outputName, + }); if (realisation) XPUSHs(sv_2mortal(newSVpv(static_cast(*realisation).dump().c_str(), 0))); else diff --git a/tests/functional/ca/common.sh b/tests/functional/ca/common.sh index dc8e650fd6..10298d9d87 100644 --- a/tests/functional/ca/common.sh +++ b/tests/functional/ca/common.sh @@ -1,6 +1,9 @@ # shellcheck shell=bash source ../common.sh +# Need backend to support revamped CA +requireDaemonNewerThan "2.34.0pre20251217" + enableFeatures "ca-derivations" TODO_NixOS diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index 2f6ebcef5c..022bea5cfc 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -49,14 +49,14 @@ fi clearStore nix build --file ../simple.nix -L --no-link --post-build-hook "$pushToStore" clearStore -rm -r "$REMOTE_STORE_DIR/realisations" +rm -r "$REMOTE_STORE_DIR/build-trace-v2" nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # There's no easy way to check whether a realisation is present on the local # store − short of manually querying the db, but the build environment doesn't # have the sqlite binary − so we instead push things again, and check that the # realisations have correctly been pushed to the remote store nix copy --to "$REMOTE_STORE" --file ../simple.nix -if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then +if [[ -z "$(ls "$REMOTE_STORE_DIR/build-trace-v2")" ]]; then echo "Realisations not rebuilt" exit 1 fi @@ -71,5 +71,5 @@ buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # Try rebuilding, but remove the realisations from the remote cache to force # using the cachecache clearStore -rm "$REMOTE_STORE_DIR"/realisations/* +rm -r "$REMOTE_STORE_DIR"/build-trace-v2/* buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 From ca42db38a3e16445bf04ee59f4124d1138807ae0 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 2 Mar 2026 21:24:05 +0300 Subject: [PATCH 020/927] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index dc1591835c..aa5388f637 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.34.0 +2.35.0 From 855208ba24badfee6e3677090a156394bf020d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 2 Mar 2026 19:22:36 +0100 Subject: [PATCH 021/927] docs: add release notes for build trace rework The previous commit changed CA derivation realisations to be keyed by store path instead of hash modulo, affecting binary cache and wire protocols. Users and tool authors need to understand the impact on binary cache layout, protocol negotiation, and what migration looks like. This documents the key changes: the new build-trace-v2/ cache directory, the split key/value JSON format, worker protocol feature negotiation, serve protocol version bump, and the fact that non-CA users are unaffected. --- doc/manual/rl-next/build-trace-rework.md | 75 ++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 doc/manual/rl-next/build-trace-rework.md diff --git a/doc/manual/rl-next/build-trace-rework.md b/doc/manual/rl-next/build-trace-rework.md new file mode 100644 index 0000000000..fd3541c5b3 --- /dev/null +++ b/doc/manual/rl-next/build-trace-rework.md @@ -0,0 +1,75 @@ +--- +synopsis: "Content-addressed derivations: realisations keyed by store path instead of hash modulo" +issues: [11897] +prs: [12464] +--- + +The experimental content-addressed (CA) derivation feature has undergone a significant change to how build traces (formerly called "realisations") are identified. This affects the **binary cache protocol** and the **wire protocols**. + +### What changed + +Previously, a build trace entry (realisation) was keyed by the **hash modulo** of the derivation. +A SHA-256 hash computed via the complex "derivation hash modulo" algorithm. +This required implementations to understand ATerm serialisation and the full derivation hashing scheme just to look up or store build results. + +Now, build trace entries are keyed by the **regular derivation store path** plus the output name. For example, instead of: + +``` +sha256:ba7816bf8f01...!out +``` + +The key is now: + +``` +/nix/store/abc...-foo.drv^out +``` + +This is simpler, more intuitive, and means that third-party tools implementing CA derivation support (e.g., Hydra) +no longer need to implement the derivation hash modulo algorithm. + +### Binary cache protocol + +- The directory for build traces moved from `realisations/` to `build-trace-v2/`. +- File paths changed from `realisations/!.doi` to `build-trace-v2//.doi`. +- The JSON format of build trace entries is now split into `key` and `value` objects: + ```json + { + "key": { + "drvPath": "abc...-foo.drv", + "outputName": "out" + }, + "value": { + "outPath": "xyz...-foo", + "signatures": ["..."] + } + } + ``` + Previously, these were flat objects with a string `id` field like `"sha256:...!out"`. +- The deprecated `dependentRealisations` field has been removed. + +Existing binary caches will need to be re-populated with the new format for CA derivation build traces. +Old build traces at the previous URLs are simply abandoned. +Non-CA builds are unaffected. + +### Wire protocols + +- **Worker protocol**: + A new feature flag `realisation-with-path-not-hash` is negotiated during the handshake. + Clients and daemons that both support this feature use the new binary serialisation for `DrvOutput`, `UnkeyedRealisation`, and related types. + Fallback to older protocol versions gracefully degrades (realisations are unavailable). +- **Serve protocol**: + Bumped from 2.7 to 2.8 with native serialisers for the new types. + Fallback to older protocol versions gracefully degrades in the same way. + +Stable code paths do use the realization fields (`BuildResult::Success::builtOutputs`), but only the output name and outpath parts of that. +For older protocols, we can fake enough of the realisation format to provide those two parts forthat map, which keeps operations like `--print-output-paths` working. + +### Impact + +- **Non-CA derivation users**: No impact. This only affects the experimental `ca-derivations` feature. +- **Binary cache operators**: + Binary caches serving CA derivation build traces will need to be repopulated. + Existing NARs and narinfo files are unaffected. +- **Tool authors**: + Implementations interfacing with the CA derivations protocol are simplified. + The derivation hash modulo algorithm is no longer required to form build trace keys. From 62a78c86b0a16e71c29856d542af5b7fbea94e2d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 2 Mar 2026 20:11:49 +0100 Subject: [PATCH 022/927] release notes 2.34: add entries for additional PRs --- doc/manual/source/release-notes/rl-2.34.md | 49 +++++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/release-notes/rl-2.34.md b/doc/manual/source/release-notes/rl-2.34.md index 854f1d1ed6..c415041120 100644 --- a/doc/manual/source/release-notes/rl-2.34.md +++ b/doc/manual/source/release-notes/rl-2.34.md @@ -153,6 +153,8 @@ When these options are configured, Nix will use this certificate/private key pair to authenticate to the server. +- `nix store gc --dry-run` and `nix-collect-garbage --dry-run` now report the number of paths that would be freed [#15229](https://github.com/NixOS/nix/pull/15229) [#5704](https://github.com/NixOS/nix/issues/5704) + ## Performance improvements - Unpacking tarballs to `~/.cache/nix/tarball-cache-v2` is now multithreaded [#12087](https://github.com/NixOS/nix/pull/12087) @@ -168,9 +170,11 @@ - `nix nar ls` and other NAR listing operations have been optimised further [#15163](https://github.com/NixOS/nix/pull/15163) +- Evaluator hot-path optimizations [#15270](https://github.com/NixOS/nix/pull/15270) [#15271](https://github.com/NixOS/nix/pull/15271) + ## C API Changes -- New store API methods [#14766](https://github.com/NixOS/nix/pull/14766) +- New store API methods [#14766](https://github.com/NixOS/nix/pull/14766) [#14768](https://github.com/NixOS/nix/pull/14768) The C API now includes additional methods: @@ -204,7 +208,7 @@ Due to a bug in how Nix handled Boost.Coroutine2 suspension and resumption, copying from `ssh-ng://` stores would drop the SSH connection for each copied path. This issue has been fixed, which improves performance by avoiding multiple SSH/Nix Worker Protocol handshakes. -- S3 binary caches now use virtual-hosted-style addressing by default [#15208](https://github.com/NixOS/nix/issues/15208) +- S3 binary caches now use virtual-hosted-style addressing by default [#15208](https://github.com/NixOS/nix/issues/15208) [#15216](https://github.com/NixOS/nix/pull/15216) S3 binary caches now use virtual-hosted-style URLs (`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs @@ -243,6 +247,34 @@ When run as root, Nix doesn't run builds via the daemon and is a parent of the forked build processes. Prior versions of Nix failed to preserve the `PR_SET_PDEATHSIG` parent-death signal across `setuid` calls. This could lead to build processes being reparented and continue running in the background. This has been fixed. +- Fix crash when interrupting `--log-format internal-json` [#15335](https://github.com/NixOS/nix/pull/15335) + + Pressing Ctrl-C during `--log-format internal-json` (used by [nix-output-monitor](https://github.com/maralorn/nix-output-monitor)) no longer causes a spurious "Nix crashed. This is a bug." report. + +- Fix percent-encoding in `file://` and `local://` store URIs [#15280](https://github.com/NixOS/nix/pull/15280) + + Store URIs with special characters like `+` in the path (e.g. `file:///tmp/a+b`) no longer incorrectly create percent-encoded directories (e.g. `/tmp/a%2Bb`). + +- Fix crash during tab completion in `nix repl` [#15255](https://github.com/NixOS/nix/pull/15255) + +- Fix "Too many open files" on macOS [#15205](https://github.com/NixOS/nix/pull/15205) + + Nix now raises the open file soft limit to the hard limit at startup, fixing "Too many open files" errors on macOS where the default soft limit is low. + +- `nix develop` no longer fails when `inputs.nixpkgs` has `flake = false` [#15175](https://github.com/NixOS/nix/pull/15175) + +- `builtins.flakeRefToString` no longer fails with "attribute is a thunk" [#15160](https://github.com/NixOS/nix/pull/15160) + +- Fix `QueryPathInfo` throwing on invalid paths in the daemon [#15134](https://github.com/NixOS/nix/pull/15134) + +- `nix-store --generate-binary-cache-key` now fsyncs key files to prevent corruption [#15107](https://github.com/NixOS/nix/pull/15107) + +- Fix `build-hook` setting in `nix.conf` being ignored [#15083](https://github.com/NixOS/nix/pull/15083) + +- Fix empty error messages when builds are cancelled due to a dependency failure [#14972](https://github.com/NixOS/nix/pull/14972) + + When a build fails without `--keep-going`, other in-progress builds are cancelled. Previously, these cancelled builds were incorrectly reported as failed with empty error messages. This affected `buildPathsWithResults` callers such as `nix flake check`. + ## Miscellaneous changes - Content-Encoding decompression is now handled by libcurl [#14324](https://github.com/NixOS/nix/issues/14324) [#15336](https://github.com/NixOS/nix/pull/15336) @@ -254,6 +286,19 @@ - Static builds now support S3 features (`libstore:s3-aws-auth` meson option) [#15076](https://github.com/NixOS/nix/pull/15076) +- Improved package-related error messages [#15349](https://github.com/NixOS/nix/pull/15349) + + Store path context is now rendered in the user-facing `hash^out` format instead of the internal `!out!hash` format. + A misleading error message in `nix-env` that incorrectly blamed content-addressed derivations has been fixed. + +- Improved error message for empty derivation files [#15298](https://github.com/NixOS/nix/pull/15298) + + Parsing an empty `.drv` file (e.g. due to store corruption after an unclean shutdown) now produces a clear error message instead of the cryptic `expected string 'D'`. + +- Relative `file:` paths for tarballs are now rejected with a clear error [#14983](https://github.com/NixOS/nix/pull/14983) + +- Continued progress on the Windows port, including build fixes, CI improvements, and platform abstractions. + ## Contributors This release was made possible by the following 43 contributors: From 5f666eff14529cc3794cc195f4ced876799d247f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 2 Mar 2026 21:18:22 +0300 Subject: [PATCH 023/927] Add release note for GHCR Docker images (cherry picked from commit 5d054fe91ef95650ba7a7925b15ed9acfc0e7006) --- doc/manual/source/release-notes/rl-2.34.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/source/release-notes/rl-2.34.md b/doc/manual/source/release-notes/rl-2.34.md index c415041120..335e88ee89 100644 --- a/doc/manual/source/release-notes/rl-2.34.md +++ b/doc/manual/source/release-notes/rl-2.34.md @@ -299,6 +299,10 @@ - Continued progress on the Windows port, including build fixes, CI improvements, and platform abstractions. +- Nix docker images are now uploaded to [GHCR](https://github.com/NixOS/nix/pkgs/container/nix) as part of the release process + + Historically, only pre-release builds of `amd64` docker images have been uploaded to ghcr.io with the `latest` tag pointing to the last built image from `master` branch. This has been fixed and going forward, will include the same images as that are built by [Hydra](https://hydra.nixos.org/project/nix) for [arm64](https://hydra.nixos.org/job/nix/maintenance-2.34/dockerImage.aarch64-linux) and [amd64](https://hydra.nixos.org/job/nix/maintenance-2.34/dockerImage.x86_64-linux). Pre-release versions are no longer pushed to the registry. + ## Contributors This release was made possible by the following 43 contributors: From 4f8f581830e74ec7502d97feeea6bdaaaad4349c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 2 Mar 2026 10:37:21 -0500 Subject: [PATCH 024/927] libutil: Reimplement `*stat` functions on Windows for symlink support Reimplement `lstat` and `maybeLstat` from first principles on Windows so they work with symlinks. Properly define `S_IFLNK` and `S_ISLNK`. Use `GetFileAttributesExW` instead of `_wstat64` since the latter doesn't properly detect symlinks (reparse points) on Windows. Key changes: - Add `S_IFLNK` (0120000) and proper `S_ISLNK` macro for Windows - Add `windows::fileTimeToUnixTime` to convert FILETIME to Unix time_t - Add `windows::statFromFileInfo` to populate PosixStat from Windows file attributes - Move `lstat` and `maybeLstat` to platform-specific files (unix/file-system.cc and windows/file-system.cc) - Move `fstat` from file-system.hh to file-system-at.hh, changing signature from `int fd` to `Descriptor fd` for cross-platform support - Windows `lstat` now detects symlinks via FILE_ATTRIBUTE_REPARSE_POINT and sets S_IFLNK in st_mode - Windows `fstat` uses GetFileInformationByHandle Co-authored-by: Sergei Zimmerman --- src/libstore/unix/pathlocks.cc | 2 +- src/libutil/file-system.cc | 42 +-------- .../include/nix/util/file-system-at.hh | 9 ++ src/libutil/include/nix/util/file-system.hh | 53 +++++++++-- src/libutil/unix/file-descriptor.cc | 1 + src/libutil/unix/file-system-at.cc | 16 +++- src/libutil/unix/file-system.cc | 19 ++++ src/libutil/windows/file-system-at.cc | 20 +++++ src/libutil/windows/file-system.cc | 90 +++++++++++++++++++ 9 files changed, 199 insertions(+), 53 deletions(-) diff --git a/src/libstore/unix/pathlocks.cc b/src/libstore/unix/pathlocks.cc index ce2adf74de..47e33853c9 100644 --- a/src/libstore/unix/pathlocks.cc +++ b/src/libstore/unix/pathlocks.cc @@ -1,5 +1,5 @@ #include "nix/store/pathlocks.hh" -#include "nix/util/file-system.hh" +#include "nix/util/file-system-at.hh" #include "nix/util/util.hh" #include "nix/util/sync.hh" #include "nix/util/signals.hh" diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 5dec2decb3..d810e4a06c 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -171,10 +171,8 @@ bool isDirOrInDir(const std::filesystem::path & path, const std::filesystem::pat #ifdef _WIN32 # define STAT _wstat64 -# define LSTAT _wstat64 #else # define STAT stat -# define LSTAT lstat #endif PosixStat stat(const std::filesystem::path & path) @@ -185,54 +183,18 @@ PosixStat stat(const std::filesystem::path & path) return st; } -PosixStat lstat(const std::filesystem::path & path) -{ - PosixStat st; - if (LSTAT(path.c_str(), &st)) - throw SysError("getting status of %s", PathFmt(path)); - return st; -} - -PosixStat fstat(int fd) -{ - PosixStat st; - if ( -#ifdef _WIN32 - _fstat64 -#else - ::fstat -#endif - (fd, &st)) - throw SysError("getting status of fd %d", fd); - return st; -} - std::optional maybeStat(const std::filesystem::path & path) { std::optional st{std::in_place}; if (STAT(path.c_str(), &*st)) { if (errno == ENOENT || errno == ENOTDIR) - st.reset(); - else - throw SysError("getting status of %s", PathFmt(path)); - } - return st; -} - -std::optional maybeLstat(const std::filesystem::path & path) -{ - std::optional st{std::in_place}; - if (LSTAT(path.c_str(), &*st)) { - if (errno == ENOENT || errno == ENOTDIR) - st.reset(); - else - throw SysError("getting status of %s", PathFmt(path)); + return std::nullopt; + throw SysError("getting status of %s", PathFmt(path)); } return st; } #undef STAT -#undef LSTAT bool pathExists(const std::filesystem::path & path) { diff --git a/src/libutil/include/nix/util/file-system-at.hh b/src/libutil/include/nix/util/file-system-at.hh index 4558f0e0ef..a8cb17b95e 100644 --- a/src/libutil/include/nix/util/file-system-at.hh +++ b/src/libutil/include/nix/util/file-system-at.hh @@ -15,6 +15,7 @@ */ #include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" #include @@ -25,6 +26,14 @@ namespace nix { +/** + * Get status of an open file/directory handle. + * + * @param fd File descriptor/handle + * @throws SystemError on I/O errors. + */ +PosixStat fstat(Descriptor fd); + /** * Read a symlink relative to a directory file descriptor. * diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 58aecade75..aae7b7bf37 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -30,11 +30,20 @@ * Polyfill for MinGW * * Windows does in fact support symlinks, but the C runtime interfaces predate this. - * - * @todo get rid of this, and stop using `stat` when we want `lstat` too. - */ + * We define S_IFLNK and S_ISLNK so that our lstat implementation can properly + * indicate symlinks by setting these mode bits when it detects a reparse point. + */ +#ifndef S_IFLNK +# ifndef _WIN32 +# error "S_IFLNK should be defined on non-Windows platforms" +# endif +# define S_IFLNK 0120000 +#endif #ifndef S_ISLNK -# define S_ISLNK(m) false +# ifndef _WIN32 +# error "S_ISLNK should be defined on non-Windows platforms" +# endif +# define S_ISLNK(m) (((m) & S_IFMT) == S_IFLNK) #endif namespace nix { @@ -97,6 +106,38 @@ using PosixStat = #endif ; +#ifdef _WIN32 +namespace windows { + +/** + * Convert Windows FILETIME to Unix time_t. + */ +time_t fileTimeToUnixTime(const FILETIME & ft); + +/** + * Fill a PosixStat structure from file attributes and timestamps. + * + * @param dwFileAttributes File attributes (FILE_ATTRIBUTE_*) + * @param ftCreationTime Creation time + * @param ftLastAccessTime Last access time + * @param ftLastWriteTime Last write time + * @param nFileSizeHigh High 32 bits of file size + * @param nFileSizeLow Low 32 bits of file size + * @param nNumberOfLinks Number of hard links (default 1) + */ +void statFromFileInfo( + PosixStat & st, + DWORD dwFileAttributes, + const FILETIME & ftCreationTime, + const FILETIME & ftLastAccessTime, + const FILETIME & ftLastWriteTime, + DWORD nFileSizeHigh, + DWORD nFileSizeLow, + DWORD nNumberOfLinks = 1); + +} // namespace windows +#endif + /** * Get status of `path`. */ @@ -105,10 +146,6 @@ PosixStat lstat(const std::filesystem::path & path); * Get status of `path` following symlinks. */ PosixStat stat(const std::filesystem::path & path); -/** - * Get status of an open file descriptor. - */ -PosixStat fstat(int fd); /** * `lstat` the given path if it exists. * @return std::nullopt if the path doesn't exist, or an optional containing the result of `lstat` otherwise diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index 05ec38a67c..f2c69f2a17 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -1,4 +1,5 @@ #include "nix/util/file-system.hh" +#include "nix/util/file-system-at.hh" #include "nix/util/signals.hh" #include "nix/util/finally.hh" #include "nix/util/serialise.hh" diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index 739cac0177..fee985fd5d 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -89,10 +89,9 @@ void unix::fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t }); } - struct ::stat st; - /* Possible since https://github.com/torvalds/linux/commit/55815f70147dcfa3ead5738fd56d3574e2e3c1c2 (3.6) */ - if (::fstat(pathFd.get(), &st) == -1) - throw SysError("statting '%s' relative to parent directory via O_PATH file descriptor", path.rel()); + /* Possible to use with O_PATH fd since + * https://github.com/torvalds/linux/commit/55815f70147dcfa3ead5738fd56d3574e2e3c1c2 (3.6) */ + auto st = fstat(pathFd.get()); if (S_ISLNK(st.st_mode)) throw SysError(EOPNOTSUPP, "can't change mode of symlink %s", PathFmt(descriptorToPath(dirFd) / path.rel())); @@ -222,4 +221,13 @@ OsString readLinkAt(Descriptor dirFd, const CanonPath & path) } } +PosixStat fstat(Descriptor fd) +{ + PosixStat st; + if (::fstat(fd, &st)) { + throw SysError([&] { return HintFmt("getting status of %s", PathFmt(descriptorToPath(fd))); }); + } + return st; +} + } // namespace nix diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index 9423b76a1e..e709c5298a 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -77,6 +77,25 @@ std::filesystem::path defaultTempDir() return getEnvNonEmpty("TMPDIR").value_or("/tmp"); } +PosixStat lstat(const std::filesystem::path & path) +{ + PosixStat st; + if (::lstat(path.c_str(), &st)) + throw SysError("getting status of %s", PathFmt(path)); + return st; +} + +std::optional maybeLstat(const std::filesystem::path & path) +{ + std::optional st{std::in_place}; + if (::lstat(path.c_str(), &*st)) { + if (errno == ENOENT || errno == ENOTDIR) + return std::nullopt; + throw SysError("getting status of %s", PathFmt(path)); + } + return st; +} + void setWriteTime( const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) { diff --git a/src/libutil/windows/file-system-at.cc b/src/libutil/windows/file-system-at.cc index ebbe840007..a10dedbb88 100644 --- a/src/libutil/windows/file-system-at.cc +++ b/src/libutil/windows/file-system-at.cc @@ -210,6 +210,26 @@ bool isReparsePoint(HANDLE handle) } // namespace windows +PosixStat fstat(Descriptor fd) +{ + BY_HANDLE_FILE_INFORMATION info; + if (!GetFileInformationByHandle(fd, &info)) + throw WinError("getting file information for %s", PathFmt(descriptorToPath(fd))); + + PosixStat st; + windows::statFromFileInfo( + st, + info.dwFileAttributes, + info.ftCreationTime, + info.ftLastAccessTime, + info.ftLastWriteTime, + info.nFileSizeHigh, + info.nFileSizeLow, + info.nNumberOfLinks); + + return st; +} + AutoCloseFD openFileEnsureBeneathNoSymlinks( Descriptor dirFd, const CanonPath & path, ACCESS_MASK desiredAccess, ULONG createOptions, ULONG createDisposition) { diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index c739ed0413..3e9680ff24 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -6,6 +6,13 @@ #include #include +#include + +// Verify that our S_IFLNK polyfill doesn't conflict with Windows file type constants +static_assert((S_IFLNK & S_IFMT) == S_IFLNK, "S_IFLNK must fit within S_IFMT mask"); +static_assert(S_IFLNK != S_IFDIR, "S_IFLNK must not equal S_IFDIR"); +static_assert(S_IFLNK != S_IFREG, "S_IFLNK must not equal S_IFREG"); +static_assert(S_IFLNK != S_IFCHR, "S_IFLNK must not equal S_IFCHR"); namespace nix { @@ -105,4 +112,87 @@ std::filesystem::path descriptorToPath(Descriptor handle) return std::filesystem::path{std::wstring{buf.data(), dw}}; } +time_t windows::fileTimeToUnixTime(const FILETIME & ft) +{ + ULARGE_INTEGER ull; + ull.LowPart = ft.dwLowDateTime; + ull.HighPart = ft.dwHighDateTime; + + // file_clock on Windows matches FILETIME's epoch (1601-01-01) and resolution (100ns) + auto fileTP = std::chrono::file_clock::time_point{std::chrono::file_clock::duration{ull.QuadPart}}; + + auto sysTP = std::chrono::clock_cast(fileTP); + return std::chrono::system_clock::to_time_t(sysTP); +} + +void windows::statFromFileInfo( + PosixStat & st, + DWORD dwFileAttributes, + const FILETIME & ftCreationTime, + const FILETIME & ftLastAccessTime, + const FILETIME & ftLastWriteTime, + DWORD nFileSizeHigh, + DWORD nFileSizeLow, + DWORD nNumberOfLinks) +{ + memset(&st, 0, sizeof(st)); + + /* Determine file type */ + if (dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + st.st_mode = S_IFLNK | 0777; + } else if (dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + st.st_mode = S_IFDIR | 0755; + } else { + st.st_mode = S_IFREG | 0644; + } + + /* File size (only meaningful for regular files) */ + st.st_size = (static_cast(nFileSizeHigh) << 32) | nFileSizeLow; + + /* Timestamps */ + st.st_atime = fileTimeToUnixTime(ftLastAccessTime); + st.st_mtime = fileTimeToUnixTime(ftLastWriteTime); + st.st_ctime = fileTimeToUnixTime(ftCreationTime); + + st.st_nlink = nNumberOfLinks; + /* https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/stat-functions?view=msvc-170 + matches the regular windows stat functions. */ + st.st_uid = 0; + st.st_gid = 0; +} + +static PosixStat statFromFileInfo(const WIN32_FILE_ATTRIBUTE_DATA & attrData) +{ + PosixStat st; + windows::statFromFileInfo( + st, + attrData.dwFileAttributes, + attrData.ftCreationTime, + attrData.ftLastAccessTime, + attrData.ftLastWriteTime, + attrData.nFileSizeHigh, + attrData.nFileSizeLow); + return st; +} + +PosixStat lstat(const std::filesystem::path & path) +{ + WIN32_FILE_ATTRIBUTE_DATA attrData; + if (!GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &attrData)) + throw WinError("getting status of %s", PathFmt(path)); + return statFromFileInfo(attrData); +} + +std::optional maybeLstat(const std::filesystem::path & path) +{ + WIN32_FILE_ATTRIBUTE_DATA attrData; + if (!GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &attrData)) { + auto lastError = GetLastError(); + if (lastError == ERROR_FILE_NOT_FOUND || lastError == ERROR_PATH_NOT_FOUND) + return std::nullopt; + throw WinError(lastError, "getting status of %s", PathFmt(path)); + } + return statFromFileInfo(attrData); +} + } // namespace nix From 6633c55f8a1f8b9943f9c5bd18693ad4f9c03b6f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 2 Mar 2026 20:40:10 +0100 Subject: [PATCH 025/927] fix(libexpr-c): pass valid EvalState to primop callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Due to an erroneous cast, the wrong pointer was passed to these callbacks, leading to a crash. We now create a lightweight temporary EvalState wrapper on the stack in each callback bridge. This also eliminates the need for unsafe_new_with_self for EvalState construction. Co-authored-by: Jörg Thalheim --- doc/manual/rl-next/fix-primop-eval-state.md | 10 +++++ src/libexpr-c/nix_api_expr.cc | 16 ++++--- src/libexpr-c/nix_api_expr_internal.h | 9 ++-- src/libexpr-c/nix_api_external.cc | 13 ++---- src/libexpr-c/nix_api_external.h | 1 + src/libexpr-c/nix_api_value.cc | 3 +- src/libexpr-tests/nix_api_expr.cc | 46 +++++++++++++++++++++ src/libexpr-tests/nix_api_external.cc | 40 ++++++++++++++++++ 8 files changed, 116 insertions(+), 22 deletions(-) create mode 100644 doc/manual/rl-next/fix-primop-eval-state.md diff --git a/doc/manual/rl-next/fix-primop-eval-state.md b/doc/manual/rl-next/fix-primop-eval-state.md new file mode 100644 index 0000000000..1a3bc28753 --- /dev/null +++ b/doc/manual/rl-next/fix-primop-eval-state.md @@ -0,0 +1,10 @@ +--- +synopsis: "C API: Fix `EvalState` pointer passed to primop callbacks" +prs: [15300, 15383] +--- + +The `EvalState *` passed to C API primop callbacks was incorrectly pointing to +the internal `nix::EvalState` rather than the C API wrapper struct. This caused +a segfault when the callback used the pointer with C API functions such as +`nix_alloc_value()`. The same issue affected `printValueAsJSON` and +`printValueAsXML` callbacks on external values. diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 61edf9bc35..97680ac6bf 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -181,15 +181,13 @@ EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder if (context) context->last_err_code = NIX_OK; try { - return unsafe_new_with_self([&](auto * self) { - return EvalState{ - .fetchSettings = std::move(builder->fetchSettings), - .settings = std::move(builder->settings), - .statePtr = std::make_shared( - builder->lookupPath, builder->store, self->fetchSettings, self->settings), - .state = *self->statePtr, - }; - }); + auto fetchSettings = std::make_unique(std::move(builder->fetchSettings)); + auto settings = std::make_unique(std::move(builder->settings)); + auto ownedState = + std::make_shared(builder->lookupPath, builder->store, *fetchSettings, *settings); + auto & stateRef = *ownedState; + void * p = ::operator new(sizeof(EvalState), static_cast(alignof(EvalState))); + return new (p) EvalState{stateRef, std::move(fetchSettings), std::move(settings), std::move(ownedState)}; } NIXC_CATCH_ERRS_NULL } diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index 633599fb41..b38aeaf7b4 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -1,6 +1,8 @@ #ifndef NIX_API_EXPR_INTERNAL_H #define NIX_API_EXPR_INTERNAL_H +#include + #include "nix/fetchers/fetch-settings.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" @@ -22,10 +24,11 @@ struct nix_eval_state_builder struct EvalState { - nix::fetchers::Settings fetchSettings; - nix::EvalSettings settings; - std::shared_ptr statePtr; nix::EvalState & state; + // Owned resources; null for temporary wrappers created in C API callbacks. + std::unique_ptr ownedFetchSettings; + std::unique_ptr ownedSettings; + std::shared_ptr ownedState; }; struct BindingsBuilder diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index ee500619fc..a874d9a086 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -137,7 +137,8 @@ class NixCExternalValue : public nix::ExternalValueBase } nix_string_context ctx{context}; nix_string_return res{""}; - desc.printValueAsJSON(v, (EvalState *) &state, strict, &ctx, copyToStore, &res); + EvalState wrapper{state}; + desc.printValueAsJSON(v, &wrapper, strict, &ctx, copyToStore, &res); if (res.str.empty()) { return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore); } @@ -160,15 +161,9 @@ class NixCExternalValue : public nix::ExternalValueBase return nix::ExternalValueBase::printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); } nix_string_context ctx{context}; + EvalState wrapper{state}; desc.printValueAsXML( - v, - (EvalState *) &state, - strict, - location, - &doc, - &ctx, - &drvsSeen, - *reinterpret_cast(&pos)); + v, &wrapper, strict, location, &doc, &ctx, &drvsSeen, *reinterpret_cast(&pos)); } virtual ~NixCExternalValue() override {}; diff --git a/src/libexpr-c/nix_api_external.h b/src/libexpr-c/nix_api_external.h index 96c479d576..1ae4c22f89 100644 --- a/src/libexpr-c/nix_api_external.h +++ b/src/libexpr-c/nix_api_external.h @@ -145,6 +145,7 @@ typedef struct NixCExternalValueDesc * Optional, the default is to throw an error * @todo The mechanisms for this call are incomplete. There are no C * bindings to work with XML, pathsets and positions. + * This callback also has no test coverage. * @param[in] self the void* passed to nix_create_external_value * @param[in] state The evaluator state * @param[in] strict boolean Whether to force the value before printing diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index c6b96213ea..589ebf9a8e 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -105,7 +105,8 @@ static void nix_c_primop_wrapper( nix_value * external_arg = new_nix_value(args[i], state.mem); external_args.push_back(external_arg); } - f(userdata, &ctx, (EvalState *) &state, external_args.data(), vTmpPtr); + EvalState wrapper{state}; + f(userdata, &ctx, &wrapper, external_args.data(), vTmpPtr); if (ctx.last_err_code != NIX_OK) { if (ctx.last_err_code == NIX_ERR_RECOVERABLE) { diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index 56363038d0..c3a3f2dd53 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -476,6 +476,52 @@ TEST_F(nix_api_expr_test, nix_expr_primop_nix_err_key_conversion) nix_gc_decref(ctx, result); } +static void +primop_alloc_value(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ + assert(context); + assert(state); + + // Regression test: nix_c_primop_wrapper previously cast the inner + // nix::EvalState* directly to EvalState* (C wrapper). C API functions + // like nix_alloc_value() then accessed state->state at the wrong offset, + // causing a segfault. + nix_value * v = nix_alloc_value(context, state); + assert(v != nullptr); + nix_init_int(context, v, 42); + nix_copy_value(context, ret, v); + nix_gc_decref(nullptr, v); +} + +TEST_F(nix_api_expr_test, nix_primop_can_use_state_in_callback) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_alloc_value, 1, "allocValue", nullptr, "test alloc_value in callback", nullptr); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * dummy = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, dummy, 0); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_value_call(ctx, state, primopValue, dummy, result); + assert_ctx_ok(); + + auto r = nix_get_int(ctx, result); + ASSERT_EQ(42, r); + + nix_gc_decref(ctx, dummy); + nix_gc_decref(ctx, result); + nix_gc_decref(ctx, primopValue); + nix_gc_decref(ctx, primop); +} + TEST_F(nix_api_expr_test, nix_value_call_multi_no_args) { nix_value * n = nix_alloc_value(ctx, state); diff --git a/src/libexpr-tests/nix_api_external.cc b/src/libexpr-tests/nix_api_external.cc index ec19f1212e..885b9c1d2d 100644 --- a/src/libexpr-tests/nix_api_external.cc +++ b/src/libexpr-tests/nix_api_external.cc @@ -66,4 +66,44 @@ TEST_F(nix_api_expr_test, nix_expr_eval_external) nix_state_free(stateFn); } +static void print_value_as_json_using_state( + void * self, EvalState * state, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res) +{ + // Regression test: same cast bug as in nix_c_primop_wrapper (see primop_alloc_value). + nix_value * v = nix_alloc_value(nullptr, state); + assert(v != nullptr); + nix_gc_decref(nullptr, v); + + nix_set_string_return(res, "42"); +} + +TEST_F(nix_api_expr_test, nix_external_printValueAsJSON_can_use_state) +{ + NixCExternalValueDesc desc{}; + desc.print = [](void *, nix_printer *) {}; + desc.showType = [](void *, nix_string_return *) {}; + desc.typeOf = [](void *, nix_string_return *) {}; + desc.printValueAsJSON = print_value_as_json_using_state; + + ExternalValue * val = nix_create_external_value(ctx, &desc, nullptr); + assert_ctx_ok(); + nix_init_external(ctx, value, val); + assert_ctx_ok(); + + nix_value * toJsonFn = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "builtins.toJSON", ".", toJsonFn); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + nix_value_call(ctx, state, toJsonFn, value, result); + assert_ctx_ok(); + + std::string json_str; + nix_get_string(ctx, result, OBSERVE_STRING(json_str)); + ASSERT_EQ("42", json_str); + + nix_gc_decref(ctx, result); + nix_gc_decref(ctx, toJsonFn); +} + } // namespace nixC From faa16841b6fbd594f5449135d92b4ebfee062ab5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 23:39:07 +0000 Subject: [PATCH 026/927] build(deps): bump actions/download-artifact from 7.0.0 to 8.0.0 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 7.0.0 to 8.0.0. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/37930b1c2abaa49bbe596cd826c3c89aef350131...70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: 8.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e37193966a..67f8c51783 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,7 +182,7 @@ jobs: steps: - uses: actions/checkout@v6 - name: Download installer tarball - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: installer-${{matrix.os}} path: out @@ -232,7 +232,7 @@ jobs: repository: NixOS/flake-regressions-data path: flake-regressions/tests - name: Download installer tarball - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: installer-linux path: out From 539c6a1aaf9ecbb538ca8803256542dcfa036578 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 23:39:12 +0000 Subject: [PATCH 027/927] build(deps): bump actions/upload-artifact from 6 to 7 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 6 to 7. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e37193966a..5df158911b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,13 +125,13 @@ jobs: cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY if: ${{ matrix.instrumented }} - name: Upload coverage reports - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: coverage-reports path: coverage-reports/ if: ${{ matrix.instrumented }} - name: Upload installer tarball - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: installer-${{matrix.os}} path: out/* From d7245ff8aca13e1ac3810199706a7fe1e4114643 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Tue, 20 Jan 2026 13:55:09 -0500 Subject: [PATCH 028/927] protocol: update JSON output with structured `Signature` type This commit updates the JSON output with a Signature type containing keyName and sig fields. JSON parsing accepts both formats for backwards compatibility. Co-authored-by: John Ericson --- doc/manual/rl-next/build-trace-rework.md | 8 ++- doc/manual/source/SUMMARY.md.in | 1 + .../protocols/json/build-trace-entry.md | 8 +-- doc/manual/source/protocols/json/meson.build | 5 +- .../json/schema/build-result-v1.yaml | 2 +- ...ld-trace-entry-v2 => build-trace-entry-v3} | 0 ...ntry-v2.yaml => build-trace-entry-v3.yaml} | 13 ++--- .../source/protocols/json/schema/nar-info-v3 | 1 + .../protocols/json/schema/signature-v2.yaml | 33 ++++++++++++ .../json/schema/signature-v2/simple.json | 4 ++ .../json/schema/store-object-info-v2 | 1 - .../json/schema/store-object-info-v3 | 1 + ...info-v2.yaml => store-object-info-v3.yaml} | 14 +++--- .../protocols/json/schema/store-v1.yaml | 4 +- doc/manual/source/protocols/json/signature.md | 9 ++++ .../protocols/json/store-object-info.md | 16 +++--- src/json-schema-checks/meson.build | 47 ++++++++++-------- src/json-schema-checks/signature/simple.json | 4 ++ .../data/common-protocol/realisation.bin | Bin 560 -> 600 bytes .../data/common-protocol/realisation.json | 10 +++- .../data/dummy-store/one-flat-file.json | 2 +- .../data/nar-info/json-3/impure.json | 31 ++++++++++++ .../data/nar-info/json-3/pure.json | 14 ++++++ .../data/path-info/json-3/empty_impure.json | 12 +++++ .../data/path-info/json-3/empty_pure.json | 8 +++ .../data/path-info/json-3/impure.json | 27 ++++++++++ .../data/path-info/json-3/pure.json | 14 ++++++ .../with-signature-structured.json | 15 ++++++ .../data/realisation/with-signature.json | 2 +- .../with-structured-signature.json | 15 ++++++ .../data/serve-protocol/realisation-2.8.json | 10 +++- .../data/serve-protocol/realisation.bin | Bin 560 -> 600 bytes .../data/serve-protocol/realisation.json | 10 +++- .../unkeyed-realisation-2.8.json | 10 +++- .../unkeyed-valid-path-info-2.3.json | 4 +- .../unkeyed-valid-path-info-2.4.json | 14 ++++-- ...sation-realisation-with-path-not-hash.json | 10 +++- .../data/worker-protocol/realisation.bin | Bin 560 -> 600 bytes .../data/worker-protocol/realisation.json | 10 +++- ...sation-realisation-with-path-not-hash.json | 10 +++- .../unkeyed-valid-path-info-1.15.json | 4 +- .../worker-protocol/valid-path-info-1.15.json | 4 +- .../worker-protocol/valid-path-info-1.16.json | 16 ++++-- .../ca-drv/store-after.json | 2 +- .../ca-drv/substituter.json | 2 +- .../issue-11928/store-after.json | 2 +- .../issue-11928/substituter.json | 4 +- .../single/substituter.json | 2 +- .../with-dep/substituter.json | 4 +- src/libstore-tests/nar-info.cc | 38 ++++++++++++++ src/libstore-tests/path-info.cc | 40 +++++++++++++++ src/libstore-tests/realisation.cc | 33 ++++++++---- src/libstore/include/nix/store/path-info.hh | 2 + src/libstore/path-info.cc | 14 ++++-- src/libutil/signature/local-keys.cc | 13 ++++- 55 files changed, 475 insertions(+), 104 deletions(-) rename doc/manual/source/protocols/json/schema/{build-trace-entry-v2 => build-trace-entry-v3} (100%) rename doc/manual/source/protocols/json/schema/{build-trace-entry-v2.yaml => build-trace-entry-v3.yaml} (92%) create mode 120000 doc/manual/source/protocols/json/schema/nar-info-v3 create mode 100644 doc/manual/source/protocols/json/schema/signature-v2.yaml create mode 100644 doc/manual/source/protocols/json/schema/signature-v2/simple.json delete mode 120000 doc/manual/source/protocols/json/schema/store-object-info-v2 create mode 120000 doc/manual/source/protocols/json/schema/store-object-info-v3 rename doc/manual/source/protocols/json/schema/{store-object-info-v2.yaml => store-object-info-v3.yaml} (97%) create mode 100644 doc/manual/source/protocols/json/signature.md create mode 100644 src/json-schema-checks/signature/simple.json create mode 100644 src/libstore-tests/data/nar-info/json-3/impure.json create mode 100644 src/libstore-tests/data/nar-info/json-3/pure.json create mode 100644 src/libstore-tests/data/path-info/json-3/empty_impure.json create mode 100644 src/libstore-tests/data/path-info/json-3/empty_pure.json create mode 100644 src/libstore-tests/data/path-info/json-3/impure.json create mode 100644 src/libstore-tests/data/path-info/json-3/pure.json create mode 100644 src/libstore-tests/data/realisation/with-signature-structured.json create mode 100644 src/libstore-tests/data/realisation/with-structured-signature.json diff --git a/doc/manual/rl-next/build-trace-rework.md b/doc/manual/rl-next/build-trace-rework.md index fd3541c5b3..1c4271b903 100644 --- a/doc/manual/rl-next/build-trace-rework.md +++ b/doc/manual/rl-next/build-trace-rework.md @@ -40,7 +40,7 @@ no longer need to implement the derivation hash modulo algorithm. }, "value": { "outPath": "xyz...-foo", - "signatures": ["..."] + "signatures": [{ "keyName": "cache.example.com-1", "sig": "..." }] } } ``` @@ -64,6 +64,12 @@ Non-CA builds are unaffected. Stable code paths do use the realization fields (`BuildResult::Success::builtOutputs`), but only the output name and outpath parts of that. For older protocols, we can fake enough of the realisation format to provide those two parts forthat map, which keeps operations like `--print-output-paths` working. +### Structured signatures + +[Signatures](@docroot@/protocols/json/signature.md) in JSON formats are now represented as structured objects with `keyName` and `sig` fields, rather than colon-separated strings. +`nix path-info --json --json-format 3` opts into the new version for this command. +JSON parsing accepts both the old string format and new structured format for backwards compatibility. + ### Impact - **Non-CA derivation users**: No impact. This only affects the experimental `ca-derivations` feature. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 18d24788ca..2f78cb2d33 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -125,6 +125,7 @@ - [Hash](protocols/json/hash.md) - [Content Address](protocols/json/content-address.md) - [Store Path](protocols/json/store-path.md) + - [Signature](protocols/json/signature.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation/index.md) - [Derivation Options](protocols/json/derivation/options.md) diff --git a/doc/manual/source/protocols/json/build-trace-entry.md b/doc/manual/source/protocols/json/build-trace-entry.md index 9eea937126..2f8a4dbeef 100644 --- a/doc/manual/source/protocols/json/build-trace-entry.md +++ b/doc/manual/source/protocols/json/build-trace-entry.md @@ -1,21 +1,21 @@ -{{#include build-trace-entry-v2-fixed.md}} +{{#include build-trace-entry-v3-fixed.md}} ## Examples ### Simple build trace entry ```json -{{#include schema/build-trace-entry-v2/simple.json}} +{{#include schema/build-trace-entry-v3/simple.json}} ``` ### Build trace entry with signature ```json -{{#include schema/build-trace-entry-v2/with-signature.json}} +{{#include schema/build-trace-entry-v3/with-structured-signature.json}} ``` diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index c7c48c4ed6..36e9220c02 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -13,11 +13,12 @@ schemas = [ 'hash-v1', 'content-address-v1', 'store-path-v1', - 'store-object-info-v2', + 'signature-v2', + 'store-object-info-v3', 'derivation-v4', 'derivation-options-v1', 'deriving-path-v1', - 'build-trace-entry-v2', + 'build-trace-entry-v3', 'build-result-v1', 'store-v1', ] diff --git a/doc/manual/source/protocols/json/schema/build-result-v1.yaml b/doc/manual/source/protocols/json/schema/build-result-v1.yaml index 7730f3d939..d0d8d8a0c7 100644 --- a/doc/manual/source/protocols/json/schema/build-result-v1.yaml +++ b/doc/manual/source/protocols/json/schema/build-result-v1.yaml @@ -83,7 +83,7 @@ properties: description: | A mapping from output names to their build trace entries. additionalProperties: - "$ref": "build-trace-entry-v2.yaml#/$defs/value" + "$ref": "build-trace-entry-v3.yaml#/$defs/value" failure: type: object diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v2 b/doc/manual/source/protocols/json/schema/build-trace-entry-v3 similarity index 100% rename from doc/manual/source/protocols/json/schema/build-trace-entry-v2 rename to doc/manual/source/protocols/json/schema/build-trace-entry-v3 diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml similarity index 92% rename from doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml rename to doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml index 2e8cd07da2..c3a27d2a6e 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml @@ -1,5 +1,5 @@ "$schema": "http://json-schema.org/draft-04/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v2.json" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v3.json" title: Build Trace Entry description: | A record of a successful build outcome for a specific derivation output. @@ -12,14 +12,17 @@ description: | > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) > and subject to change. - Verision history: + ## Version History - Version 1: Original format - Version 2: - - Use `drvPath` not `drvHash` to refer to derivation in a more conventional way. - Remove `dependentRealisations` + + - Version 3: + - Use `drvPath` not `drvHash` to refer to derivation in a more conventional way. - Separate into `key` and `value` + - Use 2nd version of signatures format (objects, not strings) type: object required: @@ -77,6 +80,4 @@ additionalProperties: false description: | A set of cryptographic signatures attesting to the authenticity of this build trace entry. items: - type: string - title: Signature - description: A single cryptographic signature + "$ref": "signature-v2.yaml" diff --git a/doc/manual/source/protocols/json/schema/nar-info-v3 b/doc/manual/source/protocols/json/schema/nar-info-v3 new file mode 120000 index 0000000000..378215b415 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/nar-info-v3 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/nar-info/json-3 \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/signature-v2.yaml b/doc/manual/source/protocols/json/schema/signature-v2.yaml new file mode 100644 index 0000000000..e2b25dbc6f --- /dev/null +++ b/doc/manual/source/protocols/json/schema/signature-v2.yaml @@ -0,0 +1,33 @@ +"$schema": "http://json-schema.org/draft-07/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/signature-v2.json" +title: Signature +description: | + A cryptographic signature along with the name of the key that produced it. + + This schema describes the JSON representation of signatures as used in various Nix JSON APIs. + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > and subject to change. + + ## Version History + + - Version 1: Colon-separated string in the format `:` + + - Version 2: Structured object with `keyName` and `sig` fields + +type: object +required: + - keyName + - sig +properties: + keyName: + type: string + title: Key Name + description: The name of the key used to produce this signature + sig: + type: string + title: Signature Data + description: The raw signature bytes, Base64-encoded diff --git a/doc/manual/source/protocols/json/schema/signature-v2/simple.json b/doc/manual/source/protocols/json/schema/signature-v2/simple.json new file mode 100644 index 0000000000..abf69dfa0f --- /dev/null +++ b/doc/manual/source/protocols/json/schema/signature-v2/simple.json @@ -0,0 +1,4 @@ +{ + "keyName": "cache.nixos.org-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +} diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v2 b/doc/manual/source/protocols/json/schema/store-object-info-v2 deleted file mode 120000 index 36ca7f13db..0000000000 --- a/doc/manual/source/protocols/json/schema/store-object-info-v2 +++ /dev/null @@ -1 +0,0 @@ -../../../../../../src/libstore-tests/data/path-info/json-2 \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v3 b/doc/manual/source/protocols/json/schema/store-object-info-v3 new file mode 120000 index 0000000000..d5dae48fcf --- /dev/null +++ b/doc/manual/source/protocols/json/schema/store-object-info-v3 @@ -0,0 +1 @@ +../../../../../../src/libstore-tests/data/path-info/json-3 \ No newline at end of file diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml similarity index 97% rename from doc/manual/source/protocols/json/schema/store-object-info-v2.yaml rename to doc/manual/source/protocols/json/schema/store-object-info-v3.yaml index 3ed7e99e28..e0be716ef7 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v2.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml @@ -1,6 +1,6 @@ "$schema": "http://json-schema.org/draft-04/schema" -"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v2.json" -title: Store Object Info v2 +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-object-info-v3.json" +title: Store Object Info v3 description: | Information about a [store object](@docroot@/store/store-object.md). @@ -50,10 +50,10 @@ $defs: properties: version: type: integer - const: 2 - title: Format version (must be 2) + const: 3 + title: Format version (must be 3) description: | - Must be `2`. + Must be `3`. This is a guard that allows us to continue evolving this format. Here is the rough version history: @@ -63,6 +63,8 @@ $defs: - Version 2: Use structured JSON type for `ca` + - Version 3: Use structured JSON type for `signatures` + path: "$ref": "./store-path-v1.yaml" title: Store Path @@ -174,7 +176,7 @@ $defs: > This is an "impure" field that may not be included in certain contexts. items: - type: string + "$ref": "./signature-v2.yaml" # Computed closure fields closureSize: diff --git a/doc/manual/source/protocols/json/schema/store-v1.yaml b/doc/manual/source/protocols/json/schema/store-v1.yaml index ebe61d9cb2..b8ed3c8cc1 100644 --- a/doc/manual/source/protocols/json/schema/store-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-v1.yaml @@ -37,7 +37,7 @@ properties: - contents properties: info: - "$ref": "./store-object-info-v2.yaml#/$defs/impure" + "$ref": "./store-object-info-v3.yaml#/$defs/impure" title: Store Object Info description: | Metadata about the [store object](@docroot@/store/store-object.md) including hash, size, references, etc. @@ -70,7 +70,7 @@ properties: "^[A-Za-z0-9+/]{43}=$": type: object additionalProperties: - "$ref": "./build-trace-entry-v2.yaml#/$defs/value" + "$ref": "./build-trace-entry-v3.yaml#/$defs/value" additionalProperties: false "$defs": diff --git a/doc/manual/source/protocols/json/signature.md b/doc/manual/source/protocols/json/signature.md new file mode 100644 index 0000000000..363b4ef11e --- /dev/null +++ b/doc/manual/source/protocols/json/signature.md @@ -0,0 +1,9 @@ +{{#include signature-v2-fixed.md}} + +## Examples + +### Simple signature + +```json +{{#include schema/signature-v2/simple.json}} +``` diff --git a/doc/manual/source/protocols/json/store-object-info.md b/doc/manual/source/protocols/json/store-object-info.md index 4ad83de00b..ea1cc3a9d2 100644 --- a/doc/manual/source/protocols/json/store-object-info.md +++ b/doc/manual/source/protocols/json/store-object-info.md @@ -1,45 +1,45 @@ -{{#include store-object-info-v2-fixed.md}} +{{#include store-object-info-v3-fixed.md}} ## Examples ### Minimal store object (content-addressed) ```json -{{#include schema/store-object-info-v2/pure.json}} +{{#include schema/store-object-info-v3/pure.json}} ``` ### Store object with impure fields ```json -{{#include schema/store-object-info-v2/impure.json}} +{{#include schema/store-object-info-v3/impure.json}} ``` ### Minimal store object (empty) ```json -{{#include schema/store-object-info-v2/empty_pure.json}} +{{#include schema/store-object-info-v3/empty_pure.json}} ``` ### Store object with all impure fields ```json -{{#include schema/store-object-info-v2/empty_impure.json}} +{{#include schema/store-object-info-v3/empty_impure.json}} ``` ### NAR info (minimal) ```json -{{#include schema/nar-info-v2/pure.json}} +{{#include schema/nar-info-v3/pure.json}} ``` ### NAR info (with binary cache fields) ```json -{{#include schema/nar-info-v2/impure.json}} +{{#include schema/nar-info-v3/impure.json}} ``` diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index a4d9461609..46078deec5 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -51,6 +51,13 @@ schemas = [ 'simple.json', ], }, + { + 'stem' : 'signature', + 'schema' : schema_dir / 'signature-v2.yaml', + 'files' : [ + 'simple.json', + ], + }, { 'stem' : 'deriving-path', 'schema' : schema_dir / 'deriving-path-v1.yaml', @@ -62,10 +69,10 @@ schemas = [ }, { 'stem' : 'build-trace-entry', - 'schema' : schema_dir / 'build-trace-entry-v2.yaml', + 'schema' : schema_dir / 'build-trace-entry-v3.yaml', 'files' : [ 'simple.json', - 'with-signature.json', + 'with-structured-signature.json', ], }, { @@ -151,20 +158,20 @@ schemas += [ # Match overall { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml', + 'schema' : schema_dir / 'store-object-info-v3.yaml', 'files' : [ - 'json-2' / 'pure.json', - 'json-2' / 'impure.json', - 'json-2' / 'empty_pure.json', - 'json-2' / 'empty_impure.json', + 'json-3' / 'pure.json', + 'json-3' / 'impure.json', + 'json-3' / 'empty_pure.json', + 'json-3' / 'empty_impure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml', + 'schema' : schema_dir / 'store-object-info-v3.yaml', 'files' : [ - 'json-2' / 'pure.json', - 'json-2' / 'impure.json', + 'json-3' / 'pure.json', + 'json-3' / 'impure.json', ], }, { @@ -179,32 +186,32 @@ schemas += [ # Match exact variant { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/base', 'files' : [ - 'json-2' / 'pure.json', - 'json-2' / 'empty_pure.json', + 'json-3' / 'pure.json', + 'json-3' / 'empty_pure.json', ], }, { 'stem' : 'store-object-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/impure', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/impure', 'files' : [ - 'json-2' / 'impure.json', - 'json-2' / 'empty_impure.json', + 'json-3' / 'impure.json', + 'json-3' / 'empty_impure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/base', 'files' : [ - 'json-2' / 'pure.json', + 'json-3' / 'pure.json', ], }, { 'stem' : 'nar-info', - 'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/narInfo', + 'schema' : schema_dir / 'store-object-info-v3.yaml#/$defs/narInfo', 'files' : [ - 'json-2' / 'impure.json', + 'json-3' / 'impure.json', ], }, ] diff --git a/src/json-schema-checks/signature/simple.json b/src/json-schema-checks/signature/simple.json new file mode 100644 index 0000000000..abf69dfa0f --- /dev/null +++ b/src/json-schema-checks/signature/simple.json @@ -0,0 +1,4 @@ +{ + "keyName": "cache.nixos.org-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +} diff --git a/src/libstore-tests/data/common-protocol/realisation.bin b/src/libstore-tests/data/common-protocol/realisation.bin index 44885cbdafdab24143f74aeef0ef65649339efb5..cbd942ed477a3f77de93e970f3033b22d456dfb8 100644 GIT binary patch delta 92 zcmdnMa)V{U4#w#dcW#xeR?1GT^h?Z5RkBh_EKW&N(orhTOb2o%H!^xop1{Z}XKSle UtAnhpuspR0Lm4|rS!^u>0LmI2jQ{`u delta 52 zcmcb?vVmp74#xb6JGZhaB^IZoSxshS3Yffrk&VmNR!K*xuspR0F2n{Dimhb;0NcC| A@c;k- diff --git a/src/libstore-tests/data/common-protocol/realisation.json b/src/libstore-tests/data/common-protocol/realisation.json index 034d620306..ad1ae3ab5f 100644 --- a/src/libstore-tests/data/common-protocol/realisation.json +++ b/src/libstore-tests/data/common-protocol/realisation.json @@ -10,8 +10,14 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } ] diff --git a/src/libstore-tests/data/dummy-store/one-flat-file.json b/src/libstore-tests/data/dummy-store/one-flat-file.json index 804bbf07da..f7563abe6a 100644 --- a/src/libstore-tests/data/dummy-store/one-flat-file.json +++ b/src/libstore-tests/data/dummy-store/one-flat-file.json @@ -23,7 +23,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/nar-info/json-3/impure.json b/src/libstore-tests/data/nar-info/json-3/impure.json new file mode 100644 index 0000000000..eb6d4d48b8 --- /dev/null +++ b/src/libstore-tests/data/nar-info/json-3/impure.json @@ -0,0 +1,31 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "compression": "xz", + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "downloadSize": 4029176, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ], + "storeDir": "/nix/store", + "ultimate": true, + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz", + "version": 3 +} diff --git a/src/libstore-tests/data/nar-info/json-3/pure.json b/src/libstore-tests/data/nar-info/json-3/pure.json new file mode 100644 index 0000000000..32fc916489 --- /dev/null +++ b/src/libstore-tests/data/nar-info/json-3/pure.json @@ -0,0 +1,14 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "storeDir": "/nix/store", + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/empty_impure.json b/src/libstore-tests/data/path-info/json-3/empty_impure.json new file mode 100644 index 0000000000..47d5030318 --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/empty_impure.json @@ -0,0 +1,12 @@ +{ + "ca": null, + "deriver": null, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 0, + "references": [], + "registrationTime": null, + "signatures": [], + "storeDir": "/nix/store", + "ultimate": false, + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/empty_pure.json b/src/libstore-tests/data/path-info/json-3/empty_pure.json new file mode 100644 index 0000000000..6cc032782a --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/empty_pure.json @@ -0,0 +1,8 @@ +{ + "ca": null, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 0, + "references": [], + "storeDir": "/nix/store", + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/impure.json b/src/libstore-tests/data/path-info/json-3/impure.json new file mode 100644 index 0000000000..f7ec907dce --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/impure.json @@ -0,0 +1,27 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ], + "storeDir": "/nix/store", + "ultimate": true, + "version": 3 +} diff --git a/src/libstore-tests/data/path-info/json-3/pure.json b/src/libstore-tests/data/path-info/json-3/pure.json new file mode 100644 index 0000000000..32fc916489 --- /dev/null +++ b/src/libstore-tests/data/path-info/json-3/pure.json @@ -0,0 +1,14 @@ +{ + "ca": { + "hash": "sha256-EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM=", + "method": "nar" + }, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "storeDir": "/nix/store", + "version": 3 +} diff --git a/src/libstore-tests/data/realisation/with-signature-structured.json b/src/libstore-tests/data/realisation/with-signature-structured.json new file mode 100644 index 0000000000..a9efc97ca9 --- /dev/null +++ b/src/libstore-tests/data/realisation/with-signature-structured.json @@ -0,0 +1,15 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] + } +} diff --git a/src/libstore-tests/data/realisation/with-signature.json b/src/libstore-tests/data/realisation/with-signature.json index cca29ef3e4..7952e43697 100644 --- a/src/libstore-tests/data/realisation/with-signature.json +++ b/src/libstore-tests/data/realisation/with-signature.json @@ -4,7 +4,7 @@ "outputName": "foo" }, "value": { - "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", "signatures": [ "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" ] diff --git a/src/libstore-tests/data/realisation/with-structured-signature.json b/src/libstore-tests/data/realisation/with-structured-signature.json new file mode 100644 index 0000000000..1b98554a0d --- /dev/null +++ b/src/libstore-tests/data/realisation/with-structured-signature.json @@ -0,0 +1,15 @@ +{ + "key": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "outputName": "foo" + }, + "value": { + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] + } +} diff --git a/src/libstore-tests/data/serve-protocol/realisation-2.8.json b/src/libstore-tests/data/serve-protocol/realisation-2.8.json index 1977d8485b..d6dde73276 100644 --- a/src/libstore-tests/data/serve-protocol/realisation-2.8.json +++ b/src/libstore-tests/data/serve-protocol/realisation-2.8.json @@ -6,8 +6,14 @@ "value": { "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } } diff --git a/src/libstore-tests/data/serve-protocol/realisation.bin b/src/libstore-tests/data/serve-protocol/realisation.bin index 44885cbdafdab24143f74aeef0ef65649339efb5..cbd942ed477a3f77de93e970f3033b22d456dfb8 100644 GIT binary patch delta 92 zcmdnMa)V{U4#w#dcW#xeR?1GT^h?Z5RkBh_EKW&N(orhTOb2o%H!^xop1{Z}XKSle UtAnhpuspR0Lm4|rS!^u>0LmI2jQ{`u delta 52 zcmcb?vVmp74#xb6JGZhaB^IZoSxshS3Yffrk&VmNR!K*xuspR0F2n{Dimhb;0NcC| A@c;k- diff --git a/src/libstore-tests/data/serve-protocol/realisation.json b/src/libstore-tests/data/serve-protocol/realisation.json index 034d620306..ad1ae3ab5f 100644 --- a/src/libstore-tests/data/serve-protocol/realisation.json +++ b/src/libstore-tests/data/serve-protocol/realisation.json @@ -10,8 +10,14 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } ] diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json index 2c2d264f9a..9c8277922c 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-realisation-2.8.json @@ -1,7 +1,13 @@ { "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json index 0f593f4248..55bbd81bf1 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json @@ -9,7 +9,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": null, @@ -23,6 +23,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json index 9c1fa3134d..5c0f436e1f 100644 --- a/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json +++ b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json @@ -11,7 +11,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": { @@ -27,11 +27,17 @@ ], "registrationTime": null, "signatures": [ - "fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "fake-sig-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "fake-sig-2", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json index 1977d8485b..d6dde73276 100644 --- a/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json +++ b/src/libstore-tests/data/worker-protocol/realisation-realisation-with-path-not-hash.json @@ -6,8 +6,14 @@ "value": { "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } } diff --git a/src/libstore-tests/data/worker-protocol/realisation.bin b/src/libstore-tests/data/worker-protocol/realisation.bin index 44885cbdafdab24143f74aeef0ef65649339efb5..cbd942ed477a3f77de93e970f3033b22d456dfb8 100644 GIT binary patch delta 92 zcmdnMa)V{U4#w#dcW#xeR?1GT^h?Z5RkBh_EKW&N(orhTOb2o%H!^xop1{Z}XKSle UtAnhpuspR0Lm4|rS!^u>0LmI2jQ{`u delta 52 zcmcb?vVmp74#xb6JGZhaB^IZoSxshS3Yffrk&VmNR!K*xuspR0F2n{Dimhb;0NcC| A@c;k- diff --git a/src/libstore-tests/data/worker-protocol/realisation.json b/src/libstore-tests/data/worker-protocol/realisation.json index 034d620306..ad1ae3ab5f 100644 --- a/src/libstore-tests/data/worker-protocol/realisation.json +++ b/src/libstore-tests/data/worker-protocol/realisation.json @@ -10,8 +10,14 @@ "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz", "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } ] diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json index 2c2d264f9a..9c8277922c 100644 --- a/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json +++ b/src/libstore-tests/data/worker-protocol/unkeyed-realisation-realisation-with-path-not-hash.json @@ -1,7 +1,13 @@ { "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", "signatures": [ - "asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "qwer", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ] } diff --git a/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json index 9cc53c6804..fb2fbeef0c 100644 --- a/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json +++ b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.json @@ -9,7 +9,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": null, @@ -23,6 +23,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json index 427c286ddf..e62fd7a236 100644 --- a/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.json @@ -10,7 +10,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": null, @@ -26,6 +26,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json index 2c377145a1..8bffd4410f 100644 --- a/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json +++ b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.json @@ -10,7 +10,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": true, - "version": 2 + "version": 3 }, { "ca": null, @@ -24,12 +24,18 @@ ], "registrationTime": 23423, "signatures": [ - "fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", - "fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + { + "keyName": "fake-sig-1", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + }, + { + "keyName": "fake-sig-2", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } ], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 }, { "ca": { @@ -48,6 +54,6 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } ] diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json index 3271cb9859..ce95439d6e 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/store-after.json @@ -30,7 +30,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json index f10653a3a7..c596ae7f88 100644 --- a/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json +++ b/src/libstore-tests/data/worker-substitution/ca-drv/substituter.json @@ -30,7 +30,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json index c4d3901236..6263797a71 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/store-after.json @@ -30,7 +30,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json index ecd2282146..41b1238679 100644 --- a/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json +++ b/src/libstore-tests/data/worker-substitution/issue-11928/substituter.json @@ -36,7 +36,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } }, "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": { @@ -58,7 +58,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/single/substituter.json b/src/libstore-tests/data/worker-substitution/single/substituter.json index f22d4c7dfb..55e16a6d38 100644 --- a/src/libstore-tests/data/worker-substitution/single/substituter.json +++ b/src/libstore-tests/data/worker-substitution/single/substituter.json @@ -23,7 +23,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/data/worker-substitution/with-dep/substituter.json b/src/libstore-tests/data/worker-substitution/with-dep/substituter.json index 3f2994dfb4..7653400310 100644 --- a/src/libstore-tests/data/worker-substitution/with-dep/substituter.json +++ b/src/libstore-tests/data/worker-substitution/with-dep/substituter.json @@ -23,7 +23,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } }, "k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": { @@ -47,7 +47,7 @@ "signatures": [], "storeDir": "/nix/store", "ultimate": false, - "version": 2 + "version": 3 } } }, diff --git a/src/libstore-tests/nar-info.cc b/src/libstore-tests/nar-info.cc index 9b0f6018cd..9c8a93e16b 100644 --- a/src/libstore-tests/nar-info.cc +++ b/src/libstore-tests/nar-info.cc @@ -31,6 +31,16 @@ class NarInfoTestV2 : public CharacterizationTest, public LibStoreTest } }; +class NarInfoTestV3 : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "nar-info" / "json-3"; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / (testStem + ".json"); + } +}; + static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { auto info = NarInfo::makeFromCA( @@ -122,6 +132,31 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) JSON_READ_TEST_V2(STEM, PURE) \ JSON_WRITE_TEST_V2(STEM, PURE) +#define JSON_READ_TEST_V3(STEM, PURE) \ + TEST_F(NarInfoTestV3, NarInfo_##STEM##_from_json) \ + { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + auto expected = makeNarInfo(*store, PURE); \ + auto got = UnkeyedNarInfo::fromJSON(nullptr, encoded); \ + ASSERT_EQ(got, expected); \ + }); \ + } + +#define JSON_WRITE_TEST_V3(STEM, PURE) \ + TEST_F(NarInfoTestV3, NarInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> json { return makeNarInfo(*store, PURE).toJSON(nullptr, PURE, PathInfoJsonFormat::V3); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } + +#define JSON_TEST_V3(STEM, PURE) \ + JSON_READ_TEST_V3(STEM, PURE) \ + JSON_WRITE_TEST_V3(STEM, PURE) + JSON_TEST_V1(pure, false) JSON_TEST_V1(impure, true) @@ -131,4 +166,7 @@ JSON_READ_TEST_V1(pure_noversion, false) JSON_TEST_V2(pure, false) JSON_TEST_V2(impure, true) +JSON_TEST_V3(pure, false) +JSON_TEST_V3(impure, true) + } // namespace nix diff --git a/src/libstore-tests/path-info.cc b/src/libstore-tests/path-info.cc index 97ad4b270a..9141d06cd6 100644 --- a/src/libstore-tests/path-info.cc +++ b/src/libstore-tests/path-info.cc @@ -30,6 +30,16 @@ class PathInfoTestV2 : public CharacterizationTest, public LibStoreTest } }; +class PathInfoTestV3 : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "path-info" / "json-3"; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / (testStem + ".json"); + } +}; + static UnkeyedValidPathInfo makeEmpty() { return { @@ -129,6 +139,31 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo JSON_READ_TEST_V2(STEM, OBJ) \ JSON_WRITE_TEST_V2(STEM, OBJ, PURE) +#define JSON_READ_TEST_V3(STEM, OBJ) \ + TEST_F(PathInfoTestV3, PathInfo_##STEM##_from_json) \ + { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON(nullptr, encoded); \ + auto expected = OBJ; \ + ASSERT_EQ(got, expected); \ + }); \ + } + +#define JSON_WRITE_TEST_V3(STEM, OBJ, PURE) \ + TEST_F(PathInfoTestV3, PathInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> json { return OBJ.toJSON(nullptr, PURE, PathInfoJsonFormat::V3); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } + +#define JSON_TEST_V3(STEM, OBJ, PURE) \ + JSON_READ_TEST_V3(STEM, OBJ) \ + JSON_WRITE_TEST_V3(STEM, OBJ, PURE) + JSON_TEST_V1(empty_pure, makeEmpty(), false) JSON_TEST_V1(empty_impure, makeEmpty(), true) JSON_TEST_V1(pure, makeFull(*store, false), false) @@ -142,6 +177,11 @@ JSON_TEST_V2(empty_impure, makeEmpty(), true) JSON_TEST_V2(pure, makeFull(*store, false), false) JSON_TEST_V2(impure, makeFull(*store, true), true) +JSON_TEST_V3(empty_pure, makeEmpty(), false) +JSON_TEST_V3(empty_impure, makeEmpty(), true) +JSON_TEST_V3(pure, makeFull(*store, false), false) +JSON_TEST_V3(impure, makeFull(*store, true), true) + TEST_F(PathInfoTestV2, PathInfo_full_shortRefs) { ValidPathInfo it = makeFullKeyed(*store, true); diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index 9c0ebbe8ab..1bc7708e7c 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -54,6 +54,20 @@ Realisation simple{ }, }; +Realisation withSignature{ + { + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + }, + }, + { + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, + .outputName = "foo", + }, +}; + INSTANTIATE_TEST_SUITE_P( RealisationJSON, RealisationJsonTest, @@ -63,15 +77,16 @@ INSTANTIATE_TEST_SUITE_P( simple, }, std::pair{ - "with-signature", - [&] { - auto r = simple; - // FIXME actually sign properly - r.signatures = { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - }; - return r; - }(), + "with-signature-structured", + withSignature, })); +/** + * Old signature format (string) should still be parseable. + */ +TEST_F(RealisationTest, with_signature_from_json) +{ + readJsonTest("with-signature", withSignature); +} + } // namespace nix diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index 44840b0e5e..c94a0d0936 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -22,6 +22,8 @@ enum class PathInfoJsonFormat { V1 = 1, /// New format with structured hashes and store path base names V2 = 2, + /// New format with structured signatures + V3 = 3, }; /** diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 1c631cf147..1ffd4d52fb 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -15,8 +15,10 @@ PathInfoJsonFormat parsePathInfoJsonFormat(uint64_t version) return PathInfoJsonFormat::V1; case 2: return PathInfoJsonFormat::V2; + case 3: + return PathInfoJsonFormat::V3; default: - throw Error("unsupported path info JSON format version %d; supported versions are 1 and 2", version); + throw Error("unsupported path info JSON format version %d; supported versions are 1, 2 and 3", version); } } @@ -211,7 +213,13 @@ UnkeyedValidPathInfo::toJSON(const StoreDirConfig * store, bool includeImpureInf jsonObject["ultimate"] = ultimate; - jsonObject["signatures"] = sigs; + if (format == PathInfoJsonFormat::V3) { + jsonObject["signatures"] = sigs; + } else { + auto & sigsObj = jsonObject["signatures"] = json::array(); + for (auto & sig : sigs) + sigsObj.push_back(sig.to_string()); + } } return jsonObject; @@ -313,7 +321,7 @@ UnkeyedValidPathInfo adl_serializer::from_json(const json void adl_serializer::to_json(json & json, const UnkeyedValidPathInfo & c) { - json = c.toJSON(nullptr, true, PathInfoJsonFormat::V2); + json = c.toJSON(nullptr, true, PathInfoJsonFormat::V3); } ValidPathInfo adl_serializer::from_json(const json & json0) diff --git a/src/libutil/signature/local-keys.cc b/src/libutil/signature/local-keys.cc index 51f94cee00..8afa3eb728 100644 --- a/src/libutil/signature/local-keys.cc +++ b/src/libutil/signature/local-keys.cc @@ -178,12 +178,21 @@ bool verifyDetached(std::string_view data, const Signature & sig, const PublicKe namespace nlohmann { void adl_serializer::to_json(json & j, const Signature & s) { - j = s.to_string(); + j = { + {"keyName", s.keyName}, + {"sig", base64::encode(std::as_bytes(std::span{s.sig}))}, + }; } Signature adl_serializer::from_json(const json & j) { - return Signature::parse(getString(j)); + if (j.is_string()) + return Signature::parse(getString(j)); + auto obj = getObject(j); + return Signature{ + .keyName = getString(valueAt(obj, "keyName")), + .sig = base64::decode(getString(valueAt(obj, "sig"))), + }; } } // namespace nlohmann From 3df8dbc07c6b1f14c1b234a20761f909ca1c47fc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 2 Mar 2026 10:22:40 -0500 Subject: [PATCH 029/927] libutil: Add dupDescriptor function Add a cross-platform function for duplicating file descriptors: - Unix: uses fcntl(F_DUPFD_CLOEXEC) - Windows: uses DuplicateHandle This is useful when code needs to take ownership of a borrowed descriptor. --- src/libutil/include/nix/util/file-descriptor.hh | 8 ++++++++ src/libutil/unix/file-descriptor.cc | 8 ++++++++ src/libutil/windows/file-descriptor.cc | 9 +++++++++ 3 files changed, 25 insertions(+) diff --git a/src/libutil/include/nix/util/file-descriptor.hh b/src/libutil/include/nix/util/file-descriptor.hh index 1e0d726adb..fbfe5e4f8a 100644 --- a/src/libutil/include/nix/util/file-descriptor.hh +++ b/src/libutil/include/nix/util/file-descriptor.hh @@ -285,6 +285,14 @@ public: void startFsync() const; }; +/** + * Duplicate a file descriptor. + * + * Returns a new file descriptor that refers to the same open file + * description as the original. + */ +AutoCloseFD dupDescriptor(Descriptor fd); + class Pipe { public: diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index f2c69f2a17..75e7e12c87 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -56,6 +56,14 @@ size_t write(Descriptor fd, std::span buffer, bool allowInterru return static_cast(n); } +AutoCloseFD dupDescriptor(Descriptor fd) +{ + int newFd = fcntl(fd, F_DUPFD_CLOEXEC, 0); + if (newFd == -1) + throw SysError("duplicating file descriptor"); + return AutoCloseFD{newFd}; +} + ////////////////////////////////////////////////////////////////////// void Pipe::create() diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 7268591351..0020623e31 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -68,6 +68,15 @@ size_t write(Descriptor fd, std::span buffer, bool allowInterru return static_cast(n); } +AutoCloseFD dupDescriptor(Descriptor fd) +{ + HANDLE newHandle; + if (!DuplicateHandle(GetCurrentProcess(), fd, GetCurrentProcess(), &newHandle, 0, FALSE, DUPLICATE_SAME_ACCESS)) { + throw WinError("duplicating handle"); + } + return AutoCloseFD{newHandle}; +} + ////////////////////////////////////////////////////////////////////// void Pipe::create() From 72abbc4164893bba7bf7d28e2be59960de4078b2 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 2 Mar 2026 17:17:00 -0500 Subject: [PATCH 030/927] libstore: use `urlPathToPath` for store URL path parsing `store-registration.hh` constructed a `std::filesystem::path` directly from `percentDecode(uri)`, which broke on Windows because file:// URLs like `/C:/foo` lack a drive-letter root. Replace with `splitString` + `urlPathToPath`, which handles Windows drive-letter stripping. --- src/libstore-tests/nix_api_store.cc | 5 +++-- src/libstore/include/nix/store/store-registration.hh | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 9bfbf8453a..b029b2b425 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -251,8 +251,9 @@ TEST_F(nix_api_util_context, nix_store_real_path_relocated) TEST_F(nix_api_util_context, nix_store_real_path_binary_cache) { - Store * store = - nix_store_open(ctx, nix::fmt("file://%s/binary-cache", nix::createTempDir().string()).c_str(), nullptr); + auto tmpDir = nix::createTempDir() / "binary-cache"; + auto url = nix::ParsedURL{.scheme = "file", .path = nix::pathToUrlPath(tmpDir)}; + Store * store = nix_store_open(ctx, url.to_string().c_str(), nullptr); assert_ctx_ok(); ASSERT_NE(store, nullptr); diff --git a/src/libstore/include/nix/store/store-registration.hh b/src/libstore/include/nix/store/store-registration.hh index 51db33e128..e805523e08 100644 --- a/src/libstore/include/nix/store/store-registration.hh +++ b/src/libstore/include/nix/store/store-registration.hh @@ -66,8 +66,11 @@ struct Implementations .experimentalFeature = TConfig::experimentalFeature(), .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { if constexpr (std::is_constructible_v) { - std::filesystem::path path = percentDecode(uri); - return make_ref(path.empty() ? std::filesystem::path{} : canonPath(path), params); + auto path = + uri.empty() + ? std::filesystem::path{} + : canonPath(urlPathToPath(splitString>(percentDecode(uri), "/"))); + return make_ref(std::move(path), params); } else if constexpr (std::is_constructible_v) { return make_ref(parseURL(concatStrings(scheme, "://", uri)), params); } else if constexpr (std::is_constructible_v) { From 08efb2b4541dbd218c7a395822a99a54e2374be9 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 2 Mar 2026 17:18:31 -0500 Subject: [PATCH 031/927] tests: skip symlink-dependent nix_api tests on Windows --- src/libstore-tests/nix_api_store.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index b029b2b425..fc9b50d2d7 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -212,6 +212,9 @@ TEST_F(nix_api_store_test, nix_store_real_path) TEST_F(nix_api_util_context, nix_store_real_path_relocated) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine/Windows does not support symlinks needed for local store gcroots"; +#endif auto tmp = nix::createTempDir(); auto storeRoot = (tmp / "store").string(); auto stateDir = (tmp / "state").string(); From 9534543860ce92fb3de0ddf49cf385401c50830a Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 2 Mar 2026 17:36:44 -0500 Subject: [PATCH 032/927] tests: skip and fix nix_api_store tests for Windows/Wine --- .../include/nix/expr/tests/nix_api_expr.hh | 9 ++++---- .../include/nix/store/tests/nix_api_store.hh | 22 +++++++++---------- src/libstore-tests/meson.build | 15 ++++++++----- src/libstore-tests/nix_api_store.cc | 15 +++++++++++++ 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh b/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh index 376761d763..4aa55688de 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/nix_api_expr.hh @@ -12,21 +12,22 @@ class nix_api_expr_test : public nix_api_store_test { protected: - nix_api_expr_test() + void SetUp() override { + nix_api_store_test::SetUp(); nix_libexpr_init(ctx); state = nix_state_create(nullptr, nullptr, store); value = nix_alloc_value(nullptr, state); } - ~nix_api_expr_test() + void TearDown() override { nix_gc_decref(nullptr, value); nix_state_free(state); } - EvalState * state; - nix_value * value; + EvalState * state = nullptr; + nix_value * value = nullptr; }; } // namespace nixC diff --git a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh index bb9e5a3038..15df329cb2 100644 --- a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh +++ b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh @@ -79,23 +79,23 @@ class nix_api_store_test : public nix_api_store_test_base { public: nix_api_store_test() - : nix_api_store_test_base{} - { - init_local_store(); - }; + : nix_api_store_test_base{} {}; - ~nix_api_store_test() override + void SetUp() override { - nix_store_free(store); +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif + store = open_local_store(); } - Store * store; - -protected: - void init_local_store() + ~nix_api_store_test() override { - store = open_local_store(); + if (store) + nix_store_free(store); } + + Store * store = nullptr; }; } // namespace nixC diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 2d12b14d3c..3d448678b9 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -105,14 +105,19 @@ this_exe = executable( cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [], ) +test_env = { + '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + 'HOME' : meson.current_build_dir() / 'test-home', +} + +if host_machine.system() != 'windows' + test_env += {'NIX_REMOTE' : meson.current_build_dir() / 'test-home' / 'store'} +endif + test( meson.project_name(), this_exe, - env : { - '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', - 'HOME' : meson.current_build_dir() / 'test-home', - 'NIX_REMOTE' : meson.current_build_dir() / 'test-home' / 'store', - }, + env : test_env, protocol : 'gtest', ) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index fc9b50d2d7..468eac1b95 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -301,6 +301,9 @@ class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base void SetUp() override { nix_api_store_test_base::SetUp(); +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); nix::settings.getWorkerSettings().substituters = {}; @@ -356,6 +359,9 @@ class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base TEST_F(nix_api_store_test_base, build_from_json) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // FIXME get rid of these nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); nix::settings.getWorkerSettings().substituters = {}; @@ -403,6 +409,9 @@ TEST_F(nix_api_store_test_base, build_from_json) TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // Test that nix_store_realise properly reports errors when the system is invalid nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); nix::settings.getWorkerSettings().substituters = {}; @@ -448,6 +457,9 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system) TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // Test that nix_store_realise properly reports errors when the builder fails nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); nix::settings.getWorkerSettings().substituters = {}; @@ -493,6 +505,9 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails) TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output) { +#ifdef _WIN32 + GTEST_SKIP() << "Wine does not support symlinks needed for local store gcroots"; +#endif // Test that nix_store_realise properly reports errors when builder succeeds but produces no output nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); nix::settings.getWorkerSettings().substituters = {}; From f17d9dee9e1584922c93020720e11bf38bd0b5ad Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 2 Mar 2026 17:49:36 -0500 Subject: [PATCH 033/927] tests: use empty temp file instead of `/dev/null` in machines test --- src/libstore-tests/machines.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index f0a334c1b6..c4cbf47e4e 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -185,7 +185,10 @@ TEST(machines, getMachinesWithCorrectFileReference) TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { - std::filesystem::path path = "/dev/null"; + auto tmpDir = nix::createTempDir(); + AutoDelete delTmpDir(tmpDir); + auto path = tmpDir / "empty-machines"; + nix::writeFile(path, ""); ASSERT_TRUE(std::filesystem::exists(path)); auto actual = Machine::parseConfig({}, "@" + path.string()); From 70c6e0a399a0efa523367788aa2ef2ec85863e75 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 2 Mar 2026 17:49:36 -0500 Subject: [PATCH 034/927] tests: use Windows-absolute paths in LocalStore/LocalOverlayStore config tests --- src/libstore-tests/local-overlay-store.cc | 18 ++++++++++++++---- src/libstore-tests/local-store.cc | 20 +++++++++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/libstore-tests/local-overlay-store.cc b/src/libstore-tests/local-overlay-store.cc index c207564255..4c12f6256a 100644 --- a/src/libstore-tests/local-overlay-store.cc +++ b/src/libstore-tests/local-overlay-store.cc @@ -6,24 +6,34 @@ namespace nix { TEST(LocalOverlayStore, constructConfig_rootQueryParam) { +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif LocalOverlayStoreConfig config{ "", { { "root", - "/foo/bar", + std::string{root}, }, }, }; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } TEST(LocalOverlayStore, constructConfig_rootPath) { - LocalOverlayStoreConfig config{"/foo/bar", {}}; +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif + LocalOverlayStoreConfig config{std::string{root}, {}}; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } } // namespace nix diff --git a/src/libstore-tests/local-store.cc b/src/libstore-tests/local-store.cc index 554d42efc8..a094774f8f 100644 --- a/src/libstore-tests/local-store.cc +++ b/src/libstore-tests/local-store.cc @@ -12,24 +12,34 @@ namespace nix { TEST(LocalStore, constructConfig_rootQueryParam) { +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif LocalStoreConfig config{ "", { { "root", - "/foo/bar", + std::string{root}, }, }, }; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } TEST(LocalStore, constructConfig_rootPath) { - LocalStoreConfig config{"/foo/bar", {}}; - - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +#ifdef _WIN32 + constexpr std::string_view root = "C:\\foo\\bar"; +#else + constexpr std::string_view root = "/foo/bar"; +#endif + LocalStoreConfig config{std::string{root}, {}}; + + EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } TEST(LocalStore, constructConfig_to_string) From 5bedf61b83fbf075dcc22abd08b522abadd11816 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 2 Mar 2026 19:21:35 -0500 Subject: [PATCH 035/927] tests: fix two assertion crashes during Windows test teardown --- src/libstore-test-support/https-store.cc | 4 +++- .../include/nix/store/tests/https-store.hh | 2 ++ src/libstore-tests/outputs-spec.cc | 9 +++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libstore-test-support/https-store.cc b/src/libstore-test-support/https-store.cc index 79548f61f3..549e3ce361 100644 --- a/src/libstore-test-support/https-store.cc +++ b/src/libstore-test-support/https-store.cc @@ -60,7 +60,7 @@ void HttpsBinaryCacheStoreTest::SetUp() openssl({"x509", "-req", "-in", (tmpDir / "client.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", clientCert.string(), "-days", "1"}); // clang-format on -#ifndef _WIN32 /* FIXME: Can't yet start processes on windows */ +#ifndef _WIN32 /* FIXME: Can't yet start background processes on windows */ auto args = serverArgs(); serverPid = startProcess( [&] { @@ -91,7 +91,9 @@ void HttpsBinaryCacheStoreTest::SetUp() void HttpsBinaryCacheStoreTest::TearDown() { +#ifndef _WIN32 /* FIXME: Can't yet start background processes on windows */ serverPid.kill(); +#endif delTmpDir.reset(); testFileTransferSettings.reset(); } diff --git a/src/libstore-test-support/include/nix/store/tests/https-store.hh b/src/libstore-test-support/include/nix/store/tests/https-store.hh index 3aa2cd3455..19ecd62e9f 100644 --- a/src/libstore-test-support/include/nix/store/tests/https-store.hh +++ b/src/libstore-test-support/include/nix/store/tests/https-store.hh @@ -65,7 +65,9 @@ protected: std::filesystem::path tmpDir, cacheDir; std::filesystem::path caCert, caKey, serverCert, serverKey; std::filesystem::path clientCert, clientKey; +#ifndef _WIN32 /* FIXME: Can't yet start background processes on windows */ Pid serverPid; +#endif uint16_t port = 8443; std::shared_ptr localCacheStore; diff --git a/src/libstore-tests/outputs-spec.cc b/src/libstore-tests/outputs-spec.cc index 1fac222fcc..d791709ac2 100644 --- a/src/libstore-tests/outputs-spec.cc +++ b/src/libstore-tests/outputs-spec.cc @@ -33,7 +33,16 @@ class ExtendedOutputsSpecTest : public virtual CharacterizationTest TEST_F(OutputsSpecTest, no_empty_names) { +#ifndef _WIN32 ASSERT_DEATH(OutputsSpec::Names{StringSet{}}, ""); +#else + // ASSERT_DEATH relies on CreateProcess, which under Wine leaks the + // child's stderr into the parent console. Test both sides of the + // invariant without a death test instead. + ASSERT_FALSE(OutputsSpec::parseOpt("")); + OutputsSpec::Names names{StringSet{"out"}}; + ASSERT_FALSE(names.empty()); +#endif } #define TEST_DONT_PARSE(NAME, STR) \ From 6bedefea9e610ba739efcc7a3ab8f9cb6618da7b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 3 Mar 2026 16:59:49 +0300 Subject: [PATCH 036/927] Improve formatting of various errors and warnings Removes unnecessary `error: warning:` prefixes by using .message() and not .msg(). Also includes attempt counts in the filetransfer warnings. Removes a redundant trace from the filetransfer errors with FileTransferError (which is already descriptive enough) and contains the same information basically. --- src/libcmd/installables.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libmain/common-args.cc | 2 +- src/libstore/daemon.cc | 2 +- src/libstore/filetransfer.cc | 34 +++++++++++++++++++++++++--------- src/libstore/local-store.cc | 2 +- src/libstore/store-api.cc | 2 +- src/nix/develop.cc | 7 ++++--- 8 files changed, 35 insertions(+), 18 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index b9366a2759..8f8309bd96 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -400,7 +400,7 @@ void completeFlakeRefWithFragment( } } } catch (Error & e) { - warn(e.msg()); + logWarning(e.info()); } } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b415a13428..0651744331 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -51,7 +51,7 @@ DownloadFileResult downloadFile( res = getFileTransfer()->download(request); } catch (FileTransferError & e) { if (cached) { - warn("%s; using cached version", e.msg()); + warn("%s; using cached version", e.message()); return useCached(); } else throw; diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 6055ec0e75..42b814ee5b 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -48,7 +48,7 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) globalConfig.set(name, value); } catch (UsageError & e) { if (!getRoot().completions) - warn(e.what()); + logWarning(e.info()); } }}, .completer = diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 3e5eac5170..38eaad1891 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -305,7 +305,7 @@ struct ClientSettings "ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { - warn(e.what()); + logWarning(e.info()); } } } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index ba86e9ebbf..099f436ae6 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -221,6 +221,8 @@ struct curlFileTransfer : public FileTransfer done = true; try { std::rethrow_exception(ex); + } catch (FileTransferError & e) { + /* Already descriptive enough. */ } catch (nix::Error & e) { /* Add more context to the error message. */ e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.uri.to_string()); @@ -628,7 +630,7 @@ struct curlFileTransfer : public FileTransfer debug( "finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", - request.noun(), + Uncolored(request.noun()), request.uri, code, httpStatus, @@ -723,14 +725,14 @@ struct curlFileTransfer : public FileTransfer Interrupted, std::move(response), "%s of '%s' was interrupted", - request.noun(), + Uncolored(request.noun()), request.uri) : httpStatus != 0 ? FileTransferError( err, std::move(response), "unable to %s '%s': HTTP error %d%s", - request.verb(), + Uncolored(request.verb()), request.uri, httpStatus, code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) @@ -738,7 +740,7 @@ struct curlFileTransfer : public FileTransfer err, std::move(response), "unable to %s '%s': %s (%d) %s", - request.verb(), + Uncolored(request.verb()), request.uri, curl_easy_strerror(code), code, @@ -753,10 +755,24 @@ struct curlFileTransfer : public FileTransfer int ms = retryTimeMs * std::pow( 2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); - if (writtenToSink) - warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); - else - warn("%s; retrying in %d ms", exc.what(), ms); + + if (writtenToSink) { + warn( + "%s; retrying from offset %d in %d ms (attempt %d/%d)", + exc.message(), + writtenToSink, + ms, + attempt, + fileTransfer.settings.tries); + } else { + warn( + "%s; retrying in %d ms (attempt %d/%d)", + exc.message(), + ms, + attempt, + fileTransfer.settings.tries); + } + errorSink.reset(); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); try { @@ -937,7 +953,7 @@ struct curlFileTransfer : public FileTransfer } for (auto & item : incoming) { - debug("starting %s of %s", item->request.noun(), item->request.uri); + debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.uri); item->init(); curl_multi_add_handle(curlm.get(), item->req); item->active = true; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8631df2184..649ad70dd3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1407,7 +1407,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (isValidPath(i)) logError(e.info()); else - warn(e.msg()); + logWarning(e.info()); errors = true; } } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 845e7bcb37..5b5d416238 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1201,7 +1201,7 @@ static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, return parseDerivation(store, std::move(contents), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { - throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); + throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.message()); } } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d093c7f27f..b898e99a4f 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -676,9 +676,10 @@ struct CmdDevelop : Common, MixEnvironment // Override SHELL with the one chosen for this environment. // This is to make sure the system shell doesn't leak into the build environment. - setEnv("SHELL", shell.string().c_str()); - // https://github.com/NixOS/nix/issues/5873 - script += fmt("SHELL=%s\n", PathFmt(shell)); + setEnvOs(OS_STR("SHELL"), shell.c_str()); + /* See: https://github.com/NixOS/nix/issues/5873 + Format via .string() and not PathFmt intentionally. */ + script += fmt("SHELL=\"%s\"\n", shell.string()); if (foundInteractive) script += fmt("PATH=\"%s${PATH:+:$PATH}\"\n", std::filesystem::path(shell).parent_path().string()); writeFull(rcFileFd.get(), script); From 5e84b575945f0344867f6294a47b43b46c852e35 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 3 Mar 2026 09:59:50 -0500 Subject: [PATCH 037/927] Introduce `Source::drainInto` with explicit length This is used to deduplicate two spots in the code: - `git::parseBlob` - `parseContents` in `archive.cc` Implementation is a bit more hardened than either of the two originals. --- src/libutil/archive.cc | 13 +------------ src/libutil/git.cc | 12 +----------- src/libutil/include/nix/util/serialise.hh | 12 ++++++++++++ src/libutil/serialise.cc | 15 +++++++++++++++ 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 847e339b6d..53aebde70a 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -135,18 +135,7 @@ static void parseContents(CreateRegularFileSink & sink, Source & source) return; } - uint64_t left = size; - std::array buf; - - while (left) { - checkInterrupt(); - auto n = buf.size(); - if ((uint64_t) n > left) - n = left; - source(buf.data(), n); - sink({buf.data(), n}); - left -= n; - } + source.drainInto(sink, size); readPadding(size, source); } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 1b6c65c2e0..2646babfd5 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -68,17 +68,7 @@ void parseBlob( crf.preallocateContents(size); - unsigned long long left = size; - std::string buf; - buf.reserve(65536); - - while (left) { - checkInterrupt(); - buf.resize(std::min((unsigned long long) buf.capacity(), left)); - source(buf); - crf(buf); - left -= buf.size(); - } + source.drainInto(crf, size); }); }; diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index f8b545f49b..0598104e87 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -96,8 +96,20 @@ struct Source return true; } + /** + * Read the rest of this `Source` into `sink`. + */ void drainInto(Sink & sink); + /** + * Read exactly 'len' bytes and write them to 'sink'. + * + * Virtual in anticipation that some `Source` implementations, like + * `FdSource` may eventually be able to provide more performant + * implementations of this function. + */ + virtual void drainInto(Sink & sink, uint64_t len); + std::string drain(); virtual void skip(size_t len); diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8adbe63841..11c9d1e3e6 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -104,6 +105,20 @@ void Source::drainInto(Sink & sink) } } +void Source::drainInto(Sink & sink, uint64_t len) +{ + std::array buf; + while (len) { + checkInterrupt(); + // Until std::saturate_cast is available (C++26) + auto lenTrunc = static_cast(std::min(len, std::numeric_limits::max())); + auto n = read(buf.data(), std::min(lenTrunc, buf.size())); + sink({buf.data(), n}); + assert(n <= len); + len -= n; + } +} + std::string Source::drain() { StringSink s; From 17ced6f80c4f1a26a26f5a11715a401d0361ebfa Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 3 Mar 2026 21:46:40 +0300 Subject: [PATCH 038/927] libutil: Add finalSymlink argument to openFileReadonly We have a couple of places where this would end up being useful. --- src/libutil/include/nix/util/file-system.hh | 2 +- src/libutil/unix/file-system.cc | 5 +++-- src/libutil/windows/file-system.cc | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index aae7b7bf37..8dcddc38e2 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -220,7 +220,7 @@ AutoCloseFD openDirectory(const std::filesystem::path & path, FinalSymlink final * * @note For directories use @ref openDirectory. */ -AutoCloseFD openFileReadonly(const std::filesystem::path & path); +AutoCloseFD openFileReadonly(const std::filesystem::path & path, FinalSymlink finalSymlink = FinalSymlink::Follow); struct OpenNewFileForWriteParams { diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index e709c5298a..cc99c9ba20 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -29,9 +29,10 @@ AutoCloseFD openDirectory(const std::filesystem::path & path, FinalSymlink final path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC | (finalSymlink == FinalSymlink::Follow ? 0 : O_NOFOLLOW))}; } -AutoCloseFD openFileReadonly(const std::filesystem::path & path) +AutoCloseFD openFileReadonly(const std::filesystem::path & path, FinalSymlink finalSymlink) { - return AutoCloseFD{open(path.c_str(), O_RDONLY | O_CLOEXEC)}; + return AutoCloseFD{ + open(path.c_str(), O_RDONLY | O_CLOEXEC | (finalSymlink == FinalSymlink::Follow ? 0 : O_NOFOLLOW))}; } AutoCloseFD openNewFileForWrite(const std::filesystem::path & path, mode_t mode, OpenNewFileForWriteParams params) diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 3e9680ff24..9dab6f2b7a 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -42,7 +42,7 @@ AutoCloseFD openDirectory(const std::filesystem::path & path, FinalSymlink final /*hTemplateFile=*/nullptr)}; } -AutoCloseFD openFileReadonly(const std::filesystem::path & path) +AutoCloseFD openFileReadonly(const std::filesystem::path & path, FinalSymlink finalSymlink) { return AutoCloseFD{CreateFileW( path.c_str(), @@ -50,7 +50,7 @@ AutoCloseFD openFileReadonly(const std::filesystem::path & path) FILE_SHARE_READ | FILE_SHARE_DELETE, /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, + FILE_ATTRIBUTE_NORMAL | (finalSymlink == FinalSymlink::Follow ? 0 : FILE_FLAG_OPEN_REPARSE_POINT), /*hTemplateFile=*/nullptr)}; } From a8d51a0c59346c9326acfcfb0b7c9f9d4e17d9ce Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 3 Mar 2026 21:49:06 +0300 Subject: [PATCH 039/927] libutil: Address TODO O_NOFOLLOW in recursiveSync --- src/libutil/file-system.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index d810e4a06c..c269e81b1b 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -338,7 +338,7 @@ void recursiveSync(const std::filesystem::path & path) /* If it's a file or symlink, just fsync and return. */ auto st = lstat(path); if (S_ISREG(st.st_mode)) { - AutoCloseFD fd = openFileReadonly(path); /* TODO: O_NOFOLLOW? */ + AutoCloseFD fd = openFileReadonly(path, FinalSymlink::DontFollow); if (!fd) throw NativeSysError("opening file %s", PathFmt(path)); fd.fsync(); @@ -359,7 +359,7 @@ void recursiveSync(const std::filesystem::path & path) if (std::filesystem::is_directory(st)) { dirsToEnumerate.emplace_back(entry.path()); } else if (std::filesystem::is_regular_file(st)) { - AutoCloseFD fd = openFileReadonly(entry.path()); /* TODO: O_NOFOLLOW? */ + AutoCloseFD fd = openFileReadonly(entry.path(), FinalSymlink::DontFollow); if (!fd) throw NativeSysError("opening file %1%", PathFmt(entry.path())); fd.fsync(); From 26679828f74ee6e82a4100904e6361f993ff5390 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 3 Mar 2026 22:05:16 +0300 Subject: [PATCH 040/927] libutil,libstore: Expose FinalSymlink in writeFile arguments, use in LocalStore This exposes the knob for controlling final symlink following in writeFile with O_TRUNC (which I've previously marked with FIXMEs, since that was quite suspicious looking). Use DontFollow in restorePath for `flat` serialisation case. Callers already have the invariant that a regular file will be written to the destination path (which has been deleted recursively via deletePath for good measure). This just adds for guardrails against any possible SNAFUs. --- src/libutil/file-content-address.cc | 2 +- src/libutil/file-system.cc | 9 +++++---- src/libutil/include/nix/util/file-system.hh | 14 ++++++++++++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index caf12f81e6..e305603490 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -79,7 +79,7 @@ void restorePath(const std::filesystem::path & path, Source & source, FileSerial { switch (method) { case FileSerialisationMethod::Flat: - writeFile(path, source, 0666, startFsync ? FsSync::Yes : FsSync::No); + writeFile(path, source, 0666, startFsync ? FsSync::Yes : FsSync::No, FinalSymlink::DontFollow); break; case FileSerialisationMethod::NixArchive: restorePath(path, source, startFsync); diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index c269e81b1b..f56000ebee 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -254,14 +254,15 @@ void readFile(const std::filesystem::path & path, Sink & sink, bool memory_map) drainFD(fd.get(), sink); } -void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode, FsSync sync) +void writeFile( + const std::filesystem::path & path, std::string_view s, mode_t mode, FsSync sync, FinalSymlink finalSymlink) { AutoCloseFD fd = openNewFileForWrite( path, mode, { .truncateExisting = true, - .followSymlinksOnTruncate = true, /* FIXME: Do we want this? */ + .followSymlinksOnTruncate = (finalSymlink == FinalSymlink::Follow), }); if (!fd) throw NativeSysError("opening file %s", PathFmt(path)); @@ -287,14 +288,14 @@ void writeFile(Descriptor fd, std::string_view s, FsSync sync, const std::filesy } } -void writeFile(const std::filesystem::path & path, Source & source, mode_t mode, FsSync sync) +void writeFile(const std::filesystem::path & path, Source & source, mode_t mode, FsSync sync, FinalSymlink finalSymlink) { AutoCloseFD fd = openNewFileForWrite( path, mode, { .truncateExisting = true, - .followSymlinksOnTruncate = true, /* FIXME: Do we want this? */ + .followSymlinksOnTruncate = (finalSymlink == FinalSymlink::Follow), }); if (!fd) throw NativeSysError("opening file %s", PathFmt(path)); diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 8dcddc38e2..5dce929077 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -256,9 +256,19 @@ enum struct FsSync { Yes, No }; /** * Write a string to a file. */ -void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No); +void writeFile( + const std::filesystem::path & path, + std::string_view s, + mode_t mode = 0666, + FsSync sync = FsSync::No, + FinalSymlink finalSymlink = FinalSymlink::Follow); -void writeFile(const std::filesystem::path & path, Source & source, mode_t mode = 0666, FsSync sync = FsSync::No); +void writeFile( + const std::filesystem::path & path, + Source & source, + mode_t mode = 0666, + FsSync sync = FsSync::No, + FinalSymlink finalSymlink = FinalSymlink::Follow); void writeFile( Descriptor fd, std::string_view s, FsSync sync = FsSync::No, const std::filesystem::path * origPath = nullptr); From ae955594d695fe6641063a1ab3b3f3133c4ec250 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 3 Mar 2026 22:16:58 +0300 Subject: [PATCH 041/927] libutil: Use Source::drainInto with explicit size parameter in PosixSourceAccessor::readFile Slightly cleaner. We dont need any of the NONBLOCK logic from drainFD here since the source is a regular file + this would allow to stuff copy_file_range/sendfile into the FdSource for more efficient copying to a file descriptor. --- src/libutil/posix-source-accessor.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index e3731c9c57..7d0b1df75a 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -58,8 +58,11 @@ void PosixSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun Date: Tue, 3 Mar 2026 22:15:07 +0100 Subject: [PATCH 042/927] fix: typographical error Modify `derivatons` to `derivations`. --- doc/manual/source/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index af5b45314d..871b9416cf 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -39,7 +39,7 @@ This sandbox by default only allows reading from store objects specified as inputs, and only allows writing to designated [outputs][output] to be [captured as store objects](@docroot@/store/building.md#processing-outputs). A derivation is typically specified as a [derivation expression] in the [Nix language], and [instantiated][instantiate] to a [store derivation]. - There are multiple ways of obtaining store objects from store derivatons, collectively called [realisation][realise]. + There are multiple ways of obtaining store objects from store derivations, collectively called [realisation][realise]. [derivation]: #gloss-derivation From 93246e6d99649f562cad4103e4f24f5fcc845e0e Mon Sep 17 00:00:00 2001 From: Sander Date: Wed, 4 Mar 2026 15:42:02 +0100 Subject: [PATCH 043/927] libstore: forward trusted client settings to fileTransferSettings in daemon Client setting overrides from trusted users were not passed to fileTransferSettings, so settings like `netrc-file` had no effect when set by the client. --- src/libstore/daemon.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 043402c950..0e5eb6aa5d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -6,6 +6,7 @@ #include "nix/store/build-result.hh" #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" +#include "nix/store/filetransfer.hh" #include "nix/store/gc-store.hh" #include "nix/store/log-store.hh" #include "nix/store/indirect-root-store.hh" @@ -296,9 +297,10 @@ struct ClientSettings trusted || name == settings.getWorkerSettings().buildTimeout.name || name == settings.getWorkerSettings().maxSilentTime.name || name == settings.getWorkerSettings().pollInterval.name || name == "connect-timeout" - || (name == "builders" && value == "")) + || (name == "builders" && value == "")) { settings.set(name, value); - else if (setSubstituters(settings.getWorkerSettings().substituters)) + fileTransferSettings.set(name, value); + } else if (setSubstituters(settings.getWorkerSettings().substituters)) ; else warn( From 96c76a6e4dcb5c849a5042835b8c50b9533abd28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20=C5=BDlender?= Date: Wed, 4 Mar 2026 15:11:24 +0100 Subject: [PATCH 044/927] fix: refresh tarball cache on nix profile upgrade Setting tarball-ttl to 0 prevent the use of stale flake inputs when upgrading a profile. Closes: #15396 --- src/nix/profile.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 4f563199c0..39d51421a5 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -12,6 +12,7 @@ #include "nix/store/names.hh" #include "nix/util/url.hh" #include "nix/flake/url-name.hh" +#include "nix/fetchers/fetch-settings.hh" #include #include @@ -688,6 +689,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf void run(ref store) override { + fetchSettings.tarballTtl = 0; ProfileManifest manifest(*getEvalState(), *profile); Installables installables; From c28203f0971295eacaebde0eb503d9df9824849e Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 5 Mar 2026 00:29:47 +0300 Subject: [PATCH 045/927] libcmd,daemon: Reap children via a self-pipe trick Somewhat better solution for https://github.com/NixOS/nix/issues/15394. Instead of reaping immediately in the signal handler we instead punt that at the poll loop. This way Pid destructor has a chance to properly run instead of being rudely interrupted by the signal handler. See: https://man7.org/tlpi/code/online/dist/altio/self_pipe.c.html --- .../include/nix/cmd/unix-socket-server.hh | 13 ++++++ src/libcmd/unix/unix-socket-server.cc | 11 ++++- src/nix/unix/daemon.cc | 42 +++++++++++++++++-- 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/libcmd/include/nix/cmd/unix-socket-server.hh b/src/libcmd/include/nix/cmd/unix-socket-server.hh index 3aeee5b55e..7a0d9fa793 100644 --- a/src/libcmd/include/nix/cmd/unix-socket-server.hh +++ b/src/libcmd/include/nix/cmd/unix-socket-server.hh @@ -53,6 +53,19 @@ struct ServeUnixSocketOptions * Mode for the created socket file. */ mode_t socketMode = 0666; + +#ifndef _WIN32 + /** + * Additional file descriptor to poll. Useful for doing a self-pipe trick + * https://cr.yp.to/docs/selfpipe.html. + */ + Descriptor auxiliaryFd = INVALID_DESCRIPTOR; + + /** + * Optional callback invoked on POLLIN event for auxiliaryFd. + */ + std::function onAuxiliaryFdPollin = nullptr; +#endif }; /** diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index 8cc2bd713d..e33f4317da 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -9,6 +9,8 @@ #include "nix/util/unix-domain-socket.hh" #include "nix/util/util.hh" +#include + #include #include #include @@ -83,6 +85,9 @@ PeerInfo getPeerInfo(Descriptor remote) for (auto & i : listeningSockets) fds.push_back({.fd = i.get(), .events = POLLIN}); + if (options.auxiliaryFd != INVALID_DESCRIPTOR) + fds.push_back({.fd = options.auxiliaryFd, .events = POLLIN}); + // Loop accepting connections. while (1) { try { @@ -95,7 +100,11 @@ PeerInfo getPeerInfo(Descriptor remote) throw SysError("polling for incoming connections"); } - for (auto & fd : fds) { + if (options.auxiliaryFd != INVALID_DESCRIPTOR && options.onAuxiliaryFdPollin && fds.back().revents & POLLIN) + /* Useful for reaping children. */ + options.onAuxiliaryFdPollin(); + + for (auto & fd : std::views::take(fds, listeningSockets.size())) { if (!fd.revents) continue; diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 7fce2e8f67..3faaba6429 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -120,13 +120,20 @@ static ssize_t splice(int fd_in, void * off_in, int fd_out, void * off_out, size } #endif +static Pipe sigChldPipe; + static void sigChldHandler(int sigNo) { // Ensure we don't modify errno of whatever we've interrupted auto saved_errno = errno; - // Reap all dead children. - while (waitpid(-1, 0, WNOHANG) > 0) - ; + /* Write to the self-pipe that gets polled in the accept loop. Pipe + is non-blocking. https://man7.org/tlpi/code/online/dist/altio/self_pipe.c.html */ + auto res = ::write(sigChldPipe.writeSide.get(), "x", 1); + if (res == -1 && errno != EAGAIN) { + /* Something is deeply wrong. Can't call std::terminate here because our terminate + handler is not safe for that. */ + abort(); + } errno = saved_errno; } @@ -243,7 +250,13 @@ static void daemonLoop(ref storeConfig, std::optional storeConfig, std::optional buf; + + /* Drain the self-pipe. */ + while (true) { + if (::read(sigChldPipe.readSide.get(), buf.data(), buf.size()) == -1) { + if (errno == EAGAIN) + break; + else + throw SysError("reading from self-pipe for SIGCHLD"); + } + } + + /* Reap all dead children. */ + pid_t pid = -1; + int status; + while (pid = ::waitpid(/*pid (any child process)=*/-1, &status, WNOHANG), pid > 0) + printInfo("reaped child process %1%, status = %2%", pid, statusToString(status)); + }, }, [&](AutoCloseFD remote, std::function closeListeners) { unix::closeOnExec(remote.get()); From c38eacae2673c60071eec1a07e2181288a13eae7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 12 Sep 2025 21:14:18 +0300 Subject: [PATCH 046/927] libexpr: Use 16-byte atomic loads/stores MOVAPS/MOVDQA for ValueStorage<8> This would have to pair with CMPXCHG16B (or other double-width CAS approaches) to be able to implement concurrent thunk evaluation. See the extensive comment in the code for the reasons why we have to do it this way. The motivation for the change is to simplify the eventual implementation of parallel evaluation. Being able to atomically update the whole Value is much easier to reason about. --- src/libexpr/include/nix/expr/value.hh | 121 +++++++++++++++++++++++++- src/libexpr/meson.build | 4 + src/libexpr/package.nix | 4 +- src/libexpr/value.cc | 21 +++++ 4 files changed, 145 insertions(+), 5 deletions(-) diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index e1b2bc4e25..28ff71db88 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -22,6 +22,10 @@ #include #include +#if defined(__x86_64__) && defined(__SSE2__) +# include +#endif + namespace nix { struct Value; @@ -555,6 +559,11 @@ protected: { return internalType; } + + static bool isAtomic() + { + return false; + } }; namespace detail { @@ -588,7 +597,11 @@ class alignas(16) using PackedPointer = typename PackedPointerTypeStruct::type; using Payload = std::array; - Payload payload = {}; +#if defined(__x86_64__) && defined(__SSE2__) + __m128i payloadWords; +#else + Payload payloadWords = {}; +#endif static constexpr int discriminatorBits = 3; static constexpr PackedPointer discriminatorMask = (PackedPointer(1) << discriminatorBits) - 1; @@ -632,6 +645,54 @@ class alignas(16) pdPairOfPointers, //< layout: Pair of pointers payload }; +#if defined(__x86_64__) && defined(__SSE2__) + /* Why do we even bother with hand-rolling these arch-specific intrinsics + * and don't use libatomic directly for 16 byte atomics? Here's why: + * - https://gcc.gnu.org/legacy-ml/gcc/2018-02/msg00224.html + * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84563 + * + * Basically, we don't really ever want to go through libatomic. As is + * so happens on x86_64 with AVX, MOVDQA/MOVAPS instructions (16-byte aligned + * 128-bit loads and stores) are atomic [^]. Note that + * these instructions are not part of AVX but rather SSE2, which is x86_64-v1. + * They are just not guaranteed to be atomic without AVX. + * + * For more details see: + * - [^] Intel® 64 and IA-32 Architectures Software Developer’s Manual (10.1.1 Guaranteed Atomic Operations). + * - https://patchwork.sourceware.org/project/gcc/patch/YhxkfzGEEQ9KHbBC@tucnak/ + * - https://ibraheem.ca/posts/128-bit-atomics/ + * - https://rigtorp.se/isatomic/ + */ + [[gnu::always_inline]] + void updatePayload(Payload payload) noexcept + { + /* This intrinsic corresponds to MOVAPS. Note that Value (and thus the first member + payloadWords) is 16 bytes aligned. */ + _mm_store_si128(&payloadWords, std::bit_cast<__m128i>(payload)); + } + + [[gnu::always_inline]] + Payload loadPayload() const noexcept + { + /* This intrinsic corresponds to MOVDQA. Note that Value (and thus the first member + payloadWords) is 16 bytes aligned. */ + __m128i res = _mm_load_si128(&payloadWords); + return std::bit_cast(res); + } +#else + [[gnu::always_inline]] + void updatePayload(Payload payload) noexcept + { + payloadWords = payload; + } + + [[gnu::always_inline]] + Payload loadPayload() const noexcept + { + return payloadWords; + } +#endif + template requires std::is_pointer_v static T untagPointer(PackedPointer val) noexcept @@ -639,9 +700,9 @@ class alignas(16) return std::bit_cast(val & ~discriminatorMask); } - PrimaryDiscriminator getPrimaryDiscriminator() const noexcept + PrimaryDiscriminator getPrimaryDiscriminator(PackedPointer firstDWord) const noexcept { - return static_cast(payload[0] & discriminatorMask); + return static_cast(firstDWord & discriminatorMask); } static void assertAligned(PackedPointer val) noexcept @@ -652,25 +713,30 @@ class alignas(16) template void setSingleDWordPayload(PackedPointer untaggedVal) noexcept { + Payload payload; /* There's plenty of free upper bits in the first dword, which is used only for the discriminator. */ payload[0] = static_cast(pdSingleDWord) | (static_cast(type) << discriminatorBits); payload[1] = untaggedVal; + updatePayload(payload); } template void setUntaggablePayload(T * firstPtrField, U untaggableField) noexcept { + Payload payload; static_assert(discriminator >= pdListN && discriminator <= pdPath); auto firstFieldPayload = std::bit_cast(firstPtrField); assertAligned(firstFieldPayload); payload[0] = static_cast(discriminator) | firstFieldPayload; payload[1] = std::bit_cast(untaggableField); + updatePayload(payload); } template void setPairOfPointersPayload(T * firstPtrField, U * secondPtrField) noexcept { + Payload payload; static_assert(type >= tFirstPairOfPointers && type <= tLastPairOfPointers); { auto firstFieldPayload = std::bit_cast(firstPtrField); @@ -682,21 +748,58 @@ class alignas(16) assertAligned(secondFieldPayload); payload[1] = (type - tFirstPairOfPointers) | secondFieldPayload; } + updatePayload(payload); } template requires std::is_pointer_v && std::is_pointer_v void getPairOfPointersPayload(T & firstPtrField, U & secondPtrField) const noexcept { + Payload payload = loadPayload(); firstPtrField = untagPointer(payload[0]); secondPtrField = untagPointer(payload[1]); } +public: + ValueStorage() + { + updatePayload({}); + } + + ValueStorage(const ValueStorage & other) + { + updatePayload(other.loadPayload()); + } + + ValueStorage & operator=(const ValueStorage & other) + { + updatePayload(other.loadPayload()); + return *this; + } + + ValueStorage(ValueStorage && other) noexcept + { + updatePayload(other.loadPayload()); + other.updatePayload({}); // Zero out rhs + } + + ValueStorage & operator=(ValueStorage && other) noexcept + { + updatePayload(other.loadPayload()); + other.updatePayload({}); // Zero out rhs + return *this; + } + + ~ValueStorage() noexcept {} + protected: + static bool isAtomic(); + /** Get internal type currently occupying the storage. */ InternalType getInternalType() const noexcept { - switch (auto pd = getPrimaryDiscriminator()) { + Payload payload = loadPayload(); + switch (auto pd = getPrimaryDiscriminator(payload[0])) { case pdUninitialized: /* Discriminator value of zero is used to distinguish uninitialized values. */ return tUninitialized; @@ -738,6 +841,7 @@ protected: void getStorage(NixInt & integer) const noexcept { + Payload payload = loadPayload(); /* PackedPointerType -> int64_t here is well-formed, since the standard requires this conversion to follow 2's complement rules. This is just a no-op. */ integer = NixInt(payload[1]); @@ -745,6 +849,7 @@ protected: void getStorage(bool & boolean) const noexcept { + Payload payload = loadPayload(); boolean = payload[1]; } @@ -752,44 +857,52 @@ protected: void getStorage(NixFloat & fpoint) const noexcept { + Payload payload = loadPayload(); fpoint = std::bit_cast(payload[1]); } void getStorage(ExternalValueBase *& external) const noexcept { + Payload payload = loadPayload(); external = std::bit_cast(payload[1]); } void getStorage(PrimOp *& primOp) const noexcept { + Payload payload = loadPayload(); primOp = std::bit_cast(payload[1]); } void getStorage(Bindings *& attrs) const noexcept { + Payload payload = loadPayload(); attrs = std::bit_cast(payload[1]); } void getStorage(List & list) const noexcept { + Payload payload = loadPayload(); list.elems = untagPointer(payload[0]); list.size = payload[1]; } void getStorage(StringWithContext & string) const noexcept { + Payload payload = loadPayload(); string.context = untagPointer(payload[0]); string.str = std::bit_cast(payload[1]); } void getStorage(Path & path) const noexcept { + Payload payload = loadPayload(); path.accessor = untagPointer(payload[0]); path.path = std::bit_cast(payload[1]); } void getStorage(Failed *& failed) const noexcept { + Payload payload = loadPayload(); failed = std::bit_cast(payload[1]); } diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 0b594dc0cf..893718cb82 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -91,6 +91,10 @@ configdata_priv.set( deps_other += toml11 +cpuid = dependency('libcpuid', 'cpuid', required : false) +configdata_priv.set('HAVE_LIBCPUID', cpuid.found().to_int()) +deps_private += cpuid + config_priv_h = configure_file( configuration : configdata_priv, output : 'expr-config-private.hh', diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index d0aef34e95..e9d12035cd 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -14,6 +14,7 @@ boehmgc, nlohmann_json, toml11, + libcpuid, # Configuration Options @@ -64,7 +65,8 @@ mkMesonLibrary (finalAttrs: { buildInputs = [ toml11 - ]; + ] + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; propagatedBuildInputs = [ nix-util diff --git a/src/libexpr/value.cc b/src/libexpr/value.cc index 07d036b0dd..8dbb277750 100644 --- a/src/libexpr/value.cc +++ b/src/libexpr/value.cc @@ -1,5 +1,11 @@ #include "nix/expr/value.hh" +#include "expr-config-private.hh" + +#if HAVE_LIBCPUID +# include +#endif + namespace nix { Value Value::vEmptyList = []() { @@ -26,4 +32,19 @@ Value Value::vFalse = []() { return res; }(); +template<> +bool ValueStorage<8>::isAtomic() +{ +#if HAVE_LIBCPUID + struct cpu_id_t data; + + if (cpu_identify(NULL, &data) < 0) + return false; + + return data.flags[CPU_FEATURE_AVX]; +#else + return false; // Can't tell +#endif +} + } // namespace nix From 877d1ef25904bf5077d16e4b8433f20db7ef906c Mon Sep 17 00:00:00 2001 From: Audrey Dutcher Date: Fri, 6 Mar 2026 16:42:27 +0000 Subject: [PATCH 047/927] Use per-platform signal for internal interrupts instead of SIGUSR1 FreeBSD bdwgc uses SIGUSR1 internally for its runtime. Use a different signal. See [1] for documentation and the suggestion to use SIGTSTP on FreeBSD. [1] https://github.com/bdwgc/bdwgc/blob/af374f921837af641f2f61698ab90767befd8996/include/private/gc_priv.h#L4557 --- src/libmain/shared.cc | 6 +++--- src/libutil/include/nix/util/signals.hh | 9 ++++++++- src/libutil/unix/include/nix/util/signals-impl.hh | 4 ++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index ae5281e428..a65fc773aa 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -164,10 +164,10 @@ void initNix(bool loadConfig) if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD"); - /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ + /* Install a dummy NIX_SIG_MULTI_INT handler for use with pthread_kill(). */ act.sa_handler = sigHandler; - if (sigaction(SIGUSR1, &act, 0)) - throw SysError("handling SIGUSR1"); + if (sigaction(NIX_SIG_MULTI_INT, &act, 0)) + throw SysError("handling multiplexed interrupt"); #endif #ifdef __APPLE__ diff --git a/src/libutil/include/nix/util/signals.hh b/src/libutil/include/nix/util/signals.hh index 0062e8dcd7..c7b9d4e132 100644 --- a/src/libutil/include/nix/util/signals.hh +++ b/src/libutil/include/nix/util/signals.hh @@ -8,6 +8,13 @@ #include +#if defined(__FreeBSD__) +// SIGUSR1 is used by bdwgc +# define NIX_SIG_MULTI_INT SIGTSTP +#elif !defined(_WIN32) +# define NIX_SIG_MULTI_INT SIGUSR1 +#endif + namespace nix { /* User interruption. */ @@ -51,7 +58,7 @@ struct InterruptCallback std::unique_ptr createInterruptCallback(fun callback); /** - * A RAII class that causes the current thread to receive SIGUSR1 when + * A RAII class that causes the current thread to receive NIX_SIG_MULTI_INT when * the signal handler thread receives SIGINT. That is, this allows * SIGINT to be multiplexed to multiple threads. * diff --git a/src/libutil/unix/include/nix/util/signals-impl.hh b/src/libutil/unix/include/nix/util/signals-impl.hh index e51ce082a7..dab0b09ed2 100644 --- a/src/libutil/unix/include/nix/util/signals-impl.hh +++ b/src/libutil/unix/include/nix/util/signals-impl.hh @@ -94,7 +94,7 @@ inline void checkInterrupt() } /** - * A RAII class that causes the current thread to receive SIGUSR1 when + * A RAII class that causes the current thread to receive NIX_SIG_MULTI_INT when * the signal handler thread receives SIGINT. That is, this allows * SIGINT to be multiplexed to multiple threads. */ @@ -105,7 +105,7 @@ struct ReceiveInterrupts ReceiveInterrupts() : target(pthread_self()) - , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + , callback(createInterruptCallback([&]() { pthread_kill(target, NIX_SIG_MULTI_INT); })) { } }; From 82704a5bfeb5e4b65f8c0c38364045dbe24c5124 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 7 Mar 2026 00:42:08 +0300 Subject: [PATCH 048/927] Disable flake-regressions with failing dependency --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02a9fe146f..fc153fa537 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,7 +245,9 @@ jobs: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} - name: Run flake regressions tests - run: MAX_FLAKES=25 flake-regressions/eval-all.sh + run: | + touch flake-regressions/tests/{0utkarsh/nixconfig,HariAmoor-professional/nixos-config}/disabled + MAX_FLAKES=25 flake-regressions/eval-all.sh profile_build: needs: tests From 77d27d8c2030c8f69fb328c67e0cbc6f523d2b8d Mon Sep 17 00:00:00 2001 From: Audrey Dutcher Date: Sun, 18 Aug 2024 16:22:47 -0700 Subject: [PATCH 049/927] tests/functional/restricted: Don't use a process substitution The <() process substitution syntax doesn't work for this one testcase in bash for FreeBSD. The exact reason for this is unknown, possibly to do with pipe vs file vs fifo EOF behavior. The prior behavior was this test hanging forever, with no children of the bash process. Change-Id: I71822a4b9dea6059b34300568256c5b7848109ac (cherry picked from commit ae628d4af29a3b89340a91187a6e7d4d242e759d) --- tests/functional/restricted.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index c86abebee3..9c2b860a9e 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -6,7 +6,7 @@ clearStoreIfPossible nix-instantiate --restrict-eval --eval -E '1 + 2' (! nix-instantiate --eval --restrict-eval ./restricted.nix) -(! nix-instantiate --eval --restrict-eval <(echo '1 + 2')) +TMPFILE=$(mktemp); echo '1 + 2' >"$TMPFILE"; (! nix-instantiate --eval --restrict-eval "$TMPFILE"); rm "$TMPFILE" mkdir -p "$TEST_ROOT/nix" cp ./simple.nix "$TEST_ROOT/nix" From 14359a1f5be1fe80e59a9ef97f16a08faa4e48c5 Mon Sep 17 00:00:00 2001 From: Audrey Dutcher Date: Wed, 31 Jan 2024 20:12:40 -0700 Subject: [PATCH 050/927] Add support for sandboxing on FreeBSD New FreeBSD sandboxes are based on jails and chroots. They provide fairly similar capabilities to sandboxes on Linux and allow for pure builds of FreeBSD nixpkgs. Although it would also be possble to use jails for Linux emulation, that is not supported with this commit. Change-Id: I619e1e34c56de7aaa64a38408210a410bb13adba Co-Authored-By: Artemis Tosini Co-Authored-By: John Ericson --- src/libstore/meson.build | 6 + src/libstore/package.nix | 16 +- .../unix/build/chroot-derivation-builder.cc | 2 +- src/libstore/unix/build/derivation-builder.cc | 12 +- .../unix/build/freebsd-derivation-builder.cc | 483 ++++++++++++++++++ .../unix/build/linux-derivation-builder.cc | 3 + src/libutil/file-system.cc | 53 +- src/libutil/freebsd/freebsd-jail.cc | 19 + .../freebsd/include/nix/util/freebsd-jail.hh | 3 +- src/libutil/include/nix/util/file-system.hh | 52 +- 10 files changed, 550 insertions(+), 99 deletions(-) create mode 100644 src/libstore/unix/build/freebsd-derivation-builder.cc diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 9e52517d48..63eba31255 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -142,6 +142,12 @@ endif configdata_priv.set('HAVE_SECCOMP', seccomp.found().to_int()) deps_private += seccomp +if host_machine.system() == 'freebsd' + # libjail is not in freebsd.libc, but there's no discovery mechanism + libjail = declare_dependency(link_args : [ '-ljail' ]) + deps_other += libjail +endif + nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 1f7b56f223..4fa2abaa72 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -5,6 +5,7 @@ unixtools, darwin, + freebsd, nix-util, boost, @@ -17,6 +18,7 @@ cmake, # for resolving aws-crt-cpp dep busybox-sandbox-shell ? null, + pkgsStatic, # Configuration Options @@ -24,6 +26,15 @@ embeddedSandboxShell ? stdenv.hostPlatform.isStatic, + withSandboxShell ? stdenv.hostPlatform.isLinux || stdenv.hostPlatform.isFreeBSD, + sandboxShell ? + if stdenv.hostPlatform.isLinux then + "${busybox-sandbox-shell}/bin/busybox" + else if stdenv.hostPlatform.isFreeBSD then + "${pkgsStatic.bash}/bin/bash" + else + null, + withAWS ? # Default is this way because there have been issues building this dependency (lib.meta.availableOn stdenv.hostPlatform aws-c-common), @@ -67,6 +78,7 @@ mkMesonLibrary (finalAttrs: { sqlite ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp + ++ lib.optional stdenv.hostPlatform.isFreeBSD freebsd.libjail ++ lib.optional withAWS aws-crt-cpp; propagatedBuildInputs = [ @@ -79,8 +91,8 @@ mkMesonLibrary (finalAttrs: { (lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell) (lib.mesonEnable "s3-aws-auth" withAWS) ] - ++ lib.optionals stdenv.hostPlatform.isLinux [ - (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox") + ++ lib.optionals withSandboxShell [ + (lib.mesonOption "sandbox-shell" sandboxShell) ]; meta = { diff --git a/src/libstore/unix/build/chroot-derivation-builder.cc b/src/libstore/unix/build/chroot-derivation-builder.cc index d54aa4bbe2..4c3ed21d86 100644 --- a/src/libstore/unix/build/chroot-derivation-builder.cc +++ b/src/libstore/unix/build/chroot-derivation-builder.cc @@ -1,4 +1,4 @@ -#ifdef __linux__ +#if defined(__linux__) || defined(__FreeBSD__) namespace nix { diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index cd92370513..5fc4d05b48 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1324,9 +1324,6 @@ void DerivationBuilderImpl::runChild(RunChildArgs args) if (chdir(tmpDirInSandbox().c_str()) == -1) throw SysError("changing into %1%", PathFmt(tmpDir)); - /* Close all other file descriptors. */ - unix::closeExtraFDs(); - /* Disable core dumps by default. */ struct rlimit limit = {0, RLIM_INFINITY}; setrlimit(RLIMIT_CORE, &limit); @@ -2045,6 +2042,7 @@ StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path) // FIXME: do this properly #include "chroot-derivation-builder.cc" #include "linux-derivation-builder.cc" +#include "freebsd-derivation-builder.cc" #include "darwin-derivation-builder.cc" #include "external-derivation-builder.cc" @@ -2093,7 +2091,7 @@ std::unique_ptr makeDerivationBuild } if (store.storeDir != store.config->realStoreDir.get()) { -#ifdef __linux__ +#if defined(__linux__) || defined(__FreeBSD__) useSandbox = true; #else throw Error("building using a diverted store is not supported on this platform"); @@ -2123,6 +2121,12 @@ std::unique_ptr makeDerivationBuild new ChrootLinuxDerivationBuilder(store, std::move(miscMethods), std::move(params))); return DerivationBuilderUnique(new LinuxDerivationBuilder(store, std::move(miscMethods), std::move(params))); +#elif defined(__FreeBSD__) + if (useSandbox) + return DerivationBuilderUnique( + new ChrootFreeBSDDerivationBuilder(store, std::move(miscMethods), std::move(params))); + + return DerivationBuilderUnique(new FreeBSDDerivationBuilder(store, std::move(miscMethods), std::move(params))); #else if (useSandbox) throw Error("sandboxing builds is not supported on this platform"); diff --git a/src/libstore/unix/build/freebsd-derivation-builder.cc b/src/libstore/unix/build/freebsd-derivation-builder.cc new file mode 100644 index 0000000000..3a42f23072 --- /dev/null +++ b/src/libstore/unix/build/freebsd-derivation-builder.cc @@ -0,0 +1,483 @@ +#ifdef __FreeBSD__ + +# include +# include + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include "nix/util/freebsd-jail.hh" +# include "nix/util/util.hh" + +namespace nix { + +namespace { + +struct PasswordEntry +{ + std::string name; + uid_t uid; + gid_t gid; + std::string description; + std::filesystem::path home; + std::filesystem::path shell; +}; + +static void free_db(DB * db) +{ + if (db != nullptr) { + (db->close)(db); + } +} + +// Database open flags from FreeBSD, in case they're necessary for compatibility +static const HASHINFO db_flags = { + .bsize = 4096, + .ffactor = 32, + .nelem = 256, + .cachesize = 2 * 1024 * 1024, + .hash = nullptr, + .lorder = BIG_ENDIAN, +}; + +// Password database version +// Version 4 has been current since 2003 +static const uint8_t dbVersion = 4; + +static void serializeString(std::vector & buf, std::string const & str) +{ + buf.reserve(buf.size() + str.size() + 1); + buf.insert(buf.end(), str.begin(), str.end()); + buf.push_back(0); +} + +static void serializeInt(std::vector & buf, uint32_t num) +{ + buf.reserve(buf.size() + sizeof(num)); + // Always big endian + buf.push_back((num >> 24) & 0xff); + buf.push_back((num >> 16) & 0xff); + buf.push_back((num >> 8) & 0xff); + buf.push_back((num >> 0) & 0xff); +} + +static std::vector byNameKey(std::string const & name) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYNAME, dbVersion)}; + buf.reserve(1 + name.size()); + // We can't use serializeString since that's null terimated + buf.insert(buf.end(), name.begin(), name.end()); + + return buf; +} + +static std::vector byNumKey(uint32_t num) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYNUM, dbVersion)}; + serializeInt(buf, num); + + return buf; +} + +static std::vector byUidKey(uid_t uid) +{ + std::vector buf{_PW_VERSIONED(_PW_KEYBYUID, dbVersion)}; + serializeInt(buf, uid); + + return buf; +} + +static void createPasswordFiles(std::filesystem::path & chrootRootDir, std::vector & users) +{ + std::unique_ptr db( + dbopen((chrootRootDir + "/etc/pwd.db").c_str(), O_CREAT | O_RDWR | O_EXCL, 0644, DB_HASH, &db_flags), &free_db); + + if (db == nullptr) { + throw SysError("Could not create password database"); + } + + auto dbInsert = [&db](std::vector key_buf, std::vector & value_buf) { + DBT key = {key_buf.data(), key_buf.size()}; + DBT value = {value_buf.data(), value_buf.size()}; + + if ((db->put)(db.get(), &key, &value, R_NOOVERWRITE) == -1) { + throw SysError("Could not write to password database"); + } + }; + + // Annoyingly DBT doesn't have const pointers so we need this whole shuffle + std::string versionKeyStr(_PWD_VERSION_KEY); + std::vector versionKey(versionKeyStr.begin(), versionKeyStr.end()); + std::vector versionValue{dbVersion}; + dbInsert(versionKey, versionValue); + + for (size_t i = 0; i < users.size(); i++) { + auto user = users[i]; + + // flags for non-empty fields + uint32_t fields = _PWF_NAME | _PWF_PASSWD | _PWF_UID | _PWF_GID | _PWF_GECOS | _PWF_DIR | _PWF_SHELL; + + std::vector buf; + serializeString(buf, user.name); + // pw_password is always "*" in the insecure database + serializeString(buf, std::string("*")); + serializeInt(buf, user.uid); + serializeInt(buf, user.gid); + // pw_change = 0 means no requirement to change password + serializeInt(buf, 0); + // pw_class is empty since we don't make a class database + serializeString(buf, std::string("")); + serializeString(buf, user.description); + serializeString(buf, user.home); + serializeString(buf, user.shell); + // pw_expire = 0 means password does not expire + serializeInt(buf, 0); + serializeInt(buf, fields); + + dbInsert(byNameKey(user.name), buf); + // _PW_KEYBYNUM is 1-indexed + dbInsert(byNumKey(i + 1), buf); + dbInsert(byUidKey(user.uid), buf); + } + + // FreeBSD libc doesn't use /etc/passwd, but some software might + std::string passwdContent = ""; + for (auto user : users) { + passwdContent.append( + fmt("%s:*:%d:%d:%s:%s:%s\n", + user.name, + user.uid, + user.gid, + user.description, + PathFmt(user.home), + PathFmt(user.shell))); + } + + writeFile(chrootRootDir + "/etc/passwd", passwdContent); + + // No need to make /etc/master.passwd or /etc/spwd.db, + // our build user wouldn't be able to read them anyway +} + +} // namespace + +struct FreeBSDDerivationBuilder : virtual DerivationBuilderImpl +{ + using DerivationBuilderImpl::DerivationBuilderImpl; +}; + +template +struct iovec iovFromStaticSizedString(const char (&array)[n]) +{ + return { + .iov_base = const_cast(static_cast(&array[0])), + .iov_len = n, + }; +} + +struct iovec iovFromDynamicSizeString(const std::string & s) +{ + return { + .iov_base = const_cast(static_cast(s.c_str())), + .iov_len = s.length() + 1, + }; +} + +struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivationBuilder +{ + std::shared_ptr autoDelJail = std::make_shared(); + + ChrootFreeBSDDerivationBuilder( + LocalStore & store, std::unique_ptr miscMethods, DerivationBuilderParams params) + : DerivationBuilderImpl{store, std::move(miscMethods), std::move(params)} + , ChrootDerivationBuilder{store, std::move(miscMethods), std::move(params)} + , FreeBSDDerivationBuilder{store, std::move(miscMethods), std::move(params)} + { + } + + virtual void cleanupBuild(bool force) override + { + autoDelJail->remove(); + ChrootDerivationBuilder::cleanupBuild(force); + } + + void prepareSandbox() override + { + ChrootDerivationBuilder::prepareSandbox(); + + std::vector users{ + { + .name = "root", + .uid = 0, + .gid = 0, + .description = "Nix build user", + .home = store.config->getLocalSettings().sandboxBuildDir, + .shell = "/noshell", + }, + { + .name = "nixbld", + .uid = buildUser->getUID(), + .gid = sandboxGid(), + .description = "Nix build user", + .home = store.config->getLocalSettings().sandboxBuildDir, + .shell = "/noshell", + }, + { + .name = "nobody", + .uid = 65534, + .gid = 65534, + .description = "Nobody", + .home = "/", + .shell = "/noshell", + }, + }; + + createPasswordFiles(chrootRootDir, users); + + // FreeBSD doesn't have a group database, just write a text file + writeFile( + chrootRootDir + "/etc/group", + fmt("root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n", + sandboxGid())); + + // Linux waits until after entering the child to start mounting so it doesn't + // pollute the root mount namespace. + // FreeBSD doesn't have mount namespaces, so there's no reason to wait. + + auto devpath = chrootRootDir + "/dev"; + mkdir(devpath.c_str(), 0555); + mkdir((chrootRootDir + "/bin").c_str(), 0555); + char errmsg[255] = ""; + struct iovec iov[8] = { + iovFromStaticSizedString("fstype"), + iovFromStaticSizedString("devfs"), + iovFromStaticSizedString("fspath"), + iovFromDynamicSizeString(devpath), + iovFromStaticSizedString("ruleset"), + iovFromStaticSizedString("4"), + iovFromStaticSizedString("errmsg"), + iovFromStaticSizedString(errmsg), + }; + if (nmount(iov, 6, 0) < 0) { + throw SysError("Failed to mount jail /dev: %1%", PathFmt(errmsg)); + } + autoDelJail->childrenMounts.emplace_back(devpath); + + for (auto & i : pathsInChroot) { + char errmsg[255]; + errmsg[0] = 0; + + if (i.second.source == "/proc") { + continue; // backwards compatibility + } + std::filesystem::path path = chrootRootDir + i.first.c_str(); + + struct stat stat_buf; + if (stat(i.second.source.c_str(), &stat_buf) < 0) { + throw SysError("stat"); + } + + // mount points must exist and be the right type + if (S_ISDIR(stat_buf.st_mode)) { + createDirs(path); + } else { + createDirs(path.parent_path()); + writeFile(path, ""); + } + + struct iovec iov[8] = { + iovFromStaticSizedString("fstype"), + iovFromStaticSizedString("nullfs"), + iovFromStaticSizedString("fspath"), + iovFromDynamicSizeString(path), + iovFromStaticSizedString("target"), + iovFromDynamicSizeString(i.second.source), + iovFromStaticSizedString("errmsg"), + iovFromStaticSizedString(errmsg), + }; + if (nmount(iov, 8, 0) < 0) { + throw SysError("Failed to mount nullfs for %1% - %2%", PathFmt(path), errmsg); + } + autoDelJail->childrenMounts.emplace_back(path); + } + + /* Fixed-output derivations typically need to access the + network, so give them access to /etc/resolv.conf and so + on. */ + if (!derivationType.isSandboxed()) { + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed-outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + + /* N.B. it is realistic that these paths might not exist. It + happens when testing Nix building fixed-output derivations + within a pure derivation. */ + for (std::filesystem::path path : {"/etc/resolv.conf", "/etc/services", "/etc/hosts"}) { + if (pathExists(path)) { + // This means if your network config changes during a FOD build, + // the DNS in the sandbox will be wrong. However, this is pretty unlikely + // to actually be a problem, because FODs are generally pretty fast, + // and machines with often-changing network configurations probably + // want to run resolved or some other local resolver anyway. + // + // There's also just no simple way to do this correctly, you have to manually + // inotify watch the files for changes on the outside and update the sandbox + // while the build is running (or at least that's what Flatpak does). + // + // I also just generally feel icky about modifying sandbox state under a build, + // even though it really shouldn't be a big deal. -K900 + copyFile(path, chrootRootDir / path.relative_path(), false, true); + } + } + + if (fileTransferSettings.caFile.get() && pathExists(fileTransferSettings.caFile.get().value())) { + // For the same reasons as above, copy the CA certificates file too. + // It should be even less likely to change during the build than resolv.conf. + createDirs(chrootRootDir + "/etc/ssl/certs"); + copyFile( + fileTransferSettings.caFile.get().value(), + chrootRootDir + "/etc/ssl/certs/ca-certificates.crt", + false, + true); + } + } + } + + void startChild() override + { + int jid; + + RunChildArgs args{ +# if NIX_WITH_AWS_AUTH + .awsCredentials = preResolveAwsCredentials(), +# endif + }; + + if (derivationType.isSandboxed()) { + jid = jail_setv( + JAIL_CREATE, + "persist", + "true", + "path", + chrootRootDir.c_str(), + "host.hostname", + "localhost", + // TODO: Make our own ruleset + "vnet", + "new", + NULL); + if (jid < 0) { + throw SysError("Failed to create jail (isolated network): %1%", jail_errmsg); + } + autoDelJail->jid = jid; + + // Everything from here to the end of the block is setting up the network + // code adapted from freebsd/sbin/ifconfig/af_inet.c, in_exec_nl + Pid helper = startProcess([&]() { + unix::closeExtraFDs(); + enterChroot(); + + struct snl_state ss = {}; + if (!snl_init(&ss, NETLINK_ROUTE)) { + throw SysError("Failed to init netlink connection"); + } + + struct snl_writer nw = {}; + snl_init_writer(&ss, &nw); + struct nlmsghdr * hdr = snl_create_msg_request(&nw, NL_RTM_NEWADDR); + struct ifaddrmsg * ifahdr = snl_reserve_msg_object(&nw, struct ifaddrmsg); + + ifahdr->ifa_family = AF_INET; + ifahdr->ifa_prefixlen = 8; + ifahdr->ifa_index = if_nametoindex("lo0"); + snl_add_msg_attr_ip4(&nw, IFA_LOCAL, (const struct in_addr *) "\x7f\x00\x00\x01"); + + int off = snl_add_msg_attr_nested(&nw, IFA_FREEBSD); + snl_add_msg_attr_u32(&nw, IFAF_FLAGS, IFF_LOOPBACK | IFF_UP); + snl_end_attr_nested(&nw, off); + + if (!(hdr = snl_finalize_msg(&nw)) || !snl_send_message(&ss, hdr)) { + snl_free(&ss); + throw SysError("Failed to sendoff netlink message"); + } + + struct snl_errmsg_data e = {}; + snl_read_reply_code(&ss, hdr->nlmsg_seq, &e); + if (e.error_str != NULL) { + snl_free(&ss); + throw SysError("Failed to configure loopback interface: %1%", e.error_str); + } + snl_free(&ss); + _exit(0); + }); + + if (helper.wait() != 0) { + throw SysError("Failed to configure loopback address"); + } + } else { + jid = jail_setv( + JAIL_CREATE, + "persist", + "true", + // 4 is the most restrictive devfs ruleset that meets our needs + // which is found in the default installation. Trying to add + // another one is a huge pain... + "devfs_ruleset", + "4", + "path", + chrootRootDir.c_str(), + "host.hostname", + "localhost", + "ip4", + "inherit", + "ip6", + "inherit", + "allow.raw_sockets", + "true", + NULL); + if (jid < 0) { + throw SysError("Failed to create jail (networked): %1%", jail_errmsg); + } + autoDelJail->jid = jid; + } + + pid = startProcess([&]() { + openSlave(); + runChild(args); + }); + } + + void enterChroot() override + { + /* Close all other file descriptors. This must happen before + jail_attach for FreeBSD. */ + unix::closeExtraFDs(); + + if (jail_attach(autoDelJail->jid) < 0) { + throw SysError("Failed to attach to jail"); + } + } + + void addDependency(const StorePath & path) + { + auto [source, target] = ChrootDerivationBuilder::addDependencyPrep(path); + throw Error("Unimplemented"); + } +}; + +} // namespace nix + +#endif diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index 0ffcf0ccb6..6af4a00280 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -171,6 +171,9 @@ struct LinuxDerivationBuilder : virtual DerivationBuilderImpl .system = drv.platform, .impersonateLinux26 = localSettings.impersonateLinux26, }); + + /* Close all other file descriptors. */ + unix::closeExtraFDs(); } }; diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index f56000ebee..d2983ec7d3 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -440,47 +440,6 @@ void AutoDelete::cancel() noexcept del = false; } -////////////////////////////////////////////////////////////////////// - -#ifdef __FreeBSD__ -AutoUnmount::AutoUnmount() - : del{false} -{ -} - -AutoUnmount::AutoUnmount(const std::filesystem::path & p) - : path(p) - , del(true) -{ -} - -AutoUnmount::~AutoUnmount() -{ - try { - unmount(); - } catch (...) { - ignoreExceptionInDestructor(); - } -} - -void AutoUnmount::cancel() noexcept -{ - del = false; -} - -void AutoUnmount::unmount() -{ - if (del) { - if (::unmount(path.c_str(), 0) < 0) { - throw SysError("Failed to unmount path %1%", PathFmt(path)); - } - } - cancel(); -} -#endif - -////////////////////////////////////////////////////////////////////// - std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const std::string & prefix, mode_t mode) { while (1) { @@ -620,7 +579,7 @@ void setWriteTime(const std::filesystem::path & path, const PosixStat & st) setWriteTime(path, st.st_atime, st.st_mtime, S_ISLNK(st.st_mode)); } -void copyFile(const std::filesystem::path & from, const std::filesystem::path & to, bool andDelete) +void copyFile(const std::filesystem::path & from, const std::filesystem::path & to, bool andDelete, bool contents) { auto fromStatus = std::filesystem::symlink_status(from); @@ -633,8 +592,14 @@ void copyFile(const std::filesystem::path & from, const std::filesystem::path & } if (std::filesystem::is_symlink(fromStatus) || std::filesystem::is_regular_file(fromStatus)) { - std::filesystem::copy( - from, to, std::filesystem::copy_options::copy_symlinks | std::filesystem::copy_options::overwrite_existing); + if (contents) { + std::filesystem::copy_file(from, to, std::filesystem::copy_options::overwrite_existing); + } else { + std::filesystem::copy( + from, + to, + std::filesystem::copy_options::copy_symlinks | std::filesystem::copy_options::overwrite_existing); + } } else if (std::filesystem::is_directory(fromStatus)) { std::filesystem::create_directory(to); for (auto & entry : DirectoryIterator(from)) { diff --git a/src/libutil/freebsd/freebsd-jail.cc b/src/libutil/freebsd/freebsd-jail.cc index 1961c9a13f..9273a50b6e 100644 --- a/src/libutil/freebsd/freebsd-jail.cc +++ b/src/libutil/freebsd/freebsd-jail.cc @@ -8,6 +8,7 @@ # include "nix/util/error.hh" # include "nix/util/util.hh" +# include "nix/util/logging.hh" namespace nix { @@ -24,6 +25,24 @@ void AutoRemoveJail::remove() } } cancel(); + + bool failed = false; + for (auto & path : childrenMounts) { + int r = unmount(path.c_str(), 0); + if (r < 0 && errno == EBUSY) { + sleep(1); + r = unmount(path.c_str(), 0); + } + if (r < 0) { + warn("Failed to unmount path %1%", PathFmt(path)); + failed = true; + } + } + childrenMounts.clear(); + + if (failed) { + throw SysError("Failed to unmount some jail paths"); + } } AutoRemoveJail::~AutoRemoveJail() diff --git a/src/libutil/freebsd/include/nix/util/freebsd-jail.hh b/src/libutil/freebsd/include/nix/util/freebsd-jail.hh index b4b1cc75a0..aad51f856a 100644 --- a/src/libutil/freebsd/include/nix/util/freebsd-jail.hh +++ b/src/libutil/freebsd/include/nix/util/freebsd-jail.hh @@ -8,8 +8,9 @@ namespace nix { class AutoRemoveJail { static constexpr int INVALID_JAIL = -1; - int jid = INVALID_JAIL; public: + int jid = INVALID_JAIL; + std::vector childrenMounts; AutoRemoveJail() = default; AutoRemoveJail(int jid); AutoRemoveJail(const AutoRemoveJail &) = delete; diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 5dce929077..2601b5fdaa 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -353,8 +353,12 @@ void moveFile(const std::filesystem::path & src, const std::filesystem::path & d * `true`, then also remove `oldPath` (making this equivalent to `moveFile`, but * with the guaranty that the destination will be “fresh”, with no stale inode * or file descriptor pointing to it). + * + * If contents is set, always create a regular file, even if the source is a + * link. */ -void copyFile(const std::filesystem::path & from, const std::filesystem::path & to, bool andDelete); +void copyFile( + const std::filesystem::path & from, const std::filesystem::path & to, bool andDelete, bool contents = false); /** * Automatic cleanup of resources. @@ -598,50 +602,4 @@ private: std::filesystem::directory_iterator it_; }; -#ifdef __FreeBSD__ -class AutoUnmount -{ - std::filesystem::path path; - bool del; -public: - AutoUnmount(); - AutoUnmount(const std::filesystem::path &); - AutoUnmount(const AutoUnmount &) = delete; - - AutoUnmount(AutoUnmount && other) noexcept - : path(std::move(other.path)) - , del(std::exchange(other.del, false)) - { - } - - AutoUnmount & operator=(AutoUnmount && other) noexcept - { - swap(*this, other); - return *this; - } - - friend void swap(AutoUnmount & lhs, AutoUnmount & rhs) noexcept - { - using std::swap; - swap(lhs.path, rhs.path); - swap(lhs.del, rhs.del); - } - - ~AutoUnmount(); - - /** - * Cancel the unmounting - */ - void cancel() noexcept; - - /** - * Unmount the mountpoint right away (if it exists), resetting the - * `AutoUnmount` - * - * The destructor calls this, but ignoring any exception. - */ - void unmount(); -}; -#endif - } // namespace nix From c608b07746f1210886f747fd56619a6aa29cb957 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 6 Mar 2026 19:40:57 +0300 Subject: [PATCH 051/927] libutil: Make ChunkedVector thread-safe --- src/libexpr/include/nix/expr/symbol-table.hh | 9 +- src/libutil-tests/chunked-vector.cc | 61 ++++++++-- .../include/nix/util/chunked-vector.hh | 112 ++++++++++++++---- 3 files changed, 141 insertions(+), 41 deletions(-) diff --git a/src/libexpr/include/nix/expr/symbol-table.hh b/src/libexpr/include/nix/expr/symbol-table.hh index f0220376c5..dab0952870 100644 --- a/src/libexpr/include/nix/expr/symbol-table.hh +++ b/src/libexpr/include/nix/expr/symbol-table.hh @@ -84,8 +84,11 @@ class SymbolStr { friend class SymbolTable; - constexpr static size_t chunkSize{8192}; - using SymbolValueStore = ChunkedVector; + constexpr static size_t chunkSize{65536}; + using SymbolValueStore = ChunkedVector< + SymbolValue, + chunkSize, + /*MaxChunks=*/std::numeric_limits::max() / chunkSize>; const SymbolValue * s; @@ -265,7 +268,7 @@ private: * During its lifetime the monotonic buffer holds all strings and nodes, if the symbol set is node based. */ std::pmr::monotonic_buffer_resource buffer; - SymbolStr::SymbolValueStore store{16}; + SymbolStr::SymbolValueStore store; /** * Transparent lookup of string view for a pointer to a ChunkedVector entry -> return offset into the store. diff --git a/src/libutil-tests/chunked-vector.cc b/src/libutil-tests/chunked-vector.cc index 52f87a0d5f..da87d7e357 100644 --- a/src/libutil-tests/chunked-vector.cc +++ b/src/libutil-tests/chunked-vector.cc @@ -2,16 +2,20 @@ #include +#include +#include +#include + namespace nix { TEST(ChunkedVector, InitEmpty) { - auto v = ChunkedVector(100); + auto v = ChunkedVector(); ASSERT_EQ(v.size(), 0u); } TEST(ChunkedVector, GrowsCorrectly) { - auto v = ChunkedVector(100); + auto v = ChunkedVector(); for (uint32_t i = 1; i < 20; i++) { v.add(i); ASSERT_EQ(v.size(), i); @@ -20,7 +24,7 @@ TEST(ChunkedVector, GrowsCorrectly) TEST(ChunkedVector, AddAndGet) { - auto v = ChunkedVector(100); + auto v = ChunkedVector(); for (auto i = 1; i < 20; i++) { auto [i2, idx] = v.add(i); auto & i3 = v[idx]; @@ -31,7 +35,7 @@ TEST(ChunkedVector, AddAndGet) TEST(ChunkedVector, ForEach) { - auto v = ChunkedVector(100); + auto v = ChunkedVector(); for (auto i = 1; i < 20; i++) { v.add(i); } @@ -40,17 +44,48 @@ TEST(ChunkedVector, ForEach) ASSERT_EQ(count, v.size()); } -TEST(ChunkedVector, OverflowOK) +TEST(ChunkedVector, NonTrivialType) { - // Similar to the AddAndGet, but intentionnally use a small - // initial ChunkedVector to force it to overflow - auto v = ChunkedVector(2); - for (auto i = 1; i < 20; i++) { - auto [i2, idx] = v.add(i); - auto & i3 = v[idx]; - ASSERT_EQ(i, i2); - ASSERT_EQ(&i2, &i3); + auto v = ChunkedVector, 2, 10>(); + v.add(std::vector{1, 2}); + v.add(std::vector{3, 4}); + v.add(std::vector{5, 6}); + ASSERT_EQ(v.size(), 3); +} + +TEST(ChunkedVector, ConcurrentAdd) +{ + ChunkedVector v; + constexpr int nThreads = 8; + constexpr int perThread = 8192; + + std::vector threads; + std::vector> indices(nThreads); + + for (int t = 0; t < nThreads; t++) { + threads.emplace_back([&, t]() { + for (int i = 0; i < perThread; ++i) { + auto [val, idx] = v.add(static_cast(t * perThread + i)); + indices[t].push_back(idx); + } + }); } + + for (auto & t : threads) + t.join(); + + ASSERT_EQ(v.size(), nThreads * perThread); + + // All indices unique + std::set all; + for (auto & vec : indices) + for (auto idx : vec) + ASSERT_TRUE(all.insert(idx).second); + + // All values correct + for (int t = 0; t < nThreads; t++) + for (int i = 0; i < perThread; i++) + ASSERT_EQ(v[indices[t][i]], static_cast(t * perThread + i)); } } // namespace nix diff --git a/src/libutil/include/nix/util/chunked-vector.hh b/src/libutil/include/nix/util/chunked-vector.hh index 38e53c7f54..f3452139f4 100644 --- a/src/libutil/include/nix/util/chunked-vector.hh +++ b/src/libutil/include/nix/util/chunked-vector.hh @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include "nix/util/error.hh" @@ -18,50 +20,101 @@ namespace nix { * on large data sets by on average (growth factor)/2, mostly * eliminates copies within the vector during resizing, and provides stable * references to its elements. + * + * Thread-safe to append and read concurrently. Caps the maximum number of chunks + * to MaxChunks for ease of implementation. */ -template +template class ChunkedVector { + static_assert(alignof(T) <= __STDCPP_DEFAULT_NEW_ALIGNMENT__, "ChunkedVector does not support over-aligned types"); + private: - uint32_t size_ = 0; - std::vector> chunks; + /* Put on a separate cache line to avoid false sharing. */ + alignas(std::hardware_destructive_interference_size) std::atomic size_ = 0; + + using ChunksArray = std::array, MaxChunks>; + + std::unique_ptr chunks = std::make_unique(); + + [[gnu::noinline]] + T * allocChunk() + { + return static_cast(::operator new(ChunkSize * sizeof(T))); + } /** * Keep this out of the ::add hot path */ [[gnu::noinline]] - auto & addChunk() + T * ensureChunk(size_t chunkIdx) { - if (size_ >= std::numeric_limits::max() - ChunkSize) + if (chunkIdx >= MaxChunks) unreachable(); - chunks.emplace_back(); - chunks.back().reserve(ChunkSize); - return chunks.back(); + + /* Want synchronises-with relation with the CAS. */ + auto * p = (*chunks)[chunkIdx].load(std::memory_order_acquire); + if (p) [[likely]] + return p; + + auto * newChunk = allocChunk(); + T * expected = nullptr; + /* Try to allocate the chunk ourselves. If CAS fails then somebody beat + us to it. Release semantics for the winner and acquire semantics for + the loser. */ + if ((*chunks)[chunkIdx].compare_exchange_strong( + expected, newChunk, /*success=*/std::memory_order_release, /*failure=*/std::memory_order_acquire)) + return newChunk; /* We succeeded. */ + + /* Somebody beat us to it. */ + ::operator delete(newChunk); + assert(expected); + return expected; + } + + std::size_t numActiveChunks() const noexcept + { + return (size_.load(std::memory_order_relaxed) + ChunkSize - 1) / ChunkSize; } public: - ChunkedVector(uint32_t reserve) + ChunkedVector() = default; + + ChunkedVector(ChunkedVector &&) = delete; + ChunkedVector(const ChunkedVector &) = delete; + ChunkedVector & operator=(ChunkedVector &&) = delete; + ChunkedVector & operator=(const ChunkedVector &) = delete; + + ~ChunkedVector() { - chunks.reserve(reserve); - addChunk(); + auto len = size(); + for (auto & chunkPtr : std::ranges::views::take(*chunks, numActiveChunks())) { + auto * chunk = chunkPtr.load(std::memory_order_relaxed); + auto count = std::min(len, ChunkSize); + if constexpr (!std::is_trivially_destructible_v) { + std::destroy_n(chunk, count); + } + ::operator delete(chunk); + len -= count; + } } + /** + * Get the current size. + */ uint32_t size() const noexcept { - return size_; + return size_.load(std::memory_order_relaxed); } template std::pair add(Args &&... args) { - const auto idx = size_++; - auto & chunk = [&]() -> auto & { - if (auto & back = chunks.back(); back.size() < ChunkSize) - return back; - return addChunk(); - }(); - auto & result = chunk.emplace_back(std::forward(args)...); - return {result, idx}; + /* Get some unique index. Doesn't need any synchronises-with relation. */ + const auto idx = size_.fetch_add(1, std::memory_order_relaxed); + auto * chunk = ensureChunk(idx / ChunkSize); + auto * p = new (&chunk[idx % ChunkSize]) T(std::forward(args)...); + return {*p, idx}; } /** @@ -69,17 +122,26 @@ public: * @pre add must have been called at least idx + 1 times. * @throws nothing */ - const T & operator[](uint32_t idx) const noexcept + const T & operator[](uint32_t idx) const & noexcept { - return chunks[idx / ChunkSize][idx % ChunkSize]; + /* Synchronises-with relation with add(). */ + auto * chunk = (*chunks)[idx / ChunkSize].load(std::memory_order_acquire); + return chunk[idx % ChunkSize]; } + /** + * Iterate over all elements. Do not use when there might be concurrent writers. + */ template void forEach(Fn fn) const { - for (const auto & c : chunks) - for (const auto & e : c) - fn(e); + auto len = size(); + for (auto & chunkPtr : std::ranges::views::take(*chunks, numActiveChunks())) { + auto * chunk = chunkPtr.load(std::memory_order_relaxed); + auto count = std::min(len, ChunkSize); + std::for_each_n(chunk, count, fn); + len -= count; + } } }; } // namespace nix From 5fbf4f90e84a21ae0a8dd081c4f4c4649d6c73bc Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 6 Mar 2026 19:41:07 +0300 Subject: [PATCH 052/927] packaging: Add build with thread sanitizer If we are going to go all-in on threading then we might as well utilise all the tooling we have to get it right. I'm not super confident in acquire/release semantics in the previous commit, but TSan would barf if I had gotten it wrong. Only enabled on 64 bit linux systems - i686 doesn't support it. If warranted, we could probably enable it on darwin too, but I haven't tried. --- packaging/components.nix | 12 +++++++++++- packaging/hydra.nix | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/packaging/components.nix b/packaging/components.nix index b74484b6e9..72707a094b 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -209,8 +209,13 @@ let enableSanitizersLayer = finalAttrs: prevAttrs: let - sanitizers = lib.optional scope.withASan "address" ++ lib.optional scope.withUBSan "undefined"; + sanitizers = + lib.optional scope.withASan "address" + ++ lib.optional scope.withUBSan "undefined" + ++ lib.optional scope.withTSan "thread"; in + # Thread sanitizer can't be used with ASan or UBSan + assert scope.withTSan -> !(scope.withASan || scope.withUBSan); { mesonFlags = (prevAttrs.mesonFlags or [ ]) @@ -277,6 +282,11 @@ in */ withUBSan = false; + /** + Whether meson components are built with [ThreadSanitizer](https://clang.llvm.org/docs/ThreadSanitizer.html). + */ + withTSan = false; + /** A user-provided extension function to apply to each component derivation. */ diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 9b9b33d679..ce50edd1ea 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -187,6 +187,26 @@ rec { ) (forAllSystems (system: components.${system}.${pkgName})) ); + # Separate build because one cannot mix ASan + UBSan with TSan. + buildWithTSan = + let + components = + system: + let + pkgs = nixpkgsFor.${system}.native; + in + pkgs.nixComponents2.overrideScope ( + self: super: { + withTSan = true; + # Dies at startup. + nix-perl-bindings = null; + # TSan has issues with fork and threads. + nix-functional-tests = super.nix-functional-tests.overrideAttrs { doCheck = false; }; + } + ); + in + forAllPackages (pkgName: lib.genAttrs linux64BitSystems (system: (components system).${pkgName})); + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-cli); # Toggles some settings for better coverage. Windows needs these From f7dd04d1002a0c309d682e06de62633321658614 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 7 Mar 2026 21:43:02 +0300 Subject: [PATCH 053/927] libstore: Fix issues with FreeBSD derivation builder Various cleanup and bugfixes on the PR that was merged in quite a rough state. In the order of significance. 1. Stop following symlinks when mounting things in the jail. This was busted. 2. Restore missing unix::closeExtraFDs() for darwin/unsandboxed freebsd builder by putting it back in runChild. This has silently regressed a lot of derivation builders like the Darwin/Linux ones which is a non-starter. 3. Handle optional chrootPaths. 4. More helpful exception in addDependency. 5. Fix nmount call ignoring errmsg argument because it had a hardcoded count of 6 iovec. 6. Various code style and std::filesystem::path cleanups. --- src/libstore/unix/build/derivation-builder.cc | 3 + .../unix/build/freebsd-derivation-builder.cc | 164 +++++++++--------- .../unix/build/linux-derivation-builder.cc | 3 - 3 files changed, 89 insertions(+), 81 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 5fc4d05b48..f81e8a42fd 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1324,6 +1324,9 @@ void DerivationBuilderImpl::runChild(RunChildArgs args) if (chdir(tmpDirInSandbox().c_str()) == -1) throw SysError("changing into %1%", PathFmt(tmpDir)); + /* Close all other file descriptors. */ + unix::closeExtraFDs(); + /* Disable core dumps by default. */ struct rlimit limit = {0, RLIM_INFINITY}; setrlimit(RLIMIT_CORE, &limit); diff --git a/src/libstore/unix/build/freebsd-derivation-builder.cc b/src/libstore/unix/build/freebsd-derivation-builder.cc index 3a42f23072..c59e96e864 100644 --- a/src/libstore/unix/build/freebsd-derivation-builder.cc +++ b/src/libstore/unix/build/freebsd-derivation-builder.cc @@ -1,10 +1,12 @@ #ifdef __FreeBSD__ +# include "nix/util/freebsd-jail.hh" +# include "nix/util/util.hh" + # include # include # include -# include # include # include # include @@ -15,9 +17,6 @@ # include # include -# include "nix/util/freebsd-jail.hh" -# include "nix/util/util.hh" - namespace nix { namespace { @@ -32,15 +31,13 @@ struct PasswordEntry std::filesystem::path shell; }; -static void free_db(DB * db) -{ - if (db != nullptr) { - (db->close)(db); - } -} +using UniqueDB = std::unique_ptr<::DB, decltype([](::DB * db) { + if (db) + (db->close)(db); + })>; // Database open flags from FreeBSD, in case they're necessary for compatibility -static const HASHINFO db_flags = { +static constexpr HASHINFO dbFlags = { .bsize = 4096, .ffactor = 32, .nelem = 256, @@ -55,14 +52,12 @@ static const uint8_t dbVersion = 4; static void serializeString(std::vector & buf, std::string const & str) { - buf.reserve(buf.size() + str.size() + 1); buf.insert(buf.end(), str.begin(), str.end()); buf.push_back(0); } static void serializeInt(std::vector & buf, uint32_t num) { - buf.reserve(buf.size() + sizeof(num)); // Always big endian buf.push_back((num >> 24) & 0xff); buf.push_back((num >> 16) & 0xff); @@ -74,7 +69,7 @@ static std::vector byNameKey(std::string const & name) { std::vector buf{_PW_VERSIONED(_PW_KEYBYNAME, dbVersion)}; buf.reserve(1 + name.size()); - // We can't use serializeString since that's null terimated + // We can't use serializeString since that's null terminated buf.insert(buf.end(), name.begin(), name.end()); return buf; @@ -98,19 +93,18 @@ static std::vector byUidKey(uid_t uid) static void createPasswordFiles(std::filesystem::path & chrootRootDir, std::vector & users) { - std::unique_ptr db( - dbopen((chrootRootDir + "/etc/pwd.db").c_str(), O_CREAT | O_RDWR | O_EXCL, 0644, DB_HASH, &db_flags), &free_db); + auto db = + UniqueDB(::dbopen((chrootRootDir / "etc/pwd.db").c_str(), O_CREAT | O_RDWR | O_EXCL, 0644, DB_HASH, &dbFlags)); - if (db == nullptr) { - throw SysError("Could not create password database"); - } + if (!db) + throw SysError("could not create password database"); - auto dbInsert = [&db](std::vector key_buf, std::vector & value_buf) { - DBT key = {key_buf.data(), key_buf.size()}; - DBT value = {value_buf.data(), value_buf.size()}; + auto dbInsert = [&db](std::vector keyBuf, std::vector & valueBuf) { + DBT key = {keyBuf.data(), keyBuf.size()}; + DBT value = {valueBuf.data(), valueBuf.size()}; if ((db->put)(db.get(), &key, &value, R_NOOVERWRITE) == -1) { - throw SysError("Could not write to password database"); + throw SysError("could not write to password database"); } }; @@ -120,9 +114,7 @@ static void createPasswordFiles(std::filesystem::path & chrootRootDir, std::vect std::vector versionValue{dbVersion}; dbInsert(versionKey, versionValue); - for (size_t i = 0; i < users.size(); i++) { - auto user = users[i]; - + for (const auto & [i, user] : enumerate(users)) { // flags for non-empty fields uint32_t fields = _PWF_NAME | _PWF_PASSWD | _PWF_UID | _PWF_GID | _PWF_GECOS | _PWF_DIR | _PWF_SHELL; @@ -150,19 +142,19 @@ static void createPasswordFiles(std::filesystem::path & chrootRootDir, std::vect } // FreeBSD libc doesn't use /etc/passwd, but some software might - std::string passwdContent = ""; - for (auto user : users) { + std::string passwdContent; + for (const auto & user : users) { passwdContent.append( fmt("%s:*:%d:%d:%s:%s:%s\n", user.name, user.uid, user.gid, user.description, - PathFmt(user.home), - PathFmt(user.shell))); + user.home.native(), + user.shell.native())); } - writeFile(chrootRootDir + "/etc/passwd", passwdContent); + writeFile(chrootRootDir / "etc/passwd", passwdContent); // No need to make /etc/master.passwd or /etc/spwd.db, // our build user wouldn't be able to read them anyway @@ -175,12 +167,21 @@ struct FreeBSDDerivationBuilder : virtual DerivationBuilderImpl using DerivationBuilderImpl::DerivationBuilderImpl; }; -template -struct iovec iovFromStaticSizedString(const char (&array)[n]) +template +struct iovec iovFromMutableBuffer(std::array & array) { return { - .iov_base = const_cast(static_cast(&array[0])), - .iov_len = n, + .iov_base = static_cast(array.data()), + .iov_len = N, + }; +} + +template +struct iovec iovFromStaticSizedString(const char (&array)[N]) +{ + return { + .iov_base = const_cast(static_cast(array)), + .iov_len = N, }; } @@ -245,7 +246,7 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati // FreeBSD doesn't have a group database, just write a text file writeFile( - chrootRootDir + "/etc/group", + chrootRootDir / "etc/group", fmt("root:x:0:\n" "nixbld:!:%1%:\n" "nogroup:x:65534:\n", @@ -255,60 +256,66 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati // pollute the root mount namespace. // FreeBSD doesn't have mount namespaces, so there's no reason to wait. - auto devpath = chrootRootDir + "/dev"; - mkdir(devpath.c_str(), 0555); - mkdir((chrootRootDir + "/bin").c_str(), 0555); - char errmsg[255] = ""; - struct iovec iov[8] = { + auto devpath = chrootRootDir / "dev"; + createDir(devpath, 0555); + createDir(chrootRootDir / "bin", 0555); + + std::array errmsg{}; + std::array<::iovec, 8> iov = { iovFromStaticSizedString("fstype"), iovFromStaticSizedString("devfs"), iovFromStaticSizedString("fspath"), - iovFromDynamicSizeString(devpath), + iovFromDynamicSizeString(devpath.native()), iovFromStaticSizedString("ruleset"), iovFromStaticSizedString("4"), iovFromStaticSizedString("errmsg"), - iovFromStaticSizedString(errmsg), + iovFromMutableBuffer(errmsg), }; - if (nmount(iov, 6, 0) < 0) { - throw SysError("Failed to mount jail /dev: %1%", PathFmt(errmsg)); - } - autoDelJail->childrenMounts.emplace_back(devpath); - for (auto & i : pathsInChroot) { - char errmsg[255]; - errmsg[0] = 0; + if (nmount(iov.data(), iov.size(), 0) < 0) + throw SysError("failed to mount jail /dev: %1%", std::string_view(errmsg.data())); - if (i.second.source == "/proc") { - continue; // backwards compatibility - } - std::filesystem::path path = chrootRootDir + i.first.c_str(); + autoDelJail->childrenMounts.emplace_back(devpath); - struct stat stat_buf; - if (stat(i.second.source.c_str(), &stat_buf) < 0) { - throw SysError("stat"); + for (const auto & [target, chrootPath] : pathsInChroot) { + std::filesystem::path path = chrootRootDir / target.relative_path(); + + auto maybeSt = maybeLstat(chrootPath.source); + if (!maybeSt) { + if (chrootPath.optional) + continue; /* Skip mounting this path. */ + else + throw SysError("getting attributes of path %1%", PathFmt(chrootPath.source)); } - // mount points must exist and be the right type - if (S_ISDIR(stat_buf.st_mode)) { + /* Mount points must exist and be the right type. */ + if (S_ISDIR(maybeSt->st_mode)) { createDirs(path); + } else if (S_ISLNK(maybeSt->st_mode)) { + createDirs(path.parent_path()); + copyFile(chrootPath.source, path, /*andDelete=*/false, /*contents=*/false); + continue; } else { createDirs(path.parent_path()); writeFile(path, ""); } - struct iovec iov[8] = { + std::array<::iovec, 8> iov = { iovFromStaticSizedString("fstype"), iovFromStaticSizedString("nullfs"), iovFromStaticSizedString("fspath"), - iovFromDynamicSizeString(path), + iovFromDynamicSizeString(path.native()), iovFromStaticSizedString("target"), - iovFromDynamicSizeString(i.second.source), + iovFromDynamicSizeString(chrootPath.source.native()), iovFromStaticSizedString("errmsg"), - iovFromStaticSizedString(errmsg), + iovFromMutableBuffer(errmsg), }; - if (nmount(iov, 8, 0) < 0) { - throw SysError("Failed to mount nullfs for %1% - %2%", PathFmt(path), errmsg); - } + + debug("setting up a nullfs mount from %1% to %2%", PathFmt(chrootPath.source), PathFmt(path)); + + if (nmount(iov.data(), iov.size(), 0) < 0) + throw SysError("failed to mount nullfs for %1%: %2%", PathFmt(path), std::string_view(errmsg.data())); + autoDelJail->childrenMounts.emplace_back(path); } @@ -320,7 +327,7 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati // services. Don’t use it for anything else that may // be configured for this system. This limits the // potential impurities introduced in fixed-outputs. - writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + writeFile(chrootRootDir / "etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); /* N.B. it is realistic that these paths might not exist. It happens when testing Nix building fixed-output derivations @@ -346,10 +353,10 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati if (fileTransferSettings.caFile.get() && pathExists(fileTransferSettings.caFile.get().value())) { // For the same reasons as above, copy the CA certificates file too. // It should be even less likely to change during the build than resolv.conf. - createDirs(chrootRootDir + "/etc/ssl/certs"); + createDirs(chrootRootDir / "etc/ssl/certs"); copyFile( fileTransferSettings.caFile.get().value(), - chrootRootDir + "/etc/ssl/certs/ca-certificates.crt", + chrootRootDir / "etc/ssl/certs/ca-certificates.crt", false, true); } @@ -378,9 +385,9 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati // TODO: Make our own ruleset "vnet", "new", - NULL); + nullptr); if (jid < 0) { - throw SysError("Failed to create jail (isolated network): %1%", jail_errmsg); + throw SysError("failed to create jail (isolated network): %1%", jail_errmsg); } autoDelJail->jid = jid; @@ -411,21 +418,22 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati if (!(hdr = snl_finalize_msg(&nw)) || !snl_send_message(&ss, hdr)) { snl_free(&ss); - throw SysError("Failed to sendoff netlink message"); + throw SysError("failed to sendoff netlink message"); } struct snl_errmsg_data e = {}; snl_read_reply_code(&ss, hdr->nlmsg_seq, &e); if (e.error_str != NULL) { snl_free(&ss); - throw SysError("Failed to configure loopback interface: %1%", e.error_str); + throw SysError("failed to configure loopback interface: %1%", e.error_str); } snl_free(&ss); _exit(0); }); + /* TODO: Capture the error from the helper? */ if (helper.wait() != 0) { - throw SysError("Failed to configure loopback address"); + throw SysError("failed to configure loopback address"); } } else { jid = jail_setv( @@ -449,7 +457,7 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati "true", NULL); if (jid < 0) { - throw SysError("Failed to create jail (networked): %1%", jail_errmsg); + throw SysError("failed to create jail (networked): %1%", jail_errmsg); } autoDelJail->jid = jid; } @@ -467,14 +475,14 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati unix::closeExtraFDs(); if (jail_attach(autoDelJail->jid) < 0) { - throw SysError("Failed to attach to jail"); + throw SysError("failed to attach to jail"); } } void addDependency(const StorePath & path) { - auto [source, target] = ChrootDerivationBuilder::addDependencyPrep(path); - throw Error("Unimplemented"); + throw UnimplementedError( + "adding store path '%s' to the sandbox is not implemented (recursive-nix)", store.printStorePath(path)); } }; diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index 6af4a00280..0ffcf0ccb6 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -171,9 +171,6 @@ struct LinuxDerivationBuilder : virtual DerivationBuilderImpl .system = drv.platform, .impersonateLinux26 = localSettings.impersonateLinux26, }); - - /* Close all other file descriptors. */ - unix::closeExtraFDs(); } }; From c0dbe8e42cee61e6249664f74cedd62c6c6256f3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 7 Mar 2026 21:48:52 +0300 Subject: [PATCH 054/927] libstore: Mount store paths readonly, nosuid in sandboxed freebsd builders This isn't done on Linux (supposedly for back-compat reasons). But since the FreeBSD sandbox is a new feature we can try to get it right from the start. --- src/libstore/unix/build/freebsd-derivation-builder.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libstore/unix/build/freebsd-derivation-builder.cc b/src/libstore/unix/build/freebsd-derivation-builder.cc index c59e96e864..cf31d4457a 100644 --- a/src/libstore/unix/build/freebsd-derivation-builder.cc +++ b/src/libstore/unix/build/freebsd-derivation-builder.cc @@ -313,7 +313,15 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati debug("setting up a nullfs mount from %1% to %2%", PathFmt(chrootPath.source), PathFmt(path)); - if (nmount(iov.data(), iov.size(), 0) < 0) + int flags = 0; + if (store.isInStore(target.native())) + /* While we are at it, enforce invariants about store paths. Anything located at the "logical" store + location must be readonly (file permission canonicalisation enforces this on the host filesystem). + Also the store must never contain setuid binaries for the same reason. This is just defense-in-depth. + */ + flags = MNT_RDONLY | MNT_NOSUID; + + if (nmount(iov.data(), iov.size(), flags) < 0) throw SysError("failed to mount nullfs for %1%: %2%", PathFmt(path), std::string_view(errmsg.data())); autoDelJail->childrenMounts.emplace_back(path); From 42a735000d9e70f0b7c108992cd7635aa9e0fd46 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 7 Mar 2026 21:51:11 +0300 Subject: [PATCH 055/927] libstore/freebsd-derivation-builder: NULL -> nullptr Let's not fix legacy C things in C++. --- src/libstore/unix/build/freebsd-derivation-builder.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/freebsd-derivation-builder.cc b/src/libstore/unix/build/freebsd-derivation-builder.cc index cf31d4457a..073daac566 100644 --- a/src/libstore/unix/build/freebsd-derivation-builder.cc +++ b/src/libstore/unix/build/freebsd-derivation-builder.cc @@ -431,7 +431,7 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati struct snl_errmsg_data e = {}; snl_read_reply_code(&ss, hdr->nlmsg_seq, &e); - if (e.error_str != NULL) { + if (e.error_str != nullptr) { snl_free(&ss); throw SysError("failed to configure loopback interface: %1%", e.error_str); } @@ -463,7 +463,7 @@ struct ChrootFreeBSDDerivationBuilder : ChrootDerivationBuilder, FreeBSDDerivati "inherit", "allow.raw_sockets", "true", - NULL); + nullptr); if (jid < 0) { throw SysError("failed to create jail (networked): %1%", jail_errmsg); } From b523564eb3b5ec5070d5de82d19906a06726c9ea Mon Sep 17 00:00:00 2001 From: Luna Nova Date: Sat, 7 Mar 2026 19:09:27 -0800 Subject: [PATCH 056/927] libstore: handle root path in RemoteFSAccessor::maybeLstat RemoteFSAccessor::fetch() crashes when called with the root path. $ nix build nixpkgs#hello --store ssh-ng://host error: path '/nix/store/' is not in the Nix store Add path.isRoot() guard like LocalStoreAccessor::maybeLstat Fixes: eb643d034 ("`Store::getFSAccessor`: Do not include the store dir") --- src/libstore/remote-fs-accessor.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index fbb7ac2230..31356a506a 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -35,6 +35,8 @@ std::shared_ptr RemoteFSAccessor::accessObject(const StorePath & std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) { + if (path.isRoot()) + return Stat{.type = tDirectory}; auto res = fetch(path); return res.first->maybeLstat(res.second); } From f84a7d2a4d8a202e30481638fb360be88a12dea3 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 20 Feb 2026 19:28:55 -0500 Subject: [PATCH 057/927] libstore: use Windows known folders API for runtime path configuration On Windows the compile-time path constants (`NIX_CONF_DIR`, `NIX_STATE_DIR`, `NIX_LOG_DIR`) are meaningless because the install drive varies, so this resolves them at runtime via `FOLDERID_ProgramData` through the existing known-folders API. Co-authored-by: John Ericson --- src/libstore/globals.cc | 78 ++++++++++++++++++++++++++-------------- src/libstore/meson.build | 10 +++--- 2 files changed, 57 insertions(+), 31 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 2e262aec8a..ece19d8525 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -53,29 +53,23 @@ namespace nix { Nix daemon by setting the mode/ownership of the directory appropriately. (This wouldn't work on the socket itself since it must be deleted and recreated on startup.) */ -#define DEFAULT_SOCKET_PATH "daemon-socket/socket" +#define DEFAULT_SOCKET_PATH "daemon-socket" / "socket" -/** - * Helper to resolve the NIX_CONF_DIR at runtime on Windows. - * On Windows, NIX_CONF_DIR is not defined at compile time, so we determine - * the path at runtime using the Windows known folders API (FOLDERID_ProgramData). - * This allows Nix to work correctly regardless of which drive Windows is installed on. - */ -static std::filesystem::path resolveNixConfDir() -{ +LogFileSettings::LogFileSettings() + : nixLogDir(getEnvOsNonEmpty(OS_STR("NIX_LOG_DIR")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { #ifdef _WIN32 -# ifdef NIX_CONF_DIR - // On Windows, NIX_CONF_DIR should not be defined at compile time -# error "NIX_CONF_DIR should not be defined on Windows" +# ifdef NIX_LOG_DIR +# error "NIX_LOG_DIR should not be defined on Windows" # endif - return windows::known_folders::getProgramData() / "nix"; + return windows::known_folders::getProgramData() / "nix" / "log"; #else - return NIX_CONF_DIR; + return NIX_LOG_DIR; #endif -} - -LogFileSettings::LogFileSettings() - : nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) + }) + .transform([](auto && s) { return canonPath(s); }) + .value()) { } @@ -84,10 +78,26 @@ Settings settings; static GlobalConfig::Register rSettings(&settings); Settings::Settings() - : nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) - , nixDaemonSocketFile(canonPath(getEnvOsNonEmpty(OS_STR("NIX_DAEMON_SOCKET_PATH")) - .transform([](auto && s) { return std::filesystem::path(s); }) - .value_or(nixStateDir / DEFAULT_SOCKET_PATH))) + : nixStateDir(getEnvOsNonEmpty(OS_STR("NIX_STATE_DIR")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { +#ifdef _WIN32 +# ifdef NIX_STATE_DIR +# error "NIX_STATE_DIR should not be defined on Windows" +# endif + return windows::known_folders::getProgramData() / "nix" / "state"; +#else + return NIX_STATE_DIR; +#endif + }) + .transform([](auto && s) { return canonPath(s); }) + .value()) + , nixDaemonSocketFile( + getEnvOsNonEmpty(OS_STR("NIX_DAEMON_SOCKET_PATH")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .or_else([this]() -> std::optional { return nixStateDir / DEFAULT_SOCKET_PATH; }) + .transform([](auto && s) { return canonPath(s); }) + .value()) { #ifndef _WIN32 buildUsersGroup = isRootUser() ? "nixbld" : ""; @@ -151,12 +161,28 @@ void loadConfFile(AbstractConfig & config) } } +/** + * On Windows, NIX_CONF_DIR (and other directories like NIX_STATE_DIR, NIX_LOG_DIR) + * are not defined at compile time, so we determine paths at runtime using the + * Windows known folders API (FOLDERID_ProgramData). This allows Nix to work + * correctly regardless of which drive Windows is installed on. + */ const std::filesystem::path & nixConfDir() { - static const std::filesystem::path dir = - canonPath(getEnvOsNonEmpty(OS_STR("NIX_CONF_DIR")) - .transform([](auto && s) { return std::filesystem::path(s); }) - .value_or(resolveNixConfDir())); + static const std::filesystem::path dir = getEnvOsNonEmpty(OS_STR("NIX_CONF_DIR")) + .transform([](auto && s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { +#ifdef _WIN32 +# ifdef NIX_CONF_DIR +# error "NIX_CONF_DIR should not be defined on Windows" +# endif + return windows::known_folders::getProgramData() / "nix" / "conf"; +#else + return NIX_CONF_DIR; +#endif + }) + .transform([](auto && s) { return canonPath(s); }) + .value(); return dir; } diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 63eba31255..96fd9f69c6 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -249,14 +249,14 @@ endif # by joining it with prefix, unless it was already an absolute path # (which is the default for store-dir, localstatedir, and log-dir). configdata_priv.set_quoted('NIX_STORE_DIR', store_dir) -configdata_priv.set_quoted('NIX_STATE_DIR', localstatedir / 'nix') -configdata_priv.set_quoted('NIX_LOG_DIR', log_dir) -# On Windows, NIX_CONF_DIR is determined at runtime using the Windows known -# folders API (FOLDERID_ProgramData), so we don't define it at compile time. +# On Windows, NIX_STATE_DIR, NIX_LOG_DIR, and NIX_CONF_DIR are determined at +# runtime using the Windows known folders API (FOLDERID_ProgramData), so we +# don't define them at compile time. if host_machine.system() != 'windows' + configdata_priv.set_quoted('NIX_STATE_DIR', localstatedir / 'nix') + configdata_priv.set_quoted('NIX_LOG_DIR', log_dir) configdata_priv.set_quoted('NIX_CONF_DIR', sysconfdir / 'nix') endif -configdata_priv.set_quoted('NIX_MAN_DIR', mandir) lsof = find_program('lsof', required : false) configdata_priv.set_quoted( From 6de78b8095099db6d7245e97350b3ffb27a1224b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Jul 2025 19:44:13 +0200 Subject: [PATCH 058/927] AllowListSourceAccessor: Make thread-safe --- src/libfetchers/filtering-source-accessor.cc | 27 ++++++++++--------- src/libfetchers/git-utils.cc | 4 +-- .../nix/fetchers/filtering-source-accessor.hh | 7 ++--- src/libutil/include/nix/util/sync.hh | 7 +++++ 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index 0daedcf7c7..6fe7d2504e 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -1,6 +1,7 @@ #include "nix/fetchers/filtering-source-accessor.hh" +#include "nix/util/sync.hh" -#include +#include namespace nix { @@ -74,39 +75,39 @@ void FilteringSourceAccessor::checkAccess(const CanonPath & path) struct AllowListSourceAccessorImpl : AllowListSourceAccessor { - std::set allowedPrefixes; - boost::unordered_flat_set allowedPaths; + SharedSync> allowedPrefixes; + boost::concurrent_flat_set allowedPaths; AllowListSourceAccessorImpl( ref next, - std::set && allowedPrefixes, - boost::unordered_flat_set && allowedPaths, + const std::set & allowedPrefixes, + const std::unordered_set & allowedPaths, MakeNotAllowedError && makeNotAllowedError) : AllowListSourceAccessor(SourcePath(next), std::move(makeNotAllowedError)) - , allowedPrefixes(std::move(allowedPrefixes)) - , allowedPaths(std::move(allowedPaths)) + , allowedPrefixes(allowedPrefixes.begin(), allowedPrefixes.end()) + , allowedPaths(allowedPaths.begin(), allowedPaths.end()) { } bool isAllowed(const CanonPath & path) override { - return allowedPaths.contains(path) || path.isAllowed(allowedPrefixes); + /* Read lock is held for the duration of the full expression if the || doesn't short-circuit. */ + return allowedPaths.contains(path) || path.isAllowed(*allowedPrefixes.readLock()); } void allowPrefix(CanonPath prefix) override { - allowedPrefixes.insert(std::move(prefix)); + allowedPrefixes.lock()->insert(std::move(prefix)); } }; ref AllowListSourceAccessor::create( ref next, - std::set && allowedPrefixes, - boost::unordered_flat_set && allowedPaths, + const std::set & allowedPrefixes, + const std::unordered_set & allowedPaths, MakeNotAllowedError && makeNotAllowedError) { - return make_ref( - next, std::move(allowedPrefixes), std::move(allowedPaths), std::move(makeNotAllowedError)); + return make_ref(next, allowedPrefixes, allowedPaths, std::move(makeNotAllowedError)); } bool CachingFilteringSourceAccessor::isAllowed(const CanonPath & path) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 456d869983..fcffdcfa8a 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1435,9 +1435,9 @@ ref GitRepoImpl::getAccessor( auto self = ref(shared_from_this()); ref fileAccessor = AllowListSourceAccessor::create( makeFSSourceAccessor(path), - std::set{wd.files}, + /*allowedPrefixes=*/wd.files, // Always allow access to the root, but not its children. - boost::unordered_flat_set{CanonPath::root}, + /*allowedPaths=*/{CanonPath::root}, std::move(makeNotAllowedError)) .cast(); if (options.exportIgnore) diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 98532c4b14..13272719fe 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -2,7 +2,8 @@ #include "nix/util/source-path.hh" -#include +#include +#include namespace nix { @@ -79,8 +80,8 @@ struct AllowListSourceAccessor : public FilteringSourceAccessor static ref create( ref next, - std::set && allowedPrefixes, - boost::unordered_flat_set && allowedPaths, + const std::set & allowedPrefixes, + const std::unordered_set & allowedPaths, MakeNotAllowedError && makeNotAllowedError); using FilteringSourceAccessor::FilteringSourceAccessor; diff --git a/src/libutil/include/nix/util/sync.hh b/src/libutil/include/nix/util/sync.hh index 6f3298324f..71f4fa0562 100644 --- a/src/libutil/include/nix/util/sync.hh +++ b/src/libutil/include/nix/util/sync.hh @@ -50,6 +50,13 @@ public: { } + template + SyncBase(Ts &&... args) + requires requires { T{std::forward(args)...}; } + : data(std::forward(args)...) + { + } + SyncBase(SyncBase && other) noexcept : data(std::move(*other.lock())) { From 30e71b1d7831a2522ba024bffb398aa8831d9326 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 9 Mar 2026 12:04:59 -0400 Subject: [PATCH 059/927] tests: add `UnkeyedRealisation` JSON characterisation tests This commit extracts `UnkeyedRealisation` test values from `Realisation` and reuses them, matching how `Realisation` composes `UnkeyedRealisation` + `DrvOutput`. The new unkeyed-simple and unkeyed-with-signature fixtures test the JSON format independently of the keyed wrapper. --- .../data/realisation/unkeyed-simple.json | 4 + .../realisation/unkeyed-with-signature.json | 9 ++ src/libstore-tests/realisation.cc | 112 +++++++++++++----- 3 files changed, 95 insertions(+), 30 deletions(-) create mode 100644 src/libstore-tests/data/realisation/unkeyed-simple.json create mode 100644 src/libstore-tests/data/realisation/unkeyed-with-signature.json diff --git a/src/libstore-tests/data/realisation/unkeyed-simple.json b/src/libstore-tests/data/realisation/unkeyed-simple.json new file mode 100644 index 0000000000..2cf61d2152 --- /dev/null +++ b/src/libstore-tests/data/realisation/unkeyed-simple.json @@ -0,0 +1,4 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo", + "signatures": [] +} diff --git a/src/libstore-tests/data/realisation/unkeyed-with-signature.json b/src/libstore-tests/data/realisation/unkeyed-with-signature.json new file mode 100644 index 0000000000..c51c6127a7 --- /dev/null +++ b/src/libstore-tests/data/realisation/unkeyed-with-signature.json @@ -0,0 +1,9 @@ +{ + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [ + { + "keyName": "asdf", + "sig": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + } + ] +} diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index 1bc7708e7c..204d338740 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -11,6 +11,43 @@ namespace nix { +using nlohmann::json; + +/* ---------------------------------------------------------------------------- + * Test data + * --------------------------------------------------------------------------*/ + +UnkeyedRealisation unkeyedSimple{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, +}; + +UnkeyedRealisation unkeyedWithSignature{ + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + .signatures = + { + Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, + }, +}; + +DrvOutput testDrvOutput{ + .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, + .outputName = "foo", +}; + +Realisation simple{ + unkeyedSimple, + testDrvOutput, +}; + +Realisation withSignature{ + unkeyedWithSignature, + testDrvOutput, +}; + +/* ---------------------------------------------------------------------------- + * Realisation JSON + * --------------------------------------------------------------------------*/ + class RealisationTest : public JsonCharacterizationTest, public LibStoreTest { std::filesystem::path unitTestData = getUnitTestData() / "realisation"; @@ -23,12 +60,6 @@ class RealisationTest : public JsonCharacterizationTest, public Lib } }; -/* ---------------------------------------------------------------------------- - * JSON - * --------------------------------------------------------------------------*/ - -using nlohmann::json; - struct RealisationJsonTest : RealisationTest, ::testing::WithParamInterface> {}; @@ -44,30 +75,6 @@ TEST_P(RealisationJsonTest, to_json) writeJsonTest(name, value); } -Realisation simple{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, - }, - { - .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, - .outputName = "foo", - }, -}; - -Realisation withSignature{ - { - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, - .signatures = - { - Signature{.keyName = "asdf", .sig = std::string(64, '\0')}, - }, - }, - { - .drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"}, - .outputName = "foo", - }, -}; - INSTANTIATE_TEST_SUITE_P( RealisationJSON, RealisationJsonTest, @@ -89,4 +96,49 @@ TEST_F(RealisationTest, with_signature_from_json) readJsonTest("with-signature", withSignature); } +/* ---------------------------------------------------------------------------- + * UnkeyedRealisation JSON + * --------------------------------------------------------------------------*/ + +class UnkeyedRealisationTest : public JsonCharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "realisation"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +struct UnkeyedRealisationJsonTest : UnkeyedRealisationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(UnkeyedRealisationJsonTest, from_json) +{ + const auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(UnkeyedRealisationJsonTest, to_json) +{ + const auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + UnkeyedRealisationJSON, + UnkeyedRealisationJsonTest, + ::testing::Values( + std::pair{ + "unkeyed-simple", + unkeyedSimple, + }, + std::pair{ + "unkeyed-with-signature", + unkeyedWithSignature, + })); + } // namespace nix From 02e4f4ad75fda707a594ffd8ad511691fc1a311f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 9 Mar 2026 15:31:14 +0300 Subject: [PATCH 060/927] libutil: Implement descriptor-based PosixFileSourceAccessor Changes makeFSSourceAccessor to open a file descriptor first and then operates on it. Symlinks get copied into a MemorySourceAccessor instead to avoid any accidental symlink following issues. This does mean that makeFSSourceAccessor now dies on a non-existent path, but all existing call-sites would already fail afterwards, so I think it's fine. --- src/libutil-tests/source-accessor.cc | 4 +- src/libutil/file-descriptor.cc | 3 +- .../include/nix/util/posix-source-accessor.hh | 54 ++++-- .../include/nix/util/source-accessor.hh | 2 + src/libutil/posix-source-accessor.cc | 179 +++++++++++++++--- 5 files changed, 201 insertions(+), 41 deletions(-) diff --git a/src/libutil-tests/source-accessor.cc b/src/libutil-tests/source-accessor.cc index fd4e2429e1..1ee2c44cc6 100644 --- a/src/libutil-tests/source-accessor.cc +++ b/src/libutil-tests/source-accessor.cc @@ -127,9 +127,7 @@ TEST_F(FSSourceAccessorTest, works) } { - auto accessor = makeFSSourceAccessor(tmpDir / "nonexistent"); - EXPECT_FALSE(accessor->maybeLstat(CanonPath::root)); - EXPECT_THROW(accessor->readFile(CanonPath::root), SystemError); + EXPECT_THROW(makeFSSourceAccessor(tmpDir / "nonexistent"), SystemError); } { diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index 61c444d4fe..acd273ca79 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -1,3 +1,4 @@ +#include "nix/util/file-system.hh" #include "nix/util/serialise.hh" #include "nix/util/util.hh" #include "nix/util/signals.hh" @@ -208,7 +209,7 @@ void copyFdRange(Descriptor fd, off_t offset, size_t nbytes, Sink & sink) auto limit = std::min(left, buf.size()); auto n = readOffset(fd, offset, std::span(buf.data(), limit)); if (n == 0) - throw EndOfFile("unexpected end-of-file"); + throw EndOfFile("unexpected end-of-file reading from %1%", PathFmt(descriptorToPath(fd))); assert(n <= left); sink(std::string_view(reinterpret_cast(buf.data()), n)); offset += n; diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index f9f0309365..d93b68e0af 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -6,10 +6,49 @@ namespace nix { struct SourcePath; +namespace detail { + +/** + * Common base helper class for deduplicating common code paths for tracking mtime. + */ +class PosixSourceAccessorBase : virtual public SourceAccessor +{ +protected: + const bool trackLastModified = false; + + /** + * The most recent mtime seen by lstat(). This is a hack to + * support dumpPathAndGetMtime(). Should remove this eventually. + */ + time_t mtime = 0; + + void maybeUpdateMtime(time_t seenMTime) + { + /* The contract is that trackLastModified implies that the caller uses the accessor + from a single thread. Thus this is not a CAS loop. */ + if (trackLastModified) + mtime = std::max(mtime, seenMTime); + } + + PosixSourceAccessorBase(bool trackLastModified) + : trackLastModified(trackLastModified) + { + } + + virtual std::optional getLastModified() override + { + if (trackLastModified) + return mtime; + return std::nullopt; + } +}; + +} // namespace detail + /** * A source accessor that uses the Unix filesystem. */ -class PosixSourceAccessor : virtual public SourceAccessor +class PosixSourceAccessor : public detail::PosixSourceAccessorBase { /** * Optional root path to prefix all operations into the native file @@ -18,19 +57,11 @@ class PosixSourceAccessor : virtual public SourceAccessor */ const std::filesystem::path root; - const bool trackLastModified = false; - public: PosixSourceAccessor(); PosixSourceAccessor(std::filesystem::path && root, bool trackLastModified = false); - /** - * The most recent mtime seen by lstat(). This is a hack to - * support dumpPathAndGetMtime(). Should remove this eventually. - */ - time_t mtime = 0; - void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; using SourceAccessor::readFile; @@ -75,11 +106,6 @@ public: */ static SourcePath createAtRoot(const std::filesystem::path & path, bool trackLastModified = false); - std::optional getLastModified() override - { - return trackLastModified ? std::optional{mtime} : std::nullopt; - } - void invalidateCache(const CanonPath & path) override; private: diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index bcc25b99a5..c458cf8b5c 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -259,6 +259,8 @@ ref getFSSourceAccessor(); * that it is not possible to escape `root` by appending `..` path * elements, and that absolute symlinks are resolved relative to * `root`. + * + * Symlinks in parents of `root` are resolved. Final symlink is not. */ ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified = false); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 7d0b1df75a..730ac8efaa 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -1,4 +1,6 @@ #include "nix/util/posix-source-accessor.hh" +#include "nix/util/file-system-at.hh" +#include "nix/util/memory-source-accessor.hh" #include "nix/util/source-path.hh" #include "nix/util/signals.hh" #include "nix/util/sync.hh" @@ -7,9 +9,125 @@ namespace nix { +static SourceAccessor::Stat sourceAccessorStatFromPosixStat(const PosixStat & st) +{ + using enum SourceAccessor::Type; + return SourceAccessor::Stat{ + .type = S_ISREG(st.st_mode) ? tRegular + : S_ISDIR(st.st_mode) ? tDirectory + : S_ISLNK(st.st_mode) ? tSymlink + : S_ISCHR(st.st_mode) ? tChar + : S_ISBLK(st.st_mode) ? tBlock + : +#ifdef S_ISSOCK + S_ISSOCK(st.st_mode) ? tSocket + : +#endif + S_ISFIFO(st.st_mode) ? tFifo + : tUnknown, + .fileSize = S_ISREG(st.st_mode) ? std::optional(st.st_size) : std::nullopt, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR, + }; +} + +namespace { + +class PosixFileSourceAccessor : public detail::PosixSourceAccessorBase +{ + AutoCloseFD fd; + std::filesystem::path fsPath; + /** + * Stat is memoised once opened. This does mean that modifying the same file + * while we are reading is busted, but caching it might be considered an + * improvement. Since we have a file descriptor for a regular file it can't + * be swapped out for another file type. Thus, we are only really caching + * the file size and mtime, which shouldn't change. Providing a consistent + * size value here is also fine. If the file becomes smaller than we expect + * then readFile will barf with an EOF exception. If it becomes larger then + * we are just going to silently ignore the extra bytes. + */ + PosixStat st; + +public: + PosixFileSourceAccessor(AutoCloseFD fd, std::filesystem::path path, bool trackLastModified, const PosixStat & st_) + : PosixSourceAccessorBase(trackLastModified) + , fd(std::move(fd)) + , fsPath(std::move(path)) + , st(st_) + { + assert(S_ISREG(st.st_mode)); + assert(fsPath.is_absolute()); /* Only used for error messages, but still nice to enforce this invariant. */ + setPathDisplay(fsPath.generic_string()); + maybeUpdateMtime(st.st_mtime); + } + + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; + + bool pathExists(const CanonPath & path) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override + { + if (path.isRoot()) + return fsPath; + return std::nullopt; /* Definitely doesn't exist. */ + } + + std::string showPath(const CanonPath & path) override + { + if (path.isRoot()) + return displayPrefix; /* No trailing slash. */ + return displayPrefix + path.abs(); + } +}; + +void PosixFileSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) +{ + if (!path.isRoot()) /* We are the parent and also a regular file. */ + throw NotADirectory("reading file '%1%': %2%", showPath(path), "Not a directory"); + + auto size = st.st_size; + sizeCallback(size); + /* The most important invariant we care about here is writing exactly size + bytes to the sink. copyFdRange should throw an EndOfFile if we fail to read + `size` bytes. */ + copyFdRange(fd.get(), /*offset=*/0, size, sink); +} + +bool PosixFileSourceAccessor::pathExists(const CanonPath & path) +{ + return path.isRoot(); +} + +std::optional PosixFileSourceAccessor::maybeLstat(const CanonPath & path) +{ + if (!path.isRoot()) + return std::nullopt; + return sourceAccessorStatFromPosixStat(st); +} + +SourceAccessor::DirEntries PosixFileSourceAccessor::readDirectory(const CanonPath & path) +{ + throw NotADirectory("reading directory '%1%': %2%", showPath(path), "Not a directory"); +} + +std::string PosixFileSourceAccessor::readLink(const CanonPath & path) +{ + if (!path.isRoot()) + throw NotADirectory("reading symlink '%1%': %2%", showPath(path), "Not a directory"); + throw NotASymlink("path '%1%' is not a symlink", showPath(path)); +} + +} // namespace + PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot, bool trackLastModified) - : root(std::move(argRoot)) - , trackLastModified(trackLastModified) + : PosixSourceAccessorBase(trackLastModified) + , root(std::move(argRoot)) { assert(root.empty() || root.is_absolute()); displayPrefix = root.string(); @@ -106,27 +224,8 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP if (!st) return std::nullopt; - /* The contract is that trackLastModified implies that the caller uses the accessor - from a single thread. Thus this is not a CAS loop. */ - if (trackLastModified) - mtime = std::max(mtime, st->st_mtime); - - return Stat{ - .type = S_ISREG(st->st_mode) ? tRegular - : S_ISDIR(st->st_mode) ? tDirectory - : S_ISLNK(st->st_mode) ? tSymlink - : S_ISCHR(st->st_mode) ? tChar - : S_ISBLK(st->st_mode) ? tBlock - : -#ifdef S_ISSOCK - S_ISSOCK(st->st_mode) ? tSocket - : -#endif - S_ISFIFO(st->st_mode) ? tFifo - : tUnknown, - .fileSize = S_ISREG(st->st_mode) ? std::optional(st->st_size) : std::nullopt, - .isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR, - }; + PosixSourceAccessorBase::maybeUpdateMtime(st->st_mtime); + return sourceAccessorStatFromPosixStat(*st); } SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) @@ -211,6 +310,40 @@ ref getFSSourceAccessor() ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified) { +#ifndef _WIN32 + /* Die if the path ends in a / or /. That is a footgun that matters + for symlink resolution. TODO: Strip them or make this a proper error if + this becomes an issue. Defense-in-depth. */ + assert(!root.native().ends_with(OS_STR("/."))); + assert(!root.native().ends_with(OS_STR("/"))); + + AutoCloseFD fd = openFileReadonly(root, FinalSymlink::DontFollow); + + if (!fd) { + if (errno == ELOOP) { + /* This branch is taken either if the final component is a symlink + or we hit a symlink loop. If that's the latter this will also + throw. This can be done with O_PATH descriptor for the symlink + itself, but it's not portable. */ + auto linkTarget = readLink(root); + auto res = make_ref(); + /* Create an in-memory accessor with the symlink at the root. */ + MemorySink sink{*res}; + sink.createSymlink(CanonPath::root, os_string_to_string(linkTarget)); + return res; + } + + throw NativeSysError("opening file %1%", PathFmt(root)); + } + + auto st = nix::fstat(fd.get()); + if (S_ISREG(st.st_mode)) + return make_ref(std::move(fd), std::move(root), trackLastModified, st); + + /* TODO: Use the file descriptor for fd-relative operations on the directory. */ +#endif + return make_ref(std::move(root), trackLastModified); } + } // namespace nix From b919a12f7bd8a9c8a588a0c68b9c44a46b006c2d Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Mon, 9 Mar 2026 09:33:54 -0700 Subject: [PATCH 061/927] libstore: Replace strcpy with std::ranges::copy for loopback interface setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both loopback interface initialization sites used strcpy() to copy the interface name "lo" into ifr.ifr_name. While safe in practice (the literal is 3 bytes, the buffer is IFNAMSIZ/16 bytes), strcpy is flagged by static analysis tools (semgrep unsafe-strcpy, CWE-120) and is a banned function in several secure coding standards. Replace with std::ranges::copy() and a string_view literal, which is idiomatic C++ and avoids C string functions entirely. Also add zero-initialization (= {}) to the ifreq struct in linux-derivation-builder.cc, which was previously uninitialized. The test-support copy already had this. The test-support path is covered by the HttpsBinaryCacheStore unit test suite, which exercises setupNetworkTests() -> enterNetworkNamespace(). The production path in linux-derivation-builder.cc runs inside a Linux sandbox namespace during derivation builds, so it is not reachable from unit tests — it is covered by the functional test suite (tests/functional/linux-sandbox.sh). Co-Authored-By: Claude Opus 4.6 --- src/libstore-test-support/libstore-network.cc | 5 ++++- src/libstore/unix/build/linux-derivation-builder.cc | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libstore-test-support/libstore-network.cc b/src/libstore-test-support/libstore-network.cc index 8aa047bdd6..800dc17ed9 100644 --- a/src/libstore-test-support/libstore-network.cc +++ b/src/libstore-test-support/libstore-network.cc @@ -5,6 +5,8 @@ #ifdef __linux__ # include "nix/util/file-system.hh" # include "nix/util/linux-namespaces.hh" +# include +# include # include # include # include @@ -34,8 +36,9 @@ static void enterNetworkNamespace() if (!fd) throw SysError("cannot open IP socket for loopback interface"); + using namespace std::string_view_literals; struct ::ifreq ifr = {}; - strcpy(ifr.ifr_name, "lo"); + std::ranges::copy("lo"sv, ifr.ifr_name); ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; if (::ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) throw SysError("cannot set loopback interface flags"); diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index 0ffcf0ccb6..9e3f707aee 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -9,6 +9,8 @@ # include "nix/util/serialise.hh" # include "linux/fchmodat2-compat.hh" +# include +# include # include # include # include @@ -472,8 +474,9 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu if (!fd) throw SysError("cannot open IP socket"); - struct ifreq ifr; - strcpy(ifr.ifr_name, "lo"); + using namespace std::string_view_literals; + struct ifreq ifr = {}; + std::ranges::copy("lo"sv, ifr.ifr_name); ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) throw SysError("cannot set loopback interface flags"); From bfeb7b874b1f93f3b1db9343732c574adf4abab7 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 9 Mar 2026 14:27:38 -0400 Subject: [PATCH 062/927] local-overlay-store: require upper-layer via isOverridden The setting framework lacks required-setting support, so upper-layer used an empty-string sentinel. This commit replaces it with an absolute dummy default and check `isOverridden()` at store construction time. --- src/libstore-tests/local-overlay-store.cc | 18 ++++++++++++++++++ .../include/nix/store/local-overlay-store.hh | 2 +- src/libstore/local-overlay-store.cc | 3 +++ src/libutil/include/nix/util/configuration.hh | 4 ++-- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/libstore-tests/local-overlay-store.cc b/src/libstore-tests/local-overlay-store.cc index 4c12f6256a..e7b392b1b0 100644 --- a/src/libstore-tests/local-overlay-store.cc +++ b/src/libstore-tests/local-overlay-store.cc @@ -36,4 +36,22 @@ TEST(LocalOverlayStore, constructConfig_rootPath) EXPECT_EQ(config.rootDir.get(), std::optional{std::string{root}}); } +TEST(LocalOverlayStore, upperLayer_notOverridden) +{ + LocalOverlayStoreConfig config{"", {}}; + EXPECT_FALSE(config.upperLayer.isOverridden()); +} + +TEST(LocalOverlayStore, upperLayer_overridden) +{ + LocalOverlayStoreConfig config{ + "", + { + {"upper-layer", "/some/upper"}, + }, + }; + EXPECT_TRUE(config.upperLayer.isOverridden()); + EXPECT_EQ(config.upperLayer.get(), std::filesystem::path{"/some/upper"}); +} + } // namespace nix diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index 5d8ec1b457..d98602ad83 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -33,7 +33,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig const Setting upperLayer{ (StoreConfig *) this, - "", + "/upper-layer-must-be-set", "upper-layer", R"( Directory containing the OverlayFS upper layer for this store's store dir. diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 705e9ddf71..6025affddf 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -46,6 +46,9 @@ LocalOverlayStore::LocalOverlayStore(ref config) , config{config} , lowerStore(openStore(config->lowerStoreUri.get()).dynamic_pointer_cast()) { + if (!config->upperLayer.isOverridden()) + throw Error("overlay store at %s requires the 'upper-layer' setting", PathFmt(config->realStoreDir.get())); + if (config->checkMount.get()) { std::smatch match; std::string mountInfo; diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 446b68f3d3..22ebbb3e55 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -189,6 +189,8 @@ public: std::optional experimentalFeature; + bool isOverridden() const; + protected: AbstractSetting( @@ -214,8 +216,6 @@ protected: virtual std::map toJSONObject() const; virtual void convertToArg(Args & args, const std::string & category); - - bool isOverridden() const; }; /** From 0d5d424ff8195663cc23e819e1e6804f3528bf93 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 9 Mar 2026 01:45:33 -0400 Subject: [PATCH 063/927] libstore: use `std::optional` for path settings that can be unset `sshKey`, `secretKeyFile`, `remountHook`, and `upperLayer` used empty strings denoting a setting that was "not configured". This is unsound for a type called `AbsolutePath` since an empty path is not absolute. This commit changes them to `Setting>`, which makes the "unset" state explicit and will allow the `AbsolutePath` constructors to enforce the absolute invariant unconditionally. `upperLayer` is effectively required, so the `LocalOverlayStore` constructor now errors immediately if it is unset. --- src/libstore-tests/machines.cc | 4 ++-- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/include/nix/store/binary-cache-store.hh | 4 ++-- src/libstore/include/nix/store/common-ssh-store-config.hh | 4 ++-- src/libstore/include/nix/store/local-overlay-store.hh | 4 ++-- src/libstore/include/nix/store/machines.hh | 2 +- src/libstore/include/nix/store/ssh.hh | 4 ++-- src/libstore/local-overlay-store.cc | 6 +++--- src/libstore/machines.cc | 6 +++--- src/libstore/ssh.cc | 6 +++--- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index c4cbf47e4e..4246a876ca 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -27,7 +27,7 @@ TEST(machines, getMachinesUriOnly) ASSERT_THAT(actual, SizeIs(1)); EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); - EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::filesystem::path{}))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::nullopt))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); @@ -49,7 +49,7 @@ TEST(machines, getMachinesDefaults) ASSERT_THAT(actual, SizeIs(1)); EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); - EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::filesystem::path{}))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq(std::nullopt))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 34a3209620..73fc6981d5 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -27,8 +27,8 @@ namespace nix { BinaryCacheStore::BinaryCacheStore(Config & config) : config{config} { - if (!config.secretKeyFile.get().empty()) - signers.push_back(std::make_unique(SecretKey{readFile(config.secretKeyFile.get())})); + if (auto & skf = config.secretKeyFile.get()) + signers.push_back(std::make_unique(SecretKey{readFile(*skf)})); if (config.secretKeyFiles != "") { std::stringstream ss(config.secretKeyFiles); diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index bc499130b0..4728dbd5e0 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -39,8 +39,8 @@ struct BinaryCacheStoreConfig : virtual StoreConfig fetch debug info on demand )"}; - Setting secretKeyFile{ - this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; + Setting> secretKeyFile{ + this, std::nullopt, "secret-key", "Path to the secret key used to sign the binary cache."}; Setting secretKeyFiles{ this, "", "secret-keys", "List of comma-separated paths to the secret keys used to sign the binary cache."}; diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index 42b3415b77..b9c21fd8ef 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -14,8 +14,8 @@ struct CommonSSHStoreConfig : virtual StoreConfig CommonSSHStoreConfig(const ParsedURL::Authority & authority, const Params & params); - Setting sshKey{ - this, "", "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; + Setting> sshKey{ + this, std::nullopt, "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; Setting sshPublicHostKey{ this, "", "base64-ssh-public-host-key", "The public host key of the remote machine."}; diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index d98602ad83..dae7925d29 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -53,9 +53,9 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig default, but can be disabled if needed. )"}; - const Setting remountHook{ + const Setting> remountHook{ (StoreConfig *) this, - "", + std::nullopt, "remount-hook", R"( Script or other executable to run when overlay filesystem needs remounting. diff --git a/src/libstore/include/nix/store/machines.hh b/src/libstore/include/nix/store/machines.hh index 6b34846994..a3e9353c80 100644 --- a/src/libstore/include/nix/store/machines.hh +++ b/src/libstore/include/nix/store/machines.hh @@ -17,7 +17,7 @@ struct Machine const StoreReference storeUri; const StringSet systemTypes; - const std::filesystem::path sshKey; + const std::optional sshKey; const unsigned int maxJobs; const float speedFactor; const StringSet supportedFeatures; diff --git a/src/libstore/include/nix/store/ssh.hh b/src/libstore/include/nix/store/ssh.hh index c64e9b1cca..e090dba231 100644 --- a/src/libstore/include/nix/store/ssh.hh +++ b/src/libstore/include/nix/store/ssh.hh @@ -18,7 +18,7 @@ private: ParsedURL::Authority authority; std::string hostnameAndUser; bool fakeSSH; - const std::filesystem::path keyFile; + const std::optional keyFile; /** * Raw bytes, not Base64 encoding. */ @@ -50,7 +50,7 @@ public: SSHMaster( const ParsedURL::Authority & authority, - std::filesystem::path keyFile, + std::optional keyFile, std::string_view sshPublicHostKey, bool useMaster, bool compress, diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 6025affddf..89c5a42d8a 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -283,10 +283,10 @@ void LocalOverlayStore::remountIfNecessary() if (!_remountRequired) return; - if (config->remountHook.get().empty()) { - warn("%s needs remounting, set remount-hook to do this automatically", PathFmt(config->realStoreDir.get())); + if (auto & hook = config->remountHook.get()) { + runProgram(*hook, false, {config->realStoreDir.get().native()}); } else { - runProgram(config->remountHook.get(), false, {config->realStoreDir.get().native()}); + warn("%s needs remounting, set remount-hook to do this automatically", PathFmt(config->realStoreDir.get())); } _remountRequired = false; diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 3b11ec9025..00a4d4d1fe 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -69,8 +69,8 @@ StoreReference Machine::completeStoreReference() const } if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) { - if (!sshKey.empty()) - storeUri.params["ssh-key"] = sshKey.string(); + if (sshKey) + storeUri.params["ssh-key"] = sshKey->string(); if (sshPublicHostKey != "") storeUri.params["base64-ssh-public-host-key"] = sshPublicHostKey; } @@ -179,7 +179,7 @@ static Machine parseBuilderLine(const StringSet & defaultSystems, const std::str // `systemTypes` isSet(1) ? tokenizeString(tokens[1], ",") : defaultSystems, // `sshKey` - isSet(2) ? tokens[2] : "", + isSet(2) && !tokens[2].empty() ? std::make_optional(tokens[2]) : std::nullopt, // `maxJobs` isSet(3) ? parseUnsignedIntField(3) : 1U, // `speedFactor` diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 849fcff2fd..0240f9cca3 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -66,7 +66,7 @@ OsStrings getNixSshOpts() SSHMaster::SSHMaster( const ParsedURL::Authority & authority, - std::filesystem::path keyFile, + std::optional keyFile, std::string_view sshPublicHostKey, bool useMaster, bool compress, @@ -95,8 +95,8 @@ void SSHMaster::addCommonSSHOpts(OsStrings & args) auto sshArgs = getNixSshOpts(); args.insert(args.end(), sshArgs.begin(), sshArgs.end()); - if (!keyFile.empty()) - args.insert(args.end(), {OS_STR("-i"), keyFile.native()}); + if (keyFile) + args.insert(args.end(), {OS_STR("-i"), keyFile->native()}); if (!sshPublicHostKey.empty()) { std::filesystem::path fileName = tmpDir->path() / "host-key"; writeFile(fileName, authority.host + " " + sshPublicHostKey + "\n"); From 53197b9cca9a0e6bf4de1172ec1eed56de56631b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 9 Mar 2026 21:56:45 +0300 Subject: [PATCH 064/927] libexpr: Get rid of std::filesystem::path in parser.y This has the exact same behaviour on unix, while fixing path resolution bugs on windows. I've checked nix up to 2.3 and all versions got the case of literal being `path/to/symlink/../otherfile` wrong, because they never resolved any symlinks at that point and always used absPath there. --- src/libexpr/parser.y | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index e5e4241ea7..77e63ed629 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -426,7 +426,7 @@ path_start /* Absolute paths are always interpreted relative to the root filesystem accessor, rather than the accessor of the current Nix expression. */ - auto path = canonPath(literal).string(); + auto path = CanonPath(literal).abs(); /* add back in the trailing '/' to the first segment */ if (literal.size() > 1 && literal.back() == '/') path += '/'; @@ -442,8 +442,7 @@ path_start return std::nullopt; }); - auto basePath = std::filesystem::path(state->basePath.path.abs()); - auto path = absPath(literal, &basePath).string(); + auto path = CanonPath(literal, state->basePath.path).abs(); /* add back in the trailing '/' to the first segment */ if (literal.size() > 1 && literal.back() == '/') path += '/'; From 73151045c2308505060155d14591e1a59a7cbcc8 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 9 Mar 2026 22:05:28 +0300 Subject: [PATCH 065/927] printVersion: Fix quoting of the output (once again) --- src/libmain/shared.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index a65fc773aa..3640efc07a 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -330,12 +330,12 @@ void printVersion(const std::string & programName) std::cout << "System type: " << settings.thisSystem << "\n"; std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; - std::cout << "System configuration file: " << nixConfFile() << "\n"; + std::cout << "System configuration file: " << os_string_to_string(nixConfFile().native()) << "\n"; std::cout << "User configuration files: " << os_string_to_string(ExecutablePath{.directories = nixUserConfFiles()}.render()) << "\n"; std::cout << "Store directory: " << resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir << "\n"; - std::cout << "State directory: " << settings.nixStateDir << "\n"; + std::cout << "State directory: " << os_string_to_string(settings.nixStateDir.native()) << "\n"; } throw Exit(); } From a322fd0f80b4c2b74deaadb65fd25d464a39d1a5 Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Mon, 9 Mar 2026 12:14:38 -0700 Subject: [PATCH 066/927] libutil: Initialize Handler::arity with in-class default Handler() = default leaves size_t arity uninitialized. Currently unreachable since all callers use parameterized constructors, but this eliminates latent undefined behavior. Consistent with the existing pattern in the same file (shortName = 0, timesUsed = 0). Found by cppcheck (uninitMemberVar). Co-Authored-By: Claude Opus 4.6 --- src/libutil/include/nix/util/args.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/include/nix/util/args.hh b/src/libutil/include/nix/util/args.hh index b4337590a3..2d4a81cafc 100644 --- a/src/libutil/include/nix/util/args.hh +++ b/src/libutil/include/nix/util/args.hh @@ -84,7 +84,7 @@ protected: struct Handler { std::function)> fun; - size_t arity; + size_t arity = 0; Handler() = default; From 1b356130787b44d5e6d443ff816f54ce2e702f03 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 9 Mar 2026 22:14:58 +0300 Subject: [PATCH 067/927] filetransfer: Fix Windows passing wide strings to curl_easy_setopt --- src/libstore/filetransfer.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 099f436ae6..e6d4caa69f 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -551,7 +551,7 @@ struct curlFileTransfer : public FileTransfer } if (auto & caFile = fileTransfer.settings.caFile.get()) - curl_easy_setopt(req, CURLOPT_CAINFO, caFile->c_str()); + curl_easy_setopt(req, CURLOPT_CAINFO, caFile->string().c_str()); #if !defined(_WIN32) curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback); @@ -569,7 +569,7 @@ struct curlFileTransfer : public FileTransfer /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ - curl_easy_setopt(req, CURLOPT_NETRC_FILE, fileTransfer.settings.netrcFile.get().c_str()); + curl_easy_setopt(req, CURLOPT_NETRC_FILE, fileTransfer.settings.netrcFile.get().string().c_str()); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); if (writtenToSink) From 9f889e813fff4493f17d3afd3b3f53c0ef4991e5 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 9 Mar 2026 02:01:39 -0400 Subject: [PATCH 068/927] configuration: assert that `AbsolutePath` is absolute on construction The inherited constructors from `std::filesystem::path` allowed creating an `AbsolutePath` from any string without validation. Now that all settings using empty strings as "unset" have been converted to `std::optional`, the constructors can enforce the invariant that every `AbsolutePath` is actually absolute. --- src/libstore/builtins/fetchurl.cc | 7 ++++--- src/libstore/filetransfer.cc | 16 +++++++++++----- .../include/nix/store/filetransfer.hh | 2 +- src/libutil/include/nix/util/configuration.hh | 19 ++++++++++++++++++- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 1ff66eb89f..767fc9b8f0 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -14,12 +14,13 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx) this to be stored in a file. It would be nice if we could just pass a pointer to the data. */ if (ctx.netrcData != "") { - fileTransferSettings.netrcFile = "netrc"; + fileTransferSettings.netrcFile = ctx.tmpDirInSandbox / "netrc"; writeFile(fileTransferSettings.netrcFile.get(), ctx.netrcData, 0600); } - fileTransferSettings.caFile = "ca-certificates.crt"; - writeFile(*fileTransferSettings.caFile.get(), ctx.caFileData, 0600); + auto caFilePath = ctx.tmpDirInSandbox / "ca-certificates.crt"; + fileTransferSettings.caFile = std::optional{caFilePath}; + writeFile(caFilePath, ctx.caFileData, 0600); auto out = get(ctx.drv.outputs, "out"); if (!out) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 099f436ae6..53f8a50198 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -34,20 +34,26 @@ namespace nix { const unsigned int RETRY_TIME_MS_DEFAULT = 250; const unsigned int RETRY_TIME_MS_TOO_MANY_REQUESTS = 60000; -std::filesystem::path FileTransferSettings::getDefaultSSLCertFile() +std::optional FileTransferSettings::getDefaultSSLCertFile() { for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) if (pathAccessible(fn)) return fn; - return ""; + return std::nullopt; } FileTransferSettings::FileTransferSettings() { - auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); - if (sslOverride != "") - caFile = sslOverride; + std::optional sslOverride = + getEnvOs(OS_STR("NIX_SSL_CERT_FILE")) + .or_else([] { return getEnvOs(OS_STR("SSL_CERT_FILE")); }) + .and_then([](OsString s) -> std::optional { + return s.empty() ? std::nullopt : std::optional{std::move(s)}; + }) + .transform([](OsString s) { return AbsolutePath{std::filesystem::path{std::move(s)}}; }); + if (sslOverride) + caFile = *sslOverride; } FileTransferSettings fileTransferSettings; diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 340ad856cc..5df5ae59e6 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -24,7 +24,7 @@ const std::filesystem::path & nixConfDir(); struct FileTransferSettings : Config { private: - static std::filesystem::path getDefaultSSLCertFile(); + static std::optional getDefaultSSLCertFile(); public: FileTransferSettings(); diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 22ebbb3e55..a8ba936d9a 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -221,21 +221,38 @@ protected: /** * For `Setting`. `parse()` calls `canonPath`, * rejecting empty and relative paths. + * + * Constructors assert that the path is absolute. */ struct AbsolutePath : std::filesystem::path { - using path::path; using path::operator=; AbsolutePath(const std::filesystem::path & p) : path(p) { + assert(is_absolute()); } AbsolutePath(std::filesystem::path && p) : path(std::move(p)) { + assert(is_absolute()); + } + + AbsolutePath(const char * s) + : path(s) + { + assert(is_absolute()); + } + +#ifdef _WIN32 + AbsolutePath(const wchar_t * s) + : path(s) + { + assert(is_absolute()); } +#endif }; template<> From 34233db951e41159e36312f0b0b97868fcc1a484 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Thu, 19 Feb 2026 19:04:46 -0500 Subject: [PATCH 069/927] libstore: preserve mount flags when remounting store writable `makeStoreWritable` passed only `MS_REMOUNT | MS_BIND` to `mount()`, telling the kernel to drop every other flag. In a user namespace, flags like `nodev` and `nosuid` are locked by the kernel, so dropping them causes `EPERM`. This commit reads the active flags from `statvfs` and translates each `ST_*` constant to its `MS_*` counterpart individually, because the two sets are not always equal (`ST_RELATIME` is 4096, which collides with `MS_BIND`; the real `MS_RELATIME` is 1 << 21). Fixes #9705 --- src/libstore/local-store.cc | 24 +++++++++++- tests/nixos/default.nix | 2 + tests/nixos/store-remount.nix | 71 +++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 tests/nixos/store-remount.nix diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 08b9f53169..2be86f2f6a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -621,7 +621,29 @@ void LocalStore::makeStoreWritable() throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - if (mount(0, config->realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + /* In a user namespace, mount flags like `nodev` and `nosuid` are + locked and dropping them causes `EPERM`, so here we translate each + `statvfs` flag to the corresponding `mount` flag individually. */ + unsigned long flags = MS_REMOUNT | MS_BIND; + if (stat.f_flag & ST_NODEV) + flags |= MS_NODEV; + if (stat.f_flag & ST_NOSUID) + flags |= MS_NOSUID; + if (stat.f_flag & ST_NOEXEC) + flags |= MS_NOEXEC; + if (stat.f_flag & ST_NOATIME) + flags |= MS_NOATIME; + if (stat.f_flag & ST_NODIRATIME) + flags |= MS_NODIRATIME; + if (stat.f_flag & ST_RELATIME) + flags |= MS_RELATIME; + if (stat.f_flag & ST_SYNCHRONOUS) + flags |= MS_SYNCHRONOUS; +# ifdef ST_NOSYMFOLLOW + if (stat.f_flag & ST_NOSYMFOLLOW) + flags |= MS_NOSYMFOLLOW; +# endif + if (mount(0, config->realStoreDir.get().c_str(), "none", flags, 0) == -1) throw SysError("remounting %s writable", PathFmt(config->realStoreDir.get())); } #endif diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index c30e872bc1..a2f6ba1184 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -205,4 +205,6 @@ in fetchersSubstitute = runNixOSTest ./fetchers-substitute.nix; chrootStore = runNixOSTest ./chroot-store.nix; + + storeRemount = runNixOSTest ./store-remount.nix; } diff --git a/tests/nixos/store-remount.nix b/tests/nixos/store-remount.nix new file mode 100644 index 0000000000..09f76dab79 --- /dev/null +++ b/tests/nixos/store-remount.nix @@ -0,0 +1,71 @@ +# Test that `makeStoreWritable` doesn't fail with EPERM when inherited +# mount flags like `nodev` are locked in a user namespace +# (https://github.com/NixOS/nix/issues/9705). + +{ lib, ... }: + +{ + name = "store-remount"; + + nodes.machine = + { pkgs, ... }: + { + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ + pkgs.nix + pkgs.util-linux + pkgs.strace + pkgs.gnugrep + pkgs.bash + ]; + nix.settings.substituters = lib.mkForce [ ]; + }; + + testScript = + { nodes }: + let + pkgs = nodes.machine.nixpkgs.pkgs; + nix = nodes.machine.nix.package; + in + '' + machine.wait_for_unit("multi-user.target") + + # Add nodev to the host store mount so it becomes locked in the + # user namespace below. + machine.succeed("mount -o remount,bind,rw,nodev /nix/store") + + # Set up a minimal container root (which needs its own db to avoid + # UID-mapping issues on the host db). + machine.succeed("mkdir -p /tmp/container/{nix/store,nix/var/nix/db,tmp,etc,proc,sys}") + machine.succeed("echo 'root:x:0:0:root:/root:/bin/sh' > /tmp/container/etc/passwd") + machine.succeed("echo 'root:x:0:' > /tmp/container/etc/group") + + # In a user namespace (`--private-users`), inherited flags like nodev + # are locked. We make the store read-only, then let `nix store info` + # trigger `makeStoreWritable`. Without the fix, the remount fails + # with EPERM because the old code implicitly dropped locked flags. + # + # We use strace to capture the mount() syscall and verify that + # MS_NODEV is present in the flags, since nix does + # unshare(CLONE_NEWNS) internally so the remount is not observable + # from outside the nix process. + (status, output) = machine.execute( + "systemd-nspawn --quiet --register=no --private-network " + "--private-users=pick " + "-D /tmp/container " + "--bind=/nix/store " + "--as-pid2 " + "${pkgs.bash}/bin/bash -c '" + " ${pkgs.util-linux}/bin/mount -o remount,bind,ro /nix/store && " + " ${pkgs.strace}/bin/strace -f -e trace=mount -o /strace.log " + " ${nix}/bin/nix store info --store local " + " --extra-experimental-features nix-command" + "'" + ) + + print(machine.succeed("cat /tmp/container/strace.log")) + + assert status == 0, f"nspawn failed (status {status}): {output}" + machine.succeed("${pkgs.gnugrep}/bin/grep -q MS_NODEV /tmp/container/strace.log") + ''; +} From 6911469734cc6d798e9407fba21bce87d15538e8 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Thu, 19 Feb 2026 19:04:46 -0500 Subject: [PATCH 070/927] cli: warn when private mount namespace setup fails The catch block silently swallowed errors from `saveMountNamespace()` and `unshare(CLONE_NEWNS)`. This commit logs a warning so the failure is visible. --- src/nix/main.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 2a790e7be9..297318ddcd 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -404,6 +404,7 @@ void mainWrapped(int argc, char ** argv) if (unshare(CLONE_NEWNS) == -1) throw SysError("setting up a private mount namespace"); } catch (Error & e) { + warn("failed to set up a private mount namespace: %s", e.msg()); } } #endif From f053019e48ac16c2e8de60bddc3d70db42e20106 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 9 Mar 2026 17:26:13 -0400 Subject: [PATCH 071/927] tests: rename signature fixture files for clarity `with-signature-structured` and `with-structured-signature` were redundant fixtures that only differed in `outPath` (with or without `.drv`). The unused `with-structured-signature` is removed, and `with-signature-structured` becomes `with-signature` since it is the canonical structured format. The old `with-signature` (unstructured string format) is renamed to `with-signature-unstructured`. Also fixes stale references to `with-structured-signature` in `meson.build` and the build-trace-entry documentation. --- .../source/protocols/json/build-trace-entry.md | 2 +- src/json-schema-checks/meson.build | 2 +- ...ured.json => with-signature-unstructured.json} | 5 +---- .../data/realisation/with-signature.json | 5 ++++- .../realisation/with-structured-signature.json | 15 --------------- src/libstore-tests/realisation.cc | 4 ++-- 6 files changed, 9 insertions(+), 24 deletions(-) rename src/libstore-tests/data/realisation/{with-signature-structured.json => with-signature-unstructured.json} (57%) delete mode 100644 src/libstore-tests/data/realisation/with-structured-signature.json diff --git a/doc/manual/source/protocols/json/build-trace-entry.md b/doc/manual/source/protocols/json/build-trace-entry.md index 2f8a4dbeef..d4a3a96edb 100644 --- a/doc/manual/source/protocols/json/build-trace-entry.md +++ b/doc/manual/source/protocols/json/build-trace-entry.md @@ -11,7 +11,7 @@ ### Build trace entry with signature ```json -{{#include schema/build-trace-entry-v3/with-structured-signature.json}} +{{#include schema/build-trace-entry-v3/with-signature.json}} ``` + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory + +- [binary cache]{#gloss-binary-cache} + + A *binary cache* is a Nix store which uses a different format: its + metadata and signatures are kept in `.narinfo` files rather than in a + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). + - [build system]{#gloss-build-system} Generic term for software that facilitates the building of software by automating the invocation of compilers, linkers, and other tools. @@ -8,6 +43,26 @@ It has no knowledge of any particular programming language or toolchain. These details are specified in [derivation expressions](#gloss-derivation-expression). +- [closure]{#gloss-closure} + + The closure of a store path is the set of store paths that are + directly or indirectly “reachable” from that store path; that is, + it’s the closure of the path under the *references* relation. For + a package, the closure of its derivation is equivalent to the + build-time dependencies, while the closure of its [output path] is + equivalent to its runtime dependencies. For correct deployment it + is necessary to deploy whole closures, since otherwise at runtime + files could be missing. The command `nix-store --query --requisites ` prints out + closures of store paths. + + As an example, if the [store object] at path `P` contains a [reference] + to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` + references `R` then `R` is also in the closure of `P`. + + See [References](@docroot@/store/store-object.md#references) for details. + + [closure]: #gloss-closure + - [content address]{#gloss-content-address} A @@ -31,6 +86,20 @@ The industry term for storage and retrieval systems using [content addressing](#gloss-content-address). A Nix store also has [input addressing](#gloss-input-addressed-store-object), and metadata. +- [content-addressed store object]{#gloss-content-addressed-store-object} + + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. + This includes derivations, the outputs of [content-addressing derivations](#gloss-content-addressing-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + +- [content-addressing derivation]{#gloss-content-addressing-derivation} + + A derivation which has the + [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) + attribute set to `true`. + - [derivation]{#gloss-derivation} A derivation can be thought of as a [pure function](https://en.wikipedia.org/wiki/Pure_function) that produces new [store objects][store object] from existing store objects. @@ -43,20 +112,11 @@ [derivation]: #gloss-derivation -- [store derivation]{#gloss-store-derivation} - - A [derivation] represented as a [store object]. - - See [Store Derivation](@docroot@/store/derivation/index.md#store-derivation) for details. - - [store derivation]: #gloss-store-derivation - -- [directed acyclic graph]{#gloss-directed-acyclic-graph} +- [derivation expression]{#gloss-derivation-expression} - A [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) (DAG) is graph whose edges are given a direction ("a to b" is not the same edge as "b to a"), and for which no possible path (created by joining together edges) forms a cycle. + A description of a [store derivation] using the [`derivation` primitive](./language/derivations.md) in the [Nix language]. - DAGs are very important to Nix. - In particular, the non-self-[references][reference] of [store object][store object] form a cycle. + [derivation expression]: #gloss-derivation-expression - [derivation path]{#gloss-derivation-path} @@ -68,79 +128,34 @@ [derivation path]: #gloss-derivation-path -- [derivation expression]{#gloss-derivation-expression} - - A description of a [store derivation] using the [`derivation` primitive](./language/derivations.md) in the [Nix language]. - - [derivation expression]: #gloss-derivation-expression - -- [instantiate]{#gloss-instantiate}, instantiation - - Translate a [derivation expression] into a [store derivation]. - - See [`nix-instantiate`](./command-ref/nix-instantiate.md), which produces a store derivation from a Nix expression that evaluates to a derivation. - - [instantiate]: #gloss-instantiate - -- [realise]{#gloss-realise}, realisation - - Ensure a [store path] is [valid][validity]. - - This can be achieved by: - - Fetching a pre-built [store object] from a [substituter] - - [Building](@docroot@/store/building.md) the corresponding [store derivation] - - Delegating to a [remote machine](@docroot@/command-ref/conf-file.md#conf-builders) and retrieving the outputs - - See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. - - See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). - - [realise]: #gloss-realise - -- [content-addressing derivation]{#gloss-content-addressing-derivation} - - A derivation which has the - [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) - attribute set to `true`. - -- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - - A [store derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. +- [deriver]{#gloss-deriver} -- [store]{#gloss-store} + The [store derivation] that produced an [output path]. - A collection of [store objects][store object], with operations to manipulate that collection. - See [Nix Store](./store/index.md) for details. + The deriver for an output path can be queried with the `--deriver` option to + [`nix-store --query`](@docroot@/command-ref/nix-store/query.md). - There are many types of stores, see [Store Types](./store/types/index.md) for details. +- [deriving path]{#gloss-deriving-path} - [store]: #gloss-store + Deriving paths are a way to refer to [store objects][store object] that might not yet be [realised][realise]. -- [Nix instance]{#gloss-nix-instance} - - 1. An installation of Nix, which includes the presence of a [store], and the Nix package manager which operates on that store. - A local Nix installation and a [remote builder](@docroot@/advanced-topics/distributed-builds.md) are two examples of Nix instances. - 2. A running Nix process, such as the `nix` command. + See [Deriving Path](./store/derivation/index.md#deriving-path) for details. -- [binary cache]{#gloss-binary-cache} + Not to be confused with [derivation path]. - A *binary cache* is a Nix store which uses a different format: its - metadata and signatures are kept in `.narinfo` files rather than in a - [Nix database]. This different format simplifies serving store objects - over the network, but cannot host builds. Examples of binary caches - include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). +- [directed acyclic graph]{#gloss-directed-acyclic-graph} -- [store path]{#gloss-store-path} + A [directed acyclic graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) (DAG) is graph whose edges are given a direction ("a to b" is not the same edge as "b to a"), and for which no possible path (created by joining together edges) forms a cycle. - The location of a [store object] in the file system, i.e., an immediate child of the Nix store directory. + DAGs are very important to Nix. + In particular, the non-self-[references][reference] of [store object][store object] form a cycle. - > **Example** - > - > `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` +- [experimental feature]{#gloss-experimental-feature} - See [Store Path](@docroot@/store/store-path.md) for details. + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - [store path]: #gloss-store-path + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). - [file system object]{#gloss-file-system-object} @@ -150,21 +165,19 @@ [file system object]: #gloss-file-system-object -- [store object]{#gloss-store-object} - - Part of the contents of a [store]. - - A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. - It can be referred to by a [store path]. - - See [Store Object](@docroot@/store/store-object.md) for details. +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - [store object]: #gloss-store-object + A [store derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [IFD]{#gloss-ifd} [Import From Derivation](./language/import-from-derivation.md) +- [impure derivation]{#gloss-impure-derivation} + + [An experimental feature](@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. + - [input-addressed store object]{#gloss-input-addressed-store-object} A store object produced by building a @@ -174,42 +187,28 @@ See [input-addressing derivation outputs](store/derivation/outputs/input-address.md) for details. -- [content-addressed store object]{#gloss-content-addressed-store-object} - - A [store object] which is [content-addressed](#gloss-content-address), - i.e. whose [store path] is determined by its contents. - This includes derivations, the outputs of [content-addressing derivations](#gloss-content-addressing-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). - - See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. - -- [substitute]{#gloss-substitute} +- [installable]{#gloss-installable} - A substitute is a command invocation stored in the [Nix database] that - describes how to build a store object, bypassing the normal build - mechanism (i.e., derivations). Typically, the substitute builds the - store object by downloading a pre-built version of the store object - from some server. + Something that can be realised in the Nix store. -- [substituter]{#gloss-substituter} + See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. - An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. - Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. +- [instantiate]{#gloss-instantiate}, instantiation - See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. + Translate a [derivation expression] into a [store derivation]. - [substituter]: #gloss-substituter + See [`nix-instantiate`](./command-ref/nix-instantiate.md), which produces a store derivation from a Nix expression that evaluates to a derivation. -- [purity]{#gloss-purity} + [instantiate]: #gloss-instantiate - The assumption that equal Nix derivations when run always produce - the same output. This cannot be guaranteed in general (e.g., a - builder can rely on external inputs such as the network or the - system time) but the Nix model assumes it. +- [Nix Archive (NAR)]{#gloss-nar} -- [impure derivation]{#gloss-impure-derivation} + A *N*ix *AR*chive. This is a serialisation of a path in the Nix + store. It can contain regular files, directories and symbolic + links. NARs are generated and unpacked using `nix-store --dump` + and `nix-store --restore`. - [An experimental feature](@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, - so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. - [Nix database]{#gloss-nix-database} @@ -235,145 +234,136 @@ > > Building and deploying software using Nix entails writing Nix expressions to describe [packages][package] and compositions thereof. -- [reference]{#gloss-reference} - - An edge from one [store object] to another. +- [Nix instance]{#gloss-nix-instance} + + 1. An installation of Nix, which includes the presence of a [store], and the Nix package manager which operates on that store. + A local Nix installation and a [remote builder](@docroot@/advanced-topics/distributed-builds.md) are two examples of Nix instances. + 2. A running Nix process, such as the `nix` command. - See [References](@docroot@/store/store-object.md#references) for details. +- [output]{#gloss-output} - [reference]: #gloss-reference + A [store object] produced by a [store derivation]. + See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. - See [References](@docroot@/store/store-object.md#references) for details. + [output]: #gloss-output -- [reachable]{#gloss-reachable} +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. - A store path `Q` is reachable from another store path `P` if `Q` - is in the *closure* of the *references* relation. +- [output path]{#gloss-output-path} - See [References](@docroot@/store/store-object.md#references) for details. + The [store path] to the [output] of a [store derivation]. -- [closure]{#gloss-closure} + [output path]: #gloss-output-path - The closure of a store path is the set of store paths that are - directly or indirectly “reachable” from that store path; that is, - it’s the closure of the path under the *references* relation. For - a package, the closure of its derivation is equivalent to the - build-time dependencies, while the closure of its [output path] is - equivalent to its runtime dependencies. For correct deployment it - is necessary to deploy whole closures, since otherwise at runtime - files could be missing. The command `nix-store --query --requisites ` prints out - closures of store paths. +- [package]{#package} - As an example, if the [store object] at path `P` contains a [reference] - to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` - references `R` then `R` is also in the closure of `P`. + A software package; files that belong together for a particular purpose, and metadata. - See [References](@docroot@/store/store-object.md#references) for details. + Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. - [closure]: #gloss-closure + The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#type-attrs) containing: + - attributes that refer to the files of a package, typically in the form of [derivation outputs](#gloss-output), + - attributes with metadata, such as information about how the package is supposed to be used. -- [requisite]{#gloss-requisite} + The exact shape of these attribute sets is up to convention. - A store object [reachable] by a path (chain of references) from a given [store object]. - The [closure] is the set of requisites. + [package]: #package - See [References](@docroot@/store/store-object.md#references) for details. +- [profile]{#gloss-profile} -- [referrer]{#gloss-referrer} + A symlink to the current *user environment* of a user, e.g., + `/nix/var/nix/profiles/default`. - A reversed edge from one [store object] to another. +- [purity]{#gloss-purity} -- [output]{#gloss-output} + The assumption that equal Nix derivations when run always produce + the same output. This cannot be guaranteed in general (e.g., a + builder can rely on external inputs such as the network or the + system time) but the Nix model assumes it. - A [store object] produced by a [store derivation]. - See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. +- [reachable]{#gloss-reachable} - [output]: #gloss-output + A store path `Q` is reachable from another store path `P` if `Q` + is in the *closure* of the *references* relation. -- [output path]{#gloss-output-path} + See [References](@docroot@/store/store-object.md#references) for details. - The [store path] to the [output] of a [store derivation]. +- [realise]{#gloss-realise}, realisation - [output path]: #gloss-output-path + Ensure a [store path] is [valid][validity]. -- [output closure]{#gloss-output-closure}\ - The [closure] of an [output path]. It only contains what is [reachable] from the output. + This can be achieved by: + - Fetching a pre-built [store object] from a [substituter] + - [Building](@docroot@/store/building.md) the corresponding [store derivation] + - Delegating to a [remote machine](@docroot@/command-ref/conf-file.md#conf-builders) and retrieving the outputs -- [deriving path]{#gloss-deriving-path} + See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. - Deriving paths are a way to refer to [store objects][store object] that might not yet be [realised][realise]. + See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). - See [Deriving Path](./store/derivation/index.md#deriving-path) for details. + [realise]: #gloss-realise - Not to be confused with [derivation path]. +- [reference]{#gloss-reference} -- [deriver]{#gloss-deriver} + An edge from one [store object] to another. - The [store derivation] that produced an [output path]. + See [References](@docroot@/store/store-object.md#references) for details. - The deriver for an output path can be queried with the `--deriver` option to - [`nix-store --query`](@docroot@/command-ref/nix-store/query.md). + [reference]: #gloss-reference -- [validity]{#gloss-validity} + See [References](@docroot@/store/store-object.md#references) for details. - A store path is valid if all [store object]s in its [closure] can be read from the [store]. +- [referrer]{#gloss-referrer} - For a [local store], this means: - - The store path leads to an existing [store object] in that [store]. - - The store path is listed in the [Nix database] as being valid. - - All paths in the store path's [closure] are valid. + A reversed edge from one [store object] to another. - [validity]: #gloss-validity - [local store]: @docroot@/store/types/local-store.md +- [requisite]{#gloss-requisite} -- [user environment]{#gloss-user-env} + A store object [reachable] by a path (chain of references) from a given [store object]. + The [closure] is the set of requisites. - An automatically generated store object that consists of a set of - symlinks to “active” applications, i.e., other store paths. These - are generated automatically by - [`nix-env`](./command-ref/nix-env.md). See *profiles*. + See [References](@docroot@/store/store-object.md#references) for details. -- [profile]{#gloss-profile} +- [store]{#gloss-store} - A symlink to the current *user environment* of a user, e.g., - `/nix/var/nix/profiles/default`. + A collection of [store objects][store object], with operations to manipulate that collection. + See [Nix Store](./store/index.md) for details. -- [installable]{#gloss-installable} + There are many types of stores, see [Store Types](./store/types/index.md) for details. - Something that can be realised in the Nix store. + [store]: #gloss-store - See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. +- [store derivation]{#gloss-store-derivation} -- [Nix Archive (NAR)]{#gloss-nar} + A [derivation] represented as a [store object]. - A *N*ix *AR*chive. This is a serialisation of a path in the Nix - store. It can contain regular files, directories and symbolic - links. NARs are generated and unpacked using `nix-store --dump` - and `nix-store --restore`. + See [Store Derivation](@docroot@/store/derivation/index.md#store-derivation) for details. - See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. + [store derivation]: #gloss-store-derivation -- [`∅`]{#gloss-empty-set} +- [store object]{#gloss-store-object} - The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. + Part of the contents of a [store]. -- [`ε`]{#gloss-epsilon} + A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. + It can be referred to by a [store path]. - The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. + See [Store Object](@docroot@/store/store-object.md) for details. -- [package]{#package} + [store object]: #gloss-store-object - A software package; files that belong together for a particular purpose, and metadata. +- [store path]{#gloss-store-path} - Nix represents files as [file system objects][file system object], and how they belong together is encoded as [references][reference] between [store objects][store object] that contain these file system objects. + The location of a [store object] in the file system, i.e., an immediate child of the Nix store directory. - The [Nix language] allows denoting packages in terms of [attribute sets](@docroot@/language/types.md#type-attrs) containing: - - attributes that refer to the files of a package, typically in the form of [derivation outputs](#gloss-output), - - attributes with metadata, such as information about how the package is supposed to be used. + > **Example** + > + > `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` - The exact shape of these attribute sets is up to convention. + See [Store Path](@docroot@/store/store-path.md) for details. - [package]: #package + [store path]: #gloss-store-path - [string interpolation]{#gloss-string-interpolation} @@ -385,31 +375,40 @@ [path]: ./language/types.md#type-path [attribute name]: ./language/types.md#type-attrs -- [base directory]{#gloss-base-directory} +- [substitute]{#gloss-substitute} - The location from which relative paths are resolved. + A substitute is a command invocation stored in the [Nix database] that + describes how to build a store object, bypassing the normal build + mechanism (i.e., derivations). Typically, the substitute builds the + store object by downloading a pre-built version of the store object + from some server. - - For expressions in a file, the base directory is the directory containing that file. - This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). - +- [substituter]{#gloss-substituter} - - - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. + Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. - [base directory]: #gloss-base-directory + See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. -- [experimental feature]{#gloss-experimental-feature} + [substituter]: #gloss-substituter - Not yet stabilized functionality guarded by named experimental feature flags. - These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. +- [user environment]{#gloss-user-env} - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). + An automatically generated store object that consists of a set of + symlinks to “active” applications, i.e., other store paths. These + are generated automatically by + [`nix-env`](./command-ref/nix-env.md). See *profiles*. + +- [validity]{#gloss-validity} + A store path is valid if all [store object]s in its [closure] can be read from the [store]. + + For a [local store], this means: + - The store path leads to an existing [store object] in that [store]. + - The store path is listed in the [Nix database] as being valid. + - All paths in the store path's [closure] are valid. + + [validity]: #gloss-validity + [local store]: @docroot@/store/types/local-store.md [Nix language]: ./language/index.md From 04ef6379cafd8f900f08c2f12571520b0df342bf Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 28 Mar 2026 16:25:35 +0300 Subject: [PATCH 179/927] libstore: Use new Worker::Waker mechanism for the substitution thread Previously we would create a new pipe for each substitution goal, but now we can reuse the new Waker coroutine resumption mechanism that writes to a single wakeup pipe and a thread-safe queue of goals to resume. Also documents and fixes some lifetimes issues in the thread captures. I haven't seen this fail, but looking at the code, I'm pretty sure that the thread can still be running when the Worker dies, depending on when the SIGUSR1 signal arrives and when the thread unwinds its stack. So use the same weak_ptr approach to optionally share the ownership of all the members (like the store, substituter and the Waker) that are needed by the substitution thread. --- src/libstore/build/goal.cc | 7 ++ src/libstore/build/substitution-goal.cc | 65 +++++++++---------- src/libstore/build/worker.cc | 11 ---- src/libstore/include/nix/store/build/goal.hh | 12 ++++ .../nix/store/build/substitution-goal.hh | 6 -- .../include/nix/store/build/worker.hh | 17 +---- 6 files changed, 53 insertions(+), 65 deletions(-) diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index ef3501b009..ce6b3e5883 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -308,6 +308,13 @@ Goal::Co Goal::waitForAWhile() co_return Return{}; } +Goal::Co Goal::waitUntilWoken() +{ + worker.waitForCompletion(shared_from_this()); + co_await Suspend{}; + co_return Return{}; +} + Goal::Co Goal::waitForBuildSlot() { worker.waitForBuildSlot(shared_from_this()); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 91ca7e1f21..2c1573b82c 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -207,53 +207,54 @@ Goal::Co PathSubstitutionGoal::tryToRun( auto maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); worker.updateProgress(); - outPipe = worker.makeMuxablePipe(); - auto promise = std::promise(); - - thr = std::thread([this, &promise, &subPath, &sub]() { + auto future = promise.get_future(); + + /* Be careful with ownership. cleanup() doesn't signal the worker thread + to cleanly shutdown, so the worker can die while the thread is still + running. That's why we use weak_ptr for everything that is owned by the + Worker. */ + thr = std::thread([weakGoal = weak_from_this(), + promise = std::move(promise), + subPath, + storePath = storePath, + repair = repair, + sub, + maybeWaker = worker.getCrossThreadWaker(), + maybeWorkerStore = worker.store.weak_from_this()]() mutable { try { ReceiveInterrupts receiveInterrupts; - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide.close(); }); + /* The Worker might have died while we were starting up. */ + auto workerStore = maybeWorkerStore.lock(); + if (!workerStore) + return; Activity act( *logger, actSubstitute, - Logger::Fields{worker.store.printStorePath(storePath), sub->config.getHumanReadableURI()}); + Logger::Fields{workerStore->printStorePath(storePath), sub->config.getHumanReadableURI()}); PushActivity pact(act.id); - copyStorePath(*sub, worker.store, subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs); + copyStorePath(*sub, *workerStore, subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { promise.set_exception(std::current_exception()); } + + /* The Worker might have already died (and the waker with it) by the + time we finished. N.B. if enqueueing to the waker throws, we better + std::terminate, since something has gone very wrong. This intentionally + lets the thread crash on exceptions for that reason. */ + if (auto waker = maybeWaker.lock()) + waker->enqueue(weakGoal); }); - worker.childStarted( - shared_from_this(), - { -#ifndef _WIN32 - outPipe.readSide.get() -#else - &outPipe -#endif - }, - true, - false); - - while (true) { - auto event = co_await WaitForChildEvent{}; - if (std::get_if(&event)) { - // Substitution doesn't process child output - } else if (std::get_if(&event)) { - break; - } else if (std::get_if(&event)) { - unreachable(); // Substitution doesn't use timeouts - } - } + /* Use up the substitution slot. */ + worker.childStarted(shared_from_this(), /*channels=*/{}, /*inBuildSlot=*/true, /*respectTimeouts=*/false); + /* Suspend until the thread finishes. */ + co_await waitUntilWoken(); trace("substitute finished"); @@ -261,7 +262,7 @@ Goal::Co PathSubstitutionGoal::tryToRun( worker.childTerminated(this); try { - promise.get_future().get(); + future.get(); } catch (std::exception & e) { /* Cause the parent build to fail unless --fallback is given, or the substitute has disappeared. The latter case behaves @@ -313,8 +314,6 @@ void PathSubstitutionGoal::cleanup() thr.join(); worker.childTerminated(this, JobCategory::Substitution); } - - outPipe.close(); } catch (...) { ignoreExceptionInDestructor(); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 9d2664305a..20131aaf3f 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -607,17 +607,6 @@ void Worker::markContentsGood(const StorePath & path) pathContentsGoodCache.insert_or_assign(path, true); } -MuxablePipe Worker::makeMuxablePipe() -{ - MuxablePipe pipe; -#ifndef _WIN32 - pipe.create(); -#else - pipe.createAsyncPipe(ioport.get()); -#endif - return pipe; -} - GoalPtr upcast_goal(std::shared_ptr subGoal) { return subGoal; diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 2efcd27a5b..3e32f4fecb 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -609,7 +609,19 @@ public: protected: Co await(Goals waitees); + /** + * Awaiting on the resulting coroutine yields the goal for several seconds. + * Used for retrying goals blocked on acquiring lockfiles. + */ Co waitForAWhile(); + + /** + * Awaiting on the resulting coroutine yields the goal until it is + * explicitly woken up via Worker::wakeUp. Wakeup can be queued from another + * thread via Worker::Waker. + */ + Co waitUntilWoken(); + Co waitForBuildSlot(); Co yield(); }; diff --git a/src/libstore/include/nix/store/build/substitution-goal.hh b/src/libstore/include/nix/store/build/substitution-goal.hh index 7ce28d1def..27585e348d 100644 --- a/src/libstore/include/nix/store/build/substitution-goal.hh +++ b/src/libstore/include/nix/store/build/substitution-goal.hh @@ -4,7 +4,6 @@ #include "nix/store/build/worker.hh" #include "nix/store/store-api.hh" #include "nix/store/build/goal.hh" -#include "nix/util/muxable-pipe.hh" #include #include #include @@ -23,11 +22,6 @@ struct PathSubstitutionGoal : public Goal */ RepairFlag repair; - /** - * Pipe for the substituter's standard output. - */ - MuxablePipe outPipe; - /** * The substituter thread. */ diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 6f58d4021b..2f5272cef5 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -150,8 +150,8 @@ private: { #ifndef _WIN32 /** - * Wakeup pipe polled alongside all other goal FDs. Gets written to by wakeUpCrossThread. - * Not needed on Windows. + * Wakeup pipe polled alongside all other goal FDs. Gets written to by + * enqueue(). Not needed on Windows. */ unix::SelfPipe wakeupPipe; #else @@ -173,13 +173,6 @@ private: #endif } -#ifdef _WIN32 - Waker(Descriptor ioport) - : ioport(ioport) - { - } -#endif - public: void enqueue(WeakGoalPtr goal); }; @@ -403,12 +396,6 @@ public: act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); } - - /** - * Create a MuxablePipe that the worker can poll. Primary exists to - * deduplicate WIN32 ifdefs. - */ - MuxablePipe makeMuxablePipe(); }; } // namespace nix From 05ff8f2cf9487d5c5cbf038d302d299ec83edc80 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sun, 29 Mar 2026 01:07:24 +0000 Subject: [PATCH 180/927] fix(libstore/aws-creds): use S3 URL region as STS WebIdentity fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, GitHub Actions OIDC), the aws-c-auth provider needs a region to select an STS endpoint. It looks in AWS_REGION, AWS_DEFAULT_REGION, then the profile config. In IRSA setups that export AWS_WEB_IDENTITY_TOKEN_FILE and AWS_ROLE_ARN but no region — and additionally set AWS_CONFIG_FILE=/dev/null to suppress SSO/profile interference — all three sources are absent. The provider returns nullptr, the credential chain silently skips it, and the user sees a misleading 'IMDS provider' failure. Thread the ?region= parameter from the parsed S3 URL through to the STS WebIdentity provider as a last-resort fallback. aws-c-auth gives options.region precedence over env vars, so we only set it when neither AWS_REGION nor AWS_DEFAULT_REGION is present, preserving existing behaviour for users who do set a region. The credential-provider cache key changes from profile to (profile, region) since the region is baked into the provider at construction time. --- .../aws-sts-webidentity-region-fallback.md | 11 ++++ src/libstore/aws-creds.cc | 56 +++++++++++++++---- 2 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 doc/manual/rl-next/aws-sts-webidentity-region-fallback.md diff --git a/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md b/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md new file mode 100644 index 0000000000..5034c65438 --- /dev/null +++ b/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md @@ -0,0 +1,11 @@ +--- +synopsis: S3 substituters fall back to the URL's region for STS WebIdentity auth +prs: [15594] +--- + +When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, +GitHub Actions OIDC), Nix now uses the `?region=` parameter from the S3 URL +as a fallback for the STS endpoint region if neither `AWS_REGION` nor +`AWS_DEFAULT_REGION` is set. Previously, IRSA setups that exported +`AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` but no region would fail +with a misleading "IMDS provider" error. diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc index b471962897..f755717eb5 100644 --- a/src/libstore/aws-creds.cc +++ b/src/libstore/aws-creds.cc @@ -4,6 +4,7 @@ # include # include "nix/store/s3-url.hh" +# include "nix/util/environment-variables.hh" # include "nix/util/logging.hh" # include @@ -170,15 +171,33 @@ static std::shared_ptr createSSOProvider( return createWrappedProvider(aws_credentials_provider_new_sso(allocator, &options), allocator); } +/** + * Check whether the AWS SDK can resolve a region from the standard + * environment variables. Mirrors aws_credentials_provider_resolve_region_from_env. + */ +static bool awsRegionSetInEnv() +{ + return getEnvNonEmpty("AWS_REGION") || getEnvNonEmpty("AWS_DEFAULT_REGION"); +} + /** * Create an STS WebIdentity credentials provider using the C library directly. * This reads AWS_WEB_IDENTITY_TOKEN_FILE, AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, * and AWS_REGION from the environment (falling back to the profile config). * Used by EKS IRSA, GitHub Actions OIDC, and other sts:AssumeRoleWithWebIdentity flows. * Returns nullptr if the required parameters can't be resolved. + * + * @param fallbackRegion Region to use when neither AWS_REGION nor + * AWS_DEFAULT_REGION is set — typically the ?region= from the S3 URL. + * Note: this also overrides any region from the profile config, since + * aws-c-auth gives options.region precedence over all implicit sources. + * Without this fallback the provider fails entirely in IRSA setups where + * the pod environment sets the token and role ARN but not the region + * (observed with AWS_CONFIG_FILE=/dev/null). */ static std::shared_ptr createSTSWebIdentityProvider( const std::string & profileName, + const std::string & fallbackRegion, Aws::Crt::Io::ClientBootstrap * bootstrap, Aws::Crt::Io::TlsContext * tlsContext, Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator()) @@ -192,6 +211,12 @@ static std::shared_ptr createSTSWebIdentit options.profile_name_override = aws_byte_cursor_from_c_str(profileName.c_str()); } + // aws-c-auth gives options.region precedence over env vars, so only set it + // when env vars are absent — otherwise we'd mask a user-supplied AWS_REGION. + if (!fallbackRegion.empty() && !awsRegionSetInEnv()) { + options.region = aws_byte_cursor_from_c_str(fallbackRegion.c_str()); + } + return createWrappedProvider(aws_credentials_provider_new_sts_web_identity(allocator, &options), allocator); } @@ -288,32 +313,38 @@ class AwsCredentialProviderImpl : public AwsCredentialProvider } } - AwsCredentials getCredentialsRaw(const std::string & profile); + AwsCredentials getCredentialsRaw(const std::string & profile, const std::string & region); AwsCredentials getCredentials(const ParsedS3URL & url) override { auto profile = url.profile.value_or(""); + auto region = url.region.value_or(""); try { - return getCredentialsRaw(profile); + return getCredentialsRaw(profile, region); } catch (AwsAuthError & e) { warn("AWS authentication failed for S3 request %s: %s", url.toHttpsUrl(), e.message()); - credentialProviderCache.erase(profile); + credentialProviderCache.erase({profile, region}); throw; } } - std::shared_ptr createProviderForProfile(const std::string & profile); + std::shared_ptr + createProviderForProfile(const std::string & profile, const std::string & region); private: Aws::Crt::ApiHandle apiHandle; std::shared_ptr tlsContext; Aws::Crt::Io::ClientBootstrap * bootstrap; - boost::concurrent_flat_map> - credentialProviderCache; + // Keyed by (profile, region). Region is part of the key because it is baked + // into the STS WebIdentity provider at construction time; two S3 URLs with + // different ?region= parameters need distinct chains. + boost:: + concurrent_flat_map, std::shared_ptr> + credentialProviderCache; }; std::shared_ptr -AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) +AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile, const std::string & region) { // profileDisplayName is only used for debug logging - SDK uses its default profile // when ProfileNameOverride is not set @@ -369,7 +400,7 @@ AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) bool ecsAdded = false; if (tlsContext) { addProviderToChain("STS WebIdentity", [&]() { - return createSTSWebIdentityProvider(profile, bootstrap, tlsContext.get(), allocator); + return createSTSWebIdentityProvider(profile, region, bootstrap, tlsContext.get(), allocator); }); ecsAdded = addProviderToChain("ECS", [&]() { return createECSProvider(bootstrap, tlsContext.get(), allocator); }); @@ -395,18 +426,19 @@ AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile) return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChain(chainConfig, allocator); } -AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile) +AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile, const std::string & region) { std::shared_ptr provider; + auto key = std::make_pair(profile, region); credentialProviderCache.try_emplace_and_cvisit( - profile, + key, nullptr, - [&](auto & kv) { provider = kv.second = createProviderForProfile(profile); }, + [&](auto & kv) { provider = kv.second = createProviderForProfile(profile, region); }, [&](const auto & kv) { provider = kv.second; }); if (!provider) { - credentialProviderCache.erase_if(profile, [](const auto & kv) { + credentialProviderCache.erase_if(key, [](const auto & kv) { [[maybe_unused]] auto [_, provider] = kv; return !provider; }); From aa96fd767aa83b14f89fce4f9251d02245fb2b93 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 29 Mar 2026 22:14:43 +0300 Subject: [PATCH 181/927] libexpr-test-support: Remove duplicate definitions of rc::Arbitrary for SingleDerivedPath This redefinition is strictly speaking UB. --- .../include/nix/expr/tests/value/context.hh | 22 +++++-------------- .../include/nix/store/tests/derived-path.hh | 21 +++++++++--------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/src/libexpr-test-support/include/nix/expr/tests/value/context.hh b/src/libexpr-test-support/include/nix/expr/tests/value/context.hh index 68a0b8dea7..b8f4b4f67a 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/value/context.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/value/context.hh @@ -4,32 +4,20 @@ #include #include "nix/expr/value/context.hh" +#include "nix/store/tests/derived-path.hh" // IWYU pragma: keep namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); -}; - -template<> -struct Arbitrary -{ - static Gen arbitrary(); -}; - -template<> -struct Arbitrary -{ - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libstore-test-support/include/nix/store/tests/derived-path.hh b/src/libstore-test-support/include/nix/store/tests/derived-path.hh index b3b43474a9..7d93af08fd 100644 --- a/src/libstore-test-support/include/nix/store/tests/derived-path.hh +++ b/src/libstore-test-support/include/nix/store/tests/derived-path.hh @@ -9,36 +9,35 @@ #include "nix/store/tests/outputs-spec.hh" namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc From 94159358f068ab47f1a2c5d3441501df1102a329 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 29 Mar 2026 23:27:44 +0300 Subject: [PATCH 182/927] Add missing pragma once to headersA Needed for unity builds. --- src/libcmd/include/nix/cmd/misc-store-flags.hh | 3 +++ src/libstore/include/nix/store/local-binary-cache-store.hh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/libcmd/include/nix/cmd/misc-store-flags.hh b/src/libcmd/include/nix/cmd/misc-store-flags.hh index 27e1390768..5c38a25934 100644 --- a/src/libcmd/include/nix/cmd/misc-store-flags.hh +++ b/src/libcmd/include/nix/cmd/misc-store-flags.hh @@ -1,3 +1,6 @@ +#pragma once +/// @file + #include "nix/util/args.hh" #include "nix/store/content-address.hh" diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index ff28b4b734..69a4bac1c8 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -1,3 +1,6 @@ +#pragma once +/// @file + #include "nix/store/binary-cache-store.hh" namespace nix { From 39827d46c1f3070c6b8afc15e0d634eae8f01f8a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 29 Mar 2026 23:33:38 +0300 Subject: [PATCH 183/927] Prep for unity builds For unity builds we have several issues. We must make sure that we don't do using namespace nix; in any translation unit in libexpr-tests and libstore-tests, because we have an ambiguity between ::Store and nix::Store (same for StorePath). We also need to disambiguate between nix::testing and ::testing namespaces for name lookup. Settings registration RAII objects also need to be called differently. --- src/libexpr-tests/dynamic-attrs-bench.cc | 4 +++- src/libexpr-tests/get-drvs-bench.cc | 5 +++-- src/libexpr-tests/main.cc | 6 ++---- src/libexpr-tests/primops.cc | 12 ++++++------ src/libstore-tests/main.cc | 4 +--- .../include/nix/store/build/derivation-builder.hh | 3 +++ src/libutil-tests/file-content-address.cc | 4 ++-- src/libutil-tests/suggestions.cc | 4 ++-- src/libutil/include/nix/util/compression-settings.hh | 5 +++++ src/nix/nix-env/nix-env.cc | 2 +- src/nix/unix/daemon.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 12 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/libexpr-tests/dynamic-attrs-bench.cc b/src/libexpr-tests/dynamic-attrs-bench.cc index 1b1c199bdf..ea0736564f 100644 --- a/src/libexpr-tests/dynamic-attrs-bench.cc +++ b/src/libexpr-tests/dynamic-attrs-bench.cc @@ -5,7 +5,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/store/store-open.hh" -using namespace nix; +namespace nix { static std::string mkDynamicAttrsExpr(size_t attrCount) { @@ -54,3 +54,5 @@ static void BM_EvalDynamicAttrs(benchmark::State & state) } BENCHMARK(BM_EvalDynamicAttrs)->Arg(100)->Arg(500)->Arg(2'000); + +} // namespace nix diff --git a/src/libexpr-tests/get-drvs-bench.cc b/src/libexpr-tests/get-drvs-bench.cc index a5cd59154f..9241e48048 100644 --- a/src/libexpr-tests/get-drvs-bench.cc +++ b/src/libexpr-tests/get-drvs-bench.cc @@ -6,8 +6,7 @@ #include "nix/store/store-open.hh" #include "nix/util/fmt.hh" -using namespace nix; - +namespace nix { namespace { struct GetDerivationsEnv @@ -64,3 +63,5 @@ static void BM_GetDerivationsAttrScan(benchmark::State & state) } BENCHMARK(BM_GetDerivationsAttrScan)->Arg(1'000)->Arg(5'000)->Arg(10'000); + +} // namespace nix diff --git a/src/libexpr-tests/main.cc b/src/libexpr-tests/main.cc index 88a9d6684d..365fa482de 100644 --- a/src/libexpr-tests/main.cc +++ b/src/libexpr-tests/main.cc @@ -3,16 +3,14 @@ #include "nix/store/tests/test-main.hh" #include "nix/util/config-global.hh" -using namespace nix; - int main(int argc, char ** argv) { - auto res = testMainForBuidingPre(argc, argv); + auto res = nix::testMainForBuidingPre(argc, argv); if (res) return res; // For pipe operator tests in trivial.cc - experimentalFeatureSettings.set("experimental-features", "pipe-operators"); + nix::experimentalFeatureSettings.set("experimental-features", "pipe-operators"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/src/libexpr-tests/primops.cc b/src/libexpr-tests/primops.cc index ac5e893bf7..fc4f6c0702 100644 --- a/src/libexpr-tests/primops.cc +++ b/src/libexpr-tests/primops.cc @@ -637,7 +637,7 @@ TEST_F(PrimOpTest, toStringLambdaThrows) } class ToStringPrimOpTest : public PrimOpTest, - public testing::WithParamInterface> + public ::testing::WithParamInterface> {}; TEST_P(ToStringPrimOpTest, toString) @@ -651,7 +651,7 @@ TEST_P(ToStringPrimOpTest, toString) INSTANTIATE_TEST_SUITE_P( toString, ToStringPrimOpTest, - testing::Values( + ::testing::Values( CASE(R"("foo")", "foo"), CASE(R"(1)", "1"), CASE(R"([1 2 3])", "1 2 3"), @@ -799,7 +799,7 @@ TEST_F(PrimOpTest, splitVersion) } class CompareVersionsPrimOpTest : public PrimOpTest, - public testing::WithParamInterface> + public ::testing::WithParamInterface> {}; TEST_P(CompareVersionsPrimOpTest, compareVersions) @@ -813,7 +813,7 @@ TEST_P(CompareVersionsPrimOpTest, compareVersions) INSTANTIATE_TEST_SUITE_P( compareVersions, CompareVersionsPrimOpTest, - testing::Values( + ::testing::Values( // The first two are weird cases. Intuition tells they should // be the same but they aren't. CASE(1.0, 1.0.0, -1), @@ -835,7 +835,7 @@ INSTANTIATE_TEST_SUITE_P( class ParseDrvNamePrimOpTest : public PrimOpTest, - public testing::WithParamInterface> + public ::testing::WithParamInterface> {}; TEST_P(ParseDrvNamePrimOpTest, parseDrvName) @@ -857,7 +857,7 @@ TEST_P(ParseDrvNamePrimOpTest, parseDrvName) INSTANTIATE_TEST_SUITE_P( parseDrvName, ParseDrvNamePrimOpTest, - testing::Values( + ::testing::Values( std::make_tuple("nix-0.12pre12876", "nix", "0.12pre12876"), std::make_tuple("a-b-c-1234pre5+git", "a-b-c", "1234pre5+git"))); diff --git a/src/libstore-tests/main.cc b/src/libstore-tests/main.cc index c45e3a7f38..93d93e4aa0 100644 --- a/src/libstore-tests/main.cc +++ b/src/libstore-tests/main.cc @@ -3,11 +3,9 @@ #include "nix/store/tests/test-main.hh" #include "nix/store/tests/libstore-network.hh" -using namespace nix; - int main(int argc, char ** argv) { - auto res = testMainForBuidingPre(argc, argv); + auto res = nix::testMainForBuidingPre(argc, argv); if (res) return res; diff --git a/src/libstore/include/nix/store/build/derivation-builder.hh b/src/libstore/include/nix/store/build/derivation-builder.hh index f521c0402a..bff249f781 100644 --- a/src/libstore/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/include/nix/store/build/derivation-builder.hh @@ -48,6 +48,9 @@ struct ChrootPath bool optional = false; }; +void to_json(nlohmann::json & j, const ChrootPath & cp); +void from_json(const nlohmann::json & j, ChrootPath & cp); + typedef std::map PathsInChroot; // maps target path to source path /** diff --git a/src/libutil-tests/file-content-address.cc b/src/libutil-tests/file-content-address.cc index a6b10d4f62..8d6ca3ce87 100644 --- a/src/libutil-tests/file-content-address.cc +++ b/src/libutil-tests/file-content-address.cc @@ -33,7 +33,7 @@ TEST(FileSerialisationMethod, testParseFileSerialisationMethodOptException) { EXPECT_THAT( []() { parseFileSerialisationMethod("narwhal"); }, - testing::ThrowsMessage(testing::HasSubstr("narwhal"))); + ::testing::ThrowsMessage(::testing::HasSubstr("narwhal"))); } /* ---------------------------------------------------------------------------- @@ -66,7 +66,7 @@ TEST(FileIngestionMethod, testParseFileIngestionMethodOptException) { EXPECT_THAT( []() { parseFileIngestionMethod("narwhal"); }, - testing::ThrowsMessage(testing::HasSubstr("narwhal"))); + ::testing::ThrowsMessage(::testing::HasSubstr("narwhal"))); } } // namespace nix diff --git a/src/libutil-tests/suggestions.cc b/src/libutil-tests/suggestions.cc index a23e5d3f43..a94ec30ee6 100644 --- a/src/libutil-tests/suggestions.cc +++ b/src/libutil-tests/suggestions.cc @@ -9,7 +9,7 @@ struct LevenshteinDistanceParam int distance; }; -class LevenshteinDistanceTest : public testing::TestWithParam +class LevenshteinDistanceTest : public ::testing::TestWithParam {}; TEST_P(LevenshteinDistanceTest, CorrectlyComputed) @@ -23,7 +23,7 @@ TEST_P(LevenshteinDistanceTest, CorrectlyComputed) INSTANTIATE_TEST_SUITE_P( LevenshteinDistance, LevenshteinDistanceTest, - testing::Values( + ::testing::Values( LevenshteinDistanceParam{"foo", "foo", 0}, LevenshteinDistanceParam{"foo", "", 3}, LevenshteinDistanceParam{"", "", 0}, diff --git a/src/libutil/include/nix/util/compression-settings.hh b/src/libutil/include/nix/util/compression-settings.hh index f9d42c298c..756d636f13 100644 --- a/src/libutil/include/nix/util/compression-settings.hh +++ b/src/libutil/include/nix/util/compression-settings.hh @@ -18,4 +18,9 @@ std::optional BaseSetting>::pars template<> std::string BaseSetting>::to_string() const; +/* Same as with all settings - empty string means std::nullopt. */ +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index 63d3eb137b..b93a51ac53 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -67,7 +67,7 @@ struct EnvSettings : Config EnvSettings envSettings; -static GlobalConfig::Register rSettings(&envSettings); +static GlobalConfig::Register rEnvSettings(&envSettings); typedef enum { srcNixExprDrvs, srcNixExprs, srcStorePaths, srcProfile, srcAttrPath, srcUnknown } InstallSourceType; diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index c9db03a912..a8cfb089fc 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -100,7 +100,7 @@ struct AuthorizationSettings : Config AuthorizationSettings authorizationSettings; -static GlobalConfig::Register rSettings(&authorizationSettings); +static GlobalConfig::Register rAuthorizationSettings(&authorizationSettings); #ifndef __linux__ # define SPLICE_F_MOVE 0 diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 513d76e9b7..17b410b325 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -43,7 +43,7 @@ struct UpgradeSettings : Config UpgradeSettings upgradeSettings; -static GlobalConfig::Register rSettings(&upgradeSettings); +static GlobalConfig::Register rUpgradeSettings(&upgradeSettings); struct CmdUpgradeNix : MixDryRun, StoreCommand { From 7ed1fa9394666fa2ce82041102f52d69184be91d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 29 Mar 2026 23:34:34 +0300 Subject: [PATCH 184/927] Move lexer-helpers.cc into a separate library for unity builds Flex and bison generated code is very tightly coupled with some include cycles that simply must be broken by compiling in separate translation units. --- src/libexpr/meson.build | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 893718cb82..510b6d696e 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -168,7 +168,6 @@ sources = files( 'function-trace.cc', 'get-drvs.cc', 'json-to-value.cc', - 'lexer-helpers.cc', 'nixexpr.cc', 'paths.cc', 'primops.cc', @@ -219,6 +218,7 @@ parser_library = static_library( 'nixexpr-parser', parser_tab, lexer_tab, + 'lexer-helpers.cc', cpp_args : parser_library_cpp_args, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, @@ -229,6 +229,8 @@ parser_library = static_library( override_options : [ 'b_ndebug=@0@'.format(not get_option('debug')), 'b_lto=@0@'.format(get_option('b_lto') and cxx.get_id() != 'gcc'), + # Unity builds mess up the recursive header dependency between flex and bison generated code. + 'unity=off', ], ) From 3f4b66013a714bec3add1cb54709f9faf7193108 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 29 Mar 2026 23:46:16 +0300 Subject: [PATCH 185/927] Make unity builds work --- nix-meson-build-support/common/meson.build | 1 + src/libexpr-tests/regex-cache-bench.cc | 4 +- src/libfetchers-tests/git-utils.cc | 4 +- src/libfetchers-tests/git.cc | 4 +- .../include/nix/store/tests/outputs-spec.hh | 5 +- .../include/nix/store/tests/path.hh | 9 +-- .../include/nix/store/tests/protocol.hh | 6 ++ src/libstore-test-support/outputs-spec.cc | 7 +- src/libstore-test-support/path.cc | 12 +-- src/libstore-tests/derivation-parser-bench.cc | 4 +- .../derivation/external-formats.cc | 73 +++++++++---------- src/libstore-tests/machines.cc | 14 ++-- src/libstore-tests/nar-info.cc | 12 +++ src/libstore-tests/path-info.cc | 12 +++ src/libstore-tests/ref-scan-bench.cc | 4 +- .../register-valid-paths-bench.cc | 4 +- src/libstore-tests/serve-protocol.cc | 14 +--- src/libstore-tests/worker-protocol.cc | 20 ++--- src/libstore-tests/write-derivation.cc | 21 +++--- .../build/derivation-trampoline-goal.cc | 19 +++-- .../include/nix/store/local-settings.hh | 6 ++ .../include/nix/store/store-reference.hh | 19 +++++ src/libutil/compression-settings.cc | 5 -- .../include/nix/util/compression-settings.hh | 1 - src/nix/nix-store/graphml.cc | 10 +-- 25 files changed, 165 insertions(+), 125 deletions(-) diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index d0dde91348..edb40635f2 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -36,6 +36,7 @@ do_pch = cxx.get_id() == 'clang' if cxx.get_id() == 'gcc' add_project_arguments( '-Wno-interference-size', # Used for C++ ABI only. We don't provide any guarantees about different march tunings. + '-Wno-subobject-linkage', # GCC doesn't like unity builds. language : 'cpp', ) endif diff --git a/src/libexpr-tests/regex-cache-bench.cc b/src/libexpr-tests/regex-cache-bench.cc index 2eb17b212a..c0c61e58e9 100644 --- a/src/libexpr-tests/regex-cache-bench.cc +++ b/src/libexpr-tests/regex-cache-bench.cc @@ -5,7 +5,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/store/store-open.hh" -using namespace nix; +namespace nix { static void BM_EvalManyBuiltinsMatchSameRegex(benchmark::State & state) { @@ -44,3 +44,5 @@ static void BM_EvalManyBuiltinsMatchSameRegex(benchmark::State & state) } BENCHMARK(BM_EvalManyBuiltinsMatchSameRegex); + +} // namespace nix diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index 0b21fd0c67..a80ff590f9 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -15,7 +15,7 @@ #include #include -namespace nix { +namespace nix::fetchers { class GitUtilsTest : public ::testing::Test { @@ -234,4 +234,4 @@ TEST(GitUtils, isLegalRefName) ASSERT_FALSE(isLegalRefName("")); } -} // namespace nix +} // namespace nix::fetchers diff --git a/src/libfetchers-tests/git.cc b/src/libfetchers-tests/git.cc index 9fb9a2ce1c..e970c3b8bd 100644 --- a/src/libfetchers-tests/git.cc +++ b/src/libfetchers-tests/git.cc @@ -98,7 +98,7 @@ static void commitAll(git_repository * repo, const char * msg) } // namespace -using namespace nix; +namespace nix::fetchers { class GitTest : public ::testing::Test { @@ -201,3 +201,5 @@ TEST_F(GitTest, submodulePeriodSupport) ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n"); } + +} // namespace nix::fetchers diff --git a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh index 5bbcc73408..a30f837702 100644 --- a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh +++ b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh @@ -9,12 +9,11 @@ #include "nix/store/tests/path.hh" namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libstore-test-support/include/nix/store/tests/path.hh b/src/libstore-test-support/include/nix/store/tests/path.hh index ff80b1299a..e2ecd23ca5 100644 --- a/src/libstore-test-support/include/nix/store/tests/path.hh +++ b/src/libstore-test-support/include/nix/store/tests/path.hh @@ -18,18 +18,17 @@ void showValue(const StorePath & p, std::ostream & os); } // namespace nix namespace rc { -using namespace nix; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; template<> -struct Arbitrary +struct Arbitrary { - static Gen arbitrary(); + static Gen arbitrary(); }; } // namespace rc diff --git a/src/libstore-test-support/include/nix/store/tests/protocol.hh b/src/libstore-test-support/include/nix/store/tests/protocol.hh index 563c8cfb6c..543facead0 100644 --- a/src/libstore-test-support/include/nix/store/tests/protocol.hh +++ b/src/libstore-test-support/include/nix/store/tests/protocol.hh @@ -122,4 +122,10 @@ public: VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) \ VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, (VERSION), VALUE) +/// Has to be a `BufferedSink` for handshake. +struct NullBufferedSink : BufferedSink +{ + void writeUnbuffered(std::string_view data) override {} +}; + } // namespace nix diff --git a/src/libstore-test-support/outputs-spec.cc b/src/libstore-test-support/outputs-spec.cc index d5128a8bd9..947f02eb53 100644 --- a/src/libstore-test-support/outputs-spec.cc +++ b/src/libstore-test-support/outputs-spec.cc @@ -3,10 +3,13 @@ #include namespace rc { -using namespace nix; -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { + using nix::OutputsSpec; + using nix::StorePathName; + using nix::StringSet; + return gen::mapcat( gen::inRange(0, std::variant_size_v), [](uint8_t n) -> Gen { switch (n) { diff --git a/src/libstore-test-support/path.cc b/src/libstore-test-support/path.cc index 1459203104..1e5f74a285 100644 --- a/src/libstore-test-support/path.cc +++ b/src/libstore-test-support/path.cc @@ -20,7 +20,6 @@ void showValue(const StorePath & p, std::ostream & os) } // namespace nix namespace rc { -using namespace nix; Gen storePathChar() { @@ -52,18 +51,19 @@ Gen storePathChar() gen::inRange(0, 10 + 2 * 26 + 6)); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { - return gen::construct( + return gen::construct( gen::suchThat(gen::container(storePathChar()), [](const std::string & s) { return !(s == "" || s == "." || s == ".." || s.starts_with(".-") || s.starts_with("..-")); })); } -Gen Arbitrary::arbitrary() +Gen Arbitrary::arbitrary() { - return gen::construct( - gen::arbitrary(), gen::apply([](StorePathName n) { return n.name; }, gen::arbitrary())); + return gen::construct( + gen::arbitrary(), + gen::apply([](nix::StorePathName n) { return n.name; }, gen::arbitrary())); } } // namespace rc diff --git a/src/libstore-tests/derivation-parser-bench.cc b/src/libstore-tests/derivation-parser-bench.cc index f0aa721cb7..93fa64f619 100644 --- a/src/libstore-tests/derivation-parser-bench.cc +++ b/src/libstore-tests/derivation-parser-bench.cc @@ -7,7 +7,7 @@ #include #include -using namespace nix; +namespace nix { // Benchmark parsing real derivation files static void BM_ParseRealDerivationFile(benchmark::State & state, const std::string & filename) @@ -54,3 +54,5 @@ BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, hello, (getUnitTestData() / "deriv BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string()); BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string()); BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string()); + +} // namespace nix diff --git a/src/libstore-tests/derivation/external-formats.cc b/src/libstore-tests/derivation/external-formats.cc index a9be99f996..6fee675e98 100644 --- a/src/libstore-tests/derivation/external-formats.cc +++ b/src/libstore-tests/derivation/external-formats.cc @@ -177,44 +177,41 @@ struct DerivationJsonAtermTest : DerivationTest, MAKE_TEST_P(DerivationJsonAtermTest); -Derivation makeSimpleDrv() -{ - Derivation drv; - drv.name = "simple-derivation"; - drv.inputSrcs = { - StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputDrvs = { - .map = - { - { - StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), - { - .value = - { - "cat", - "dog", - }, - }, - }, - }, - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = StringPairs{ - { - "BIG_BAD", - "WOLF", - }, - }; - return drv; -} - -INSTANTIATE_TEST_SUITE_P(DerivationJSONATerm, DerivationJsonAtermTest, ::testing::Values(makeSimpleDrv())); +INSTANTIATE_TEST_SUITE_P(DerivationJSONATerm, DerivationJsonAtermTest, ::testing::Values([]() { + Derivation drv; + drv.name = "simple-derivation"; + drv.inputSrcs = { + StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = + { + { + StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + .value = + { + "cat", + "dog", + }, + }, + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = StringPairs{ + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; + }())); struct DynDerivationJsonAtermTest : DynDerivationTest, JsonCharacterizationTest, diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index 4246a876ca..e412376323 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -7,13 +7,13 @@ #include #include -using testing::Contains; -using testing::ElementsAre; -using testing::Eq; -using testing::Field; -using testing::SizeIs; +using ::testing::Contains; +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::Field; +using ::testing::SizeIs; -using namespace nix; +namespace nix { TEST(machines, getMachinesWithEmptyBuilders) { @@ -210,3 +210,5 @@ TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) {}, "@" + std::filesystem::weakly_canonical(getUnitTestData() / "machines" / "bad_format").string()), FormatError); } + +} // namespace nix diff --git a/src/libstore-tests/nar-info.cc b/src/libstore-tests/nar-info.cc index 9c8a93e16b..21a1d63a7e 100644 --- a/src/libstore-tests/nar-info.cc +++ b/src/libstore-tests/nar-info.cc @@ -169,4 +169,16 @@ JSON_TEST_V2(impure, true) JSON_TEST_V3(pure, false) JSON_TEST_V3(impure, true) +#undef JSON_TEST_V1 +#undef JSON_READ_TEST_V1 +#undef JSON_WRITE_TEST_V1 + +#undef JSON_TEST_V2 +#undef JSON_READ_TEST_V2 +#undef JSON_WRITE_TEST_V2 + +#undef JSON_TEST_V3 +#undef JSON_READ_TEST_V3 +#undef JSON_WRITE_TEST_V3 + } // namespace nix diff --git a/src/libstore-tests/path-info.cc b/src/libstore-tests/path-info.cc index 9141d06cd6..7734ee1755 100644 --- a/src/libstore-tests/path-info.cc +++ b/src/libstore-tests/path-info.cc @@ -182,6 +182,18 @@ JSON_TEST_V3(empty_impure, makeEmpty(), true) JSON_TEST_V3(pure, makeFull(*store, false), false) JSON_TEST_V3(impure, makeFull(*store, true), true) +#undef JSON_TEST_V1 +#undef JSON_READ_TEST_V1 +#undef JSON_WRITE_TEST_V1 + +#undef JSON_TEST_V2 +#undef JSON_READ_TEST_V2 +#undef JSON_WRITE_TEST_V2 + +#undef JSON_TEST_V3 +#undef JSON_READ_TEST_V3 +#undef JSON_WRITE_TEST_V3 + TEST_F(PathInfoTestV2, PathInfo_full_shortRefs) { ValidPathInfo it = makeFullKeyed(*store, true); diff --git a/src/libstore-tests/ref-scan-bench.cc b/src/libstore-tests/ref-scan-bench.cc index ff0aa18150..05264fd290 100644 --- a/src/libstore-tests/ref-scan-bench.cc +++ b/src/libstore-tests/ref-scan-bench.cc @@ -6,7 +6,7 @@ #include -using namespace nix; +namespace nix { template static void randomReference(std::mt19937 & urng, OIt outIter) @@ -91,3 +91,5 @@ static void BM_RefScanSinkRandom(benchmark::State & state) } BENCHMARK(BM_RefScanSinkRandom)->Arg(10'000)->Arg(100'000)->Arg(1'000'000)->Arg(5'000'000)->Arg(10'000'000); + +} // namespace nix diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 1d79581836..400a705556 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -12,7 +12,7 @@ # include # include -using namespace nix; +namespace nix { static void BM_RegisterValidPathsDerivations(benchmark::State & state) { @@ -76,4 +76,6 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) BENCHMARK(BM_RegisterValidPathsDerivations)->Arg(10); +} // namespace nix + #endif diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 3eeecd70f4..6418cb55dc 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -17,8 +17,6 @@ namespace nix { const char serveProtoDir[] = "serve-protocol"; -static constexpr std::string_view defaultStoreDir = "/nix/store"; - struct ServeProtoTest : VersionedProtoTest { /** @@ -326,12 +324,12 @@ VERSIONED_CHARACTERIZATION_TEST( }), (std::tuple{ ({ - UnkeyedValidPathInfo info{std::string{defaultStoreDir}, Hash::dummy}; + UnkeyedValidPathInfo info{"/nix/store", Hash::dummy}; info.narSize = 34878; info; }), ({ - UnkeyedValidPathInfo info{std::string{defaultStoreDir}, Hash::dummy}; + UnkeyedValidPathInfo info{"/nix/store", Hash::dummy}; info.deriver = StorePath{ "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", }; @@ -356,7 +354,7 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple{ ({ UnkeyedValidPathInfo info{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }; info.deriver = StorePath{ @@ -547,12 +545,6 @@ TEST_F(ServeProtoTest, handshake_log) }); } -/// Has to be a `BufferedSink` for handshake. -struct NullBufferedSink : BufferedSink -{ - void writeUnbuffered(std::string_view data) override {} -}; - TEST_F(ServeProtoTest, handshake_client_replay) { CharacterizationTest::readTest("handshake-to-client.bin", [&](std::string toClientLog) { diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index 044fbcc920..ed288a7443 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -91,8 +91,6 @@ TEST(WorkerProtoVersion, partialOrderingEmptyFeatures) const char workerProtoDir[] = "worker-protocol"; -static constexpr std::string_view defaultStoreDir = "/nix/store"; - struct WorkerProtoTest : VersionedProtoTest { /** @@ -597,7 +595,7 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple{ ({ UnkeyedValidPathInfo info{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }; info.registrationTime = 23423; @@ -606,7 +604,7 @@ VERSIONED_CHARACTERIZATION_TEST( }), ({ UnkeyedValidPathInfo info{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }; info.deriver = StorePath{ @@ -641,7 +639,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -655,7 +653,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -696,7 +694,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -711,7 +709,7 @@ VERSIONED_CHARACTERIZATION_TEST( "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", }, UnkeyedValidPathInfo{ - std::string{defaultStoreDir}, + "/nix/store", Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), }, }; @@ -979,12 +977,6 @@ TEST_F(WorkerProtoTest, handshake_features) })); } -/// Has to be a `BufferedSink` for handshake. -struct NullBufferedSink : BufferedSink -{ - void writeUnbuffered(std::string_view data) override {} -}; - TEST_F(WorkerProtoTest, handshake_client_replay) { CharacterizationTest::readTest("handshake-to-client.bin", [&](std::string toClientLog) { diff --git a/src/libstore-tests/write-derivation.cc b/src/libstore-tests/write-derivation.cc index c68753823d..43060cb697 100644 --- a/src/libstore-tests/write-derivation.cc +++ b/src/libstore-tests/write-derivation.cc @@ -27,22 +27,19 @@ class WriteDerivationTest : public LibStoreTest ref config; }; -static Derivation makeSimpleDrv() -{ - Derivation drv; - drv.name = "simple-derivation"; - drv.platform = "system"; - drv.builder = "foo"; - drv.args = {"bar", "baz"}; - drv.env = StringPairs{{"BIG_BAD", "WOLF"}}; - return drv; -} - } // namespace TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce) { - auto drv = makeSimpleDrv(); + auto drv = []() { + Derivation drv; + drv.name = "simple-derivation"; + drv.platform = "system"; + drv.builder = "foo"; + drv.args = {"bar", "baz"}; + drv.env = StringPairs{{"BIG_BAD", "WOLF"}}; + return drv; + }(); auto path1 = store->writeDerivation(drv, NoRepair); config->readOnly = true; diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index 58a43c043a..4c6553a521 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -46,18 +46,17 @@ void DerivationTrampolineGoal::commonInit() DerivationTrampolineGoal::~DerivationTrampolineGoal() {} -static StorePath pathPartOfReq(const SingleDerivedPath & req) -{ - return std::visit( - overloaded{ - [&](const SingleDerivedPath::Opaque & bo) { return bo.path; }, - [&](const SingleDerivedPath::Built & bfd) { return pathPartOfReq(*bfd.drvPath); }, - }, - req.raw()); -} - std::string DerivationTrampolineGoal::key() { + auto pathPartOfReq = [](this const auto & self, const SingleDerivedPath & req) -> StorePath { + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) { return bo.path; }, + [&](const SingleDerivedPath::Built & bfd) { return self(*bfd.drvPath); }, + }, + req.raw()); + }; + return "da$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + DerivedPath::Built{ .drvPath = drvReq, .outputs = wantedOutputs, diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index cac6dc1b0c..962e831a3e 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -721,4 +721,10 @@ public: const ExternalBuilder * findExternalDerivationBuilderIfSupported(const Derivation & drv); }; +template<> +LocalSettings::ExternalBuilders BaseSetting::parse(const std::string & str) const; + +template<> +std::string BaseSetting::to_string() const; + } // namespace nix diff --git a/src/libstore/include/nix/store/store-reference.hh b/src/libstore/include/nix/store/store-reference.hh index 37ff2cdee3..49b41404ee 100644 --- a/src/libstore/include/nix/store/store-reference.hh +++ b/src/libstore/include/nix/store/store-reference.hh @@ -4,6 +4,7 @@ #include #include "nix/util/types.hh" +#include "nix/util/configuration.hh" #include "nix/util/json-impls.hh" #include "nix/util/json-non-null.hh" @@ -128,6 +129,24 @@ template<> struct json_avoids_null : std::true_type {}; +template<> +StoreReference BaseSetting::parse(const std::string & str) const; + +template<> +std::string BaseSetting::to_string() const; + +template<> +std::vector BaseSetting>::parse(const std::string & str) const; + +template<> +std::string BaseSetting>::to_string() const; + +template<> +std::set BaseSetting>::parse(const std::string & str) const; + +template<> +std::string BaseSetting>::to_string() const; + } // namespace nix JSON_IMPL(StoreReference) diff --git a/src/libutil/compression-settings.cc b/src/libutil/compression-settings.cc index 6e30811ef1..14fdfd76b4 100644 --- a/src/libutil/compression-settings.cc +++ b/src/libutil/compression-settings.cc @@ -52,11 +52,6 @@ std::string BaseSetting>::to_string() const return ""; } -/* Same as with all settings - empty string means std::nullopt. */ -template<> -struct json_avoids_null : std::true_type -{}; - #define NIX_COMPRESSION_JSON(name, value) {CompressionAlgo::value, name}, NLOHMANN_JSON_SERIALIZE_ENUM(CompressionAlgo, {NIX_FOR_EACH_COMPRESSION_ALGO(NIX_COMPRESSION_JSON)}); #undef NIX_COMPRESSION_JSON diff --git a/src/libutil/include/nix/util/compression-settings.hh b/src/libutil/include/nix/util/compression-settings.hh index 756d636f13..942f3dd9c8 100644 --- a/src/libutil/include/nix/util/compression-settings.hh +++ b/src/libutil/include/nix/util/compression-settings.hh @@ -18,7 +18,6 @@ std::optional BaseSetting>::pars template<> std::string BaseSetting>::to_string() const; -/* Same as with all settings - empty string means std::nullopt. */ template<> struct json_avoids_null : std::true_type {}; diff --git a/src/nix/nix-store/graphml.cc b/src/nix/nix-store/graphml.cc index 009db05d41..3863247e12 100644 --- a/src/nix/nix-store/graphml.cc +++ b/src/nix/nix-store/graphml.cc @@ -20,11 +20,6 @@ static std::string symbolicName(std::string_view p) return std::string(p.substr(0, p.find('-') + 1)); } -static std::string makeEdge(std::string_view src, std::string_view dst) -{ - return fmt(" \n", xmlQuote(src), xmlQuote(dst)); -} - static std::string makeNode(const ValidPathInfo & info) { return fmt( @@ -67,6 +62,11 @@ void printGraphML(ref store, StorePathSet && roots) for (auto & p : info->references) { if (p != path) { workList.insert(p); + + auto makeEdge = [](std::string_view src, std::string_view dst) -> std::string { + return fmt(" \n", xmlQuote(src), xmlQuote(dst)); + }; + cout << makeEdge(path.to_string(), p.to_string()); } } From 1e22d60be664469b7d2fe9bf9e0527bf9279c286 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 30 Mar 2026 00:01:09 +0300 Subject: [PATCH 186/927] Clean up BaseOption to_string/parse declarations with a macro Rename to NIX_ to avoid polluting the identifiers and change all of the declaration sites to make them much less verbose. --- src/libexpr/include/nix/expr/diagnose.hh | 6 +--- .../nix/expr/eval-profiler-settings.hh | 6 +--- src/libstore/include/nix/store/s3-url.hh | 6 +--- .../include/nix/store/store-reference.hh | 20 ++----------- .../include/nix/util/compression-settings.hh | 13 ++------- src/libutil/include/nix/util/config-impl.hh | 28 ++++++++----------- src/libutil/include/nix/util/configuration.hh | 6 ++++ 7 files changed, 25 insertions(+), 60 deletions(-) diff --git a/src/libexpr/include/nix/expr/diagnose.hh b/src/libexpr/include/nix/expr/diagnose.hh index 8dfe052134..68c8f6543f 100644 --- a/src/libexpr/include/nix/expr/diagnose.hh +++ b/src/libexpr/include/nix/expr/diagnose.hh @@ -28,11 +28,7 @@ enum struct Diagnose { Fatal, }; -template<> -Diagnose BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(Diagnose) /** * Check a diagnostic setting and either do nothing, log a warning, or throw an error. diff --git a/src/libexpr/include/nix/expr/eval-profiler-settings.hh b/src/libexpr/include/nix/expr/eval-profiler-settings.hh index 32138e7f13..4fec151b80 100644 --- a/src/libexpr/include/nix/expr/eval-profiler-settings.hh +++ b/src/libexpr/include/nix/expr/eval-profiler-settings.hh @@ -7,10 +7,6 @@ namespace nix { enum struct EvalProfilerMode { disabled, flamegraph }; -template<> -EvalProfilerMode BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(EvalProfilerMode) } // namespace nix diff --git a/src/libstore/include/nix/store/s3-url.hh b/src/libstore/include/nix/store/s3-url.hh index 5337107452..2bd2b55e64 100644 --- a/src/libstore/include/nix/store/s3-url.hh +++ b/src/libstore/include/nix/store/s3-url.hh @@ -30,11 +30,7 @@ MakeError(InvalidS3AddressingStyle, Error); S3AddressingStyle parseS3AddressingStyle(std::string_view style); std::string_view showS3AddressingStyle(S3AddressingStyle style); -template<> -S3AddressingStyle BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(S3AddressingStyle) /** * Parsed S3 URL. diff --git a/src/libstore/include/nix/store/store-reference.hh b/src/libstore/include/nix/store/store-reference.hh index 49b41404ee..5beeb0c070 100644 --- a/src/libstore/include/nix/store/store-reference.hh +++ b/src/libstore/include/nix/store/store-reference.hh @@ -129,23 +129,9 @@ template<> struct json_avoids_null : std::true_type {}; -template<> -StoreReference BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; - -template<> -std::vector BaseSetting>::parse(const std::string & str) const; - -template<> -std::string BaseSetting>::to_string() const; - -template<> -std::set BaseSetting>::parse(const std::string & str) const; - -template<> -std::string BaseSetting>::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(StoreReference) +NIX_DECLARE_CONFIG_SERIALISER(std::vector) +NIX_DECLARE_CONFIG_SERIALISER(std::set) } // namespace nix diff --git a/src/libutil/include/nix/util/compression-settings.hh b/src/libutil/include/nix/util/compression-settings.hh index 942f3dd9c8..9f0245d077 100644 --- a/src/libutil/include/nix/util/compression-settings.hh +++ b/src/libutil/include/nix/util/compression-settings.hh @@ -6,17 +6,8 @@ namespace nix { -template<> -CompressionAlgo BaseSetting::parse(const std::string & str) const; - -template<> -std::string BaseSetting::to_string() const; - -template<> -std::optional BaseSetting>::parse(const std::string & str) const; - -template<> -std::string BaseSetting>::to_string() const; +NIX_DECLARE_CONFIG_SERIALISER(CompressionAlgo) +NIX_DECLARE_CONFIG_SERIALISER(std::optional) template<> struct json_avoids_null : std::true_type diff --git a/src/libutil/include/nix/util/config-impl.hh b/src/libutil/include/nix/util/config-impl.hh index 88d82394f3..c719f86530 100644 --- a/src/libutil/include/nix/util/config-impl.hh +++ b/src/libutil/include/nix/util/config-impl.hh @@ -122,23 +122,17 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) }); } -#define DECLARE_CONFIG_SERIALISER(TY) \ - template<> \ - TY BaseSetting::parse(const std::string & str) const; \ - template<> \ - std::string BaseSetting::to_string() const; - -DECLARE_CONFIG_SERIALISER(std::string) -DECLARE_CONFIG_SERIALISER(std::optional) -DECLARE_CONFIG_SERIALISER(bool) -DECLARE_CONFIG_SERIALISER(Strings) -DECLARE_CONFIG_SERIALISER(StringSet) -DECLARE_CONFIG_SERIALISER(StringMap) -DECLARE_CONFIG_SERIALISER(std::set) -DECLARE_CONFIG_SERIALISER(std::filesystem::path) -DECLARE_CONFIG_SERIALISER(AbsolutePath) -DECLARE_CONFIG_SERIALISER(std::set) -DECLARE_CONFIG_SERIALISER(std::optional) +NIX_DECLARE_CONFIG_SERIALISER(std::string) +NIX_DECLARE_CONFIG_SERIALISER(std::optional) +NIX_DECLARE_CONFIG_SERIALISER(bool) +NIX_DECLARE_CONFIG_SERIALISER(Strings) +NIX_DECLARE_CONFIG_SERIALISER(StringSet) +NIX_DECLARE_CONFIG_SERIALISER(StringMap) +NIX_DECLARE_CONFIG_SERIALISER(std::set) +NIX_DECLARE_CONFIG_SERIALISER(std::filesystem::path) +NIX_DECLARE_CONFIG_SERIALISER(AbsolutePath) +NIX_DECLARE_CONFIG_SERIALISER(std::set) +NIX_DECLARE_CONFIG_SERIALISER(std::optional) template T BaseSetting::parse(const std::string & str) const diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 595b9948c4..19d678601b 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -598,6 +598,12 @@ struct ExperimentalFeatureSettings : Config void require(const std::optional &) const; }; +#define NIX_DECLARE_CONFIG_SERIALISER(TY) \ + template<> \ + TY BaseSetting::parse(const std::string & str) const; \ + template<> \ + std::string BaseSetting::to_string() const; + // FIXME: don't use a global variable. extern ExperimentalFeatureSettings experimentalFeatureSettings; From a60d54c1a53eafd93756b2f1e24338fff02a5922 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 30 Mar 2026 01:13:54 +0300 Subject: [PATCH 187/927] packaging: Enable unity builds Best reviewed with git show -w --color-moved. --- flake.nix | 18 +- packaging/components.nix | 12 ++ packaging/dev-shell.nix | 451 ++++++++++++++++++++------------------- packaging/hydra.nix | 2 + 4 files changed, 254 insertions(+), 229 deletions(-) diff --git a/flake.nix b/flake.nix index dc3214d1e0..97db5eef93 100644 --- a/flake.nix +++ b/flake.nix @@ -512,6 +512,16 @@ devShells = let makeShell = import ./packaging/dev-shell.nix { inherit lib devFlake; }; + makeShell' = + { pkgs }: + makeShell { + inherit pkgs; + nixComponents = pkgs.nixComponents2.overrideScope ( + finalScope: prevScope: { + withUnityBuild = false; + } + ); + }; prefixAttrs = prefix: lib.concatMapAttrs (k: v: { "${prefix}-${k}" = v; }); in forAllSystems ( @@ -519,7 +529,7 @@ prefixAttrs "native" ( forAllStdenvs ( stdenvName: - makeShell { + makeShell' { pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}; } ) @@ -528,7 +538,7 @@ prefixAttrs "static" ( forAllStdenvs ( stdenvName: - makeShell { + makeShell' { pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.pkgsStatic; } ) @@ -536,7 +546,7 @@ // prefixAttrs "llvm" ( forAllStdenvs ( stdenvName: - makeShell { + makeShell' { pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.pkgsLLVM; } ) @@ -544,7 +554,7 @@ // prefixAttrs "cross" ( forAllCrossSystems ( crossSystem: - makeShell { + makeShell' { pkgs = nixpkgsFor.${system}.cross.${crossSystem}; } ) diff --git a/packaging/components.nix b/packaging/components.nix index 09e767cf08..112791c27d 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -153,6 +153,13 @@ let ninja ] ++ prevAttrs.nativeBuildInputs or [ ]; + mesonFlags = + prevAttrs.mesonFlags or [ ] + ++ (lib.optionals scope.withUnityBuild [ + "-Dunity=on" + "-Dunity_size=8192" + "-Db_pch=false" + ]); mesonCheckFlags = prevAttrs.mesonCheckFlags or [ ] ++ [ "--print-errorlogs" ]; @@ -319,6 +326,11 @@ in */ withClangTidy = false; + /** + Whether to use [unity builds](https://mesonbuild.com/Unity-builds.html#unity-builds). + */ + withUnityBuild = true; + /** A user-provided extension function to apply to each component derivation. */ diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 72d517d876..b058bbe53d 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -110,245 +110,246 @@ let in -{ pkgs }: - -# TODO: don't use nix-util for this? -pkgs.nixComponents2.nix-util.overrideAttrs ( - finalAttrs: prevAttrs: - - let - stdenv = pkgs.nixDependencies2.stdenv; - buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; - modular = devFlake.getSystem stdenv.buildPlatform.system; - transformFlag = - prefix: flag: - assert builtins.isString flag; - let - rest = builtins.substring 2 (builtins.stringLength flag) flag; - in - "-D${prefix}:${rest}"; - havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; - ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; - - availableComponents = lib.filterAttrs ( - k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v - ) allComponents; - - activeComponents = buildInputsClosureCond isInternal ( - lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents) - ); +{ pkgs, nixComponents }: - allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2; - internalDrvs = byDrvPath ( - # Drop the attr names (not present in buildInputs anyway) - lib.attrValues availableComponents - ++ lib.concatMap (c: lib.filter (v: !v.meta.broken) (lib.attrValues (c.tests or { }))) ( - lib.attrValues availableComponents - ) - ); +nixComponents.callPackage ( + { stdenv }: + (stdenv.mkDerivation ( + finalAttrs: - isInternal = - dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"}; + let + buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = + prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + + availableComponents = lib.filterAttrs ( + k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v + ) allComponents; + + activeComponents = buildInputsClosureCond isInternal ( + lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents) + ); - activeComponentNames = lib.listToAttrs ( - map (c: { - name = c.pname or c.name; - value = null; - }) activeComponents - ); + allComponents = lib.filterAttrs (k: v: lib.isDerivation v) nixComponents; + internalDrvs = byDrvPath ( + # Drop the attr names (not present in buildInputs anyway) + lib.attrValues availableComponents + ++ lib.concatMap (c: lib.filter (v: !v.meta.broken) (lib.attrValues (c.tests or { }))) ( + lib.attrValues availableComponents + ) + ); - isActiveComponent = name: activeComponentNames ? ${name}; + isInternal = + dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"}; - in - { - pname = "shell-for-nix"; + activeComponentNames = lib.listToAttrs ( + map (c: { + name = c.pname or c.name; + value = null; + }) activeComponents + ); - passthru = { - inherit activeComponents; + isActiveComponent = name: activeComponentNames ? ${name}; - # We use this attribute to store non-derivation values like functions and - # perhaps other things that are primarily for overriding and not the shell. - config = { - # Default getComponents - getComponents = - c: - builtins.removeAttrs c ( - lib.optionals (!havePerl) [ "nix-perl-bindings" ] - ++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ] + in + { + pname = "shell-for-nix"; + + passthru = { + inherit activeComponents; + + # We use this attribute to store non-derivation values like functions and + # perhaps other things that are primarily for overriding and not the shell. + config = { + # Default getComponents + getComponents = + c: + builtins.removeAttrs c ( + lib.optionals (!havePerl) [ "nix-perl-bindings" ] + ++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ] + ); + }; + + /** + Produce a devShell for a given set of nix components + + Example: + + ```nix + shell.withActiveComponents (c: { + inherit (c) nix-util; + }) + ``` + */ + withActiveComponents = + f2: + finalAttrs.finalPackage.overrideAttrs ( + finalAttrs: prevAttrs: { + passthru = prevAttrs.passthru // { + config = prevAttrs.passthru.config // { + getComponents = f2; + }; + }; + } ); - }; - /** - Produce a devShell for a given set of nix components - - Example: - - ```nix - shell.withActiveComponents (c: { - inherit (c) nix-util; - }) - ``` - */ - withActiveComponents = - f2: - finalAttrs.finalPackage.overrideAttrs ( - finalAttrs: prevAttrs: { - passthru = prevAttrs.passthru // { - config = prevAttrs.passthru.config // { - getComponents = f2; - }; - }; - } + small = finalAttrs.finalPackage.withActiveComponents ( + c: + lib.intersectAttrs (lib.genAttrs [ + "nix-cli" + "nix-util-tests" + "nix-store-tests" + "nix-expr-tests" + "nix-fetchers-tests" + "nix-flake-tests" + "nix-functional-tests" + "nix-perl-bindings" + ] (_: null)) c ); + }; - small = finalAttrs.finalPackage.withActiveComponents ( - c: - lib.intersectAttrs (lib.genAttrs [ - "nix-cli" - "nix-util-tests" - "nix-store-tests" - "nix-expr-tests" - "nix-fetchers-tests" - "nix-flake-tests" - "nix-functional-tests" - "nix-perl-bindings" - ] (_: null)) c - ); - }; - - # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop - version = lib.fileContents ../.version; - name = finalAttrs.pname; - - installFlags = "sysconfdir=$(out)/etc"; - shellHook = '' - PATH=$prefix/bin:$PATH - unset PYTHONPATH - export MANPATH=$out/share/man:$MANPATH - - # Make bash completion work. - XDG_DATA_DIRS+=:$out/share - - # Make the default phases do the right thing. - # FIXME: this wouldn't be needed if the ninja package set buildPhase() instead of $buildPhase. - # FIXME: mesonConfigurePhase shouldn't cd to the build directory. It would be better to pass '-C ' to ninja. - - cdToBuildDir() { - if [[ ! -e build.ninja ]]; then - cd build - fi - } - - configurePhase() { - mesonConfigurePhase - } - - buildPhase() { - cdToBuildDir - ninjaBuildPhase - } - - checkPhase() { - cdToBuildDir - mesonCheckPhase - } - - installPhase() { - cdToBuildDir - ninjaInstallPhase + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ../.version; + name = finalAttrs.pname; + + installFlags = "sysconfdir=$(out)/etc"; + shellHook = '' + PATH=$prefix/bin:$PATH + unset PYTHONPATH + export MANPATH=$out/share/man:$MANPATH + + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share + + # Make the default phases do the right thing. + # FIXME: this wouldn't be needed if the ninja package set buildPhase() instead of $buildPhase. + # FIXME: mesonConfigurePhase shouldn't cd to the build directory. It would be better to pass '-C ' to ninja. + + cdToBuildDir() { + if [[ ! -e build.ninja ]]; then + cd build + fi + } + + configurePhase() { + mesonConfigurePhase + } + + buildPhase() { + cdToBuildDir + ninjaBuildPhase + } + + checkPhase() { + cdToBuildDir + mesonCheckPhase + } + + installPhase() { + cdToBuildDir + ninjaInstallPhase + } + ''; + + # We use this shell with the local checkout, not unpackPhase. + src = null; + + # Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html + # Remove when gdb fix is rolled out everywhere. + separateDebugInfo = false; + + mesonBuildType = "debugoptimized"; + + env = { + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" + modular.pre-commit.settings.rawConfig + }"; } - ''; - - # We use this shell with the local checkout, not unpackPhase. - src = null; - - # Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html - # Remove when gdb fix is rolled out everywhere. - separateDebugInfo = false; + // lib.optionalAttrs stdenv.hostPlatform.isLinux { + CC_LD = "mold"; + CXX_LD = "mold"; + }; - mesonBuildType = "debugoptimized"; + dontUseCmakeConfigure = true; - env = { - # For `make format`, to work without installing pre-commit - _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" - modular.pre-commit.settings.rawConfig - }"; - } - // lib.optionalAttrs stdenv.hostPlatform.isLinux { - CC_LD = "mold"; - CXX_LD = "mold"; - }; - - dontUseCmakeConfigure = true; - - mesonFlags = [ - (lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks")) - ] - ++ map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags) - ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags) - ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags) - ++ lib.optionals havePerl ( - map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags) - ) - ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags) - ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags); - - nativeBuildInputs = - let - inputs = - dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) ( - lib.lists.concatMap ( - # Nix manual has a build-time dependency on nix, but we - # don't want to do a native build just to enter the cross - # dev shell. - # - # TODO: think of a more principled fix for this. - c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs - ) activeComponents - ) - ) - ++ lib.optional ( - !buildCanExecuteHost - # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 - && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) - && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages - && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages) - ) pkgs.buildPackages.mesonEmulatorHook - ++ [ - pkgs.buildPackages.gnused - modular.pre-commit.settings.package - (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) - pkgs.buildPackages.nixfmt-rfc-style - pkgs.buildPackages.shellcheck - pkgs.buildPackages.include-what-you-use - ] - ++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb - ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) ( - lib.hiPrio pkgs.buildPackages.clang-tools - ) - ++ lib.optional stdenv.hostPlatform.isLinux pkgs.buildPackages.mold-wrapped; - in - # FIXME: separateDebugInfo = false doesn't actually prevent -Wa,--compress-debug-sections - # from making its way into NIX_CFLAGS_COMPILE. - lib.filter (p: !lib.hasInfix "separate-debug-info" p) inputs; - - propagatedNativeBuildInputs = dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) ( - lib.lists.concatMap (c: c.propagatedNativeBuildInputs) activeComponents + mesonFlags = [ + (lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks")) + ] + ++ map (transformFlag "libutil") (ignoreCrossFile nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl ( + map (transformFlag "perl") (ignoreCrossFile nixComponents.nix-perl-bindings.mesonFlags) ) - ); + ++ map (transformFlag "libexpr") (ignoreCrossFile nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile nixComponents.nix-cmd.mesonFlags); + + nativeBuildInputs = + let + inputs = + dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) ( + lib.lists.concatMap ( + # Nix manual has a build-time dependency on nix, but we + # don't want to do a native build just to enter the cross + # dev shell. + # + # TODO: think of a more principled fix for this. + c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs + ) activeComponents + ) + ) + ++ lib.optional ( + !buildCanExecuteHost + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages) + ) pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.gnused + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) + pkgs.buildPackages.nixfmt-rfc-style + pkgs.buildPackages.shellcheck + pkgs.buildPackages.include-what-you-use + ] + ++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb + ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) ( + lib.hiPrio pkgs.buildPackages.clang-tools + ) + ++ lib.optional stdenv.hostPlatform.isLinux pkgs.buildPackages.mold-wrapped; + in + # FIXME: separateDebugInfo = false doesn't actually prevent -Wa,--compress-debug-sections + # from making its way into NIX_CFLAGS_COMPILE. + lib.filter (p: !lib.hasInfix "separate-debug-info" p) inputs; + + propagatedNativeBuildInputs = dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) ( + lib.lists.concatMap (c: c.propagatedNativeBuildInputs) activeComponents + ) + ); - buildInputs = - # TODO change Nixpkgs to mark gbenchmark as building on Windows - lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark - ++ dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) - ) - ++ lib.optional havePerl pkgs.perl; + buildInputs = + # TODO change Nixpkgs to mark gbenchmark as building on Windows + lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark + ++ dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) + ) + ++ lib.optional havePerl pkgs.perl; - propagatedBuildInputs = dedupByString (v: "${v}") ( - lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) - ); - } -) + propagatedBuildInputs = dedupByString (v: "${v}") ( + lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) + ); + } + )) +) { } diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 75830b98cf..fa41b070cc 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -173,6 +173,8 @@ rec { # Boost coroutines fail with ASAN on darwin. withASan = !pkgs.stdenv.buildPlatform.isDarwin; withUBSan = true; + # Build without unity to catch include issues. + withUnityBuild = false; nix-expr = super.nix-expr.override { enableGC = false; }; # Unclear how to make Perl bindings work with a dynamically linked ASAN. nix-perl-bindings = null; From 0f67c95181a2ecc371160573d6e1f97268e3c1b4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Feb 2026 22:54:15 +0100 Subject: [PATCH 188/927] Drop unnecessary nlohmann include --- src/libcmd/include/nix/cmd/installable-attr-path.hh | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libcmd/include/nix/cmd/installable-attr-path.hh b/src/libcmd/include/nix/cmd/installable-attr-path.hh index 474bb358ec..ef9dac8133 100644 --- a/src/libcmd/include/nix/cmd/installable-attr-path.hh +++ b/src/libcmd/include/nix/cmd/installable-attr-path.hh @@ -21,8 +21,6 @@ #include #include -#include - namespace nix { class InstallableAttrPath : public InstallableValue From 9e903deff686bf653c56156f5af5514f3d14e38c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2026 18:38:37 +0200 Subject: [PATCH 189/927] Drop another unnecessary nlohmann/json.hpp include --- src/libstore/include/nix/store/derivation-options.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index 4e6ec22f9c..d96c4892e1 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -2,7 +2,6 @@ ///@file #include -#include #include #include From 75cda3c7d7624df0dca2ad77e57ad1c71250e3fe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 27 Mar 2026 14:19:57 -0400 Subject: [PATCH 190/927] doxygen: suppress warnings in dev builds They are probably good to keep in standalone release builds, but they clutter scroll back massively in dev shells. Co-authored-by: Amaan Qureshi --- src/external-api-docs/doxygen.cfg.in | 1 + src/external-api-docs/meson.build | 1 + src/internal-api-docs/doxygen.cfg.in | 1 + src/internal-api-docs/meson.build | 1 + 4 files changed, 4 insertions(+) diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in index 3af2f5b813..18d94e5eee 100644 --- a/src/external-api-docs/doxygen.cfg.in +++ b/src/external-api-docs/doxygen.cfg.in @@ -61,3 +61,4 @@ USE_MDFILE_AS_MAINPAGE = @src@/src/external-api-docs/README.md WARN_IF_UNDOCUMENTED = NO WARN_IF_INCOMPLETE_DOC = NO QUIET = YES +WARNINGS = @WARNINGS@ diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build index cba6f646b9..1903b36e58 100644 --- a/src/external-api-docs/meson.build +++ b/src/external-api-docs/meson.build @@ -14,6 +14,7 @@ doxygen_cfg = configure_file( 'PROJECT_NUMBER' : meson.project_version(), 'OUTPUT_DIRECTORY' : meson.current_build_dir(), 'src' : fs.parent(fs.parent(meson.project_source_root())), + 'WARNINGS' : meson.is_subproject() ? 'NO' : 'YES', }, ) diff --git a/src/internal-api-docs/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in index 2769edd9f7..aa81d77c56 100644 --- a/src/internal-api-docs/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -112,3 +112,4 @@ PREDEFINED = DOXYGEN_SKIP WARN_IF_UNDOCUMENTED = NO WARN_IF_INCOMPLETE_DOC = NO QUIET = YES +WARNINGS = @WARNINGS@ diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build index daab4c93c3..844cb262ee 100644 --- a/src/internal-api-docs/meson.build +++ b/src/internal-api-docs/meson.build @@ -15,6 +15,7 @@ doxygen_cfg = configure_file( 'OUTPUT_DIRECTORY' : meson.current_build_dir(), 'BUILD_ROOT' : meson.build_root(), 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + 'WARNINGS' : meson.is_subproject() ? 'NO' : 'YES', }, ) From 0a8b126661ba657cc7960d5f589d7ddcb4f6a7f4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Mar 2026 14:16:51 -0400 Subject: [PATCH 191/927] Add `BasicDerivation` JSON serialization Add JSON serialization for `BasicDerivation` with a flat "inputs" array format (vs `Derivation`'s nested `{srcs, drvs}` format), and a corresponding JSON schema (`derivation-resolved-v4.yaml`) that references common field definitions factored out of `derivation-v4.yaml`. Co-authored-by: Amaan Qureshi --- doc/manual/source/SUMMARY.md.in | 1 + .../protocols/json/derivation/resolved.md | 1 + .../json/fixup-json-schema-generated-doc.sed | 3 + doc/manual/source/protocols/json/meson.build | 1 + .../json/schema/derivation-resolved-v4.yaml | 58 ++++++ .../protocols/json/schema/derivation-v4.yaml | 185 ++++++++++-------- src/json-schema-checks/meson.build | 11 ++ src/libstore/derivations.cc | 123 +++++++----- src/libstore/include/nix/store/derivations.hh | 1 + 9 files changed, 255 insertions(+), 129 deletions(-) create mode 100644 doc/manual/source/protocols/json/derivation/resolved.md create mode 100644 doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 1a81bad10b..5a17426b90 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -128,6 +128,7 @@ - [Signature](protocols/json/signature.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation/index.md) + - [Resolved Derivation](protocols/json/derivation/resolved.md) - [Derivation Options](protocols/json/derivation/options.md) - [Deriving Path](protocols/json/deriving-path.md) - [Build Trace Entry](protocols/json/build-trace-entry.md) diff --git a/doc/manual/source/protocols/json/derivation/resolved.md b/doc/manual/source/protocols/json/derivation/resolved.md new file mode 100644 index 0000000000..45e18d9af0 --- /dev/null +++ b/doc/manual/source/protocols/json/derivation/resolved.md @@ -0,0 +1 @@ +{{#include ../derivation-resolved-v4-fixed.md}} diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed index 96b6f1801a..0162036d42 100644 --- a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -16,3 +16,6 @@ s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](@docroot@/proto s^\(./hash-v1.yaml\)^[JSON format for `Hash`](@docroot@/protocols/json/hash.html)^g s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](@docroot@/protocols/json/content-address.html#method)^g s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](@docroot@/protocols/json/content-address.html)^g +s^\(./store-path-v1.yaml\)^[JSON format for `StorePath`](@docroot@/protocols/json/store-path.html)^g +s^\(./derivation-v4.yaml\)\?#/\$defs/common/properties/[a-zA-Z]*^[JSON format for `Derivation`](@docroot@/protocols/json/derivation/index.html)^g +s^\(./derivation-v4.yaml\)^[JSON format for `Derivation`](@docroot@/protocols/json/derivation/index.html)^g diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index 36e9220c02..9fcacb2038 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -16,6 +16,7 @@ schemas = [ 'signature-v2', 'store-object-info-v3', 'derivation-v4', + 'derivation-resolved-v4', 'derivation-options-v1', 'deriving-path-v1', 'build-trace-entry-v3', diff --git a/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml b/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml new file mode 100644 index 0000000000..781f07fb37 --- /dev/null +++ b/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml @@ -0,0 +1,58 @@ +"$schema": "http://json-schema.org/draft-04/schema" +"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-resolved-v4.json" +title: Resolved Derivation +description: | + Experimental JSON representation of a resolved Nix derivation (version 4). + + This schema describes the JSON representation of Nix's `BasicDerivation` type, + which is a derivation with all input derivation dependencies resolved to store paths. + This is the result of `Derivation::tryResolve`. + + This is called "version 4" because we wish to keep it in sync with the + [primary (not necessarily resolved) JSON schema](@docroot@/protocols/json/derivation/index.md), + but actually it is the first version for this format. + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > and subject to change. + +type: object +required: + - name + - version + - outputs + - inputs + - system + - builder + - args + - env +properties: + name: { "$ref": "derivation-v4.yaml#/$defs/common/properties/name" } + version: { "$ref": "derivation-v4.yaml#/$defs/common/properties/version" } + outputs: { "$ref": "derivation-v4.yaml#/$defs/common/properties/outputs" } + system: { "$ref": "derivation-v4.yaml#/$defs/common/properties/system" } + builder: { "$ref": "derivation-v4.yaml#/$defs/common/properties/builder" } + args: { "$ref": "derivation-v4.yaml#/$defs/common/properties/args" } + env: { "$ref": "derivation-v4.yaml#/$defs/common/properties/env" } + structuredAttrs: { "$ref": "derivation-v4.yaml#/$defs/common/properties/structuredAttrs" } + + inputs: + type: array + title: Input source paths + description: | + List of store paths on which this resolved derivation depends. + Since all derivation inputs have been resolved, only source paths remain. + + > **Example** + > + > ```json + > "inputs": [ + > "b8nwz167km1yciqpwzjj24f8jcy8pq1h-separate-debug-info.sh", + > "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out" + > ] + > ``` + items: + $ref: "store-path-v1.yaml" +additionalProperties: false diff --git a/doc/manual/source/protocols/json/schema/derivation-v4.yaml b/doc/manual/source/protocols/json/schema/derivation-v4.yaml index c41eef31bf..170c145c23 100644 --- a/doc/manual/source/protocols/json/schema/derivation-v4.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-v4.yaml @@ -23,57 +23,14 @@ required: - args - env properties: - name: - type: string - title: Derivation name - description: | - The name of the derivation. - Used when calculating store paths for the derivation’s outputs. - - version: - const: 4 - title: Format version (must be 4) - description: | - Must be `4`. - This is a guard that allows us to continue evolving this format. - The choice of `3` is fairly arbitrary, but corresponds to this informal version: - - - Version 0: ATerm format - - - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from ATerm format. - - - Version 2: Separate `method` and `hashAlgo` fields in output specs - - - Version 3: Drop store dir from store paths, just include base name. - - - Version 4: Two cleanups, batched together to lesson churn: - - - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) - - - Use canonical content address JSON format for floating content addressed derivation outputs. - - Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. - - outputs: - type: object - title: Output specifications - description: | - Information about the output paths of the derivation. - This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described. - - > **Example** - > - > ```json - > "outputs": { - > "out": { - > "method": "nar", - > "hashAlgo": "sha256", - > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" - > } - > } - > ``` - additionalProperties: - "$ref": "#/$defs/output/overall" + name: { "$ref": "#/$defs/common/properties/name" } + version: { "$ref": "#/$defs/common/properties/version" } + outputs: { "$ref": "#/$defs/common/properties/outputs" } + system: { "$ref": "#/$defs/common/properties/system" } + builder: { "$ref": "#/$defs/common/properties/builder" } + args: { "$ref": "#/$defs/common/properties/args" } + env: { "$ref": "#/$defs/common/properties/env" } + structuredAttrs: { "$ref": "#/$defs/common/properties/structuredAttrs" } inputs: type: object @@ -126,47 +83,105 @@ properties: - "$ref": "#/$defs/dynamicOutputs" additionalProperties: false additionalProperties: false +additionalProperties: false - system: - type: string - title: Build system type +"$defs": + common: + title: Common Derivation Fields description: | - The system type on which this derivation is to be built - (e.g. `x86_64-linux`). + Fields shared between `Derivation` and `BasicDerivation`. + properties: + name: + type: string + title: Derivation name + description: | + The name of the derivation. + Used when calculating store paths for the derivation's outputs. - builder: - type: string - title: Build program path - description: | - Absolute path of the program used to perform the build. - Typically this is the `bash` shell - (e.g. `/nix/store/p4xlj4imjbnm4v0x5jf4qysvyjjlgq1d-bash-4.4-p23/bin/bash`). + version: + const: 4 + title: Format version (must be 4) + description: | + Must be `4`. + This is a guard that allows us to continue evolving this format. + The choice of `3` is fairly arbitrary, but corresponds to this informal version: - args: - type: array - title: Builder arguments - description: | - Command-line arguments passed to the `builder`. - items: - type: string + - Version 0: ATerm format - env: - type: object - title: Environment variables - description: | - Environment variables passed to the `builder`. - additionalProperties: - type: string + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from ATerm format. - structuredAttrs: - title: Structured attributes - description: | - [Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them. - Structured attributes are JSON, and thus embedded as-is. - type: object - additionalProperties: true + - Version 2: Separate `method` and `hashAlgo` fields in output specs + + - Version 3: Drop store dir from store paths, just include base name. + + - Version 4: Two cleanups, batched together to lesson churn: + + - Reorganize inputs into nested structure (`inputs.srcs` and `inputs.drvs`) + + - Use canonical content address JSON format for floating content addressed derivation outputs. + + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. + + outputs: + type: object + title: Output specifications + description: | + Information about the output paths of the derivation. + This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described. + + > **Example** + > + > ```json + > "outputs": { + > "out": { + > "method": "nar", + > "hashAlgo": "sha256", + > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + > } + > } + > ``` + additionalProperties: + "$ref": "#/$defs/output/overall" + + system: + type: string + title: Build system type + description: | + The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + + builder: + type: string + title: Build program path + description: | + Absolute path of the program used to perform the build. + Typically this is the `bash` shell + (e.g. `/nix/store/p4xlj4imjbnm4v0x5jf4qysvyjjlgq1d-bash-4.4-p23/bin/bash`). + + args: + type: array + title: Builder arguments + description: | + Command-line arguments passed to the `builder`. + items: + type: string + + env: + type: object + title: Environment variables + description: | + Environment variables passed to the `builder`. + additionalProperties: + type: string + + structuredAttrs: + title: Structured attributes + description: | + [Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them. + Structured attributes are JSON, and thus embedded as-is. + type: object + additionalProperties: true -"$defs": output: overall: title: Derivation Output diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 921709b9d3..8a0bde04b8 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -98,6 +98,17 @@ schemas += [ 'files' : [ 'dyn-dep-derivation.json', 'simple-derivation.json', + 'try-resolve' / 'no-inputs-before.json', + 'try-resolve' / 'with-inputs-before.json', + 'try-resolve' / 'resolution-failure-before.json', + ], + }, + { + 'stem' : 'derivation', + 'schema' : schema_dir / 'derivation-resolved-v4.yaml', + 'files' : [ + 'try-resolve' / 'no-inputs-after.json', + 'try-resolve' / 'with-inputs-after.json', ], }, { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 16eaff1811..b618e98180 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1526,32 +1526,52 @@ adl_serializer::from_json(const json & _json, const Experiment } } -void adl_serializer::to_json(json & res, const Derivation & d) +static void inputSrcsToJson(json & res, const StorePathSet & inputSrcs) +{ + res = nlohmann::json::array(); + for (auto & input : inputSrcs) + res.emplace_back(input); +} + +static void basicDerivationToJson(json & res, const BasicDerivation & d) { res = nlohmann::json::object(); res["name"] = d.name; - res["version"] = expectedJsonVersionDerivation; { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); - for (auto & [outputName, output] : d.outputs) { + for (auto & [outputName, output] : d.outputs) outputsObj[outputName] = output; - } } + res["system"] = d.platform; + res["builder"] = d.builder; + res["args"] = d.args; + res["env"] = d.env; + + if (d.structuredAttrs) + res["structuredAttrs"] = d.structuredAttrs->structuredAttrs; +} + +void adl_serializer::to_json(json & res, const BasicDerivation & d) +{ + basicDerivationToJson(res, d); + + inputSrcsToJson(res["inputs"], d.inputSrcs); +} + +void adl_serializer::to_json(json & res, const Derivation & d) +{ + basicDerivationToJson(res, d); + { auto & inputsObj = res["inputs"]; inputsObj = nlohmann::json::object(); - { - auto & inputsList = inputsObj["srcs"]; - inputsList = nlohmann::json::array(); - for (auto & input : d.inputSrcs) - inputsList.emplace_back(input); - } + inputSrcsToJson(inputsObj["srcs"], d.inputSrcs); auto doInput = [&](this const auto & doInput, const auto & inputNode) -> nlohmann::json { auto value = nlohmann::json::object(); @@ -1567,28 +1587,21 @@ void adl_serializer::to_json(json & res, const Derivation & d) auto & inputDrvsObj = inputsObj["drvs"]; inputDrvsObj = nlohmann::json::object(); - for (auto & [inputDrv, inputNode] : d.inputDrvs.map) { + for (auto & [inputDrv, inputNode] : d.inputDrvs.map) inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); - } } - - res["system"] = d.platform; - res["builder"] = d.builder; - res["args"] = d.args; - res["env"] = d.env; - - if (d.structuredAttrs) - res["structuredAttrs"] = d.structuredAttrs->structuredAttrs; } -Derivation adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +static void inputSrcsFromJson(const json & inputSrcsJson, StorePathSet & inputSrcs) { - using nlohmann::detail::value_t; - - Derivation res; - - auto & json = getObject(_json); + auto arr = getArray(inputSrcsJson); + for (auto & input : arr) + inputSrcs.insert(input); +} +static void basicDerivationFromJson( + const json::object_t & json, BasicDerivation & res, const ExperimentalFeatureSettings & xpSettings) +{ res.name = getString(valueAt(json, "name")); { @@ -1610,13 +1623,50 @@ Derivation adl_serializer::from_json(const json & _json, const Exper throw; } + res.platform = getString(valueAt(json, "system")); + res.builder = getString(valueAt(json, "builder")); + res.args = getStringList(valueAt(json, "args")); + + auto envJson = valueAt(json, "env"); + try { + res.env = getStringMap(envJson); + } catch (Error & e) { + e.addTrace({}, "while reading key 'env'"); + throw; + } + + if (auto structuredAttrs = get(json, "structuredAttrs")) + res.structuredAttrs = StructuredAttrs{*structuredAttrs}; +} + +BasicDerivation +adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +{ + BasicDerivation res; + auto & json = getObject(_json); + basicDerivationFromJson(json, res, xpSettings); + + try { + inputSrcsFromJson(valueAt(json, "inputs"), res.inputSrcs); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputs'"); + throw; + } + + return res; +} + +Derivation adl_serializer::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings) +{ + Derivation res; + auto & json = getObject(_json); + basicDerivationFromJson(json, res, xpSettings); + try { auto inputsObj = getObject(valueAt(json, "inputs")); try { - auto inputSrcs = getArray(valueAt(inputsObj, "srcs")); - for (auto & input : inputSrcs) - res.inputSrcs.insert(input); + inputSrcsFromJson(valueAt(inputsObj, "srcs"), res.inputSrcs); } catch (Error & e) { e.addTrace({}, "while reading key 'srcs'"); throw; @@ -1647,21 +1697,6 @@ Derivation adl_serializer::from_json(const json & _json, const Exper throw; } - res.platform = getString(valueAt(json, "system")); - res.builder = getString(valueAt(json, "builder")); - res.args = getStringList(valueAt(json, "args")); - - auto envJson = valueAt(json, "env"); - try { - res.env = getStringMap(envJson); - } catch (Error & e) { - e.addTrace({}, "while reading key 'env'"); - throw; - } - - if (auto structuredAttrs = get(json, "structuredAttrs")) - res.structuredAttrs = StructuredAttrs{*structuredAttrs}; - return res; } diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index b74fb420e6..4cfc79acbe 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -620,4 +620,5 @@ constexpr unsigned expectedJsonVersionDerivation = 4; } // namespace nix JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput) +JSON_IMPL_WITH_XP_FEATURES(nix::BasicDerivation) JSON_IMPL_WITH_XP_FEATURES(nix::Derivation) From 0624f2260ea81126219dfb8c2e88e7145b1c42c5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Mar 2026 14:17:09 -0400 Subject: [PATCH 192/927] Add `tryResolve` unit tests Add dedicated unit tests for `Derivation::tryResolve` covering: - No-input derivations - Multi-input/multi-output derivations with structured attrs and embedded placeholders - Resolution failure when inputs cannot be resolved As usual, the tests are JSON based so they are less implementation-specific, gettting use closer to an implementation-agnostic "conformance test suite". Co-authored-by: Amaan Qureshi --- .../try-resolve/no-inputs-after.json | 17 ++ .../try-resolve/no-inputs-before.json | 20 ++ .../try-resolve/no-inputs-buildTrace.json | 1 + .../resolution-failure-before.json | 25 ++ .../resolution-failure-buildTrace.json | 1 + .../try-resolve/with-inputs-after.json | 33 +++ .../try-resolve/with-inputs-before.json | 46 ++++ .../try-resolve/with-inputs-buildTrace.json | 23 ++ src/libstore-tests/derivations.cc | 251 ++++++++++++++++++ src/libstore-tests/meson.build | 1 + 10 files changed, 418 insertions(+) create mode 100644 src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json create mode 100644 src/libstore-tests/derivations.cc diff --git a/src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json b/src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json new file mode 100644 index 0000000000..ed9b9bc48a --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/no-inputs-after.json @@ -0,0 +1,17 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "FOO": "bar" + }, + "inputs": [], + "name": "no-inputs", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json b/src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json new file mode 100644 index 0000000000..8ade9a6d90 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/no-inputs-before.json @@ -0,0 +1,20 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "FOO": "bar" + }, + "inputs": { + "drvs": {}, + "srcs": [] + }, + "name": "no-inputs", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/no-inputs-buildTrace.json @@ -0,0 +1 @@ +[] diff --git a/src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json new file mode 100644 index 0000000000..cb57937ff5 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-before.json @@ -0,0 +1,25 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": {}, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "resolution-failure", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/resolution-failure-buildTrace.json @@ -0,0 +1 @@ +[] diff --git a/src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json b/src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json new file mode 100644 index 0000000000..5c318ff840 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/with-inputs-after.json @@ -0,0 +1,33 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "DEP1_DEV": "/nix/store/j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev", + "DEP1_OUT": "prefix-/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out-suffix", + "DEP2": "/nix/store/i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out" + }, + "inputs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out", + "i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out", + "j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev" + ], + "name": "with-inputs", + "outputs": { + "dev": { + "hashAlgo": "sha256", + "method": "nar" + }, + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "dep1out": "/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out", + "nested": { + "dep2": "before /nix/store/i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out after" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json b/src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json new file mode 100644 index 0000000000..6b39e5fc06 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/with-inputs-before.json @@ -0,0 +1,46 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "DEP1_DEV": "/1r6jj1sjvjx4d1jqlnak71d8vzysdl2gyz2m01spjf31nhr1g55f", + "DEP1_OUT": "prefix-/10qd59vslmmj9w7hx8lpr0yxazwvhf3gfrnvbbc2s9071rhl3l55-suffix", + "DEP2": "/1zwa38661ns8faagbp31qzc8vx94hxsxcncs7hvcxhjvbcfyv9wb" + }, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "h1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "with-inputs", + "outputs": { + "dev": { + "hashAlgo": "sha256", + "method": "nar" + }, + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "dep1out": "/10qd59vslmmj9w7hx8lpr0yxazwvhf3gfrnvbbc2s9071rhl3l55", + "nested": { + "dep2": "before /1zwa38661ns8faagbp31qzc8vx94hxsxcncs7hvcxhjvbcfyv9wb after" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json new file mode 100644 index 0000000000..a15a69869a --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/with-inputs-buildTrace.json @@ -0,0 +1,23 @@ +[ + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv", + "output": "dev" + }, + "j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev" + ], + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv", + "output": "out" + }, + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out" + ], + [ + { + "drvPath": "h1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2.drv", + "output": "out" + }, + "i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out" + ] +] diff --git a/src/libstore-tests/derivations.cc b/src/libstore-tests/derivations.cc new file mode 100644 index 0000000000..a898b8ee9c --- /dev/null +++ b/src/libstore-tests/derivations.cc @@ -0,0 +1,251 @@ +#include +#include + +#include "nix/store/derivations.hh" +#include "nix/store/downstream-placeholder.hh" +#include "nix/store/parsed-derivations.hh" +#include "nix/store/tests/libstore.hh" +#include "nix/util/tests/json-characterization.hh" + +namespace nix { + +class TryResolveTest : public LibStoreTest, public CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "derivation" / "try-resolve"; + +public: + + /** + * A simple in-memory *derived* build trace. (e.g. we have + * `SingleDerivedPath::Built` not `StorePath` keys, because that is + * how the `tryResolved` callback works.) + * + * For a real-world build trace, derived build traces are much more + * work to get correct, and should be at most a caching layer atop + * an underlying source-of-truth build trace. (This is as described + * in the manual.) + * + * However, this is just for some simple in-memory unit tests, with + * tiny amounts of data we can hand-review, so it's fine. The point + * is unit testing `tryResolve`, not unit testing "deep queries" + * from a non-derived build trace (what is done in + * `outputs-query.cc`) anyways --- that would be a separate unit + * test. + */ + struct BuildTrace + { + std::map dict; + + bool operator==(const BuildTrace &) const = default; + }; + +protected: + + EnableExperimentalFeature caFeature{"ca-derivations"}; + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } + + /** + * Just here because we do this a few times in a tests. + */ + static DerivationOutput caFloatingOutput() + { + return DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}; + } + + /** + * Build a callback from a BuildTrace lookup. + */ + static auto makeCallback(const BuildTrace & table) + { + return + [&table](ref drvPath, const std::string & outputName) -> std::optional { + if (auto p = get(table.dict, SingleDerivedPath::Built{drvPath, outputName})) + return *p; + return std::nullopt; + }; + } + + /** + * Checkpoint before/buildTrace/after and assert the resolved derivation + * matches expected. + */ + void resolveExpect( + std::string_view stem, const Derivation & drv, const BuildTrace & buildTrace, const BasicDerivation & expected) + { + nix::checkpointJson(*this, std::string{stem} + "-before", drv); + nix::checkpointJson(*this, std::string{stem} + "-buildTrace", buildTrace); + + auto resolved = drv.tryResolve(*store, makeCallback(buildTrace)); + ASSERT_TRUE(resolved); + + nix::checkpointJson(*this, std::string{stem} + "-after", *resolved); + + EXPECT_EQ(*resolved, expected); + } +}; + +} // namespace nix + +JSON_IMPL(nix::TryResolveTest::BuildTrace); + +namespace nlohmann { + +using nix::SingleDerivedPath; +using nix::StorePath; +using nix::TryResolveTest; + +void adl_serializer::to_json(json & j, const TryResolveTest::BuildTrace & t) +{ + j = t.dict; +} + +TryResolveTest::BuildTrace adl_serializer::from_json(const json & j) +{ + return TryResolveTest::BuildTrace{ + .dict = j.get>(), + }; +} + +} // namespace nlohmann + +namespace nix { + +TEST_F(TryResolveTest, noInputs) +{ + resolveExpect( + "no-inputs", + [&] { + Derivation drv; + drv.name = "no-inputs"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.env = {{"FOO", "bar"}}; + return drv; + }(), + {}, + [&] { + BasicDerivation expected; + expected.name = "no-inputs"; + expected.platform = "x86_64-linux"; + expected.builder = "/bin/bash"; + expected.outputs = {{"out", caFloatingOutput()}}; + expected.env = {{"FOO", "bar"}}; + return expected; + }()); +} + +TEST_F(TryResolveTest, withInputs) +{ + // dep1 has two outputs (out, dev), dep2 has one (out) + StorePath dep1DrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1.drv"}; + StorePath dep1OutPath{"f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-out"}; + StorePath dep1DevPath{"j1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep1-dev"}; + StorePath dep2DrvPath{"h1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2.drv"}; + StorePath dep2OutPath{"i1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep2-out"}; + + auto placeholder1Out = DownstreamPlaceholder::unknownCaOutput(dep1DrvPath, "out").render(); + auto placeholder1Dev = DownstreamPlaceholder::unknownCaOutput(dep1DrvPath, "dev").render(); + auto placeholder2Out = DownstreamPlaceholder::unknownCaOutput(dep2DrvPath, "out").render(); + + DerivationOutputs multiOutputs = { + {"out", caFloatingOutput()}, + {"dev", caFloatingOutput()}, + }; + + resolveExpect( + "with-inputs", + [&] { + Derivation drv; + drv.name = "with-inputs"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = multiOutputs; + drv.inputDrvs = { + .map = { + {dep1DrvPath, {.value = {"out", "dev"}}}, + {dep2DrvPath, {.value = {"out"}}}, + }}; + drv.env = { + {"DEP1_OUT", "prefix-" + placeholder1Out + "-suffix"}, + {"DEP1_DEV", placeholder1Dev}, + {"DEP2", placeholder2Out}, + }; + drv.structuredAttrs = StructuredAttrs{{ + {"dep1out", placeholder1Out}, + {"nested", nlohmann::json::object({{"dep2", "before " + placeholder2Out + " after"}})}, + }}; + return drv; + }(), + {.dict{ + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(dep1DrvPath), + .output = "out", + }, + dep1OutPath, + }, + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(dep1DrvPath), + .output = "dev", + }, + dep1DevPath, + }, + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(dep2DrvPath), + .output = "out", + }, + dep2OutPath, + }, + }}, + [&] { + BasicDerivation expected; + expected.name = "with-inputs"; + expected.platform = "x86_64-linux"; + expected.builder = "/bin/bash"; + expected.outputs = multiOutputs; + expected.inputSrcs = {dep1OutPath, dep1DevPath, dep2OutPath}; + expected.env = { + {"DEP1_OUT", "prefix-" + store->printStorePath(dep1OutPath) + "-suffix"}, + {"DEP1_DEV", store->printStorePath(dep1DevPath)}, + {"DEP2", store->printStorePath(dep2OutPath)}, + }; + expected.structuredAttrs = StructuredAttrs{{ + {"dep1out", store->printStorePath(dep1OutPath)}, + {"nested", + nlohmann::json::object({{"dep2", "before " + store->printStorePath(dep2OutPath) + " after"}})}, + }}; + return expected; + }()); +} + +TEST_F(TryResolveTest, resolutionFailure) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + + Derivation drv; + drv.name = "resolution-failure"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + + BuildTrace buildTrace; + + checkpointJson(*this, "resolution-failure-before", drv); + checkpointJson(*this, "resolution-failure-buildTrace", buildTrace); + + auto resolved = drv.tryResolve(*store, makeCallback(buildTrace)); + EXPECT_FALSE(resolved); +} + +} // namespace nix diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index fd1f0a05e7..a126a87aca 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -60,6 +60,7 @@ sources = files( 'derivation-advanced-attrs.cc', 'derivation/external-formats.cc', 'derivation/invariants.cc', + 'derivations.cc', 'derived-path.cc', 'downstream-placeholder.cc', 'dummy-store.cc', From edd2e2f6d8ad7a95224efbd5523a94b2a85a200a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 30 Mar 2026 23:46:35 +0300 Subject: [PATCH 193/927] libstore: Use std::weak_ptr in ItemHandle I noticed cole-h came across this issue in detnix. Silly mistake on my part, the TransferItem can die by the time we might want to unpause it. Haven't seen this fail in in the wild, but the weak_ptr approach is the correct one. The enqueueing thread mustn't take shared ownership. Same for enqueueing for wakeup. Only the worker thread must have the ownership of the TransferItem. --- src/libstore/filetransfer.cc | 21 +++++++++++++------ .../include/nix/store/filetransfer.hh | 4 ++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index d5eb1c538b..88853620fa 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -925,7 +925,7 @@ struct curlFileTransfer : public FileTransfer }; std::priority_queue, std::vector>, EmbargoComparator> incoming; - std::vector> unpause; + std::vector> unpause; private: bool quitting = false; public: @@ -1098,8 +1098,15 @@ struct curlFileTransfer : public FileTransfer return res; }(); - for (auto & item : unpause) - item->unpause(); + for (auto & item : unpause) { + /* The transfer might have completed (failed) between it getting + enqueued for unpause and by the time the worker thread picked + it up. */ + auto ptr = item.lock(); + if (!ptr) + continue; + static_cast(*ptr).unpause(); + } } debug("download thread shutting down"); @@ -1139,7 +1146,7 @@ struct curlFileTransfer : public FileTransfer } wakeupMulti(); - return ItemHandle(static_cast(*item)); + return ItemHandle(item.get_ptr()); } ItemHandle enqueueFileTransfer(const FileTransferRequest & request, Callback callback) override @@ -1154,7 +1161,7 @@ struct curlFileTransfer : public FileTransfer return enqueueItem(make_ref(*this, request, std::move(callback))); } - void unpauseTransfer(ref item) + void unpauseTransfer(std::weak_ptr item) { auto state(state_.lock()); state->unpause.push_back(std::move(item)); @@ -1163,7 +1170,9 @@ struct curlFileTransfer : public FileTransfer void unpauseTransfer(ItemHandle handle) override { - unpauseTransfer(ref{static_cast(handle.item.get()).shared_from_this()}); + /* The transfer might have completed (more likely failed) when we want + to wake it up. That's why we must use a weak_ptr throughout. */ + unpauseTransfer(handle.item); } }; diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 64f1cadf56..a423249e66 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -401,10 +401,10 @@ public: */ struct ItemHandle { - std::reference_wrapper item; + std::weak_ptr item; friend struct FileTransfer; - ItemHandle(Item & item) + explicit ItemHandle(std::weak_ptr item) : item(item) { } From 3672f815d69357b7afa43c797a51078fd135067d Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 31 Mar 2026 01:08:02 +0300 Subject: [PATCH 194/927] libutil: Fix FreeBSD build --- src/libutil/file-system.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 16e4e6ce8d..dfba75e500 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -462,7 +462,7 @@ std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const will be owned by "wheel"; but if the user is not in "wheel", then "tar" will fail to unpack archives that have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + if (::chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) throw SysError("setting group of directory %1%", PathFmt(tmpDir)); #endif return tmpDir; From 2239fa6d7b4d7d44b32f7209f76b1cd7ceae8856 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 22:42:43 +0000 Subject: [PATCH 195/927] build(deps): bump cachix/install-nix-action from 31.10.2 to 31.10.3 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.10.2 to 31.10.3. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/51f3067b56fe8ae331890c77d4e454f6d60615ff...96951a368ba55167b55f1c916f7d416bac6505fe) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.10.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21f567f7ca..51cd9b48c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,7 +192,7 @@ jobs: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@51f3067b56fe8ae331890c77d4e454f6d60615ff # v31.10.2 + - uses: cachix/install-nix-action@96951a368ba55167b55f1c916f7d416bac6505fe # v31.10.3 if: ${{ !matrix.experimental-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} @@ -258,7 +258,7 @@ jobs: id: installer-tarball-url run: | echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@51f3067b56fe8ae331890c77d4e454f6d60615ff # v31.10.2 + - uses: cachix/install-nix-action@96951a368ba55167b55f1c916f7d416bac6505fe # v31.10.3 with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} From c1c1d477db466eb0b0fc69dba3eccab6081f1136 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 22:42:48 +0000 Subject: [PATCH 196/927] build(deps): bump korthout/backport-action from 4.2.0 to 4.3.0 Bumps [korthout/backport-action](https://github.com/korthout/backport-action) from 4.2.0 to 4.3.0. - [Release notes](https://github.com/korthout/backport-action/releases) - [Commits](https://github.com/korthout/backport-action/compare/4aaf0e03a94ff0a619c9a511b61aeb42adea5b02...3c06f323a58619da1e8522229ebc8d5de2633e46) --- updated-dependencies: - dependency-name: korthout/backport-action dependency-version: 4.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 3fe089df53..2d01cfefc2 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -26,7 +26,7 @@ jobs: # required to find all branches fetch-depth: 0 - name: Create backport PRs - uses: korthout/backport-action@4aaf0e03a94ff0a619c9a511b61aeb42adea5b02 # v4.2.0 + uses: korthout/backport-action@3c06f323a58619da1e8522229ebc8d5de2633e46 # v4.3.0 id: backport with: # Config README: https://github.com/korthout/backport-action#backport-action From edd0af196d1ed7052adced04a71c0f771aae6c5b Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Tue, 31 Mar 2026 23:23:21 +0800 Subject: [PATCH 197/927] doc: render C API doc for nix-main-c and nix-fetchers-c --- src/external-api-docs/doxygen.cfg.in | 2 ++ src/external-api-docs/package.nix | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in index 3af2f5b813..df6d1c5b7d 100644 --- a/src/external-api-docs/doxygen.cfg.in +++ b/src/external-api-docs/doxygen.cfg.in @@ -42,6 +42,8 @@ INPUT = \ @src@/src/libexpr-c \ @src@/src/libflake-c \ @src@/src/libstore-c \ + @src@/src/libfetchers-c \ + @src@/src/libmain-c \ @src@/src/external-api-docs/README.md FILE_PATTERNS = nix_api_*.h *.md diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index b194e16d46..df6aa9838d 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -31,7 +31,9 @@ mkMesonDerivation (finalAttrs: { # Source is not compiled, but still must be available for Doxygen # to gather comments. (cpp ../libexpr-c) + (cpp ../libfetchers-c) (cpp ../libflake-c) + (cpp ../libmain-c) (cpp ../libstore-c) (cpp ../libutil-c) ]; From 26213c114f94fdd282b05564203fd9c0cd6667ef Mon Sep 17 00:00:00 2001 From: Pierre Penninckx Date: Tue, 17 Mar 2026 13:14:28 +0100 Subject: [PATCH 198/927] doc: add example for git tags --- src/nix/flake.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/flake.md b/src/nix/flake.md index 290e4694eb..88d0e1be58 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -95,6 +95,8 @@ Here are some examples of flake references in their URL-like representation: branch of a Git repository. * `git+https://github.com/NixOS/patchelf?ref=master&rev=f34751b88bd07d7f44f5cd3200fb4122bf916c7e`: A specific branch *and* revision of a Git repository. +* `git+https://github.com/NixOS/patchelf?ref=refs/tags/0.18.0`: A specific + tag of a Git repository. * `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball flake. From 09de76b98f2f0736e23105a5316318cf2569bcd5 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Apr 2026 01:03:23 +0300 Subject: [PATCH 199/927] libstore: Memoise Goal::key() --- src/libstore/build/goal.cc | 4 +--- src/libstore/include/nix/store/build/goal.hh | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index ce6b3e5883..946314afe1 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -157,9 +157,7 @@ std::coroutine_handle<> nix::Goal::Co::await_suspend(handle_type caller) bool CompareGoalPtrs::operator()(const GoalPtr & a, const GoalPtr & b) const { - std::string s1 = a->key(); - std::string s2 = b->key(); - return s1 < s2; + return a->keyCached() < b->keyCached(); } void addToWeakGoals(WeakGoals & goals, GoalPtr p) diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 3e32f4fecb..0b7367ff7e 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -80,6 +80,11 @@ private: */ Goals waitees; + /** + * Memoised result of key(). + */ + std::optional cachedKey; + public: typedef enum { ecBusy, ecSuccess, ecFailed, ecNoSubstituters } ExitCode; @@ -600,6 +605,17 @@ public: */ virtual std::string key() = 0; + /** + * Memoising variant of key(). We really don't want to pay the overhead of + * allocating strings just to compare Goals. + */ + std::string_view keyCached() & + { + if (cachedKey) + return *cachedKey; + return *(cachedKey = key()); + } + /** * @brief Hint for the scheduler, which concurrency limit applies. * @see JobCategory From 6fae3a25f1a07472b45c56862c0ffe2050159e0f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Apr 2026 01:03:27 +0300 Subject: [PATCH 200/927] libstore: Optimise memory usage of the Worker Several things to consider: - Derivation is quite big and we used to copy it unnecessarily. We can just have a single instance and pass immutable shared_ptr around. - childTerminated would wake all goals that are waiting for a build slot. This would trigger a lot of coroutine frame allocations in DerivationBuildingGoal::buildLocally. We should wake at most one waiter for each terminated child. Everything else is wasted work. --- src/libstore-tests/worker-substitution.cc | 4 +- .../build/derivation-building-goal.cc | 4 +- src/libstore/build/derivation-goal.cc | 15 ++++-- .../build/derivation-trampoline-goal.cc | 4 +- src/libstore/build/worker.cc | 49 +++++++++++++------ .../store/build/derivation-building-goal.hh | 8 ++- .../nix/store/build/derivation-goal.hh | 4 +- .../include/nix/store/build/worker.hh | 9 +++- 8 files changed, 65 insertions(+), 32 deletions(-) diff --git a/src/libstore-tests/worker-substitution.cc b/src/libstore-tests/worker-substitution.cc index 02b65054ea..3534d44d0d 100644 --- a/src/libstore-tests/worker-substitution.cc +++ b/src/libstore-tests/worker-substitution.cc @@ -241,7 +241,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput) // Create a derivation goal for the CA derivation output // The worker should substitute the output rather than building - auto goal = worker.makeDerivationGoal(drvPath, drv, "out", bmNormal, true); + auto goal = worker.makeDerivationGoal(drvPath, make_ref(drv), "out", bmNormal, true); // Run the worker Goals goals; @@ -403,7 +403,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv) // Create a derivation goal for the root derivation output // The worker should substitute the output rather than building - auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false); + auto goal = worker.makeDerivationGoal(rootDrvPath, make_ref(rootDrv), "out", bmNormal, false); // Run the worker Goals goals; diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 64801e4be3..2235cf872b 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -30,10 +30,10 @@ namespace nix { DerivationBuildingGoal::DerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode, bool storeDerivation) + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode, bool storeDerivation) : Goal(worker, gaveUpOnSubstitution(storeDerivation)) , drvPath(drvPath) - , drv{std::make_unique(drv)} + , drv{std::move(drv)} , buildMode(buildMode) { name = fmt("building derivation '%s'", worker.store.printStorePath(drvPath)); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index e824dfdc9c..115e48612f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -29,7 +29,7 @@ namespace nix { DerivationGoal::DerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, Worker & worker, BuildMode buildMode, @@ -37,7 +37,7 @@ DerivationGoal::DerivationGoal( : Goal(worker, haveDerivation(storeDerivation)) , drvPath(drvPath) , wantedOutput(wantedOutput) - , drv{std::make_unique(drv)} + , drv{std::move(drv)} , buildMode(buildMode) { @@ -167,8 +167,13 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) if (resolutionGoal->resolvedDrv) { auto & [pathResolved, drvResolved] = *resolutionGoal->resolvedDrv; - auto resolvedDrvGoal = - worker.makeDerivationGoal(pathResolved, drvResolved, wantedOutput, buildMode, /*storeDerivation=*/true); + auto resolvedDrvGoal = worker.makeDerivationGoal( + pathResolved, + make_ref(drvResolved), + wantedOutput, + buildMode, + /*storeDerivation=*/true); + { Goals waitees{resolvedDrvGoal}; co_await await(std::move(waitees)); @@ -227,7 +232,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* Give up on substitution for the output we want, actually build this derivation */ - auto g = worker.makeDerivationBuildingGoal(drvPath, *drv, buildMode, storeDerivation); + auto g = worker.makeDerivationBuildingGoal(drvPath, drv, buildMode, storeDerivation); /* We will finish with it ourselves, as if we were the derivational goal. */ g->preserveFailure = true; diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index 4c6553a521..edf8d1e86e 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -148,8 +148,10 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation /* Build this step! */ + auto sharedDrv = make_ref(std::move(drv)); + for (auto & output : resolvedWantedOutputs) { - auto g = upcast_goal(worker.makeDerivationGoal(drvPath, drv, output, buildMode, false)); + auto g = upcast_goal(worker.makeDerivationGoal(drvPath, sharedDrv, output, buildMode, false)); g->preserveFailure = true; /* We will finish with it ourselves, as if we were the derivational goal. */ concreteDrvGoals.insert(std::move(g)); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 20131aaf3f..de23487ac8 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -88,13 +88,19 @@ std::shared_ptr Worker::makeDerivationTrampolineGoal( std::shared_ptr Worker::makeDerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, BuildMode buildMode, bool storeDerivation) { return initGoalIfNeeded( - derivationGoals[drvPath][wantedOutput], drvPath, drv, wantedOutput, *this, buildMode, storeDerivation); + derivationGoals[drvPath][wantedOutput], + drvPath, + std::move(drv), + wantedOutput, + *this, + buildMode, + storeDerivation); } std::shared_ptr @@ -104,9 +110,10 @@ Worker::makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation } std::shared_ptr Worker::makeDerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, BuildMode buildMode, bool storeDerivation) + const StorePath & drvPath, ref drv, BuildMode buildMode, bool storeDerivation) { - return initGoalIfNeeded(derivationBuildingGoals[drvPath], drvPath, drv, *this, buildMode, storeDerivation); + return initGoalIfNeeded( + derivationBuildingGoals[drvPath], drvPath, std::move(drv), *this, buildMode, storeDerivation); } std::shared_ptr @@ -287,27 +294,37 @@ void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSlee children.erase(i); if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) + auto & waiting = jobCategory == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild; + + /* Wake up goals waiting for a build slot. Wake at most one waiter to avoid + starting unnecessary work (that is accompanied by coroutine frame allocation). */ + auto it = waiting.begin(); + while (it != waiting.end()) { + if (auto goal = it->lock()) { + waiting.erase(it); wakeUp(goal); + break; + } + it = waiting.erase(it); } - - wantingToBuild.clear(); } } void Worker::waitForBuildSlot(GoalPtr goal) { goal->trace("wait for build slot"); - bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution; - if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) - || (isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs)) - wakeUp(goal); /* we can do it right away */ + + bool slotAvailable = [&] { + if (goal->jobCategory() == JobCategory::Substitution) + return getNrSubstitutions() < settings.maxSubstitutionJobs; + else + return getNrLocalBuilds() < settings.maxBuildJobs; + }(); + + if (slotAvailable) + wakeUp(goal); /* Can do it right away. */ else - addToWeakGoals(wantingToBuild, goal); + addToWeakGoals(goal->jobCategory() == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild, goal); } void Worker::waitForAnyGoal(GoalPtr goal) diff --git a/src/libstore/include/nix/store/build/derivation-building-goal.hh b/src/libstore/include/nix/store/build/derivation-building-goal.hh index 826cf2c50f..50621d97d2 100644 --- a/src/libstore/include/nix/store/build/derivation-building-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-building-goal.hh @@ -41,7 +41,11 @@ struct DerivationBuildingGoal : public Goal * faithfully reconstruct the build history. */ DerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode, bool storeDerivation); + const StorePath & drvPath, + ref drv, + Worker & worker, + BuildMode buildMode, + bool storeDerivation); ~DerivationBuildingGoal(); private: @@ -52,7 +56,7 @@ private: /** * The derivation stored at drvPath. */ - const std::unique_ptr drv; + const ref drv; /** * The remainder is state held during the build. diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index c6bd412182..94aba5c316 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -45,7 +45,7 @@ struct DerivationGoal : public Goal */ DerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, Worker & worker, BuildMode buildMode, @@ -64,7 +64,7 @@ private: /** * The derivation stored at drvPath. */ - std::unique_ptr drv; + ref drv; const BuildMode buildMode; diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 2f5272cef5..104ef64acc 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -92,6 +92,11 @@ private: */ WeakGoals wantingToBuild; + /** + * Goals waiting for a substitution slot. + */ + WeakGoals wantingToSubstitute; + /** * Child processes currently running. */ @@ -256,7 +261,7 @@ public: std::shared_ptr makeDerivationGoal( const StorePath & drvPath, - const Derivation & drv, + ref drv, const OutputName & wantedOutput, BuildMode buildMode, bool storeDerivation); @@ -271,7 +276,7 @@ public: * @ref DerivationBuildingGoal "derivation building goal" */ std::shared_ptr makeDerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, BuildMode buildMode, bool storeDerivation); + const StorePath & drvPath, ref drv, BuildMode buildMode, bool storeDerivation); /** * @ref PathSubstitutionGoal "substitution goal" From 9db1f7c8af673776a9c18ec31c4be432d0e60d72 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Apr 2026 01:03:29 +0300 Subject: [PATCH 201/927] libstore: Get rid of unused wakeSleepers parameter to Worker::childTerminated It was never actually used. Also Administration goals never make their way into this code path (it would make no sense if they did). --- src/libstore/build/worker.cc | 38 +++++++++---------- .../include/nix/store/build/worker.hh | 10 ++--- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index de23487ac8..154cb80158 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -254,21 +254,21 @@ void Worker::childStarted( nrLocalBuilds++; break; case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; default: + /* Doesn't make sense, since there are only building and substitution slots. */ unreachable(); } } } -void Worker::childTerminated(Goal * goal, bool wakeSleepers) +void Worker::childTerminated(Goal * goal) { - childTerminated(goal, goal->jobCategory(), wakeSleepers); + childTerminated(goal, goal->jobCategory()); } -void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers) +void Worker::childTerminated(Goal * goal, JobCategory jobCategory) { + // FIXME: Inefficient. Make children a map from Goal -> Child instead. auto i = std::find_if(children.begin(), children.end(), [&](const Child & child) { return child.goal2 == goal; }); if (i == children.end()) return; @@ -284,29 +284,25 @@ void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSlee nrLocalBuilds--; break; case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; default: + /* Doesn't make sense, since there are only building and substitution slots. */ unreachable(); } } children.erase(i); - - if (wakeSleepers) { - auto & waiting = jobCategory == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild; - - /* Wake up goals waiting for a build slot. Wake at most one waiter to avoid - starting unnecessary work (that is accompanied by coroutine frame allocation). */ - auto it = waiting.begin(); - while (it != waiting.end()) { - if (auto goal = it->lock()) { - waiting.erase(it); - wakeUp(goal); - break; - } - it = waiting.erase(it); + auto & waiting = jobCategory == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild; + + /* Wake up goals waiting for a build slot. Wake at most one waiter to avoid + starting unnecessary work (that is accompanied by coroutine frame allocation). */ + auto it = waiting.begin(); + while (it != waiting.end()) { + if (auto goal = it->lock()) { + waiting.erase(it); + wakeUp(goal); + break; } + it = waiting.erase(it); } } diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 104ef64acc..b582498232 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -331,15 +331,13 @@ public: bool respectTimeouts); /** - * Unregisters a running child process. `wakeSleepers` should be - * false if there is no sense in waking up goals that are sleeping - * because they can't run yet (e.g., there is no free build slot, - * or the hook would still say `postpone`). + * Unregisters a running child process. Wakes at most a single goal that is + * awaiting on the corresponding build slot type (building or substitution). * * This overload requires `goal` to point to a fully constructed, * valid goal object, as it calls `goal->jobCategory()`. */ - void childTerminated(Goal * goal, bool wakeSleepers = true); + void childTerminated(Goal * goal); /** * Unregisters a running child process, like the other overload. @@ -348,7 +346,7 @@ public: * weak goal references, so it is safe to call from destructors * where the goal object may be partially destroyed. */ - void childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers = true); + void childTerminated(Goal * goal, JobCategory jobCategory); /** * Put `goal` to sleep until a build slot becomes available (which From 9ec16e28dd8363e0a921295079dac38ad0ffd26c Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Apr 2026 01:03:31 +0300 Subject: [PATCH 202/927] libstore: Get rid of unused Worker::waitForAnyGoal Nothing in-tree actually uses this. --- src/libstore/build/worker.cc | 15 --------------- src/libstore/include/nix/store/build/worker.hh | 11 ----------- 2 files changed, 26 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 154cb80158..7f63b790a4 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -207,15 +207,6 @@ void Worker::removeGoal(GoalPtr goal) if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) topGoals.clear(); } - - /* Wake up goals waiting for any goal to finish. */ - for (auto & i : waitingForAnyGoal) { - GoalPtr goal = i.lock(); - if (goal) - wakeUp(goal); - } - - waitingForAnyGoal.clear(); } void Worker::wakeUp(GoalPtr goal) @@ -323,12 +314,6 @@ void Worker::waitForBuildSlot(GoalPtr goal) addToWeakGoals(goal->jobCategory() == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild, goal); } -void Worker::waitForAnyGoal(GoalPtr goal) -{ - goal->trace("wait for any goal"); - addToWeakGoals(waitingForAnyGoal, goal); -} - void Worker::waitForAWhile(GoalPtr goal) { goal->trace("wait for a while"); diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index b582498232..4c83698681 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -126,11 +126,6 @@ private: std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; - /** - * Goals waiting for busy paths to be unlocked. - */ - WeakGoals waitingForAnyGoal; - /** * Goals sleeping for a few seconds (polling a lock). */ @@ -354,12 +349,6 @@ public: */ void waitForBuildSlot(GoalPtr goal); - /** - * Wait for any goal to finish. Pretty indiscriminate way to - * wait for some resource that some other goal is holding. - */ - void waitForAnyGoal(GoalPtr goal); - /** * Wait for a few seconds and then retry this goal. Used when * waiting for a lock held by another process. This kind of From d5a64770a4635799c7f283480561ae2da02198ae Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Apr 2026 01:03:34 +0300 Subject: [PATCH 203/927] libstore: Get rid of quardatic complexity in the Worker::removeGoal When the number of derivations is huge, derivationTrampolineGoals is also huge. removeGoal would then iterate over all of the goals and try to prune them one-by-one. This would slow down the scheduler to a crawl on just something like 50k derivations. That is totally unacceptable, especially when we can make the removal O(log n) instead of O(n). --- src/libstore/build/worker.cc | 79 ++++++------------- src/libstore/derived-path-map.cc | 28 +++++++ .../store/build/derivation-building-goal.hh | 2 + .../store/build/derivation-resolution-goal.hh | 2 + .../build/drv-output-substitution-goal.hh | 3 +- .../include/nix/store/derived-path-map.hh | 13 +++ 6 files changed, 70 insertions(+), 57 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 7f63b790a4..4666725ed0 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -141,64 +141,31 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) req.raw()); } -/** - * This function is polymorphic (both via type parameters and - * overloading) and recursive in order to work on a various types of - * trees - * - * @return Whether the tree node we are processing is not empty / should - * be kept alive. In the case of this overloading the node in question - * is the leaf, the weak reference itself. If the weak reference points - * to the goal we are looking for, our caller can delete it. In the - * inductive case where the node is an interior node, we'll likewise - * return whether the interior node is non-empty. If it is empty - * (because we just deleted its last child), then our caller can - * likewise delete it. - */ -template -static bool removeGoal(std::shared_ptr goal, std::weak_ptr & gp) -{ - return gp.lock() != goal; -} - -template -static bool removeGoal(std::shared_ptr goal, std::map & goalMap) -{ - /* !!! inefficient */ - for (auto i = goalMap.begin(); i != goalMap.end();) { - if (!removeGoal(goal, i->second)) - i = goalMap.erase(i); - else - ++i; - } - return !goalMap.empty(); -} - -template -static bool -removeGoal(std::shared_ptr goal, typename DerivedPathMap>>::ChildNode & node) -{ - bool valueKeep = removeGoal(goal, node.value); - bool childMapKeep = removeGoal(goal, node.childMap); - return valueKeep || childMapKeep; -} - void Worker::removeGoal(GoalPtr goal) { - if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, derivationTrampolineGoals.map); - else if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, derivationGoals); - else if (auto drvResolutionGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvResolutionGoal, derivationResolutionGoals); - else if (auto drvBuildingGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvBuildingGoal, derivationBuildingGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(subGoal, substitutionGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(subGoal, drvOutputSubstitutionGoals); - else - assert(false); + if (auto drvGoal = std::dynamic_pointer_cast(goal)) { + derivationTrampolineGoals.removeSlot(*drvGoal->drvReq, [&](auto & node) { + node.value.erase(drvGoal->wantedOutputs); + /* Return true if ancestors don't need to be pruned. */ + return !node.value.empty(); + }); + } else if (auto drvGoal = std::dynamic_pointer_cast(goal)) { + if (auto it = derivationGoals.find(drvGoal->drvPath); it != derivationGoals.end()) { + it->second.erase(drvGoal->wantedOutput); + if (it->second.empty()) + derivationGoals.erase(it); + } + } else if (auto drvResolutionGoal = std::dynamic_pointer_cast(goal)) { + derivationResolutionGoals.erase(drvResolutionGoal->drvPath); + } else if (auto drvBuildingGoal = std::dynamic_pointer_cast(goal)) { + derivationBuildingGoals.erase(drvBuildingGoal->drvPath); + } else if (auto subGoal = std::dynamic_pointer_cast(goal)) { + substitutionGoals.erase(subGoal->storePath); + } else if (auto subGoal = std::dynamic_pointer_cast(goal)) { + drvOutputSubstitutionGoals.erase(subGoal->id); + } else { + unreachable(); + } if (topGoals.find(goal) != topGoals.end()) { topGoals.erase(goal); diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 3d799ab1c5..684c4462fb 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -48,6 +48,34 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single return initIter(k); } +template +void DerivedPathMap::removeSlot(const SingleDerivedPath & k, fun callback) +{ + auto removeIter = [&map = + map](this auto & self, const SingleDerivedPath & k, fun onNode) -> void { + std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) { + if (auto it = map.find(bo.path); it != map.end() && !onNode(it->second)) + map.erase(it); + }, + [&](const SingleDerivedPath::Built & bfd) { + self(*bfd.drvPath, [&](ChildNode & parent) -> bool { + auto it = parent.childMap.find(bfd.output); + if (it == parent.childMap.end()) + return !parent.value.empty() || !parent.childMap.empty(); + if (!onNode(it->second)) + parent.childMap.erase(it); + return !parent.value.empty() || !parent.childMap.empty(); + }); + }, + }, + k.raw()); + }; + + removeIter(k, [&](ChildNode & node) -> bool { return callback(node) || !node.childMap.empty(); }); +} + } // namespace nix // instantiations diff --git a/src/libstore/include/nix/store/build/derivation-building-goal.hh b/src/libstore/include/nix/store/build/derivation-building-goal.hh index 50621d97d2..6a17f73eeb 100644 --- a/src/libstore/include/nix/store/build/derivation-building-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-building-goal.hh @@ -31,6 +31,8 @@ typedef enum { rpAccept, rpDecline, rpPostpone } HookReply; */ struct DerivationBuildingGoal : public Goal { + friend class Worker; + /** * @param storeDerivation Whether to store the derivation in * `worker.store`. This is useful for newly-resolved derivations. In this diff --git a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh index b79e6bbb79..843e4031aa 100644 --- a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh @@ -35,6 +35,8 @@ struct BuilderFailureError; */ struct DerivationResolutionGoal : public Goal { + friend class Worker; + DerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode); /** diff --git a/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh b/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh index 5f36bbf06d..6652da6a20 100644 --- a/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh +++ b/src/libstore/include/nix/store/build/drv-output-substitution-goal.hh @@ -20,11 +20,12 @@ class Worker; * If the output store object itself should also be substituted, that is * the responsibility of the caller to do so. * - * @todo rename this `BuidlTraceEntryGoal`, which will make sense + * @todo rename this `BuildTraceEntryGoal`, which will make sense * especially once `Realisation` is renamed to `BuildTraceEntry`. */ class DrvOutputSubstitutionGoal : public Goal { + friend class Worker; /** * The drv output we're trying to substitute diff --git a/src/libstore/include/nix/store/derived-path-map.hh b/src/libstore/include/nix/store/derived-path-map.hh index c10af84ca2..65be5df3c4 100644 --- a/src/libstore/include/nix/store/derived-path-map.hh +++ b/src/libstore/include/nix/store/derived-path-map.hh @@ -93,6 +93,19 @@ struct DerivedPathMap * `ChildNode::value`. */ ChildNode * findSlot(const SingleDerivedPath & k); + + /** + * Find the node for `k` and invoke @ref callback on it, pruning empty + * ancestors afterwards. + * + * @param callback Invoked on the found node. Should return true if + * the node's value is still non-empty (i.e. the node should be kept). + * If it returns false and the node has no children, the node is erased + * and empty ancestors are pruned recursively. + * + * No-op if the node does not exist. + */ + void removeSlot(const SingleDerivedPath & k, fun callback); }; template<> From 4c9afebaaa0707caf85ff69cb687cc1f9a118794 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Apr 2026 01:03:36 +0300 Subject: [PATCH 204/927] libstore: Add useful vomit prints in the Worker::run loop Useful for getting traces like this out of the worker: worker event loop worked goal 'building derivation '/nix/store/llw9vkyi5gj0jv01lrvyj4r0s7pahdmn-X-Reload-Triggers-caddy.drv'' for 182.477ms worker event loop worked goal 'building derivation '/nix/store/jinkpxfafkwflh5sg781knqv7x07w5yx-X-Restart-Triggers-dbus.drv'' for 252.866ms worker event loop worked goal 'building derivation '/nix/store/7jkb4bk87vlb32vzd36hs5iymgrpfyx4-X-Restart-Triggers-nginx-config-reload.drv'' for 196.006ms worker event loop worked goal 'building derivation '/nix/store/267q376i5hk2kzwqiybb8j11387jkdf6-closure-info.drv'' for 221.300ms worker event loop worked goal 'building derivation '/nix/store/5kbnzasxm28qsplwzg78jnzfcvbbkpyl-limesurvey-phpfpm-exec-pre.drv'' for 172.403ms worker event loop worked goal 'building derivation '/nix/store/4ms5j2x70kn1hsj93wkzpwv7ibgwhz9c-mailman-hyperkitty.cfg.drv'' for 177.715ms worker event loop worked goal 'building derivation '/nix/store/hix8qi0cpl19cz6qviwz20hf5lirv57c-nixos-26.05pre-git.drv'' for 170.551ms --- src/libstore/build/worker.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 4666725ed0..0c63beb4dd 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -333,9 +333,24 @@ void Worker::run(const Goals & _topGoals) awake2.insert(goal); } awake.clear(); + for (auto & goal : awake2) { checkInterrupt(); + + std::chrono::time_point startTime; + if (verbosity >= lvlVomit) + startTime = std::chrono::steady_clock::now(); + goal->work(); + + /* Useful for tracing which goals hod the event loop. */ + vomit( + "worker event loop worked goal '%1%' for %2$.3fms", + goal->name, + std::chrono::duration_cast>( + std::chrono::steady_clock::now() - startTime) + .count()); + if (topGoals.empty()) break; // stuff may have been cancelled } From 4a09c1f083679b179d3fc4c42d663ff3832469e6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Apr 2026 22:01:13 +0200 Subject: [PATCH 205/927] Don't detach the auto-GC thread It could still be running after `this` is destroyed. So we need to join it in the LocalStore destructor. --- src/libstore/gc.cc | 9 +++++++-- src/libstore/include/nix/store/local-store.hh | 1 + src/libstore/local-store.cc | 6 ++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index a88fe6a64a..2d918a1391 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -852,12 +852,17 @@ void LocalStore::autoGC(bool sync) if (avail > state->availAfterGC * 0.97) return; + /* Note: since gcRunning is false here, any previous GC thread has exited / is exiting so the join() should be + * almost instantenous. */ + if (state->gcThread.joinable()) + state->gcThread.join(); + state->gcRunning = true; std::promise promise; future = state->gcFuture = promise.get_future().share(); - std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { + state->gcThread = std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { try { /* Wake up any threads waiting for the auto-GC to finish. */ @@ -884,7 +889,7 @@ void LocalStore::autoGC(bool sync) // future, but we don't really care. (what??) ignoreExceptionInDestructor(); } - }).detach(); + }); } sync: diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 63a1da67d8..b5c80ba9e5 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -212,6 +212,7 @@ private: */ bool gcRunning = false; std::shared_future gcFuture; + std::thread gcThread; /** * How much disk space was available after the previous diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e9eb48bfe6..ef8407532b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -432,6 +432,12 @@ LocalStore::~LocalStore() future.get(); } + { + auto state(_state->lock()); + if (state->gcThread.joinable()) + state->gcThread.join(); + } + try { auto fdTempRoots(_fdTempRoots.lock()); if (*fdTempRoots) { From f4759088bf26b674294c9652d9f65da0ff6d0eaf Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 2 Apr 2026 02:29:32 +0300 Subject: [PATCH 206/927] libutil: Make `MemorySourceAccessor` exceptions more precise, `makeFSSourceAccessor` tracks metadata Changes `makeFSSourceAccessor` to behave better on symlinks. We had an unfortunate footgun of `PosixSourceAccessor` following interior links if root was a symlink. Prior commits changed that to first load the symlink in `MemorySourceAccessor`, but its exceptions were imprecise and `pathExists` didn't reject interior symlinks and instead returned false in such cases. `mtime` tracking of symlinks is now also restored, with more precise `showPath` and `getPhysicalPath`. Use some new `fstatat` wrappers. I, @ericson2314, forgot they weren't actaully commited yet, so I added them. Co-authored-by: John Ericson --- src/libutil-tests/memory-source-accessor.cc | 40 ++++++- .../include/nix/util/file-system-at.hh | 29 +++++ src/libutil/memory-source-accessor.cc | 107 +++++++++++++----- src/libutil/posix-source-accessor.cc | 73 ++++++++++-- src/libutil/unix/file-system-at.cc | 27 ++++- src/libutil/unix/file-system.cc | 10 +- 6 files changed, 233 insertions(+), 53 deletions(-) diff --git a/src/libutil-tests/memory-source-accessor.cc b/src/libutil-tests/memory-source-accessor.cc index d9a48da0ce..e80bfaeb9a 100644 --- a/src/libutil-tests/memory-source-accessor.cc +++ b/src/libutil-tests/memory-source-accessor.cc @@ -144,7 +144,7 @@ TEST_F(MemorySourceAccessorTestErrors, addFileParentNotDirectory) [&] { accessor->addFile(CanonPath("file/child"), "contents"); }, ThrowsMessage(AllOf( HasSubstrIgnoreANSIMatcher("somepath/file/child"), - HasSubstrIgnoreANSIMatcher("cannot be created because some parent file is not a directory")))); + HasSubstrIgnoreANSIMatcher("file 'somepath/file' is not a directory")))); } TEST_F(MemorySourceAccessorTestErrors, addFileNotARegularFile) @@ -165,7 +165,7 @@ TEST_F(MemorySourceAccessorTestErrors, createDirectoryParentNotDirectory) [&] { sink.createDirectory(CanonPath("file/child")); }, ThrowsMessage(AllOf( HasSubstrIgnoreANSIMatcher("somepath/file/child"), - HasSubstrIgnoreANSIMatcher("cannot be created because some parent file is not a directory")))); + HasSubstrIgnoreANSIMatcher("file 'somepath/file' is not a directory")))); } TEST_F(MemorySourceAccessorTestErrors, createDirectoryNotADirectory) @@ -185,8 +185,8 @@ TEST_F(MemorySourceAccessorTestErrors, createRegularFileParentNotDirectory) EXPECT_THAT( [&] { sink.createRegularFile(CanonPath("file/child"), [](CreateRegularFileSink &) {}); }, ThrowsMessage(AllOf( - HasSubstrIgnoreANSIMatcher("file/child"), - HasSubstrIgnoreANSIMatcher("cannot be created because some parent file is not a directory")))); + HasSubstrIgnoreANSIMatcher("somepath/file/child"), + HasSubstrIgnoreANSIMatcher("file 'somepath/file' is not a directory")))); } TEST_F(MemorySourceAccessorTestErrors, createRegularFileNotARegularFile) @@ -207,7 +207,7 @@ TEST_F(MemorySourceAccessorTestErrors, createSymlinkParentNotDirectory) [&] { sink.createSymlink(CanonPath("file/child"), "target"); }, ThrowsMessage(AllOf( HasSubstrIgnoreANSIMatcher("somepath/file/child"), - HasSubstrIgnoreANSIMatcher("cannot be created because some parent file is not a directory")))); + HasSubstrIgnoreANSIMatcher("file 'somepath/file' is not a directory")))); } TEST_F(MemorySourceAccessorTestErrors, createSymlinkNotASymlink) @@ -220,6 +220,36 @@ TEST_F(MemorySourceAccessorTestErrors, createSymlinkNotASymlink) AllOf(HasSubstrIgnoreANSIMatcher("somepath/file"), HasSubstrIgnoreANSIMatcher("is not a symbolic link")))); } +TEST_F(MemorySourceAccessorTestErrors, pathExistsThroughRegularFile) +{ + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink &) {}); + EXPECT_FALSE(accessor->pathExists(CanonPath("file/child"))); +} + +TEST_F(MemorySourceAccessorTestErrors, maybeLstatThroughRegularFile) +{ + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink &) {}); + EXPECT_FALSE(accessor->maybeLstat(CanonPath("file/child"))); +} + +TEST_F(MemorySourceAccessorTestErrors, pathExistsThroughSymlink) +{ + sink.createSymlink(CanonPath("link"), "target"); + EXPECT_THAT( + [&] { accessor->pathExists(CanonPath("link/child")); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/link"), HasSubstrIgnoreANSIMatcher("is a symlink")))); +} + +TEST_F(MemorySourceAccessorTestErrors, createFileThroughSymlink) +{ + sink.createSymlink(CanonPath("link"), "target"); + EXPECT_THAT( + [&] { sink.createRegularFile(CanonPath("link/child"), [](CreateRegularFileSink &) {}); }, + ThrowsMessage( + AllOf(HasSubstrIgnoreANSIMatcher("somepath/link"), HasSubstrIgnoreANSIMatcher("is a symlink")))); +} + /* ---------------------------------------------------------------------------- * JSON * --------------------------------------------------------------------------*/ diff --git a/src/libutil/include/nix/util/file-system-at.hh b/src/libutil/include/nix/util/file-system-at.hh index a8cb17b95e..c78cafe6a5 100644 --- a/src/libutil/include/nix/util/file-system-at.hh +++ b/src/libutil/include/nix/util/file-system-at.hh @@ -34,6 +34,35 @@ namespace nix { */ PosixStat fstat(Descriptor fd); +#ifndef _WIN32 + +/** + * Get status of a file relative to a directory file descriptor. + * + * @param dirFd Directory file descriptor + * @param path Relative path to stat + * + * @return nullopt if the path does not exist. + * @throws SystemError on other I/O errors. + * + * @pre `path` must be relative (not absolute) and non-empty. + */ +std::optional maybeFstatat(Descriptor dirFd, const std::filesystem::path & path); + +/** + * Get status of a file relative to a directory file descriptor. + * + * @param dirFd Directory file descriptor + * @param path Relative path to stat + * + * @throws SystemError if the path does not exist or on other I/O errors. + * + * @pre `path` must be relative (not absolute) and non-empty. + */ +PosixStat fstatat(Descriptor dirFd, const std::filesystem::path & path); + +#endif + /** * Read a symlink relative to a directory file descriptor. * diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index 06d81ba558..f0f7a95229 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -1,6 +1,8 @@ #include "nix/util/memory-source-accessor.hh" #include "nix/util/json-utils.hh" +#include + namespace nix { MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path, std::optional create) @@ -24,10 +26,20 @@ MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path, bool newF = false; + unsigned j = 0; for (std::string_view name : path) { auto * curDirP = std::get_if(&cur->raw); - if (!curDirP) + if (!curDirP) { + CanonPath curDirPath = CanonPath::root; + for (auto name2 : std::views::take(path, j)) + curDirPath.push(name2); + if (std::holds_alternative(cur->raw)) + throw SymlinkNotAllowed(curDirPath, "file '%s' is a symlink", showPath(curDirPath)); + if (create) + throw NotADirectory("file '%s' is not a directory", showPath(curDirPath)); return nullptr; + } + auto & curDir = *curDirP; auto i = curDir.entries.find(name); @@ -45,6 +57,7 @@ MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path, } } cur = &i->second; + ++j; } if (newF && create) @@ -58,12 +71,17 @@ void MemorySourceAccessor::readFile(const CanonPath & path, Sink & sink, fun(&f->raw)) { - sizeCallback(r->contents.size()); - StringSource source{r->contents}; - source.drainInto(sink); - } else - throw NotARegularFile("file '%s' is not a regular file", showPath(path)); + std::visit( + overloaded{ + [&](const File::Regular & r) { + sizeCallback(r.contents.size()); + StringSource source{r.contents}; + source.drainInto(sink); + }, + [&](const File::Directory &) { throw NotARegularFile("file '%s' is not a regular file", showPath(path)); }, + [&](const File::Symlink &) { throw SymlinkNotAllowed(path, "file '%s' is a symlink", showPath(path)); }, + }, + f->raw); } bool MemorySourceAccessor::pathExists(const CanonPath & path) @@ -108,14 +126,22 @@ MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const Canon auto * f = open(path, std::nullopt); if (!f) throw FileNotFound("file '%s' does not exist", showPath(path)); - if (auto * d = std::get_if(&f->raw)) { - DirEntries res; - for (auto & [name, file] : d->entries) - res.insert_or_assign(name, file.lstat().type); - return res; - } else - throw NotADirectory("file '%s' is not a directory", showPath(path)); - return {}; + return std::visit( + overloaded{ + [&](const File::Directory & d) { + DirEntries res; + for (auto & [name, file] : d.entries) + res.insert_or_assign(name, file.lstat().type); + return res; + }, + [&](const File::Regular &) -> DirEntries { + throw NotADirectory("file '%s' is not a directory", showPath(path)); + }, + [&](const File::Symlink &) -> DirEntries { + throw SymlinkNotAllowed(path, "file '%s' is a symlink", showPath(path)); + }, + }, + f->raw); } std::string MemorySourceAccessor::readLink(const CanonPath & path) @@ -135,9 +161,15 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents if (!root && !path.isRoot()) open(CanonPath::root, File::Directory{}); - auto * f = open(path, File{File::Regular{}}); - if (!f) - throw Error("file '%s' cannot be created because some parent file is not a directory", showPath(path)); + MemorySourceAccessor::File * f = nullptr; + try { + f = open(path, File{File::Regular{}}); + if (!f) + throw Error("file '%s' cannot be created because some parent directories don't exist", showPath(path)); + } catch (SourceAccessorError & e) { + e.addTrace({}, "while creating file '%s'", showPath(path)); + throw; + } if (auto * r = std::get_if(&f->raw)) r->contents = std::move(contents); else @@ -150,10 +182,16 @@ using File = MemorySourceAccessor::File; void MemorySink::createDirectory(const CanonPath & path) { - auto * f = dst.open(path, File{File::Directory{}}); - if (!f) - throw Error("directory '%s' cannot be created because some parent file is not a directory", dst.showPath(path)); - + MemorySourceAccessor::File * f = nullptr; + try { + f = dst.open(path, File{File::Directory{}}); + if (!f) + throw Error( + "directory '%s' cannot be created because some parent directories don't exist", dst.showPath(path)); + } catch (SourceAccessorError & e) { + e.addTrace({}, "while creating directory '%s'", dst.showPath(path)); + throw; + } if (!std::holds_alternative(f->raw)) throw NotADirectory("file '%s' is not a directory", dst.showPath(path)); }; @@ -174,9 +212,15 @@ struct CreateMemoryRegularFile : CreateRegularFileSink void MemorySink::createRegularFile(const CanonPath & path, fun func) { - auto * f = dst.open(path, File{File::Regular{}}); - if (!f) - throw Error("file '%s' cannot be created because some parent file is not a directory", path); + MemorySourceAccessor::File * f = nullptr; + try { + f = dst.open(path, File{File::Regular{}}); + if (!f) + throw Error("file '%s' cannot be created because some parent directories don't exist", dst.showPath(path)); + } catch (SourceAccessorError & e) { + e.addTrace({}, "while creating regular file '%s'", dst.showPath(path)); + throw; + } if (auto * rp = std::get_if(&f->raw)) { CreateMemoryRegularFile crf{*rp}; func(crf); @@ -201,9 +245,16 @@ void CreateMemoryRegularFile::operator()(std::string_view data) void MemorySink::createSymlink(const CanonPath & path, const std::string & target) { - auto * f = dst.open(path, File{File::Symlink{}}); - if (!f) - throw Error("symlink '%s' cannot be created because some parent file is not a directory", dst.showPath(path)); + MemorySourceAccessor::File * f = nullptr; + try { + f = dst.open(path, File{File::Symlink{}}); + if (!f) + throw Error( + "symlink '%s' cannot be created because some parent directories don't exist", dst.showPath(path)); + } catch (SourceAccessorError & e) { + e.addTrace({}, "while creating symlink '%s'", dst.showPath(path)); + throw; + } if (auto * s = std::get_if(&f->raw)) s->target = target; else diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 534151dfcf..45af4e88ff 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -315,20 +315,69 @@ ref makeFSSourceAccessor(std::filesystem::path root, bool trackL AutoCloseFD fd = openFileReadonly(root, FinalSymlink::DontFollow); if (!fd) { - if (errno == ELOOP) { - /* This branch is taken either if the final component is a symlink - or we hit a symlink loop. If that's the latter this will also - throw. This can be done with O_PATH descriptor for the symlink - itself, but it's not portable. */ - auto linkTarget = readLink(root); - auto res = make_ref(); - /* Create an in-memory accessor with the symlink at the root. */ - MemorySink sink{*res}; - sink.createSymlink(CanonPath::root, os_string_to_string(linkTarget)); - return res; + if (errno != ELOOP) + throw NativeSysError("opening file %1%", PathFmt(root)); + + /* A helper class that holds the symlink destination in memory. */ + class SymlinkSourceAccessor : public MemorySourceAccessor + { + bool trackLastModified; + std::time_t mtime; + std::filesystem::path fsPath; + + public: + SymlinkSourceAccessor( + std::string target, std::filesystem::path fsPath_, bool trackLastModified, std::time_t mtime) + : trackLastModified(trackLastModified) + , mtime(mtime) + , fsPath(std::move(fsPath_)) + { + MemorySink sink{*this}; + sink.createSymlink(CanonPath::root, target); + displayPrefix = fsPath.native(); + } + + std::optional getLastModified() override + { + return trackLastModified ? std::optional{mtime} : std::nullopt; + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + if (path.isRoot()) + return fsPath; + return fsPath / path.rel(); /* RHS must be a relative path. */ + } + + std::string showPath(const CanonPath & path) override + { + /* When rendering the file itself omit the trailing slash. */ + return path.isRoot() ? displayPrefix : SourceAccessor::showPath(path); + } + }; + + assert(root.has_parent_path()); + assert(root.has_filename()); + + /* This branch is taken either if the final component is a symlink + or we hit a symlink loop. If that's the latter this will also + throw. This can be done with O_PATH descriptor for the symlink + itself, but it's not portable. Note that file must have a parent directory + (it's required to be absolute and root directories can't fail with ELOOP). */ + + auto parentFd = openDirectory(root.parent_path(), FinalSymlink::Follow); + std::time_t mtime = 0; + if (!parentFd) + throw SysError("opening %1%", PathFmt(root)); + + auto relPath = CanonPath::fromFilename(root.filename().native()); + if (trackLastModified) { + auto st = fstatat(parentFd.get(), root.filename()); + mtime = st.st_mtime; } - throw NativeSysError("opening file %1%", PathFmt(root)); + auto linkTarget = readLinkAt(parentFd.get(), relPath); + return make_ref(std::move(linkTarget), std::move(root), trackLastModified, mtime); } auto st = nix::fstat(fd.get()); diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index fee985fd5d..d2401382ef 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -168,8 +168,7 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat }); if (errno == ENOTDIR) /* Path component might be a symlink. */ { - struct ::stat st; - if (::fstatat(getParentFd(), component.c_str(), &st, AT_SYMLINK_NOFOLLOW) == 0 && S_ISLNK(st.st_mode)) + if (auto st = maybeFstatat(getParentFd(), component); st && S_ISLNK(st->st_mode)) throw SymlinkNotAllowed(path2); errno = ENOTDIR; /* Restore the errno. */ } else if (errno == ELOOP) { @@ -230,4 +229,28 @@ PosixStat fstat(Descriptor fd) return st; } +PosixStat fstatat(Descriptor dirFd, const std::filesystem::path & path) +{ + assert(path.is_relative()); + assert(!path.empty()); + PosixStat st; + if (::fstatat(dirFd, path.c_str(), &st, AT_SYMLINK_NOFOLLOW)) { + throw SysError([&] { return HintFmt("getting status of %s", PathFmt(descriptorToPath(dirFd) / path)); }); + } + return st; +} + +std::optional maybeFstatat(Descriptor dirFd, const std::filesystem::path & path) +{ + assert(path.is_relative()); + assert(!path.empty()); + PosixStat st; + if (::fstatat(dirFd, path.c_str(), &st, AT_SYMLINK_NOFOLLOW)) { + if (errno == ENOENT || errno == ENOTDIR) + return std::nullopt; + throw SysError([&] { return HintFmt("getting status of %s", PathFmt(descriptorToPath(dirFd) / path)); }); + } + return st; +} + } // namespace nix diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index 1e6c4e3d88..b154ee2152 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -169,12 +169,10 @@ static void _deletePath( auto name = CanonPath::fromFilename(path.filename().native()); - PosixStat st; - if (fstatat(parentfd, name.rel_c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { - if (errno == ENOENT) - return; - throw SysError("getting status of %1%", PathFmt(path)); - } + auto st_ = maybeFstatat(parentfd, name.rel()); + if (!st_) + return; + auto & st = *st_; if (!S_ISDIR(st.st_mode)) { /* We are about to delete a file. Will it likely free space? */ From 90db12ceb17880dec8eacaa746d75ca409c750e9 Mon Sep 17 00:00:00 2001 From: dramforever Date: Thu, 2 Apr 2026 16:01:05 +0800 Subject: [PATCH 207/927] eval-cache: Add derivation result as temproot Closes the window where the derivation could be garbage collected after we checked isValidPath. --- src/libexpr/eval-cache.cc | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index a97cef550e..a419530c6d 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -707,13 +707,16 @@ StorePath AttrCursor::forceDerivation() auto aDrvPath = getAttr(root->state.s.drvPath); auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); drvPath.requireDerivation(); - if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { - /* The eval cache contains 'drvPath', but the actual path has - been garbage-collected. So force it to be regenerated. */ - aDrvPath->forceValue(); - if (!root->state.store->isValidPath(drvPath)) - throw Error( - "don't know how to recreate store derivation '%s'!", root->state.store->printStorePath(drvPath)); + if (!settings.readOnlyMode) { + root->state.store->addTempRoot(drvPath); + if (!root->state.store->isValidPath(drvPath)) { + /* The eval cache contains 'drvPath', but the actual path has + been garbage-collected. So force it to be regenerated. */ + aDrvPath->forceValue(); + if (!root->state.store->isValidPath(drvPath)) + throw Error( + "don't know how to recreate store derivation '%s'!", root->state.store->printStorePath(drvPath)); + } } return drvPath; } From 0e3412a93f43a017342c267000c152e5c45327e7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 3 Apr 2026 00:21:21 +0300 Subject: [PATCH 208/927] libstore: Make temporary in-store directory not world-readable --- src/libstore/local-store.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e9eb48bfe6..d5e4577312 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1304,8 +1304,9 @@ std::pair LocalStore::createTempDirInStore() do { /* There is a slight possibility that `tmpDir' gets deleted by the GC between createTempDir() and when we acquire a lock on it. - We'll repeat until 'tmpDir' exists and we've locked it. */ - tmpDirFn = createTempDir(std::filesystem::path{config->realStoreDir.get()}, "tmp"); + We'll repeat until 'tmpDir' exists and we've locked it. + Make the directory accessible only to the current user. */ + tmpDirFn = createTempDir(std::filesystem::path{config->realStoreDir.get()}, "tmp", /*mode=*/0700); tmpDirFd = openDirectory(tmpDirFn, FinalSymlink::DontFollow); if (!tmpDirFd) { continue; From 8dbb3daee0d435ec54441146f46ecfb1a45c8d83 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Sun, 29 Mar 2026 21:50:48 +0000 Subject: [PATCH 209/927] build: link mimalloc into the nix binary by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mimalloc is a compact, general-purpose allocator from Microsoft that consistently outperforms glibc malloc on allocation-heavy workloads. Since the nix evaluator is extremely allocation-heavy (strings, maps, symbol tables, derivation parsing), this is a cheap win. The allocator is linked into the nix executable where it overrides malloc for all components via symbol interposition. Boehm GC's heap (the bulk of evaluator memory) is unaffected since GC_malloc bypasses the system allocator entirely — but the ~5% of allocations that do go through malloc get noticeably faster. Benchmarked on x86_64-linux against nixpkgs master: | workload | thunks | glibc | mimalloc | wall | |-----------------------|--------|-------------|-------------|--------| | hello | 275K | 185.4 ms | 176.4 ms | 1.05× | | chromium | 1.9M | 760.3 ms | 709.3 ms | 1.07× | | firefox-unwrapped | 2.4M | 836.2 ms | 782.0 ms | 1.07× | | texliveFull | 5.5M | 1.604 s | 1.466 s | 1.09× | | large NixOS config | 39M | 12.79 s | 11.90 s | 1.07× | | nix-env -qa | 100M+ | 118.96 s | 105.86 s | 1.12× | Implementation: - new meson feature option 'mimalloc' (default: auto) - package.nix wires it with withMimalloc ? !isWindows - dev-shell includes mimalloc so local builds pick it up - static builds work via pkgsStatic.mimalloc's dev output pkg-config Also compared against tcmalloc and jemalloc: tcmalloc matches mimalloc on wall-clock; jemalloc lags slightly. mimalloc has the smallest footprint (no libunwind dep) so it's the pick here. --- doc/manual/rl-next/mimalloc.md | 15 +++++++++++++++ src/nix/meson.build | 5 +++++ src/nix/meson.options | 7 +++++++ src/nix/package.nix | 11 ++++++++++- 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 doc/manual/rl-next/mimalloc.md diff --git a/doc/manual/rl-next/mimalloc.md b/doc/manual/rl-next/mimalloc.md new file mode 100644 index 0000000000..dfcde0cab0 --- /dev/null +++ b/doc/manual/rl-next/mimalloc.md @@ -0,0 +1,15 @@ +--- +synopsis: "Link mimalloc for faster evaluation" +prs: [15596] +--- + +The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) +by default on non-Windows platforms, replacing glibc's malloc for all +non-GC allocations. + +This yields a **5–12% wall-clock improvement** on evaluation workloads, +ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS +configurations. + +The allocator can be disabled at build time with `-Dmimalloc=disabled` +or by passing `withMimalloc = false` to the Nix package. diff --git a/src/nix/meson.build b/src/nix/meson.build index 039b202b93..3327b846c9 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -31,6 +31,11 @@ deps_private_maybe_subproject = [ deps_public_maybe_subproject = [] subdir('nix-meson-build-support/subprojects') +mimalloc = dependency('mimalloc', required : get_option('mimalloc')) +if mimalloc.found() + deps_private += mimalloc +endif + subdir('nix-meson-build-support/export-all-symbols') subdir('nix-meson-build-support/windows-version') diff --git a/src/nix/meson.options b/src/nix/meson.options index 0fc680cfe4..ae391bfd8a 100644 --- a/src/nix/meson.options +++ b/src/nix/meson.options @@ -7,3 +7,10 @@ option( value : 'etc/profile.d', description : 'the path to install shell profile files', ) + +option( + 'mimalloc', + type : 'feature', + value : 'auto', + description : 'Link against mimalloc to override the default memory allocator', +) diff --git a/src/nix/package.nix b/src/nix/package.nix index 8195e6c6ff..68d1e379ab 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -8,9 +8,16 @@ nix-main, nix-cmd, + mimalloc, + # Configuration Options version, + + # Whether to link against mimalloc for malloc override. + # Significantly improves evaluation performance on allocation-heavy + # workloads (~10-15% on large evaluations). + withMimalloc ? !stdenv.hostPlatform.isWindows, }: let @@ -69,9 +76,11 @@ mkMesonExecutable (finalAttrs: { nix-expr nix-main nix-cmd - ]; + ] + ++ lib.optional withMimalloc mimalloc; mesonFlags = [ + (lib.mesonEnable "mimalloc" withMimalloc) ]; postInstall = lib.optionalString stdenv.hostPlatform.isStatic '' From a760af86b3a42aa5ac9d9002929107fe357bf128 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 3 Apr 2026 00:21:31 +0300 Subject: [PATCH 210/927] derivation-builder: Don't use copyFile for FOD output copying, put the output in a temporary directory in the store Puts the temporary FOD output copies in a temporary directory inside the store instead of the (for Linux sandboxed builds) chroot. This prevents file overwrite due to symlink following that std::filesystem::copy_file does. Also applies the same output copying approach for impure derivations that don't have network sandboxing and thus are subject to FD smuggling. Fixes GHSA-g3g9-5vj6-r3gj. --- src/libstore/include/nix/store/local-store.hh | 2 ++ src/libstore/unix/build/derivation-builder.cc | 36 ++++++++++++++----- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 63a1da67d8..bf3437e957 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -512,6 +512,8 @@ private: friend struct PathSubstitutionGoal; friend struct DerivationGoal; + /* Only used for createTempDirInStore. */ + friend class DerivationBuilderImpl; }; } // namespace nix diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 8f6343e0f4..8288a4a318 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1597,6 +1597,13 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() assert(output && scratchPath); auto actualPath = realPathInHost(store.printStorePath(*scratchPath)); + /* An optional file descriptor of a directory used for intermediate + operations. */ + AutoCloseFD tempDirFd; + /* RAII cleanup of a temporary directory inside the store that is used + for intermediate operations. */ + AutoDelete delTempDir; + auto finish = [&](StorePath finalStorePath) { /* Store the final path */ finalOutputs.insert_or_assign(outputName, finalStorePath); @@ -1744,6 +1751,25 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() return newInfo0; }; + auto moveOutputToTempDir = [&]() -> void { + std::filesystem::path tempDir; + std::tie(tempDir, tempDirFd) = store.createTempDirInStore(); + delTempDir = AutoDelete(tempDir); + + auto tmpOutput = tempDir / "x"; + + /* Serialise and create a fresh copy of the output to break + any stale writable file descriptors. Copy through the + serialisation/deserialisation. TODO: Use copyRecursive here and + make use of reflinking. */ + auto source = sinkToSource([&](Sink & nextSink) { dumpPath(actualPath, nextSink); }); + restorePath(tmpOutput, *source, store.config->getLocalSettings().fsyncStorePaths); + /* This makes it slightly harder to make sense of the control flow. The rule + of thumb is that actualPath points to the current location of the stuff + that we'll end up registering. */ + actualPath = std::move(tmpOutput); + }; + ValidPathInfo newInfo = std::visit( overloaded{ @@ -1771,14 +1797,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() [&](const DerivationOutput::CAFixed & dof) { auto & wanted = dof.ca.hash; - - // Replace the output by a fresh copy of itself to make sure - // that there's no stale file descriptor pointing to it - std::filesystem::path tmpOutput = actualPath.native() + ".tmp"; - copyFile(actualPath, tmpOutput, true); - - std::filesystem::rename(tmpOutput, actualPath); - + moveOutputToTempDir(); return newInfoFromCA( DerivationOutput::CAFloating{ .method = dof.ca.method, @@ -1795,6 +1814,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() }, [&](const DerivationOutput::Impure & doi) { + moveOutputToTempDir(); return newInfoFromCA( DerivationOutput::CAFloating{ .method = doi.method, From d43bb18ac272ba25c3040bc26fdbb5f548a6839e Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:51:17 +1000 Subject: [PATCH 211/927] packaging/dependencies: bump mimalloc to 3.1.6 3.1.5 seems to be broken with freebsd cross https://hydra.nixos.org/build/325420458/nixlog/1 --- packaging/dependencies.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 5338f70e57..9903a93ba9 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -16,6 +16,16 @@ in scope: { inherit stdenv; + mimalloc = pkgs.mimalloc.overrideAttrs rec { + version = "3.1.6"; + src = pkgs.fetchFromGitHub { + owner = "microsoft"; + repo = "mimalloc"; + tag = "v${version}"; + hash = "sha256-7zG0Sqloanz/b+fkJ4wzO86uBmtf9fdYNAT9ixLouyY="; + }; + }; + boehmgc = (pkgs.boehmgc.override { enableLargeConfig = true; From 323b8cfcb4b2a30da480570ded8f6eee1c18f8dd Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Sat, 4 Apr 2026 02:02:38 -0700 Subject: [PATCH 212/927] doc: fix typo: A -> The --- doc/manual/source/development/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index dd965862a3..3b80e8a266 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -80,7 +80,7 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. -A environment variables that Google Test accepts are also worth knowing: +The environment variables that Google Test accepts are also worth knowing: 1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) From 79e2d2f4943e42661062e3bd2acc81ca2a664378 Mon Sep 17 00:00:00 2001 From: Antonio Nuno Monteiro Date: Sun, 5 Apr 2026 23:02:22 -0700 Subject: [PATCH 213/927] libmain/progress-bar: invalidate redraw cache when clearing the bar --- src/libmain/progress-bar.cc | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 71c7789de2..9d6c8f1575 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -131,7 +131,7 @@ class ProgressBar : public Logger auto state(state_.lock()); if (state->active) { state->active = false; - writeToStderr("\r\e[K"); + clearProgressDisplay(); updateCV.notify_one(); quitCV.notify_one(); } @@ -150,7 +150,7 @@ class ProgressBar : public Logger } if (state->active) - writeToStderr("\r\e[K"); + clearProgressDisplay(); } void resume() override @@ -164,7 +164,7 @@ class ProgressBar : public Logger } if (state->suspensions == 0) { if (state->active) - writeToStderr("\r\e[K"); + clearProgressDisplay(); state->haveUpdate = true; updateCV.notify_one(); } @@ -196,6 +196,7 @@ class ProgressBar : public Logger void log(State & state, Verbosity lvl, std::string_view s) { if (state.active) { + invalidateRedrawCache(); writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); draw(state); } else { @@ -408,6 +409,17 @@ class ProgressBar : public Logger } } + void invalidateRedrawCache() + { + *lastOutput_.lock() = ""; + } + + void clearProgressDisplay() + { + invalidateRedrawCache(); + writeToStderr("\r\e[K"); + } + std::chrono::milliseconds draw(State & state) { auto nextWakeup = std::chrono::milliseconds::max(); @@ -647,6 +659,7 @@ class ProgressBar : public Logger { auto state(state_.lock()); if (state->active) { + invalidateRedrawCache(); std::cerr << "\r\e[K"; Logger::writeToStdout(s); draw(*state); @@ -660,6 +673,7 @@ class ProgressBar : public Logger auto state(state_.lock()); if (!state->active) return {}; + invalidateRedrawCache(); std::cerr << fmt("\r\e[K%s ", msg); auto s = trim(readLine(getStandardInput(), true)); if (s.size() != 1) From 44017ca497c8b44d5dac179f5afc63e91fe45ed6 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 5 Apr 2026 16:39:58 +0300 Subject: [PATCH 214/927] libstore: Use landlock with LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET for new enough kernels This partially fixes the issue with cooperating processes being able to communicate via abstract sockets. The fix is partial, because processes outside the landlock domain of the sandboxed process can still connect to a socket created by the FOD. There's no equivalent way of restricting inbound connections. This closes the gap when there's no cooperating process on the host (i.e. 2 separate FODs). >= 6.12 kernel is widespread enough (NixOS 25.11 ships it by default) that we have no reason not to apply this hardening, even though it's incomplete. ca-fd-leak test exercises this exact code path and now the smuggling process fails with (on new enough kernels that have landlock support enabled): vm-test-run-ca-fd-leak> machine # sandbox setup: applied landlock sandboxing vm-test-run-ca-fd-leak> machine # building '/nix/store/s7brgi6pdr5f3n8yqlgmdlz8blb89njc-smuggled.drv'... vm-test-run-ca-fd-leak> machine # building derivation '/nix/store/s7brgi6pdr5f3n8yqlgmdlz8blb89njc-smuggled.drv': woken up vm-test-run-ca-fd-leak> machine # connect: Operation not permitted vm-test-run-ca-fd-leak> machine # sendmsg: Socket not connected --- src/libstore/meson.build | 5 + .../unix/build/linux-derivation-builder.cc | 101 ++++++++++++++++++ tests/nixos/ca-fd-leak/default.nix | 8 +- 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 445798544f..753c786876 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -79,6 +79,11 @@ foreach funcspec : check_funcs configdata_priv.set(define_name, define_value) endforeach +if host_machine.system() == 'linux' + has_landlock = cxx.has_header('linux/landlock.h') + configdata_priv.set('HAVE_LANDLOCK', has_landlock.to_int()) +endif + has_acl_support = cxx.has_header('sys/xattr.h') \ and cxx.has_function('llistxattr') \ and cxx.has_function('lremovexattr') diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index 9cfd3cbcd2..c71d23e15e 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -1,5 +1,7 @@ #ifdef __linux__ +# include "store-config-private.hh" + # include "nix/store/globals.hh" # include "nix/store/personality.hh" # include "nix/store/filetransfer.hh" @@ -11,6 +13,8 @@ # include # include +# include + # include # include # include @@ -19,11 +23,16 @@ # include # include # include +# include # if HAVE_SECCOMP # include # endif +# if HAVE_LANDLOCK +# include +# endif + # define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) namespace nix { @@ -129,6 +138,77 @@ static void setupSeccomp(const LocalSettings & localSettings) # endif } +# if HAVE_LANDLOCK && defined(LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET) + +# define DO_LANDLOCK 1 + +/* We are using LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET on best-effort basis. There are no glibc wrappers for now. */ + +static int landlockCreateRuleset(const ::landlock_ruleset_attr * attr, std::size_t size, std::uint32_t flags) +{ + return ::syscall(__NR_landlock_create_ruleset, attr, size, flags); +} + +static int landlockRestrictSelf(Descriptor rulesetFd, std::uint32_t flags) +{ + return ::syscall(__NR_landlock_restrict_self, rulesetFd, flags); +} + +static int getLandlockAbiVersion() +{ + int abiVersion = landlockCreateRuleset(nullptr, 0, LANDLOCK_CREATE_RULESET_VERSION); + return abiVersion; +} + +static void setupLandlock() +{ + bool landlockSupportsScopeAbstractUnixSocket = []() { + int abiVersion = getLandlockAbiVersion(); + if (abiVersion >= 6) + /* All good, we can use LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET. See + https://docs.kernel.org/userspace-api/landlock.html#abstract-unix-socket-abi-6 */ + return true; + + if (abiVersion == -1) { + debug("landlock is not available"); + return false; + } + + debug("landlock version %d does not support LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET", abiVersion); + return false; + }(); + + /* Bail out early if landlock is not enabled or LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET wouldn't work. + TODO: Consider adding more landlock rules for filesystem access as defense-in-depth on top. */ + if (!landlockSupportsScopeAbstractUnixSocket) + return; + + ::landlock_ruleset_attr attr = { + /* This prevents multiple FODs from communicating with each other + via abstract sockets. Note that cooperating processes outside the + sandbox can still connect to an abstract socket created by the FOD. To + mitigate that issue entirely we'd still need network namespaces. */ + .scoped = LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET, + }; + + /* This better not fail - if the kernel reports a new enough ABI version we + should treat any errors as fatal from now on. */ + AutoCloseFD rulesetFd = landlockCreateRuleset(&attr, sizeof(attr), 0); + if (!rulesetFd) + throw SysError("failed to create a landlock ruleset"); + + if (landlockRestrictSelf(rulesetFd.get(), 0) == -1) + throw SysError("failed to apply landlock"); + + debug("applied landlock sandboxing"); +} + +# else + +# define DO_LANDLOCK 0 + +# endif + static void doBind(const std::filesystem::path & source, const std::filesystem::path & target, bool optional = false) { debug("bind mounting %1% to %2%", PathFmt(source), PathFmt(target)); @@ -169,8 +249,27 @@ struct LinuxDerivationBuilder : virtual DerivationBuilderImpl { auto & localSettings = store.config->getLocalSettings(); + /* Set the NO_NEW_PRIVS before doing seccomp/landlock setup. + landlock_restrict_self requires either NO_NEW_PRIVS or CAP_SYS_ADMIN. + With user namespaces we do get CAP_SYS_ADMIN. */ + if (!localSettings.allowNewPrivileges) + if (::prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) == -1) + throw SysError("failed to set PR_SET_NO_NEW_PRIVS"); + setupSeccomp(localSettings); +# if DO_LANDLOCK + try { + setupLandlock(); + } catch (SysError & e) { + if (e.errNo != EPERM) + throw; + /* If allowNewPrivileges is true and we don't have CAP_SYS_ADMIN + this code path might be hit. */ + warn("setting up landlock: %s", e.message()); + } +# endif + linux::setPersonality({ .system = drv.platform, .impersonateLinux26 = localSettings.impersonateLinux26, @@ -765,4 +864,6 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu } // namespace nix +# undef DO_LANDLOCK + #endif diff --git a/tests/nixos/ca-fd-leak/default.nix b/tests/nixos/ca-fd-leak/default.nix index 902aacdc65..dc944290f7 100644 --- a/tests/nixos/ca-fd-leak/default.nix +++ b/tests/nixos/ca-fd-leak/default.nix @@ -78,7 +78,7 @@ in # Build the smuggled derivation. # This will connect to the smuggler server and send it the file descriptor - machine.succeed(r""" + sender_output = machine.succeed(r""" nix-build -E ' builtins.derivation { name = "smuggled"; @@ -89,9 +89,13 @@ in outputHash = builtins.hashString "sha256" "hello, world\n"; builder = "${pkgs.busybox-sandbox-shell}/bin/sh"; args = [ "-c" "echo \"hello, world\" > $out; ''${${sender}} ${socketName}" ]; - }' + }' 2>&1 """.strip()) + # Landlock's LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET prevents a sandboxed process + # from connecting to an abstract socket created in an unrelated landlock domain. + # There's no such flag for preventing inbound connections. + assert "connect: Operation not permitted" in sender_output # Tell the smuggler server that we're done machine.execute("echo done | ${pkgs.socat}/bin/socat - ABSTRACT-CONNECT:${socketName}") From f4bde1f72d3f7172366d485891b67e56eff3ed40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 8 Apr 2026 10:54:00 +0200 Subject: [PATCH 215/927] Split release artifacts into a dedicated Hydra jobset Cutting a release currently blocks on the full hydraJobs evaluation (~900 builds including sanitizers, clang-tidy, static, NixOS VM and installer tests), even though upload-release only consumes ~25 of them. On recent maintenance evals the long CI tail and darwin queue depth pushed the wait into the multi-day range while the actual artifacts were ready within an hour. Hydra hard-codes flake jobsets to outputs.hydraJobs, so the subset is exposed through a legacy jobset expression that re-enters the flake via builtins.getFlake on the locked GitHub ref. Going through the ref rather than the checked-out store path preserves rev/lastModified and thus the version suffix, keeping derivations bit-identical to the flake jobset so both share builds through the binary cache. A release aggregate job provides a single gating signal for upload-release. The release process now creates a release-$VERSION jobset alongside maintenance-$VERSION and waits on that instead of the full matrix. Requires adding https://releases.nixos.org/ to allowed-uris on hydra.nixos.org, since the locked nixpkgs input is a tarball from there and legacy jobsets run under restrict-eval. --- maintainers/release-process.md | 49 ++++++++++++---------- packaging/release-jobs.nix | 76 ++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 23 deletions(-) create mode 100644 packaging/release-jobs.nix diff --git a/maintainers/release-process.md b/maintainers/release-process.md index f8b6b6bec5..19e5e05053 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -81,27 +81,30 @@ release: $ git push --set-upstream origin $VERSION-maintenance ``` -* Create a jobset for the release branch on Hydra as follows: - - * Go to the jobset of the previous release - (e.g. https://hydra.nixos.org/jobset/nix/maintenance-2.11). - - * Select `Actions -> Clone this jobset`. - - * Set identifier to `maintenance-$VERSION`. - - * Set description to `$VERSION release branch`. - - * Set flake URL to `github:NixOS/nix/$VERSION-maintenance`. - - * Hit `Create jobset`. - -* Wait for the new jobset to evaluate and build. If impatient, go to - the evaluation and select `Actions -> Bump builds to front of - queue`. - -* When the jobset evaluation has succeeded building, take note of the - evaluation ID (e.g. `1780832` in +* Create two jobsets for the release branch on Hydra: + + `maintenance-$VERSION` runs the full `hydraJobs` CI matrix. + `release-$VERSION` builds only the artifacts consumed by + `upload-release`, so a release can be cut without waiting on the full + matrix. + + * Clone the previous `maintenance-*` jobset, set identifier + `maintenance-$VERSION`, description `$VERSION release branch`, flake + URL `github:NixOS/nix/$VERSION-maintenance`. + + * Clone the previous `release-*` jobset (or create a new **legacy** + jobset), set identifier `release-$VERSION`, description `$VERSION + release artifacts`, Nix expression `packaging/release-jobs.nix` in + input `src`, and add input `src` of type *Git checkout* pointing at + `https://github.com/NixOS/nix $VERSION-maintenance`. + +* Wait for the `release-$VERSION` jobset to evaluate and build. If + impatient, go to the evaluation and select `Actions -> Bump builds to + front of queue`. The aggregate job `release` turns green once every + required artifact is available. + +* When the release jobset evaluation has succeeded building, take note of + the evaluation ID (e.g. `1780832` in `https://hydra.nixos.org/eval/1780832`). * Tag the release: @@ -174,8 +177,8 @@ release: $ git push ``` -* Wait for the desired evaluation of the maintenance jobset to finish - building. +* Wait for the desired evaluation of the `release-$VERSION` jobset to + finish building (the `release` aggregate job is the gating signal). * Tag the release diff --git a/packaging/release-jobs.nix b/packaging/release-jobs.nix new file mode 100644 index 0000000000..3da8c80de3 --- /dev/null +++ b/packaging/release-jobs.nix @@ -0,0 +1,76 @@ +# Hydra jobset containing only the artifacts consumed by +# `maintainers/upload-release.{pl,py}`, so a release can be cut without +# waiting on the full `hydraJobs` CI matrix. +# +# Evaluated as a legacy (non-flake) jobset because Hydra hard-codes flake +# jobsets to `outputs.hydraJobs`; we re-enter the flake via +# `builtins.getFlake` so derivations stay identical to the flake jobset +# and share builds through the binary cache. +# +# Hydra jobset configuration: +# Type: Legacy +# Nix expression: packaging/release-jobs.nix in input `src` +# Inputs: +# src (Git checkout) https://github.com/NixOS/nix +{ + src ? { + outPath = ./..; + }, +}: +let + # Fetch by GitHub ref rather than the bare store path Hydra hands us, + # so `rev`/`lastModified` (and thus the version suffix) match the flake + # jobset and derivations are shared. + flake = builtins.getFlake ( + if src ? rev then + "github:NixOS/nix/${src.rev}" + else + # Local evaluation / testing. + builtins.unsafeDiscardStringContext (toString src) + ); + inherit (flake) hydraJobs; + inherit (flake.inputs.nixpkgs) lib; + + jobs = { + # `nix-everything` per system: provides the store paths for + # `fallback-paths.nix` and (on x86_64-linux) the rendered manual via + # its `doc` output. + build.nix-everything = hydraJobs.build.nix-everything; + buildCross.nix-everything.riscv64-unknown-linux-gnu = + hydraJobs.buildCross.nix-everything.riscv64-unknown-linux-gnu; + + inherit (hydraJobs) + manual + binaryTarball + binaryTarballCross + installerScript + installerScriptForGHA + dockerImage + ; + + # Aggregate gating job: green ⇒ every artifact the upload script + # needs is available. `upload-release` can wait on this single job + # instead of the whole evaluation. Constituents are referenced by + # job *name* so that an evaluation failure in one of them does not + # take down the aggregate's own evaluation. + release = flake.inputs.nixpkgs.legacyPackages.x86_64-linux.releaseTools.aggregate { + name = "nix-release-${flake.packages.x86_64-linux.nix-everything.version}"; + meta.description = "Artifacts required for a Nix release"; + constituents = + let + collectJobNames = + prefix: x: + if lib.isDerivation x then + [ prefix ] + else if lib.isAttrs x then + lib.concatLists ( + lib.mapAttrsToList (n: collectJobNames (if prefix == "" then n else "${prefix}.${n}")) x + ) + else + [ ]; + in + collectJobNames "" (builtins.removeAttrs jobs [ "release" ]); + }; + }; +in +jobs From d0cbf284efe80241c78fe3850b0745b1984b8fb7 Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 8 Apr 2026 21:45:41 +0000 Subject: [PATCH 216/927] tests/functional: cover `nix-store --restore` with bare relative destination (#15645) Regression test for `restorePath` when the destination has no directory component. --- tests/functional/nars.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index 68bd191a25..ecae66c141 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -42,6 +42,11 @@ expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | gre mkdir -p "$TEST_ROOT/out2" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" +# Check that a bare relative destination (no directory component) works. +rm -rf "$TEST_ROOT/out" +(cd "$TEST_ROOT" && nix-store --restore out < tmp.nar) +[[ -f "$TEST_ROOT/out" ]] + # The same, but for a symlink. ln -sfn foo "$TEST_ROOT/symlink" nix-store --dump "$TEST_ROOT/symlink" > "$TEST_ROOT/tmp.nar" @@ -62,6 +67,12 @@ expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | gre mkdir -p "$TEST_ROOT/out2" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" +# Likewise for a bare relative destination. +rm -rf "$TEST_ROOT/out" +(cd "$TEST_ROOT" && nix-store --restore out < tmp.nar) +[[ -L "$TEST_ROOT/out" ]] +[[ $(readlink "$TEST_ROOT/out") = foo ]] + # Check whether restoring and dumping a NAR that contains case # collisions is round-tripping, even on a case-insensitive system. rm -rf "$TEST_ROOT/case" @@ -77,6 +88,11 @@ nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > "$TEST_ROOT/case.nar" cmp case.nar "$TEST_ROOT/case.nar" [ "$(nix-hash "${opts[@]}" --type sha256 "$TEST_ROOT/case")" = "$(nix-hash --flat --type sha256 case.nar)" ] +# Likewise for a bare relative destination. +rm -rf "$TEST_ROOT/case" +(cd "$TEST_ROOT" && nix-store "${opts[@]}" --restore case) < case.nar +[[ -e "$TEST_ROOT/case/xt_CONNMARK.h" ]] + # Check whether we detect true collisions (e.g. those remaining after # removal of the suffix). touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" From 96752a0f640114c4cc8a089148af48d3e40bdb27 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 8 Apr 2026 23:59:17 -0400 Subject: [PATCH 217/927] Fix #15003 (#15637) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix some missing `std::less<>` with `std::map` We need a `clang-tidy` for this, or similar. * libstore: instantiate `DerivationOptions` JSON serializers for both input types The `to_json`/`from_json` definitions in `derivation-options.cc` were only specialized for `DerivationOptions`, even though `derivation-options.hh` declared `JSON_IMPL` for both `DerivationOptions` and `DerivationOptions` (and likewise for the nested `OutputChecks`). The `StorePath` declarations were dangling — referencing them caused link errors. Refactor the per-type specializations into shared template helpers (`derivationOptionsFromJson` / `derivationOptionsToJson` / `outputChecksFromJson` / `outputChecksToJson`) and instantiate both `SingleDerivedPath` and `StorePath` `adl_serializer` specializations as thin wrappers. Add JSON characterization tests for `DerivationOptions` mirroring the existing `SingleDerivedPath` tests. The macro now takes the input type as a parameter. As noted in the comment near the new test invocations, the IA test data files are reused as-is because `DrvRef` and `DrvRef` (when only the `Opaque` case is used) JSON-encode identically. * libstore: handle CA placeholder + subpath in `exportReferencesGraph` Fixes #15003. When parsing the `exportReferencesGraph` derivation attribute, `parseSingleDerivedPath` looked up the path string in the placeholder map by exact match. For CA derivations, `dep` renders as a bare `/HASH` placeholder; if the user appended a subpath (e.g. `"${dep}/foo"`), the resulting `/HASH/foo` failed the exact-match lookup. (The fallback to `StoreDirConfig::toStorePath` then threw because placeholders don't start with `/nix/store/`.) This regressed `contentAddressedByDefault = true;` Nixpkgs usage in Nix 2.33, which as those Nixpkgs drvs use `${closureInfo}/...` style references in `exportReferencesGraph`. Fix the issue by extracting the leading path component (everything before the second `/`) before the lookup, mirroring how `toStorePath` strips subpaths from real store paths. Apply the same treatment to `parseRef` via a shared `findPlaceholder` helper. Add `TryResolveTest::exportReferencesGraphPlaceholderSubpath` (and a structured-attrs counterpart) covering both the parse step and a subsequent `Derivation::tryResolve` to confirm the placeholder gets substituted with the concrete output path end-to-end. --- .../export-ref-subpath-before-options.json | 31 ++++ .../export-ref-subpath-before.json | 27 +++ .../export-ref-subpath-buildTrace.json | 9 + .../export-ref-subpath-resolved-options.json | 28 ++++ .../export-ref-subpath-resolved.json | 19 +++ .../export-ref-subpath-sa-before-options.json | 23 +++ .../export-ref-subpath-sa-before.json | 34 ++++ .../export-ref-subpath-sa-buildTrace.json | 9 + ...xport-ref-subpath-sa-resolved-options.json | 20 +++ .../export-ref-subpath-sa-resolved.json | 26 +++ .../derivation-advanced-attrs.cc | 155 +++++++++++++++--- src/libstore-tests/derivations.cc | 114 +++++++++++++ src/libstore/build/derivation-check.cc | 3 +- src/libstore/derivation-options.cc | 120 ++++++++++---- .../include/nix/store/derivation-options.hh | 6 +- 15 files changed, 567 insertions(+), 57 deletions(-) create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json create mode 100644 src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json new file mode 100644 index 0000000000..44006012fb --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before-options.json @@ -0,0 +1,31 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + } + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json new file mode 100644 index 0000000000..0706e1b64a --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-before.json @@ -0,0 +1,27 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "exportReferencesGraph": "refs /19r5f6rm1xa6ii9sam7fqbmaj8skmasskfj0rixxb1zjpqcy470p/foo" + }, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "export-ref-subpath", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json new file mode 100644 index 0000000000..3f15525837 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-buildTrace.json @@ -0,0 +1,9 @@ +[ + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + }, + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] +] diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json new file mode 100644 index 0000000000..2bfe56fc8d --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved-options.json @@ -0,0 +1,28 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "forAllOutputs": { + "allowedReferences": null, + "allowedRequisites": null, + "disallowedReferences": [], + "disallowedRequisites": [], + "ignoreSelfRefs": true, + "maxClosureSize": null, + "maxSize": null + } + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json new file mode 100644 index 0000000000..453efae619 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-resolved.json @@ -0,0 +1,19 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "exportReferencesGraph": "refs /nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out/foo" + }, + "inputs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ], + "name": "export-ref-subpath", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json new file mode 100644 index 0000000000..7bc4cad36c --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before-options.json @@ -0,0 +1,23 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + } + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "perOutput": {} + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json new file mode 100644 index 0000000000..3e92638f9d --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-before.json @@ -0,0 +1,34 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "__json": "{\"exportReferencesGraph\":{\"refs\":[\"/19r5f6rm1xa6ii9sam7fqbmaj8skmasskfj0rixxb1zjpqcy470p/foo\"]}}" + }, + "inputs": { + "drvs": { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "srcs": [] + }, + "name": "export-ref-subpath-sa", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "exportReferencesGraph": { + "refs": [ + "/19r5f6rm1xa6ii9sam7fqbmaj8skmasskfj0rixxb1zjpqcy470p/foo" + ] + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json new file mode 100644 index 0000000000..3f15525837 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-buildTrace.json @@ -0,0 +1,9 @@ +[ + [ + { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv", + "output": "out" + }, + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] +] diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json new file mode 100644 index 0000000000..b3970f4a37 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved-options.json @@ -0,0 +1,20 @@ +{ + "additionalSandboxProfile": "", + "allowLocalNetworking": false, + "allowSubstitutes": true, + "exportReferencesGraph": { + "refs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ] + }, + "impureEnvVars": [], + "impureHostDeps": [], + "noChroot": false, + "outputChecks": { + "perOutput": {} + }, + "passAsFile": [], + "preferLocalBuild": false, + "requiredSystemFeatures": [], + "unsafeDiscardReferences": {} +} diff --git a/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json new file mode 100644 index 0000000000..d9befe3c96 --- /dev/null +++ b/src/libstore-tests/data/derivation/try-resolve/export-ref-subpath-sa-resolved.json @@ -0,0 +1,26 @@ +{ + "args": [], + "builder": "/bin/bash", + "env": { + "__json": "{\"exportReferencesGraph\":{\"refs\":[\"/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out/foo\"]}}" + }, + "inputs": [ + "f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out" + ], + "name": "export-ref-subpath-sa", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "structuredAttrs": { + "exportReferencesGraph": { + "refs": [ + "/nix/store/f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out/foo" + ] + } + }, + "system": "x86_64-linux", + "version": 4 +} diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 16988613c6..aa24943a72 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -307,7 +307,7 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) }; DerivationOptions advancedAttributes_structuredAttrs_defaults = { - .outputChecks = std::map::OutputChecks>{}, + .outputChecks = std::map::OutputChecks, std::less<>>{}, .unsafeDiscardReferences = {}, .passAsFile = {}, .exportReferencesGraph = {}, @@ -352,7 +352,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) { DerivationOptions expected = { .outputChecks = - std::map::OutputChecks>{ + std::map::OutputChecks, std::less<>>{ {"dev", DerivationOptions::OutputChecks{ .maxSize = 789, @@ -384,7 +384,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) { // Delete all keys but "dev" in options.outputChecks auto * outputChecksMapP = - std::get_if::OutputChecks>>( + std::get_if::OutputChecks, std::less<>>>( &options.outputChecks); ASSERT_TRUE(outputChecksMapP); auto & outputChecksMap = *outputChecksMapP; @@ -405,7 +405,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) DerivationOptions advancedAttributes_structuredAttrs_ia = { .outputChecks = - std::map::OutputChecks>{ + std::map::OutputChecks, std::less<>>{ {"out", DerivationOptions::OutputChecks{ .allowedReferences = std::set>{pathFoo}, @@ -447,7 +447,7 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) DerivationOptions advancedAttributes_structuredAttrs_ca = { .outputChecks = - std::map::OutputChecks>{ + std::map::OutputChecks, std::less<>>{ {"out", DerivationOptions::OutputChecks{ .allowedReferences = std::set>{placeholderFoo}, @@ -489,24 +489,137 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) {"rainbow", "uid-range", "ca-derivations"}); }; -#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2) \ - TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json) \ - { \ - nix::readJsonTest>( \ - *this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ - } \ - TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json) \ - { \ - nix::readJsonTest>( \ - *this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ +#define TEST_JSON_OPTIONS(FIXUTURE, INPUT, VAR, VAR2) \ + TEST_F(FIXUTURE, DerivationOptions_##INPUT##_##VAR##_from_json) \ + { \ + nix::readJsonTest>(*this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ + } \ + TEST_F(FIXUTURE, DerivationOptions_##INPUT##_##VAR##_to_json) \ + { \ + nix::readJsonTest>(*this, "derivation-options/" #VAR, advancedAttributes_##VAR2); \ } -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults) -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, all_set, ia) -TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, all_set, ca) -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, structuredAttrs_defaults, structuredAttrs_defaults) -TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, structuredAttrs_all_set, structuredAttrs_ia) -TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, structuredAttrs_all_set, structuredAttrs_ca) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, defaults, defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, all_set, ia) +TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, SingleDerivedPath, all_set, ca) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, structuredAttrs_defaults, structuredAttrs_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, SingleDerivedPath, structuredAttrs_all_set, structuredAttrs_ia) +TEST_JSON_OPTIONS(CaDerivationAdvancedAttrsTest, SingleDerivedPath, structuredAttrs_all_set, structuredAttrs_ca) + +/** + * `DerivationOptions` versions of the IA fixtures, used to + * exercise the `StorePath` JSON serializers. The IA test data files are + * reused as-is (see comment near the test invocations). + */ +static const StorePath spFoo{"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}, + spFooDev{"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}, spBar{"r5cff30838majxk5mp3ip2diffi8vpaj-bar"}, + spBarDev{"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}, spBarDrv{"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}; + +static const DerivationOptions advancedAttributes_sp_defaults = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "", + .noChroot = false, + .impureHostDeps = {}, + .impureEnvVars = {}, + .allowLocalNetworking = false, + .requiredSystemFeatures = {}, + .preferLocalBuild = false, + .allowSubstitutes = true, +}; + +static const DerivationOptions advancedAttributes_sp_all_set = { + .outputChecks = + DerivationOptions::OutputChecks{ + .ignoreSelfRefs = true, + .allowedReferences = std::set>{spFoo}, + .disallowedReferences = std::set>{spBar, OutputName{"dev"}}, + .allowedRequisites = std::set>{spFooDev, OutputName{"bin"}}, + .disallowedRequisites = std::set>{spBarDev}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph{ + {"refs1", {spFoo}}, + {"refs2", {spBarDrv}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, +}; + +static const DerivationOptions advancedAttributes_sp_structuredAttrs_defaults = { + .outputChecks = std::map::OutputChecks, std::less<>>{}, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = {}, + .additionalSandboxProfile = "", + .noChroot = false, + .impureHostDeps = {}, + .impureEnvVars = {}, + .allowLocalNetworking = false, + .requiredSystemFeatures = {}, + .preferLocalBuild = false, + .allowSubstitutes = true, +}; + +static const DerivationOptions advancedAttributes_sp_structuredAttrs_all_set = { + .outputChecks = + std::map::OutputChecks, std::less<>>{ + {"out", + DerivationOptions::OutputChecks{ + .allowedReferences = std::set>{spFoo}, + .allowedRequisites = std::set>{spFooDev, OutputName{"bin"}}, + }}, + {"bin", + DerivationOptions::OutputChecks{ + .disallowedReferences = std::set>{spBar, OutputName{"dev"}}, + .disallowedRequisites = std::set>{spBarDev}, + }}, + {"dev", + DerivationOptions::OutputChecks{ + .maxSize = 789, + .maxClosureSize = 5909, + }}, + }, + .unsafeDiscardReferences = {}, + .passAsFile = {}, + .exportReferencesGraph = + { + {"refs1", {spFoo}}, + {"refs2", {spBarDrv}}, + }, + .additionalSandboxProfile = "sandcastle", + .noChroot = true, + .impureHostDeps = {"/usr/bin/ditto"}, + .impureEnvVars = {"UNICORN"}, + .allowLocalNetworking = true, + .requiredSystemFeatures = {"rainbow", "uid-range"}, + .preferLocalBuild = true, + .allowSubstitutes = false, +}; + +/** + * Same JSON characterization tests, but for `DerivationOptions`. + * + * Since `DrvRef` and `DrvRef` (when only + * the `Opaque` case is used) JSON-encode identically, the IA test data + * files can be reused as-is. + */ +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, defaults, sp_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, all_set, sp_all_set) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, structuredAttrs_defaults, sp_structuredAttrs_defaults) +TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, StorePath, structuredAttrs_all_set, sp_structuredAttrs_all_set) #undef TEST_JSON_OPTIONS diff --git a/src/libstore-tests/derivations.cc b/src/libstore-tests/derivations.cc index a898b8ee9c..60b86f5712 100644 --- a/src/libstore-tests/derivations.cc +++ b/src/libstore-tests/derivations.cc @@ -2,6 +2,7 @@ #include #include "nix/store/derivations.hh" +#include "nix/store/derivation-options.hh" #include "nix/store/downstream-placeholder.hh" #include "nix/store/parsed-derivations.hh" #include "nix/store/tests/libstore.hh" @@ -72,6 +73,11 @@ class TryResolveTest : public LibStoreTest, public CharacterizationTest }; } + /** + * Helper for the `exportReferencesGraph` + placeholder subpath tests. + */ + void exportRefGraphSubpathTest(std::string_view stem, const Derivation & drv, const StructuredAttrs * parsed); + /** * Checkpoint before/buildTrace/after and assert the resolved derivation * matches expected. @@ -248,4 +254,112 @@ TEST_F(TryResolveTest, resolutionFailure) EXPECT_FALSE(resolved); } +/** + * Test that `derivationOptionsFromStructuredAttrs` can parse + * `exportReferencesGraph` entries that reference a CA derivation output + * with a subpath appended (e.g. `${dep}/foo`), and that the parsed + * options resolve correctly. + * + * Regression test for #15003: the placeholder + subpath was not found + * in the placeholder map (exact-match only), and the fallback to + * `toStorePath()` failed because placeholders don't start with + * `/nix/store/`. + */ +void TryResolveTest::exportRefGraphSubpathTest( + std::string_view stem, const Derivation & drv, const StructuredAttrs * parsed) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + StorePath depOutPath{"f1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep-out"}; + + nix::checkpointJson(*this, std::string{stem} + "-before", drv); + + auto options = derivationOptionsFromStructuredAttrs(*store, drv.inputDrvs, drv.env, parsed, true); + + nix::checkpointJson(*this, std::string{stem} + "-before-options", options); + + SingleDerivedPath expectedPath = SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(depDrvPath), + .output = "out", + }; + + ASSERT_EQ(options.exportReferencesGraph.size(), 1); + auto it = options.exportReferencesGraph.find("refs"); + ASSERT_NE(it, options.exportReferencesGraph.end()); + ASSERT_EQ(it->second.size(), 1); + EXPECT_EQ(*it->second.begin(), expectedPath); + + // Also test that resolution works + BuildTrace buildTrace{.dict{ + { + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(depDrvPath), + .output = "out", + }, + depOutPath, + }, + }}; + + nix::checkpointJson(*this, std::string{stem} + "-buildTrace", buildTrace); + + auto resolved = drv.tryResolve(*store, makeCallback(buildTrace)); + ASSERT_TRUE(resolved); + + nix::checkpointJson(*this, std::string{stem} + "-resolved", *resolved); + + // Re-parse options from the resolved derivation, where placeholders + // have been substituted with concrete store paths. + auto resolvedOptions = derivationOptionsFromStructuredAttrs( + *store, + /* inputDrvs */ {}, + resolved->env, + resolved->structuredAttrs ? &*resolved->structuredAttrs : nullptr, + true); + + nix::checkpointJson(*this, std::string{stem} + "-resolved-options", resolvedOptions); + + EXPECT_EQ( + resolvedOptions.exportReferencesGraph, + (decltype(resolvedOptions.exportReferencesGraph){ + {"refs", std::set{SingleDerivedPath::Opaque{depOutPath}}}})); +} + +TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + auto placeholder = DownstreamPlaceholder::unknownCaOutput(depDrvPath, "out").render(); + + Derivation drv; + drv.name = "export-ref-subpath"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.env = { + {"exportReferencesGraph", "refs " + placeholder + "/foo"}, + }; + + exportRefGraphSubpathTest("export-ref-subpath", drv, nullptr); +} + +TEST_F(TryResolveTest, exportReferencesGraphPlaceholderSubpath_structuredAttrs) +{ + StorePath depDrvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-dep.drv"}; + auto placeholder = DownstreamPlaceholder::unknownCaOutput(depDrvPath, "out").render(); + + Derivation drv; + drv.name = "export-ref-subpath-sa"; + drv.platform = "x86_64-linux"; + drv.builder = "/bin/bash"; + drv.outputs = {{"out", caFloatingOutput()}}; + drv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}}; + drv.structuredAttrs = StructuredAttrs{{ + {"exportReferencesGraph", nlohmann::json::object({{"refs", nlohmann::json::array({placeholder + "/foo"})}})}, + }}; + drv.env = { + {std::string{StructuredAttrs::envVarName}, nlohmann::json(drv.structuredAttrs->structuredAttrs).dump()}, + }; + + exportRefGraphSubpathTest("export-ref-subpath-sa", drv, &*drv.structuredAttrs); +} + } // namespace nix diff --git a/src/libstore/build/derivation-check.cc b/src/libstore/build/derivation-check.cc index c422897e62..dffbbc3beb 100644 --- a/src/libstore/build/derivation-check.cc +++ b/src/libstore/build/derivation-check.cc @@ -186,7 +186,8 @@ void checkOutputs( std::visit( overloaded{ [&](const DerivationOptions::OutputChecks & checks) { applyChecks(checks); }, - [&](const std::map::OutputChecks> & checksPerOutput) { + [&](const std::map::OutputChecks, std::less<>> & + checksPerOutput) { if (auto outputChecks = get(checksPerOutput, outputName)) applyChecks(*outputChecks); diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 5208440c7d..86543bc363 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -73,7 +73,8 @@ template using OutputChecks = DerivationOptions::OutputChecks; template -using OutputChecksVariant = std::variant, std::map>>; +using OutputChecksVariant = + std::variant, std::map, std::less<>>>; DerivationOptions derivationOptionsFromStructuredAttrs( const StoreDirConfig & store, @@ -124,7 +125,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( { DerivationOptions defaults = {}; - std::map placeholders; + std::map> placeholders; if (mockXpSettings.isEnabled(Xp::CaDerivations)) { /* Initialize placeholder map from inputDrvs */ auto initPlaceholders = [&](this const auto & initPlaceholders, @@ -156,16 +157,27 @@ DerivationOptions derivationOptionsFromStructuredAttrs( } } + /* Extract the placeholder key from a path that may have a subpath + appended (e.g. `/HASH/foo` → `/HASH`), mirroring how + `StoreDirConfig::toStorePath` strips subpaths from store paths. */ + auto findPlaceholder = [&](std::string_view pathS) -> const SingleDerivedPath::Built * { + auto slash = pathS.find('/', 1); + auto key = pathS.substr(0, slash); + if (auto it = placeholders.find(key); it != placeholders.end()) + return &it->second; + return nullptr; + }; + auto parseSingleDerivedPath = [&](const std::string & pathS) -> SingleDerivedPath { - if (auto it = placeholders.find(pathS); it != placeholders.end()) - return it->second; + if (auto * built = findPlaceholder(pathS)) + return *built; else return SingleDerivedPath::Opaque{store.toStorePath(pathS).first}; }; auto parseRef = [&](const std::string & pathS) -> DrvRef { - if (auto it = placeholders.find(pathS); it != placeholders.end()) - return it->second; + if (auto * built = findPlaceholder(pathS)) + return *built; if (store.isStorePath(pathS)) return SingleDerivedPath::Opaque{store.toStorePath(pathS).first}; else @@ -206,7 +218,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( if (parsed) { auto & structuredAttrs = parsed->structuredAttrs; - std::map> res; + std::map, std::less<>> res; if (auto * outputChecks = get(structuredAttrs, "outputChecks")) { for (auto & [outputName, output_] : getObject(*outputChecks)) { auto & output = getObject(output_); @@ -265,7 +277,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( }(), .unsafeDiscardReferences = [&] { - std::map res; + std::map> res; if (parsed) { if (auto * udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) { @@ -298,7 +310,7 @@ DerivationOptions derivationOptionsFromStructuredAttrs( }(), .exportReferencesGraph = [&] { - std::map> ret; + std::map, std::less<>> ret; if (parsed) { auto * e = get(parsed->structuredAttrs, "exportReferencesGraph"); @@ -444,9 +456,10 @@ std::optional> tryResolve( }; // Helper function to resolve exportReferencesGraph using functional style - auto tryResolveExportReferencesGraph = [&](const std::map> & exportGraph) - -> std::optional>> { - std::map> resolved; + auto tryResolveExportReferencesGraph = + [&](const std::map, std::less<>> & exportGraph) + -> std::optional, std::less<>>> { + std::map, std::less<>> resolved; for (const auto & [name, inputPaths] : exportGraph) { std::set resolvedPaths; for (const auto & inputPath : inputPaths) { @@ -466,19 +479,20 @@ std::optional> tryResolve( [&](const DerivationOptions::OutputChecks & checks) -> std::optional::OutputChecks, - std::map::OutputChecks>>> { + std::map::OutputChecks, std::less<>>>> { auto resolved = tryResolveOutputChecks(checks); if (!resolved) return std::nullopt; return std::variant< DerivationOptions::OutputChecks, - std::map::OutputChecks>>(*resolved); + std::map::OutputChecks, std::less<>>>(*resolved); }, - [&](const std::map::OutputChecks> & checksMap) + [&](const std::map::OutputChecks, std::less<>> & + checksMap) -> std::optional::OutputChecks, - std::map::OutputChecks>>> { - std::map::OutputChecks> resolvedMap; + std::map::OutputChecks, std::less<>>>> { + std::map::OutputChecks, std::less<>> resolvedMap; for (const auto & [outputName, checks] : checksMap) { auto resolved = tryResolveOutputChecks(checks); if (!resolved) @@ -487,7 +501,7 @@ std::optional> tryResolve( } return std::variant< DerivationOptions::OutputChecks, - std::map::OutputChecks>>(resolvedMap); + std::map::OutputChecks, std::less<>>>(resolvedMap); }}, drvOptions.outputChecks); @@ -525,21 +539,22 @@ namespace nlohmann { using namespace nix; -DerivationOptions adl_serializer>::from_json(const json & json_) +template +static DerivationOptions derivationOptionsFromJson(const nlohmann::json & json_) { auto & json = getObject(json_); return { - .outputChecks = [&]() -> OutputChecksVariant { + .outputChecks = [&]() -> OutputChecksVariant { auto outputChecks = getObject(valueAt(json, "outputChecks")); auto forAllOutputsOpt = get(outputChecks, "forAllOutputs"); auto perOutputOpt = get(outputChecks, "perOutput"); if (forAllOutputsOpt && !perOutputOpt) { - return static_cast>(*forAllOutputsOpt); + return static_cast>(*forAllOutputsOpt); } else if (perOutputOpt && !forAllOutputsOpt) { - return static_cast>>(*perOutputOpt); + return static_cast, std::less<>>>(*perOutputOpt); } else { throw Error("Exactly one of 'perOutput' or 'forAllOutputs' is required"); } @@ -561,17 +576,17 @@ DerivationOptions adl_serializer>::to_json( - json & json, const DerivationOptions & o) +template +static void derivationOptionsToJson(nlohmann::json & json, const DerivationOptions & o) { json["outputChecks"] = std::visit( overloaded{ - [&](const OutputChecks & checks) { + [&](const OutputChecks & checks) { nlohmann::json outputChecks; outputChecks["forAllOutputs"] = checks; return outputChecks; }, - [&](const std::map> & checksPerOutput) { + [&](const std::map, std::less<>> & checksPerOutput) { nlohmann::json outputChecks; outputChecks["perOutput"] = checksPerOutput; return outputChecks; @@ -594,7 +609,8 @@ void adl_serializer>::to_json( json["allowSubstitutes"] = o.allowSubstitutes; } -OutputChecks adl_serializer>::from_json(const json & json_) +template +static OutputChecks outputChecksFromJson(const nlohmann::json & json_) { auto & json = getObject(json_); @@ -602,16 +618,15 @@ OutputChecks adl_serializer>: .ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")), .maxSize = ptrToOwned(getNullable(valueAt(json, "maxSize"))), .maxClosureSize = ptrToOwned(getNullable(valueAt(json, "maxClosureSize"))), - .allowedReferences = - ptrToOwned>>(getNullable(valueAt(json, "allowedReferences"))), + .allowedReferences = ptrToOwned>>(getNullable(valueAt(json, "allowedReferences"))), .disallowedReferences = valueAt(json, "disallowedReferences"), - .allowedRequisites = - ptrToOwned>>(getNullable(valueAt(json, "allowedRequisites"))), + .allowedRequisites = ptrToOwned>>(getNullable(valueAt(json, "allowedRequisites"))), .disallowedRequisites = valueAt(json, "disallowedRequisites"), }; } -void adl_serializer>::to_json(json & json, const OutputChecks & c) +template +static void outputChecksToJson(nlohmann::json & json, const OutputChecks & c) { json["ignoreSelfRefs"] = c.ignoreSelfRefs; json["maxSize"] = c.maxSize; @@ -622,4 +637,45 @@ void adl_serializer>::to_json(json & json, const json["disallowedRequisites"] = c.disallowedRequisites; } +DerivationOptions adl_serializer>::from_json(const json & json_) +{ + return derivationOptionsFromJson(json_); +} + +void adl_serializer>::to_json( + json & json, const DerivationOptions & o) +{ + derivationOptionsToJson(json, o); +} + +DerivationOptions adl_serializer>::from_json(const json & json_) +{ + return derivationOptionsFromJson(json_); +} + +void adl_serializer>::to_json(json & json, const DerivationOptions & o) +{ + derivationOptionsToJson(json, o); +} + +OutputChecks adl_serializer>::from_json(const json & json_) +{ + return outputChecksFromJson(json_); +} + +void adl_serializer>::to_json(json & json, const OutputChecks & c) +{ + outputChecksToJson(json, c); +} + +OutputChecks adl_serializer>::from_json(const json & json_) +{ + return outputChecksFromJson(json_); +} + +void adl_serializer>::to_json(json & json, const OutputChecks & c) +{ + outputChecksToJson(json, c); +} + } // namespace nlohmann diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index d96c4892e1..e29f660c48 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -84,12 +84,12 @@ struct DerivationOptions * Either one set of checks for all outputs, or separate checks * per-output. */ - std::variant> outputChecks = OutputChecks{}; + std::variant>> outputChecks = OutputChecks{}; /** * Whether to avoid scanning for references for a given output. */ - std::map unsafeDiscardReferences; + std::map> unsafeDiscardReferences; /** * In non-structured mode, all bindings specified in the derivation @@ -122,7 +122,7 @@ struct DerivationOptions * attributes give to the builder. The set of paths in the original JSON * is replaced with a list of `PathInfo` in JSON format. */ - std::map> exportReferencesGraph; + std::map, std::less<>> exportReferencesGraph; /** * env: __sandboxProfile From 7b642b5ab3702fa6f9efa6090aaed4dea3109b8b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 9 Apr 2026 00:35:58 -0400 Subject: [PATCH 218/927] Chmod cleanup (#15647) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * libutil-test-support: Add `ThrowsSysError` gmock matcher helper Pulls out the `::testing::Throws(::testing::Field(&SysError::errNo, N))` pattern into a small inline helper so tests can write `EXPECT_THAT(..., ThrowsSysError(ENOENT))` for specific-errno assertions on `SysError` exceptions. This will be used in subsequent commits. * libutil: Use `AT_SYMLINK_NOFOLLOW` in `fchmodatTryNoFollow` on all non-Linux Unixes Previously the final `::fchmodat` fallback only passed `AT_SYMLINK_NOFOLLOW` on macOS and FreeBSD. The flag is equally supported on NetBSD, OpenBSD, and DragonFly BSD — chmod of a symlink's own inode is a standard BSD feature — so extend the guard to `!defined(__linux__)`. On Linux we deliberately keep passing `0` rather than `AT_SYMLINK_NOFOLLOW`, even though we would *prefer* not to follow symlinks here either. Ideally, on a Linux symlink we'd want this fallback to fail outright rather than silently chmod the target. Unfortunately, if we reach this path at all (i.e. both the `fchmodat2` and `/proc/self/fd` fallbacks above were unavailable), glibc's `AT_SYMLINK_NOFOLLOW` compat shim will make the call fail *even for non-symlink targets* — so we can't use it as a belt-and-suspenders check. A comment at the call site explains this. Also refresh the `fchmodatTryNoFollow` test: - Replace the `EXPECT_NO_THROW(try { ... } catch { if (...) throw; })` gymnastic with a per-OS whitelist via a small `expectSymlinkChmod` lambda: Linux asserts `ThrowsSysError(EOPNOTSUPP)`; Darwin / FreeBSD / NetBSD / OpenBSD / DragonFly BSD assert `EXPECT_NO_THROW`; any other platform logs a `GTEST_LOG_(WARNING)` that chmod-on-symlink behaviour isn't verified for this OS and just exercises the call. The real invariant — the target file's mode is unchanged — is asserted unconditionally afterwards in every branch. - Add an `ENOENT` expectation via `ThrowsSysError` for the nonexistent-path failure mode. - Drop the odd `(expr, 0)` wrapping around one of the no-throw calls — it was a leftover from an older style. * Update src/libutil/unix/file-system-at.cc Co-authored-by: Sergei Zimmerman --------- Co-authored-by: Sergei Zimmerman --- .../include/nix/util/tests/gmock-matchers.hh | 13 +++++ src/libutil-tests/unix/file-system-at.cc | 55 +++++++++++++++---- src/libutil/unix/file-system-at.cc | 9 ++- 3 files changed, 64 insertions(+), 13 deletions(-) diff --git a/src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh b/src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh index 27d765e9e8..e48faca7c6 100644 --- a/src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh +++ b/src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "nix/util/error.hh" #include "nix/util/terminal.hh" #include @@ -53,4 +54,16 @@ HasSubstrIgnoreANSIMatcher(const std::string & substring) return ::testing::MakePolymorphicMatcher(internal::HasSubstrIgnoreANSIMatcher(substring)); } +/** + * Matches a callable that throws `SysError` whose `errNo` equals `expected`. + * + * Example: + * + * EXPECT_THAT([&]{ openFile("nope"); }, ThrowsSysError(ENOENT)); + */ +inline auto ThrowsSysError(int expected) +{ + return ::testing::Throws(::testing::Field(&SysError::errNo, expected)); +} + } // namespace nix::testing diff --git a/src/libutil-tests/unix/file-system-at.cc b/src/libutil-tests/unix/file-system-at.cc index b72851358a..5fb0d26b9c 100644 --- a/src/libutil-tests/unix/file-system-at.cc +++ b/src/libutil-tests/unix/file-system-at.cc @@ -1,9 +1,11 @@ #include +#include #include "nix/util/file-system-at.hh" #include "nix/util/file-system.hh" #include "nix/util/fs-sink.hh" #include "nix/util/processes.hh" +#include "nix/util/tests/gmock-matchers.hh" #ifdef __linux__ # include "nix/util/linux-namespaces.hh" @@ -42,21 +44,48 @@ TEST(fchmodatTryNoFollow, works) struct ::stat st; - /* Check that symlinks are not followed and targets are not changed. */ + using nix::testing::ThrowsSysError; + + /* Check that symlinks are not followed and targets are not changed. + + Whitelist per OS rather than "not Linux", so that an unrecognised + platform fails loudly at compile time instead of silently getting the + wrong expectation: + + - Linux: always throws `SysError(EOPNOTSUPP)` — Linux symlinks have no + mutable mode bits and `fchmodat` with `AT_SYMLINK_NOFOLLOW` is + unsupported. + + - BSDs and descendants (Darwin, FreeBSD, NetBSD, OpenBSD, DragonFly BSD): + succeeds, modifying the symlink's own inode mode without touching the + target. + + - Anything else: unverified — warn and exercise the call so we at least + detect crashes, but don't assert an outcome. + + The `ASSERT_EQ`/`EXPECT_EQ` below run in every branch and enforce the + invariant that the *target* file's mode is unchanged, which is the + property we actually care about. */ + auto expectSymlinkChmod = [&](std::string_view path, mode_t mode) { +#if defined(__linux__) + EXPECT_THAT([&] { fchmodatTryNoFollow(dirFd.get(), CanonPath(path), mode); }, ThrowsSysError(EOPNOTSUPP)); +#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) \ + || defined(__DragonFly__) + EXPECT_NO_THROW(fchmodatTryNoFollow(dirFd.get(), CanonPath(path), mode)); +#else + GTEST_LOG_(WARNING) << "unknown platform: chmod-on-symlink behaviour is not verified for this OS"; + try { + fchmodatTryNoFollow(dirFd.get(), CanonPath(path), mode); + } catch (SysError &) { + } +#endif + }; - EXPECT_NO_THROW( - try { fchmodatTryNoFollow(dirFd.get(), CanonPath("filelink"), 0777); } catch (SysError & e) { - if (e.errNo != EOPNOTSUPP) - throw; - }); + expectSymlinkChmod("filelink", 0777); ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); EXPECT_EQ(st.st_mode & 0777, 0644); - EXPECT_NO_THROW( - try { fchmodatTryNoFollow(dirFd.get(), CanonPath("dirlink"), 0777); } catch (SysError & e) { - if (e.errNo != EOPNOTSUPP) - throw; - }); + expectSymlinkChmod("dirlink", 0777); ASSERT_EQ(stat((tmpDir / "dir").c_str(), &st), 0); EXPECT_EQ(st.st_mode & 0777, 0755); @@ -66,9 +95,11 @@ TEST(fchmodatTryNoFollow, works) ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); EXPECT_EQ(st.st_mode & 0777, 0600); - EXPECT_NO_THROW((fchmodatTryNoFollow(dirFd.get(), CanonPath("dir"), 0700), 0)); + EXPECT_NO_THROW(fchmodatTryNoFollow(dirFd.get(), CanonPath("dir"), 0700)); ASSERT_EQ(stat((tmpDir / "dir").c_str(), &st), 0); EXPECT_EQ(st.st_mode & 0777, 0700); + + EXPECT_THAT([&] { fchmodatTryNoFollow(dirFd.get(), CanonPath("nonexistent"), 0600); }, ThrowsSysError(ENOENT)); } #ifdef __linux__ diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index d2401382ef..d8be6fdeb7 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -121,9 +121,16 @@ void unix::fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t dirFd, path.rel_c_str(), mode, -#if defined(__APPLE__) || defined(__FreeBSD__) +#if defined(AT_SYMLINK_NOFOLLOW) && !defined(__linux__) AT_SYMLINK_NOFOLLOW #else + /* We would like to avoid following symlinks on Linux too. (Even though + Linux doesn't support chmoding symlinks, we should still fail if we + try, and not falsely succeed by following.) However, if we reach + this point, rather than the Linux-specific cases above, it means we + will likely hit glibc compat paths that will make using + AT_SYMLINK_NOFOLLOW cause failures even if there is no symlink + being followed! */ 0 #endif ); From 84acfc03f6af30042714d82d79eebb799b64f7a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Apr 2026 10:37:26 +0200 Subject: [PATCH 219/927] dev-shell.nix: Propagate mesonFlags from the nix component (#15646) This ensures that meson flags defined in src/nix/package.nix are respected in the dev shell. --- packaging/dev-shell.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index b058bbe53d..a29d65cb2a 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -291,7 +291,8 @@ nixComponents.callPackage ( map (transformFlag "perl") (ignoreCrossFile nixComponents.nix-perl-bindings.mesonFlags) ) ++ map (transformFlag "libexpr") (ignoreCrossFile nixComponents.nix-expr.mesonFlags) - ++ map (transformFlag "libcmd") (ignoreCrossFile nixComponents.nix-cmd.mesonFlags); + ++ map (transformFlag "libcmd") (ignoreCrossFile nixComponents.nix-cmd.mesonFlags) + ++ map (transformFlag "nix") (ignoreCrossFile nixComponents.nix-cli.mesonFlags); nativeBuildInputs = let From 6d1d4c323550fb21426edaa1d19a626eb6c717b3 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Thu, 9 Apr 2026 10:13:21 -0400 Subject: [PATCH 220/927] Add _WIN32 guard to spawn.cc Signed-off-by: Lisanna Dettwyler --- src/libutil-tests/spawn.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libutil-tests/spawn.cc b/src/libutil-tests/spawn.cc index 715c0d17fb..db6502390c 100644 --- a/src/libutil-tests/spawn.cc +++ b/src/libutil-tests/spawn.cc @@ -1,11 +1,11 @@ -#include +#ifdef _WIN32 +# include -#include "nix/util/os-string.hh" -#include "nix/util/processes.hh" +# include "nix/util/os-string.hh" +# include "nix/util/processes.hh" namespace nix { -#ifdef _WIN32 TEST(SpawnTest, spawnEcho) { auto output = @@ -34,5 +34,5 @@ TEST(SpawnTest, windowsEscape) auto space = windowsEscape(L"hello world", false); ASSERT_EQ(space, LR"("hello world")"); } -#endif } // namespace nix +#endif From 303fc45c71cb7ac6f366ac8901dab92e428c9355 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Thu, 9 Apr 2026 14:20:48 -0400 Subject: [PATCH 221/927] Cleanup unnecessary includes Signed-off-by: Lisanna Dettwyler --- src/libcmd/built-path.cc | 2 -- src/libcmd/common-eval-args.cc | 2 -- src/libcmd/installable-attr-path.cc | 11 ----------- src/libcmd/installable-flake.cc | 12 ------------ src/libcmd/installables.cc | 5 ----- src/libcmd/repl-interacter.cc | 2 -- src/libcmd/unix/unix-socket-server.cc | 1 - src/libexpr-c/nix_api_external.cc | 3 --- src/libexpr-c/nix_api_value.cc | 3 --- src/libexpr-test-support/tests/value/context.cc | 5 +++-- src/libexpr-tests/derived-path.cc | 4 +++- src/libexpr-tests/eval.cc | 1 - src/libexpr-tests/main.cc | 2 +- src/libexpr-tests/nix_api_external.cc | 2 -- src/libexpr-tests/nix_api_value.cc | 1 - src/libexpr-tests/value/context.cc | 3 +-- src/libexpr/eval-error.cc | 1 - src/libexpr/eval-gc.cc | 2 -- src/libexpr/get-drvs.cc | 1 - src/libexpr/json-to-value.cc | 1 - src/libexpr/primops/fetchClosure.cc | 1 - src/libexpr/primops/fetchMercurial.cc | 2 -- src/libexpr/primops/fetchTree.cc | 1 - src/libexpr/value-to-json.cc | 1 - src/libexpr/value/context.cc | 2 -- src/libfetchers-tests/access-tokens.cc | 2 -- src/libfetchers-tests/git-utils.cc | 1 - src/libfetchers-tests/git.cc | 1 - src/libfetchers-tests/nix_api_fetchers.cc | 1 - src/libfetchers-tests/public-key.cc | 1 - src/libfetchers/attrs.cc | 1 - src/libfetchers/fetchers.cc | 2 +- src/libfetchers/git-utils.cc | 2 -- src/libfetchers/git.cc | 7 +------ src/libfetchers/github.cc | 2 -- src/libfetchers/input-cache.cc | 1 - src/libfetchers/mercurial.cc | 2 -- src/libfetchers/registry.cc | 2 +- src/libflake/config.cc | 2 -- src/libflake/flake-primops.cc | 6 ------ src/libflake/flake.cc | 2 -- src/libflake/lockfile.cc | 5 ----- src/libmain-c/nix_api_main.cc | 2 -- src/libmain/progress-bar.cc | 1 - src/libmain/shared.cc | 2 -- src/libstore-test-support/derived-path.cc | 5 +++-- src/libstore-test-support/path.cc | 8 +++----- src/libstore-tests/common-protocol.cc | 3 --- src/libstore-tests/derivation-advanced-attrs.cc | 3 --- src/libstore-tests/derivation-parser-bench.cc | 1 - src/libstore-tests/derivation/invariants.cc | 2 -- src/libstore-tests/derived-path.cc | 2 -- src/libstore-tests/http-binary-cache-store.cc | 1 - src/libstore-tests/machines.cc | 2 +- src/libstore-tests/realisation.cc | 2 -- src/libstore-tests/register-valid-paths-bench.cc | 1 - src/libstore-tests/s3-binary-cache-store.cc | 1 - src/libstore-tests/serve-protocol.cc | 1 - src/libstore-tests/store-open.cc | 1 - src/libstore-tests/worker-protocol.cc | 1 - src/libstore/binary-cache-store.cc | 4 +--- src/libstore/build/derivation-building-goal.cc | 1 - src/libstore/build/derivation-goal.cc | 11 ----------- src/libstore/build/drv-output-substitution-goal.cc | 2 -- src/libstore/build/entry-points.cc | 1 - src/libstore/build/substitution-goal.cc | 5 +---- src/libstore/builtins/fetchurl.cc | 1 - src/libstore/common-protocol.cc | 3 +-- src/libstore/daemon.cc | 2 -- src/libstore/derivation-options.cc | 1 - src/libstore/derivations.cc | 2 -- src/libstore/derived-path.cc | 4 +--- src/libstore/filetransfer.cc | 2 -- src/libstore/gc.cc | 4 ++-- src/libstore/globals.cc | 4 ---- src/libstore/local-binary-cache-store.cc | 3 +-- src/libstore/local-fs-store.cc | 2 -- src/libstore/local-gc.cc | 1 - src/libstore/local-overlay-store.cc | 1 - src/libstore/machines.cc | 1 - src/libstore/misc.cc | 2 -- src/libstore/nar-info.cc | 3 +-- src/libstore/optimise-store.cc | 4 +--- src/libstore/path-references.cc | 4 ---- src/libstore/path-with-outputs.cc | 2 -- src/libstore/pathlocks.cc | 2 -- src/libstore/posix-fs-canonicalise.cc | 3 --- src/libstore/profiles.cc | 3 --- src/libstore/references.cc | 2 -- src/libstore/remote-store.cc | 1 - src/libstore/s3-url.cc | 1 - src/libstore/serve-protocol.cc | 1 - src/libstore/sqlite.cc | 3 +-- src/libstore/ssh-store.cc | 1 - src/libstore/ssh.cc | 1 - src/libstore/store-api.cc | 2 -- src/libstore/uds-remote-store.cc | 2 +- src/libstore/unix/pathlocks.cc | 2 -- src/libstore/unix/user-lock.cc | 2 +- src/libstore/worker-protocol.cc | 1 - src/libutil-test-support/hash.cc | 6 +++--- .../include/nix/util/tests/json-characterization.hh | 1 + src/libutil-tests/args.cc | 1 - src/libutil-tests/config.cc | 2 -- src/libutil-tests/file-system.cc | 7 ------- src/libutil-tests/hash.cc | 2 -- src/libutil-tests/monitorfdhup.cc | 1 - src/libutil-tests/nar-listing.cc | 1 - src/libutil-tests/nix_api_util.cc | 1 - src/libutil-tests/nix_api_util_internal.cc | 4 ---- src/libutil-tests/terminal.cc | 5 ----- src/libutil-tests/topo-sort.cc | 1 - src/libutil-tests/util.cc | 6 ------ src/libutil/archive.cc | 2 -- src/libutil/args.cc | 1 - src/libutil/compression.cc | 1 - src/libutil/current-process.cc | 3 --- src/libutil/environment-variables.cc | 1 - src/libutil/error.cc | 1 - src/libutil/executable-path.cc | 1 - src/libutil/file-system.cc | 1 - src/libutil/git.cc | 2 -- src/libutil/hash.cc | 2 -- src/libutil/linux/linux-namespaces.cc | 4 ---- src/libutil/logging.cc | 2 -- src/libutil/memory-source-accessor.cc | 1 - src/libutil/nar-accessor.cc | 1 - src/libutil/posix-source-accessor.cc | 1 - src/libutil/signature/signer.cc | 1 - src/libutil/strings.cc | 3 --- src/libutil/suggestions.cc | 1 - src/libutil/terminal.cc | 1 - src/libutil/unix/file-descriptor.cc | 3 --- src/libutil/unix/file-path.cc | 6 ------ src/libutil/unix/muxable-pipe.cc | 1 - src/libutil/unix/processes.cc | 2 -- src/libutil/unix/users.cc | 2 +- src/libutil/unix/xdg-dirs.cc | 2 +- src/libutil/users.cc | 2 -- src/libutil/util.cc | 3 --- src/libutil/windows/current-process.cc | 1 - src/nix/add-to-store.cc | 2 -- src/nix/build-remote/build-remote.cc | 1 - src/nix/build.cc | 1 - src/nix/bundle.cc | 1 - src/nix/cat.cc | 1 - src/nix/config.cc | 2 -- src/nix/derivation-add.cc | 1 - src/nix/derivation-show.cc | 1 - src/nix/develop.cc | 3 +-- src/nix/diff-closures.cc | 1 - src/nix/formatter.cc | 2 -- src/nix/hash.cc | 2 -- src/nix/log.cc | 1 - src/nix/main.cc | 1 - src/nix/nix-build/nix-build.cc | 1 - src/nix/nix-channel/nix-channel.cc | 1 - src/nix/nix-collect-garbage/nix-collect-garbage.cc | 1 - src/nix/nix-env/user-env.cc | 2 -- src/nix/nix-instantiate/nix-instantiate.cc | 2 -- src/nix/nix-store/nix-store.cc | 1 - src/nix/optimise-store.cc | 2 -- src/nix/path-info.cc | 1 - src/nix/prefetch.cc | 2 -- src/nix/registry.cc | 2 -- src/nix/repl.cc | 2 -- src/nix/run.cc | 4 ---- src/nix/search.cc | 2 -- src/nix/self-exe.cc | 3 +-- src/nix/store-copy-log.cc | 4 ---- src/nix/store-delete.cc | 1 - src/nix/store-gc.cc | 1 - src/nix/unix/daemon.cc | 5 +---- 173 files changed, 39 insertions(+), 355 deletions(-) diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index fae65fca2b..2c52678105 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -6,8 +6,6 @@ #include -#include - namespace nix { // Custom implementation to avoid `ref` ptr equality diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 7e4bd7a162..984bed3488 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,9 +1,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/expr/eval-settings.hh" #include "nix/cmd/common-eval-args.hh" -#include "nix/main/shared.hh" #include "nix/util/config-global.hh" -#include "nix/store/filetransfer.hh" #include "nix/expr/eval.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/registry.hh" diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 28c3db3fc7..aa5da2e566 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -1,24 +1,13 @@ -#include "nix/store/globals.hh" #include "nix/cmd/installable-attr-path.hh" #include "nix/store/outputs-spec.hh" #include "nix/util/util.hh" #include "nix/cmd/command.hh" #include "nix/expr/attr-path.hh" #include "nix/cmd/common-eval-args.hh" -#include "nix/store/derivations.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" #include "nix/expr/get-drvs.hh" -#include "nix/store/store-api.hh" -#include "nix/main/shared.hh" #include "nix/flake/flake.hh" -#include "nix/expr/eval-cache.hh" -#include "nix/util/url.hh" -#include "nix/fetchers/registry.hh" -#include "nix/store/build-result.hh" - -#include -#include #include diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 77a7c8d6ec..1ef34a1ce0 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -1,25 +1,13 @@ -#include "nix/store/globals.hh" #include "nix/cmd/installable-flake.hh" -#include "nix/cmd/installable-derived-path.hh" #include "nix/store/outputs-spec.hh" #include "nix/util/util.hh" #include "nix/cmd/command.hh" #include "nix/expr/attr-path.hh" #include "nix/cmd/common-eval-args.hh" -#include "nix/store/derivations.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" -#include "nix/expr/get-drvs.hh" -#include "nix/store/store-api.hh" -#include "nix/main/shared.hh" #include "nix/flake/flake.hh" #include "nix/expr/eval-cache.hh" -#include "nix/util/url.hh" -#include "nix/fetchers/registry.hh" -#include "nix/store/build-result.hh" - -#include -#include #include diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 8f8309bd96..a25971ad7f 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -13,18 +13,13 @@ #include "nix/expr/eval-inline.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" -#include "nix/expr/get-drvs.hh" #include "nix/store/store-api.hh" #include "nix/main/shared.hh" #include "nix/flake/flake.hh" #include "nix/expr/eval-cache.hh" -#include "nix/util/url.hh" #include "nix/fetchers/registry.hh" #include "nix/store/build-result.hh" -#include -#include - #include #include "nix/util/strings-inline.hh" diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 58d55e0a2d..81240af7f5 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -18,12 +18,10 @@ extern "C" { } #endif -#include "nix/util/signals.hh" #include "nix/util/finally.hh" #include "nix/cmd/repl-interacter.hh" #include "nix/util/file-system.hh" #include "nix/util/serialise.hh" -#include "nix/cmd/repl.hh" #include "nix/util/environment-variables.hh" namespace nix { diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index e33f4317da..5d1fba4622 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -5,7 +5,6 @@ #include "nix/util/file-system.hh" #include "nix/util/logging.hh" #include "nix/util/signals.hh" -#include "nix/util/strings.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/util.hh" diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index a874d9a086..98b68d9f6b 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -1,7 +1,4 @@ -#include "nix/expr/attr-set.hh" -#include "nix/util/configuration.hh" #include "nix/expr/eval.hh" -#include "nix/store/globals.hh" #include "nix/expr/value.hh" #include "nix_api_expr.h" diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 589ebf9a8e..3cb5705898 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -1,8 +1,6 @@ #include "nix/expr/attr-set.hh" #include "nix/expr/eval-error.hh" -#include "nix/util/configuration.hh" #include "nix/expr/eval.hh" -#include "nix/store/globals.hh" #include "nix/store/path.hh" #include "nix/expr/primops.hh" #include "nix/expr/value.hh" @@ -13,7 +11,6 @@ #include "nix_api_util_internal.h" #include "nix_api_store_internal.h" #include "nix_api_value.h" -#include "nix/expr/value/context.hh" // Internal helper functions to check [in] and [out] `Value *` parameters static const nix::Value & check_value_not_null(const nix_value * value) diff --git a/src/libexpr-test-support/tests/value/context.cc b/src/libexpr-test-support/tests/value/context.cc index d6036601a9..ca7996acc1 100644 --- a/src/libexpr-test-support/tests/value/context.cc +++ b/src/libexpr-test-support/tests/value/context.cc @@ -1,7 +1,8 @@ -#include // Needed by rapidcheck on Darwin +#ifdef __APPLE__ +# include // Needed by rapidcheck on Darwin +#endif #include -#include "nix/store/tests/path.hh" #include "nix/expr/tests/value/context.hh" namespace rc { diff --git a/src/libexpr-tests/derived-path.cc b/src/libexpr-tests/derived-path.cc index e9f9fcd072..c685f6a094 100644 --- a/src/libexpr-tests/derived-path.cc +++ b/src/libexpr-tests/derived-path.cc @@ -1,6 +1,8 @@ #include #include -#include // Needed by rapidcheck on Darwin +#ifdef __APPLE__ +# include // Needed by rapidcheck on Darwin +#endif #include #include "nix/store/tests/derived-path.hh" diff --git a/src/libexpr-tests/eval.cc b/src/libexpr-tests/eval.cc index 7562a9da21..985564f13e 100644 --- a/src/libexpr-tests/eval.cc +++ b/src/libexpr-tests/eval.cc @@ -3,7 +3,6 @@ #include "nix/expr/eval.hh" #include "nix/expr/tests/libexpr.hh" -#include "nix/util/memory-source-accessor.hh" namespace nix { diff --git a/src/libexpr-tests/main.cc b/src/libexpr-tests/main.cc index 365fa482de..5a717d6bb0 100644 --- a/src/libexpr-tests/main.cc +++ b/src/libexpr-tests/main.cc @@ -1,7 +1,7 @@ #include #include "nix/store/tests/test-main.hh" -#include "nix/util/config-global.hh" +#include "nix/util/configuration.hh" int main(int argc, char ** argv) { diff --git a/src/libexpr-tests/nix_api_external.cc b/src/libexpr-tests/nix_api_external.cc index 885b9c1d2d..e17a52c31d 100644 --- a/src/libexpr-tests/nix_api_external.cc +++ b/src/libexpr-tests/nix_api_external.cc @@ -1,5 +1,3 @@ -#include "nix_api_store.h" -#include "nix_api_util.h" #include "nix_api_expr.h" #include "nix_api_value.h" #include "nix_api_external.h" diff --git a/src/libexpr-tests/nix_api_value.cc b/src/libexpr-tests/nix_api_value.cc index 830637f3ec..01d15744a7 100644 --- a/src/libexpr-tests/nix_api_value.cc +++ b/src/libexpr-tests/nix_api_value.cc @@ -1,4 +1,3 @@ -#include "nix_api_store.h" #include "nix_api_util.h" #include "nix_api_expr.h" #include "nix_api_value.h" diff --git a/src/libexpr-tests/value/context.cc b/src/libexpr-tests/value/context.cc index fe3072b64f..544f03f3f2 100644 --- a/src/libexpr-tests/value/context.cc +++ b/src/libexpr-tests/value/context.cc @@ -2,9 +2,8 @@ #include #include -#include "nix/store/tests/path.hh" -#include "nix/expr/tests/libexpr.hh" #include "nix/expr/tests/value/context.hh" +#include "nix/store/store-dir-config.hh" namespace nix { diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 38a60883bc..45cb1e409d 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -1,7 +1,6 @@ #include "nix/expr/eval-error.hh" #include "nix/expr/eval.hh" #include "nix/expr/value.hh" -#include "nix/store/store-api.hh" namespace nix { diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 0d25f38f64..9344b0405b 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -1,8 +1,6 @@ -#include "nix/util/error.hh" #include "nix/util/environment-variables.hh" #include "nix/expr/eval-settings.hh" #include "nix/util/config-global.hh" -#include "nix/util/serialise.hh" #include "nix/expr/eval-gc.hh" #include "nix/expr/value.hh" diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 03a1aa455c..693b1946ee 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -5,7 +5,6 @@ #include "nix/store/path-with-outputs.hh" #include -#include namespace nix { diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 4a68308c64..08393435b4 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -3,7 +3,6 @@ #include "nix/expr/eval.hh" #include -#include #include using json = nlohmann::json; diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 2e01d67172..db182ab499 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -2,7 +2,6 @@ #include "nix/store/store-open.hh" #include "nix/store/realisation.hh" #include "nix/store/make-content-addressed.hh" -#include "nix/util/url.hh" #include "nix/util/environment-variables.hh" namespace nix { diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 9347645cc2..97aeb0a035 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,9 +1,7 @@ #include "nix/expr/primops.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" -#include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" -#include "nix/util/url.hh" #include "nix/util/url-parts.hh" namespace nix { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 557717d2e4..afd61e90fb 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -16,7 +16,6 @@ #include #include -#include namespace nix { diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 4fac29a667..8d48a48391 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -4,7 +4,6 @@ #include "nix/util/signals.hh" #include -#include #include namespace nix { diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 4a17bbdc3a..60ba535207 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -2,8 +2,6 @@ #include "nix/expr/value/context.hh" #include "nix/store/store-dir-config.hh" -#include - namespace nix { NixStringContextElem NixStringContextElem::parse(std::string_view s0, const ExperimentalFeatureSettings & xpSettings) diff --git a/src/libfetchers-tests/access-tokens.cc b/src/libfetchers-tests/access-tokens.cc index 7127434db9..5e92a9bc7b 100644 --- a/src/libfetchers-tests/access-tokens.cc +++ b/src/libfetchers-tests/access-tokens.cc @@ -3,8 +3,6 @@ #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/fetch-settings.hh" -#include "nix/util/json-utils.hh" -#include "nix/util/tests/characterization.hh" namespace nix::fetchers { diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index a80ff590f9..580769936d 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -10,7 +10,6 @@ #include #include "nix/util/fs-sink.hh" #include "nix/util/serialise.hh" -#include "nix/fetchers/git-lfs-fetch.hh" #include #include diff --git a/src/libfetchers-tests/git.cc b/src/libfetchers-tests/git.cc index e970c3b8bd..e41bbe520f 100644 --- a/src/libfetchers-tests/git.cc +++ b/src/libfetchers-tests/git.cc @@ -1,4 +1,3 @@ -#include "nix/store/store-open.hh" #include "nix/store/globals.hh" #include "nix/store/dummy-store.hh" #include "nix/fetchers/fetch-settings.hh" diff --git a/src/libfetchers-tests/nix_api_fetchers.cc b/src/libfetchers-tests/nix_api_fetchers.cc index 8f3e6e3c58..ff2c5d8ff3 100644 --- a/src/libfetchers-tests/nix_api_fetchers.cc +++ b/src/libfetchers-tests/nix_api_fetchers.cc @@ -1,4 +1,3 @@ -#include "gmock/gmock.h" #include #include "nix_api_fetchers.h" diff --git a/src/libfetchers-tests/public-key.cc b/src/libfetchers-tests/public-key.cc index 2991223f6b..dabbd46442 100644 --- a/src/libfetchers-tests/public-key.cc +++ b/src/libfetchers-tests/public-key.cc @@ -1,6 +1,5 @@ #include #include "nix/fetchers/fetchers.hh" -#include "nix/util/json-utils.hh" #include "nix/util/tests/json-characterization.hh" namespace nix { diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 841808bd16..cc9e72af46 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -1,5 +1,4 @@ #include "nix/fetchers/attrs.hh" -#include "nix/fetchers/fetchers.hh" #include diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 29b01e45a7..fb87f9b945 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,12 +1,12 @@ #include "nix/fetchers/fetchers.hh" #include "nix/store/store-api.hh" +#include "nix/util/fs-sink.hh" #include "nix/util/source-path.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/json-utils.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/url.hh" -#include "nix/util/archive.hh" #include diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index fcffdcfa8a..eaea4b6d59 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -42,9 +42,7 @@ #include #include #include -#include #include -#include #include namespace std { diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 736d2e39f7..3c191d32f5 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,25 +1,20 @@ +#include "nix/util/environment-variables.hh" #include "nix/util/error.hh" #include "nix/fetchers/fetchers.hh" #include "nix/util/users.hh" #include "nix/fetchers/cache.hh" -#include "nix/store/globals.hh" -#include "nix/util/tarfile.hh" #include "nix/store/store-api.hh" -#include "nix/util/url-parts.hh" #include "nix/store/pathlocks.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" #include "nix/util/git.hh" #include "nix/fetchers/git-utils.hh" #include "nix/util/logging.hh" -#include "nix/util/finally.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/util/json-utils.hh" #include "nix/util/archive.hh" #include "nix/util/mounted-source-accessor.hh" -#include -#include #include #ifndef _WIN32 diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ac8c02418d..855475d7fc 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -1,6 +1,5 @@ #include "nix/store/filetransfer.hh" #include "nix/fetchers/cache.hh" -#include "nix/store/globals.hh" #include "nix/store/store-api.hh" #include "nix/util/types.hh" #include "nix/util/url-parts.hh" @@ -13,7 +12,6 @@ #include #include -#include namespace nix::fetchers { diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 652d5ce797..85a611355e 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -1,7 +1,6 @@ #include "nix/fetchers/input-cache.hh" #include "nix/fetchers/registry.hh" #include "nix/util/sync.hh" -#include "nix/util/source-path.hh" namespace nix::fetchers { diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 38fe31fcad..db0333e908 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -7,8 +7,6 @@ #include "nix/util/environment-variables.hh" #include "nix/util/users.hh" #include "nix/fetchers/cache.hh" -#include "nix/store/globals.hh" -#include "nix/util/tarfile.hh" #include "nix/store/store-api.hh" #include "nix/util/url-parts.hh" #include "nix/fetchers/fetch-settings.hh" diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 9911586fa0..0aad8ef48a 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,8 +1,8 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/registry.hh" #include "nix/fetchers/tarball.hh" +#include "nix/store/filetransfer.hh" #include "nix/util/users.hh" -#include "nix/store/globals.hh" #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" diff --git a/src/libflake/config.cc b/src/libflake/config.cc index b7d5cd8999..6367d7b4ec 100644 --- a/src/libflake/config.cc +++ b/src/libflake/config.cc @@ -11,14 +11,12 @@ #include #include #include -#include #include "nix/util/users.hh" #include "nix/util/config-global.hh" #include "nix/flake/settings.hh" #include "nix/flake/flake.hh" #include "nix/util/ansicolor.hh" -#include "nix/util/configuration.hh" #include "nix/util/file-system.hh" #include "nix/util/fmt.hh" #include "nix/util/logging.hh" diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 4b84c081e6..897739ed17 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -1,12 +1,8 @@ #include -#include #include #include -#include #include -#include #include -#include #include "nix/flake/flake-primops.hh" #include "nix/expr/eval.hh" @@ -21,12 +17,10 @@ #include "nix/expr/value.hh" #include "nix/fetchers/attrs.hh" #include "nix/fetchers/fetchers.hh" -#include "nix/util/configuration.hh" #include "nix/util/error.hh" #include "nix/util/experimental-features.hh" #include "nix/util/pos-idx.hh" #include "nix/util/pos-table.hh" -#include "nix/util/source-path.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 67522d2c47..277adc092f 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include "nix/util/terminal.hh" #include "nix/util/ref.hh" @@ -57,7 +56,6 @@ #include "nix/util/logging.hh" #include "nix/util/pos-idx.hh" #include "nix/util/pos-table.hh" -#include "nix/util/position.hh" #include "nix/util/source-path.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc index 4eeba3bd3e..75127521b9 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/lockfile.cc @@ -8,10 +8,7 @@ #include #include #include -#include #include -#include -#include #include #include #include @@ -32,10 +29,8 @@ #include "nix/flake/flakeref.hh" #include "nix/store/path.hh" #include "nix/util/ansicolor.hh" -#include "nix/util/configuration.hh" #include "nix/util/error.hh" #include "nix/util/fmt.hh" -#include "nix/util/hash.hh" #include "nix/util/logging.hh" #include "nix/util/ref.hh" #include "nix/util/types.hh" diff --git a/src/libmain-c/nix_api_main.cc b/src/libmain-c/nix_api_main.cc index 0ee965dc82..872949e423 100644 --- a/src/libmain-c/nix_api_main.cc +++ b/src/libmain-c/nix_api_main.cc @@ -1,5 +1,3 @@ -#include "nix_api_store.h" -#include "nix_api_store_internal.h" #include "nix_api_util.h" #include "nix_api_util_internal.h" diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 71c7789de2..d177a1508b 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -5,7 +5,6 @@ #include "nix/store/store-api.hh" #include "nix/store/names.hh" -#include #include #include #include diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 3640efc07a..f3293060f1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -6,12 +6,10 @@ #include "nix/store/store-open.hh" #include "nix/store/gc-store.hh" #include "nix/main/loggers.hh" -#include "nix/main/progress-bar.hh" #include "nix/util/signals.hh" #include "nix/util/util.hh" #include -#include #include #include diff --git a/src/libstore-test-support/derived-path.cc b/src/libstore-test-support/derived-path.cc index cb1d23ac52..ee8018c326 100644 --- a/src/libstore-test-support/derived-path.cc +++ b/src/libstore-test-support/derived-path.cc @@ -1,6 +1,7 @@ -#include -#include // Needed by rapidcheck on Darwin +#ifdef __APPLE__ +# include // Needed by rapidcheck on Darwin +#endif #include #include "nix/store/tests/derived-path.hh" diff --git a/src/libstore-test-support/path.cc b/src/libstore-test-support/path.cc index 1e5f74a285..98a255ccc0 100644 --- a/src/libstore-test-support/path.cc +++ b/src/libstore-test-support/path.cc @@ -1,12 +1,10 @@ -#include // Needed by rapidcheck on Darwin -#include +#ifdef __APPLE__ +# include // Needed by rapidcheck on Darwin +#endif #include #include -#include "nix/store/path-regex.hh" -#include "nix/store/store-api.hh" - #include "nix/util/tests/hash.hh" #include "nix/store/tests/path.hh" diff --git a/src/libstore-tests/common-protocol.cc b/src/libstore-tests/common-protocol.cc index 9146af219c..5afde2bdf7 100644 --- a/src/libstore-tests/common-protocol.cc +++ b/src/libstore-tests/common-protocol.cc @@ -1,12 +1,9 @@ -#include - #include #include #include "nix/util/json-utils.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" -#include "nix/store/build-result.hh" #include "nix/store/tests/protocol.hh" #include "nix/util/tests/characterization.hh" diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index aa24943a72..bb7ad28f3d 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -1,14 +1,11 @@ #include -#include -#include "nix/util/experimental-features.hh" #include "nix/store/derivations.hh" #include "nix/store/derived-path.hh" #include "nix/store/derivation-options.hh" #include "nix/store/globals.hh" #include "nix/store/parsed-derivations.hh" #include "nix/util/types.hh" -#include "nix/util/json-utils.hh" #include "nix/store/tests/libstore.hh" #include "nix/util/tests/json-characterization.hh" diff --git a/src/libstore-tests/derivation-parser-bench.cc b/src/libstore-tests/derivation-parser-bench.cc index 93fa64f619..5bc4cec9f3 100644 --- a/src/libstore-tests/derivation-parser-bench.cc +++ b/src/libstore-tests/derivation-parser-bench.cc @@ -1,7 +1,6 @@ #include #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" -#include "nix/util/experimental-features.hh" #include "nix/util/tests/test-data.hh" #include "nix/store/store-open.hh" #include diff --git a/src/libstore-tests/derivation/invariants.cc b/src/libstore-tests/derivation/invariants.cc index 115d5bc4bb..e825655d66 100644 --- a/src/libstore-tests/derivation/invariants.cc +++ b/src/libstore-tests/derivation/invariants.cc @@ -6,8 +6,6 @@ #include "nix/store/dummy-store-impl.hh" #include "nix/util/tests/json-characterization.hh" -#include "derivation/test-support.hh" - namespace nix { class FillInOutputPathsTest : public LibStoreTest, public JsonCharacterizationTest diff --git a/src/libstore-tests/derived-path.cc b/src/libstore-tests/derived-path.cc index 70e789c0c4..541d729e03 100644 --- a/src/libstore-tests/derived-path.cc +++ b/src/libstore-tests/derived-path.cc @@ -1,5 +1,3 @@ -#include - #include #include diff --git a/src/libstore-tests/http-binary-cache-store.cc b/src/libstore-tests/http-binary-cache-store.cc index aee6f7f4b1..74f3b93cd3 100644 --- a/src/libstore-tests/http-binary-cache-store.cc +++ b/src/libstore-tests/http-binary-cache-store.cc @@ -3,7 +3,6 @@ #include "nix/store/http-binary-cache-store.hh" #include "nix/store/tests/https-store.hh" -#include "nix/util/fs-sink.hh" namespace nix { diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index e412376323..7c9a17ed6b 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -2,7 +2,7 @@ #include "nix/util/file-system.hh" #include "nix/util/util.hh" -#include "nix/util/tests/characterization.hh" +#include "nix/util/tests/test-data.hh" #include #include diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc index b3584d6a1e..814b671366 100644 --- a/src/libstore-tests/realisation.cc +++ b/src/libstore-tests/realisation.cc @@ -1,5 +1,3 @@ -#include - #include #include #include diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 400a705556..51bcb29aa9 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -5,7 +5,6 @@ #include "nix/store/store-open.hh" #include "nix/util/file-system.hh" #include "nix/util/hash.hh" -#include "nix/util/tests/test-data.hh" #ifndef _WIN32 diff --git a/src/libstore-tests/s3-binary-cache-store.cc b/src/libstore-tests/s3-binary-cache-store.cc index 9aa9b2dd1a..126d7a522a 100644 --- a/src/libstore-tests/s3-binary-cache-store.cc +++ b/src/libstore-tests/s3-binary-cache-store.cc @@ -1,7 +1,6 @@ #include "nix/store/s3-binary-cache-store.hh" #include "nix/store/http-binary-cache-store.hh" #include "nix/store/filetransfer.hh" -#include "nix/store/s3-url.hh" #include diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 6418cb55dc..e34f916cbb 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -1,5 +1,4 @@ #include -#include #include #include diff --git a/src/libstore-tests/store-open.cc b/src/libstore-tests/store-open.cc index 4e82b358af..cadeb3f040 100644 --- a/src/libstore-tests/store-open.cc +++ b/src/libstore-tests/store-open.cc @@ -3,7 +3,6 @@ #include "nix/store/store-open.hh" #include "nix/store/store-reference.hh" #include "nix/store/local-store.hh" -#include "nix/store/uds-remote-store.hh" #include "nix/store/globals.hh" #include "nix/util/file-system.hh" #include "nix/util/finally.hh" diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index ed288a7443..91acde7380 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -1,4 +1,3 @@ -#include #include #include diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 73fc6981d5..624bbd6496 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -3,11 +3,10 @@ #include "nix/util/compression.hh" #include "nix/store/derivations.hh" #include "nix/util/source-accessor.hh" -#include "nix/store/globals.hh" +#include "nix/store/nar-info-disk-cache.hh" #include "nix/store/nar-info.hh" #include "nix/util/sync.hh" #include "nix/store/remote-fs-accessor.hh" -#include "nix/store/nar-info-disk-cache.hh" #include "nix/util/nar-accessor.hh" #include "nix/util/thread-pool.hh" #include "nix/util/callback.hh" @@ -17,7 +16,6 @@ #include #include #include -#include #include #include diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 2235cf872b..93a13e5c8d 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -18,7 +18,6 @@ #include "nix/store/globals.hh" #include -#include #include #include #include diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 115e48612f..6e2d3223b1 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2,29 +2,18 @@ #include "nix/store/build/drv-output-substitution-goal.hh" #include "nix/store/build/derivation-building-goal.hh" #include "nix/store/build/derivation-resolution-goal.hh" -#ifndef _WIN32 // TODO enable build hook on Windows -# include "nix/store/build/hook-instance.hh" -# include "nix/store/build/derivation-builder.hh" -#endif -#include "nix/util/processes.hh" -#include "nix/util/config-global.hh" #include "nix/store/build/worker.hh" #include "nix/util/util.hh" -#include "nix/util/compression.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" // Don't remove is actually needed #include "nix/store/outputs-query.hh" -#include "nix/store/globals.hh" -#include #include #include #include #include -#include "nix/util/strings.hh" - namespace nix { DerivationGoal::DerivationGoal( diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 212c767a4c..ed653fed75 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -1,9 +1,7 @@ #include "goal-impl.hh" #include "nix/store/build/drv-output-substitution-goal.hh" -#include "nix/util/finally.hh" #include "nix/store/build/worker.hh" -#include "nix/store/build/substitution-goal.hh" #include "nix/util/callback.hh" namespace nix { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index e6d1f62d78..5b97966847 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -2,7 +2,6 @@ #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/build/derivation-trampoline-goal.hh" -#include "nix/store/local-store.hh" #include "nix/util/strings.hh" namespace nix { diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 2c1573b82c..4cb42975fe 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -3,12 +3,9 @@ #include "nix/store/build/worker.hh" #include "nix/store/build/substitution-goal.hh" #include "nix/store/nar-info.hh" -#include "nix/util/finally.hh" +#include "nix/store/worker-settings.hh" #include "nix/util/signals.hh" #include "nix/util/callback.hh" -#include "nix/store/globals.hh" - -#include namespace nix { diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 767fc9b8f0..d412adbb62 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -1,7 +1,6 @@ #include "nix/store/builtins.hh" #include "nix/store/filetransfer.hh" #include "nix/store/store-api.hh" -#include "nix/store/globals.hh" #include "nix/util/archive.hh" #include "nix/util/compression.hh" #include "nix/util/file-system.hh" diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index fc88ee8890..dfddd042df 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -1,11 +1,10 @@ #include "nix/util/serialise.hh" #include "nix/store/path-with-outputs.hh" -#include "nix/store/store-api.hh" #include "nix/store/build-result.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" -#include "nix/util/archive.hh" #include "nix/store/derivations.hh" +#include "nix/store/store-dir-config.hh" #include "nix/util/signature/local-keys.hh" #include diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 08802e953e..31d30097fc 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -3,7 +3,6 @@ #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-connection.hh" #include "nix/store/worker-protocol-impl.hh" -#include "nix/store/build-result.hh" #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" #include "nix/store/filetransfer.hh" @@ -16,7 +15,6 @@ #include "nix/util/archive.hh" #include "nix/store/derivations.hh" #include "nix/util/args.hh" -#include "nix/util/git.hh" #include "nix/util/logging.hh" #include "nix/store/globals.hh" diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 86543bc363..b3d4261b54 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -6,7 +6,6 @@ #include "nix/store/store-api.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" -#include "nix/util/variant-wrapper.hh" #include #include diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b618e98180..8475532bca 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1,10 +1,8 @@ #include "nix/store/derivations.hh" #include "nix/store/downstream-placeholder.hh" #include "nix/store/store-api.hh" -#include "nix/store/globals.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" -#include "nix/util/split.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" #include "nix/util/strings-inline.hh" diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 251e112514..131674aa55 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,11 +1,9 @@ #include "nix/store/derived-path.hh" #include "nix/store/derivations.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-dir-config.hh" #include "nix/util/comparator.hh" #include "nix/util/json-utils.hh" -#include - namespace nix { // Custom implementation to avoid `ref` ptr equality diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 88853620fa..85bc650dc8 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -2,13 +2,11 @@ #include "nix/store/filetransfer-impl.hh" #include "nix/store/globals.hh" #include "nix/util/config-global.hh" -#include "nix/store/store-api.hh" #include "nix/util/finally.hh" #include "nix/util/callback.hh" #include "nix/util/signals.hh" #include "nix/util/util.hh" -#include "store-config-private.hh" #include "nix/store/s3-url.hh" #include #if NIX_WITH_AWS_AUTH diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index a88fe6a64a..f3bded97e4 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,9 +1,9 @@ -#include "nix/store/derivations.hh" -#include "nix/store/globals.hh" #include "nix/store/local-gc.hh" +#include "nix/store/local-settings.hh" #include "nix/store/local-store.hh" #include "nix/store/path.hh" #include "nix/util/configuration.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/finally.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/signals.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8427bffaf6..8beb68aa8e 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -4,12 +4,10 @@ #include "nix/util/config-global.hh" #include "nix/util/current-process.hh" #include "nix/util/executable-path.hh" -#include "nix/util/archive.hh" #include "nix/util/args.hh" #include "nix/util/abstract-setting-to-json.hh" #include "nix/util/compute-levels.hh" #include "nix/util/executable-path.hh" -#include "nix/util/signals.hh" #include "nix/store/filetransfer.hh" #include @@ -34,8 +32,6 @@ # include "nix/util/processes.hh" #endif -#include "nix/util/config-impl.hh" - #ifdef __APPLE__ # include #endif diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index b79e271c27..60298efdbb 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -1,6 +1,5 @@ #include "nix/store/local-binary-cache-store.hh" -#include "nix/store/globals.hh" -#include "nix/store/nar-info-disk-cache.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/signals.hh" #include "nix/store/store-registration.hh" diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index c58b90d323..9f5864ee44 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,8 +1,6 @@ -#include "nix/util/archive.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" -#include "nix/store/globals.hh" #include "nix/util/compression.hh" #include "nix/store/derivations.hh" diff --git a/src/libstore/local-gc.cc b/src/libstore/local-gc.cc index e3c74f3575..8bbbdc8366 100644 --- a/src/libstore/local-gc.cc +++ b/src/libstore/local-gc.cc @@ -2,7 +2,6 @@ #include "nix/store/store-dir-config.hh" #include "nix/util/file-system.hh" #include "nix/util/signals.hh" -#include "nix/util/types.hh" #include "nix/store/local-gc.hh" #include #include diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 89c5a42d8a..afbd47b5bd 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -5,7 +5,6 @@ #include "nix/util/os-string.hh" #include "nix/store/realisation.hh" #include "nix/util/processes.hh" -#include "nix/util/url.hh" #include "nix/store/store-open.hh" #include "nix/store/store-registration.hh" diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 00a4d4d1fe..17864ca5ac 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,6 +1,5 @@ #include "nix/util/base-n.hh" #include "nix/store/machines.hh" -#include "nix/store/globals.hh" #include "nix/store/store-open.hh" #include diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index f446b1cf00..51708a2cbc 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,12 +1,10 @@ #include "nix/store/derivations.hh" -#include "nix/util/fun.hh" #include "nix/store/outputs-query.hh" #include "nix/store/parsed-derivations.hh" #include "nix/store/derivation-options.hh" #include "nix/store/globals.hh" #include "nix/store/store-open.hh" #include "nix/store/nar-info.hh" -#include "nix/util/thread-pool.hh" #include "nix/store/realisation.hh" #include "nix/util/topo-sort.hh" #include "nix/util/callback.hh" diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index b912467d8a..d470569ea2 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,6 +1,5 @@ -#include "nix/store/globals.hh" #include "nix/store/nar-info.hh" -#include "nix/store/store-api.hh" +#include "nix/store/store-dir-config.hh" #include "nix/util/strings.hh" #include "nix/util/json-utils.hh" diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 7d37dbf9d4..220e37fe9e 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,5 +1,5 @@ #include "nix/store/local-store.hh" -#include "nix/store/globals.hh" +#include "nix/store/local-settings.hh" #include "nix/util/signals.hh" #include "nix/store/posix-fs-canonicalise.hh" #include "nix/util/posix-source-accessor.hh" @@ -11,8 +11,6 @@ #include #include #include -#include -#include #include "store-config-private.hh" diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 409f6de647..78b64a2101 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -1,5 +1,4 @@ #include "nix/store/path-references.hh" -#include "nix/util/hash.hh" #include "nix/util/archive.hh" #include "nix/util/source-accessor.hh" #include "nix/util/canon-path.hh" @@ -7,9 +6,6 @@ #include #include -#include -#include -#include namespace nix { diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 4309ceac5f..e489a8054b 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,5 +1,3 @@ -#include - #include "nix/store/path-with-outputs.hh" #include "nix/store/store-api.hh" #include "nix/util/strings.hh" diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index a8e828655a..d20fe91b05 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -1,7 +1,5 @@ #include "nix/store/pathlocks.hh" #include "nix/util/util.hh" -#include "nix/util/sync.hh" -#include "nix/util/signals.hh" #include #include diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index 455cc7c4f5..9317d30f4e 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -2,10 +2,7 @@ #include "nix/store/build-result.hh" #include "nix/util/file-system.hh" #include "nix/util/signals.hh" -#include "nix/util/util.hh" #include "nix/store/store-api.hh" -#include "nix/store/globals.hh" -#include "store-config-private.hh" #if NIX_SUPPORT_ACL # include diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 4a60ee5596..519c2abc98 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,15 +1,12 @@ #include "nix/store/profiles.hh" #include "nix/util/signals.hh" #include "nix/store/globals.hh" -#include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" #include "nix/util/users.hh" #include #include #include -#include -#include namespace nix { diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 1620af26e4..4bc15a71de 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -3,9 +3,7 @@ #include "nix/util/hash.hh" #include "nix/util/base-nix-32.hh" -#include #include -#include #include namespace nix { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 2562e65d88..3bb5b07557 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -13,7 +13,6 @@ #include "nix/store/derivations.hh" #include "nix/util/pool.hh" #include "nix/util/finally.hh" -#include "nix/util/git.hh" #include "nix/util/logging.hh" #include "nix/util/callback.hh" #include "nix/store/filetransfer.hh" diff --git a/src/libstore/s3-url.cc b/src/libstore/s3-url.cc index e6b5553661..d1530ceb0d 100644 --- a/src/libstore/s3-url.cc +++ b/src/libstore/s3-url.cc @@ -4,7 +4,6 @@ #include "nix/util/error.hh" #include "nix/util/logging.hh" #include "nix/util/json-impls.hh" -#include "nix/util/split.hh" #include "nix/util/strings-inline.hh" #include diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index bddd17da8b..8099f7b8ba 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -5,7 +5,6 @@ #include "nix/store/common-protocol.hh" #include "nix/store/serve-protocol.hh" #include "nix/store/serve-protocol-impl.hh" -#include "nix/util/archive.hh" #include "nix/store/path-info.hh" #include "nix/util/json-utils.hh" diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 5f6119a427..3fd1f798ea 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,5 +1,5 @@ #include "nix/store/sqlite.hh" -#include "nix/store/globals.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/util.hh" #include "nix/util/url.hh" #include "nix/util/signals.hh" @@ -10,7 +10,6 @@ #include -#include #include namespace nix { diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 9fe1fdbe3f..945271b1a9 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -2,7 +2,6 @@ #include "nix/store/local-fs-store.hh" #include "nix/store/remote-store-connection.hh" #include "nix/util/source-accessor.hh" -#include "nix/util/archive.hh" #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-impl.hh" #include "nix/util/pool.hh" diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 0240f9cca3..30ed7397cb 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,5 +1,4 @@ #include "nix/store/ssh.hh" -#include "nix/util/finally.hh" #include "nix/util/current-process.hh" #include "nix/util/environment-variables.hh" #include "nix/util/os-string.hh" diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f465c959c8..3f367b19a1 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -24,8 +24,6 @@ #include #include -#include "nix/util/strings.hh" - #ifdef _WIN32 # include "nix/util/windows-known-folders.hh" #endif diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 1b3c09ba4a..e0b2d68ec3 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,8 +1,8 @@ #include "nix/store/uds-remote-store.hh" +#include "nix/util/environment-variables.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/store/worker-protocol.hh" #include "nix/store/store-registration.hh" -#include "nix/store/globals.hh" #include #include diff --git a/src/libstore/unix/pathlocks.cc b/src/libstore/unix/pathlocks.cc index 47e33853c9..5dc6cd236c 100644 --- a/src/libstore/unix/pathlocks.cc +++ b/src/libstore/unix/pathlocks.cc @@ -1,7 +1,5 @@ #include "nix/store/pathlocks.hh" #include "nix/util/file-system-at.hh" -#include "nix/util/util.hh" -#include "nix/util/sync.hh" #include "nix/util/signals.hh" #include diff --git a/src/libstore/unix/user-lock.cc b/src/libstore/unix/user-lock.cc index c9abdddcea..d5a959be2e 100644 --- a/src/libstore/unix/user-lock.cc +++ b/src/libstore/unix/user-lock.cc @@ -3,8 +3,8 @@ #include #include "nix/store/user-lock.hh" +#include "nix/store/local-settings.hh" #include "nix/util/file-system.hh" -#include "nix/store/globals.hh" #include "nix/store/pathlocks.hh" #include "nix/util/users.hh" #include "nix/util/logging.hh" diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 3da966a620..7f9bd163d9 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -6,7 +6,6 @@ #include "nix/store/common-protocol.hh" #include "nix/store/worker-protocol.hh" #include "nix/store/worker-protocol-impl.hh" -#include "nix/util/archive.hh" #include "nix/store/path-info.hh" #include "nix/util/json-utils.hh" diff --git a/src/libutil-test-support/hash.cc b/src/libutil-test-support/hash.cc index 853da8e907..fdc95a6f81 100644 --- a/src/libutil-test-support/hash.cc +++ b/src/libutil-test-support/hash.cc @@ -1,6 +1,6 @@ -#include - -#include // Needed by rapidcheck on Darwin +#ifdef __APPLE__ +# include // Needed by rapidcheck on Darwin +#endif #include #include "nix/util/hash.hh" diff --git a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh index d0f4f9c495..9bd4d7fbf1 100644 --- a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh @@ -7,6 +7,7 @@ #include "nix/util/types.hh" #include "nix/util/ref.hh" #include "nix/util/file-system.hh" +#include "nix/util/nar-accessor.hh" #include "nix/util/tests/characterization.hh" diff --git a/src/libutil-tests/args.cc b/src/libutil-tests/args.cc index 7aa996233a..f2af485bf4 100644 --- a/src/libutil-tests/args.cc +++ b/src/libutil-tests/args.cc @@ -1,5 +1,4 @@ #include "nix/util/args.hh" -#include "nix/util/fs-sink.hh" #include #include diff --git a/src/libutil-tests/config.cc b/src/libutil-tests/config.cc index 6e90dcf11b..b9eec0c417 100644 --- a/src/libutil-tests/config.cc +++ b/src/libutil-tests/config.cc @@ -1,7 +1,5 @@ #include "nix/util/configuration.hh" -#include "nix/util/args.hh" -#include #include #include diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 95bfa16b14..995b75814f 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -1,17 +1,10 @@ -#include "nix/util/util.hh" #include "nix/util/serialise.hh" -#include "nix/util/types.hh" #include "nix/util/file-system.hh" -#include "nix/util/processes.hh" -#include "nix/util/terminal.hh" -#include "nix/util/strings.hh" #include #include #include -#include - using namespace std::string_view_literals; #ifdef _WIN32 diff --git a/src/libutil-tests/hash.cc b/src/libutil-tests/hash.cc index a6bb52d195..51924f47f9 100644 --- a/src/libutil-tests/hash.cc +++ b/src/libutil-tests/hash.cc @@ -1,5 +1,3 @@ -#include - #include #include diff --git a/src/libutil-tests/monitorfdhup.cc b/src/libutil-tests/monitorfdhup.cc index 02cd7e22ca..f8c94dcdce 100644 --- a/src/libutil-tests/monitorfdhup.cc +++ b/src/libutil-tests/monitorfdhup.cc @@ -1,7 +1,6 @@ // TODO: investigate why this is hanging on cygwin #if !defined(_WIN32) && !defined(__CYGWIN__) -# include "nix/util/util.hh" # include "nix/util/monitor-fd.hh" # include diff --git a/src/libutil-tests/nar-listing.cc b/src/libutil-tests/nar-listing.cc index a2b8650481..3d2fbe9757 100644 --- a/src/libutil-tests/nar-listing.cc +++ b/src/libutil-tests/nar-listing.cc @@ -1,6 +1,5 @@ #include -#include "nix/util/nar-accessor.hh" #include "nix/util/tests/json-characterization.hh" namespace nix { diff --git a/src/libutil-tests/nix_api_util.cc b/src/libutil-tests/nix_api_util.cc index 48f85c403d..14e23138b7 100644 --- a/src/libutil-tests/nix_api_util.cc +++ b/src/libutil-tests/nix_api_util.cc @@ -1,5 +1,4 @@ #include "nix/util/config-global.hh" -#include "nix/util/args.hh" #include "nix_api_util.h" #include "nix/util/tests/nix_api_util.hh" #include "nix/util/tests/string_callback.hh" diff --git a/src/libutil-tests/nix_api_util_internal.cc b/src/libutil-tests/nix_api_util_internal.cc index 6fb0a623f6..ad61574186 100644 --- a/src/libutil-tests/nix_api_util_internal.cc +++ b/src/libutil-tests/nix_api_util_internal.cc @@ -1,5 +1,3 @@ -#include "nix/util/config-global.hh" -#include "nix/util/args.hh" #include "nix_api_util.h" #include "nix_api_util_internal.h" #include "nix/util/tests/nix_api_util.hh" @@ -9,8 +7,6 @@ #include -#include "util-tests-config.hh" - namespace nixC { TEST_F(nix_api_util_context, nix_context_error) diff --git a/src/libutil-tests/terminal.cc b/src/libutil-tests/terminal.cc index 198b209515..7f63dfdc58 100644 --- a/src/libutil-tests/terminal.cc +++ b/src/libutil-tests/terminal.cc @@ -1,13 +1,8 @@ -#include "nix/util/util.hh" -#include "nix/util/types.hh" #include "nix/util/terminal.hh" -#include "nix/util/strings.hh" #include #include -#include - namespace nix { /* ---------------------------------------------------------------------------- diff --git a/src/libutil-tests/topo-sort.cc b/src/libutil-tests/topo-sort.cc index 91030247e6..eadbe3e506 100644 --- a/src/libutil-tests/topo-sort.cc +++ b/src/libutil-tests/topo-sort.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include diff --git a/src/libutil-tests/util.cc b/src/libutil-tests/util.cc index a299cd9782..031dd7a444 100644 --- a/src/libutil-tests/util.cc +++ b/src/libutil-tests/util.cc @@ -1,15 +1,9 @@ #include "nix/util/util.hh" #include "nix/util/types.hh" -#include "nix/util/file-system.hh" -#include "nix/util/terminal.hh" -#include "nix/util/strings.hh" -#include "nix/util/base-n.hh" #include #include -#include - namespace nix { /* ----------- tests for util.hh --------------------------------------------*/ diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 53aebde70a..f704acdf78 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -1,6 +1,4 @@ #include -#include -#include #include #include // for strcasecmp diff --git a/src/libutil/args.cc b/src/libutil/args.cc index c025f119f7..0fa356f689 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -2,7 +2,6 @@ #include "nix/util/args/root.hh" #include "nix/util/hash.hh" #include "nix/util/environment-variables.hh" -#include "nix/util/signals.hh" #include "nix/util/users.hh" #include "nix/util/json-utils.hh" diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index cf2e26ba97..f07b22dcab 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,7 +1,6 @@ #include "nix/util/compression.hh" #include "nix/util/signals.hh" #include "nix/util/tarfile.hh" -#include "nix/util/finally.hh" #include "nix/util/logging.hh" #include diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 37afda89bb..177728a633 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -3,9 +3,7 @@ #include "nix/util/current-process.hh" #include "nix/util/util.hh" -#include "nix/util/finally.hh" #include "nix/util/file-system.hh" -#include "nix/util/processes.hh" #include "nix/util/signals.hh" #include "nix/util/environment-variables.hh" #include @@ -15,7 +13,6 @@ #endif #ifdef __linux__ -# include # include "nix/util/cgroup.hh" # include "nix/util/linux-namespaces.hh" #endif diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc index 9d44ad65d8..8c99e67155 100644 --- a/src/libutil/environment-variables.cc +++ b/src/libutil/environment-variables.cc @@ -1,4 +1,3 @@ -#include "nix/util/util.hh" #include "nix/util/environment-variables.hh" namespace nix { diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e36f46ee80..f43df1de52 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -10,7 +10,6 @@ #include #include #include -#include "nix/util/serialise.hh" #include namespace nix { diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index ec3520fd0c..4da0999715 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -1,7 +1,6 @@ #include "nix/util/environment-variables.hh" #include "nix/util/executable-path.hh" #include "nix/util/strings-inline.hh" -#include "nix/util/util.hh" #include "nix/util/file-path-impl.hh" namespace nix { diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index dfba75e500..b7e504e13a 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -1,4 +1,3 @@ -#include "nix/util/environment-variables.hh" #include "nix/util/file-system.hh" #include "nix/util/file-path.hh" #include "nix/util/file-path-impl.hh" diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 2646babfd5..96c6dd2879 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -1,7 +1,5 @@ #include #include -#include -#include #include #include // for strcasecmp diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 1f89114c66..2832de73c9 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -1,4 +1,3 @@ -#include #include #include @@ -8,7 +7,6 @@ #include "nix/util/args.hh" #include "nix/util/hash.hh" -#include "nix/util/archive.hh" #include "nix/util/configuration.hh" #include "nix/util/split.hh" #include "nix/util/base-n.hh" diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index 39dad9268b..9c96c5bcb7 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -1,14 +1,10 @@ #include "nix/util/linux-namespaces.hh" -#include "nix/util/current-process.hh" #include "nix/util/util.hh" -#include "nix/util/finally.hh" #include "nix/util/file-system.hh" #include "nix/util/processes.hh" -#include "nix/util/signals.hh" #include #include -#include "nix/util/cgroup.hh" #include diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index f1c8190f7d..58c78df349 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -4,7 +4,6 @@ #include "nix/util/terminal.hh" #include "nix/util/util.hh" #include "nix/util/config-global.hh" -#include "nix/util/source-path.hh" #include "nix/util/position.hh" #include "nix/util/sync.hh" #include "nix/util/unix-domain-socket.hh" @@ -12,7 +11,6 @@ #include #include #include -#include namespace nix { diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index f0f7a95229..6657291bb0 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -1,5 +1,4 @@ #include "nix/util/memory-source-accessor.hh" -#include "nix/util/json-utils.hh" #include diff --git a/src/libutil/nar-accessor.cc b/src/libutil/nar-accessor.cc index 6404128e9d..f27d1390f9 100644 --- a/src/libutil/nar-accessor.cc +++ b/src/libutil/nar-accessor.cc @@ -1,7 +1,6 @@ #include "nix/util/nar-accessor.hh" #include "nix/util/file-descriptor.hh" #include "nix/util/error.hh" -#include "nix/util/signals.hh" namespace nix { diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 45af4e88ff..6098552813 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -3,7 +3,6 @@ #include "nix/util/memory-source-accessor.hh" #include "nix/util/source-path.hh" #include "nix/util/signals.hh" -#include "nix/util/sync.hh" #include diff --git a/src/libutil/signature/signer.cc b/src/libutil/signature/signer.cc index fff03fc30d..9294b2a180 100644 --- a/src/libutil/signature/signer.cc +++ b/src/libutil/signature/signer.cc @@ -1,5 +1,4 @@ #include "nix/util/signature/signer.hh" -#include "nix/util/error.hh" #include diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index 91a0f73ec1..83ccf8b396 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -1,9 +1,6 @@ -#include #include -#include #include "nix/util/strings-inline.hh" -#include "nix/util/os-string.hh" #include "nix/util/error.hh" #include "nix/util/util.hh" diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 2367a12bf6..778b17cf0b 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -3,7 +3,6 @@ #include "nix/util/terminal.hh" #include -#include namespace nix { diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index 48d6928727..ebdc783e4a 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -13,7 +13,6 @@ #endif #include #include -#include #include // for ptsname and ptsname_r namespace { diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index 21339afe88..317bc33fff 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -1,14 +1,11 @@ #include "nix/util/file-system.hh" #include "nix/util/file-system-at.hh" #include "nix/util/signals.hh" -#include "nix/util/finally.hh" -#include "nix/util/serialise.hh" #include #include #include -#include "util-config-private.hh" #include "util-unix-config-private.hh" namespace nix { diff --git a/src/libutil/unix/file-path.cc b/src/libutil/unix/file-path.cc index 55ccddb456..01e89e1437 100644 --- a/src/libutil/unix/file-path.cc +++ b/src/libutil/unix/file-path.cc @@ -1,10 +1,4 @@ -#include -#include -#include -#include - #include "nix/util/file-path.hh" -#include "nix/util/util.hh" namespace nix { diff --git a/src/libutil/unix/muxable-pipe.cc b/src/libutil/unix/muxable-pipe.cc index 12c971f94d..3362fc34a6 100644 --- a/src/libutil/unix/muxable-pipe.cc +++ b/src/libutil/unix/muxable-pipe.cc @@ -1,6 +1,5 @@ #include -#include "nix/util/logging.hh" #include "nix/util/util.hh" #include "nix/util/muxable-pipe.hh" diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 9ae42498de..9f23da876a 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -13,7 +13,6 @@ #include #include #include -#include #include using namespace std::chrono_literals; @@ -31,7 +30,6 @@ using namespace std::chrono_literals; # include #endif -#include "util-config-private.hh" #include "util-unix-config-private.hh" namespace nix { diff --git a/src/libutil/unix/users.cc b/src/libutil/unix/users.cc index 5d05dc79f3..24ef6151c8 100644 --- a/src/libutil/unix/users.cc +++ b/src/libutil/unix/users.cc @@ -1,7 +1,7 @@ -#include "nix/util/util.hh" #include "nix/util/users.hh" #include "nix/util/environment-variables.hh" #include "nix/util/file-system.hh" +#include "nix/util/logging.hh" #include #include diff --git a/src/libutil/unix/xdg-dirs.cc b/src/libutil/unix/xdg-dirs.cc index cc66f59f3a..4384583cad 100644 --- a/src/libutil/unix/xdg-dirs.cc +++ b/src/libutil/unix/xdg-dirs.cc @@ -1,4 +1,4 @@ -#include "nix/util/util.hh" +#include "nix/util/strings.hh" #include "nix/util/users.hh" #include "nix/util/environment-variables.hh" diff --git a/src/libutil/users.cc b/src/libutil/users.cc index f05dfcf760..cc3fbb0e54 100644 --- a/src/libutil/users.cc +++ b/src/libutil/users.cc @@ -1,7 +1,5 @@ -#include "nix/util/util.hh" #include "nix/util/users.hh" #include "nix/util/environment-variables.hh" -#include "nix/util/executable-path.hh" #include "nix/util/file-system.hh" #ifndef _WIN32 diff --git a/src/libutil/util.cc b/src/libutil/util.cc index d75aa4d67d..8ed058d761 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,12 +1,9 @@ #include "nix/util/util.hh" #include "nix/util/fmt.hh" -#include "nix/util/file-path.hh" #include "nix/util/signals.hh" #include #include -#include -#include #include #include diff --git a/src/libutil/windows/current-process.cc b/src/libutil/windows/current-process.cc index 3d252257e1..b6805417be 100644 --- a/src/libutil/windows/current-process.cc +++ b/src/libutil/windows/current-process.cc @@ -1,5 +1,4 @@ #include "nix/util/current-process.hh" -#include "nix/util/error.hh" #include #ifdef _WIN32 diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 134d31fac3..2a24c4199f 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -1,8 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" #include "nix/store/store-api.hh" -#include "nix/util/archive.hh" -#include "nix/util/git.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/cmd/misc-store-flags.hh" diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index 332a3955d8..6436466374 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -5,7 +5,6 @@ #include #include #include -#include #ifdef __APPLE__ # include #endif diff --git a/src/nix/build.cc b/src/nix/build.cc index 4d2adfd5a6..2de7fdf423 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -2,7 +2,6 @@ #include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" -#include "nix/store/local-fs-store.hh" #include diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index a9266a11bd..85c8cdb587 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -1,6 +1,5 @@ #include "nix/cmd/installable-flake.hh" #include "nix/cmd/command-installable-value.hh" -#include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 09416b1f44..8a816f5288 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,7 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" #include "nix/util/archive.hh" -#include "nix/util/nar-accessor.hh" #include "nix/util/serialise.hh" #include "nix/util/source-accessor.hh" diff --git a/src/nix/config.cc b/src/nix/config.cc index c2a9fd8e2f..3fd0cde808 100644 --- a/src/nix/config.cc +++ b/src/nix/config.cc @@ -1,7 +1,5 @@ #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" -#include "nix/main/shared.hh" -#include "nix/store/store-api.hh" #include "nix/util/config-global.hh" #include diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc index 2dbc89d6f8..2046ecca38 100644 --- a/src/nix/derivation-add.cc +++ b/src/nix/derivation-add.cc @@ -3,7 +3,6 @@ #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" #include "nix/store/store-api.hh" -#include "nix/util/archive.hh" #include "nix/store/derivations.hh" #include "nix/store/globals.hh" #include diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc index ce2594ddca..1a6f00c8cd 100644 --- a/src/nix/derivation-show.cc +++ b/src/nix/derivation-show.cc @@ -4,7 +4,6 @@ #include "nix/cmd/command.hh" #include "nix/main/common-args.hh" #include "nix/store/store-api.hh" -#include "nix/util/archive.hh" #include "nix/store/derivations.hh" #include diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 29b44d7809..301e35b55f 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,7 +1,7 @@ +#include "nix/cmd/command.hh" #include "nix/util/config-global.hh" #include "nix/expr/eval.hh" #include "nix/cmd/installable-flake.hh" -#include "nix/cmd/command-installable-value.hh" #include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" @@ -14,7 +14,6 @@ # include "run.hh" #endif -#include #include #include #include diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index d36a21d746..ff37100651 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -1,7 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" -#include "nix/main/common-args.hh" #include "nix/store/names.hh" #include diff --git a/src/nix/formatter.cc b/src/nix/formatter.cc index 08f0b5f053..619dcf4fad 100644 --- a/src/nix/formatter.cc +++ b/src/nix/formatter.cc @@ -2,8 +2,6 @@ #include "nix/cmd/installable-flake.hh" #include "nix/cmd/installable-value.hh" #include "nix/expr/eval.hh" -#include "nix/store/local-fs-store.hh" -#include "nix/cmd/installable-derived-path.hh" #include "nix/util/environment-variables.hh" #include "nix/store/globals.hh" diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 43ace0d36c..bc1ce313b0 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -1,10 +1,8 @@ #include "nix/cmd/command.hh" #include "nix/util/hash.hh" -#include "nix/store/content-address.hh" #include "nix/cmd/legacy.hh" #include "nix/main/shared.hh" #include "nix/store/references.hh" -#include "nix/util/archive.hh" #include "nix/util/git.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/cmd/misc-store-flags.hh" diff --git a/src/nix/log.cc b/src/nix/log.cc index 8f251ddab1..acc03cbb67 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -1,6 +1,5 @@ #include "nix/cmd/command.hh" #include "nix/cmd/get-build-log.hh" -#include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/globals.hh" diff --git a/src/nix/main.cc b/src/nix/main.cc index 297318ddcd..c19aa26925 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -22,7 +22,6 @@ #include "nix/expr/eval-cache.hh" #include "nix/flake/flake.hh" #include "nix/flake/settings.hh" -#include "nix/util/json-utils.hh" #include "self-exe.hh" #include "crash-handler.hh" diff --git a/src/nix/nix-build/nix-build.cc b/src/nix/nix-build/nix-build.cc index 6f691faeb4..d9e2f2dc8d 100644 --- a/src/nix/nix-build/nix-build.cc +++ b/src/nix/nix-build/nix-build.cc @@ -1,5 +1,4 @@ #include -#include #include #include #include diff --git a/src/nix/nix-channel/nix-channel.cc b/src/nix/nix-channel/nix-channel.cc index 05adc3d2cd..911aabd438 100644 --- a/src/nix/nix-channel/nix-channel.cc +++ b/src/nix/nix-channel/nix-channel.cc @@ -9,7 +9,6 @@ #include "nix/util/os-string.hh" #include "nix/util/users.hh" #include "nix/fetchers/tarball.hh" -#include "nix/fetchers/fetch-settings.hh" #include "self-exe.hh" #include "man-pages.hh" diff --git a/src/nix/nix-collect-garbage/nix-collect-garbage.cc b/src/nix/nix-collect-garbage/nix-collect-garbage.cc index 3d84ca13cd..a9f8b7fb6b 100644 --- a/src/nix/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix/nix-collect-garbage/nix-collect-garbage.cc @@ -10,7 +10,6 @@ #include "nix/cmd/legacy.hh" #include "man-pages.hh" -#include #include using namespace nix; diff --git a/src/nix/nix-env/user-env.cc b/src/nix/nix-env/user-env.cc index f87e7f6103..cb74a36d0d 100644 --- a/src/nix/nix-env/user-env.cc +++ b/src/nix/nix-env/user-env.cc @@ -3,7 +3,6 @@ #include "nix/store/store-api.hh" #include "nix/store/path-with-outputs.hh" #include "nix/store/local-fs-store.hh" -#include "nix/store/globals.hh" #include "nix/main/shared.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-inline.hh" @@ -11,7 +10,6 @@ #include "nix/expr/print-ambiguous.hh" #include "nix/expr/static-string-data.hh" -#include #include namespace nix { diff --git a/src/nix/nix-instantiate/nix-instantiate.cc b/src/nix/nix-instantiate/nix-instantiate.cc index 82838e2ae8..b368346eea 100644 --- a/src/nix/nix-instantiate/nix-instantiate.cc +++ b/src/nix/nix-instantiate/nix-instantiate.cc @@ -5,7 +5,6 @@ #include "nix/expr/eval-inline.hh" #include "nix/expr/get-drvs.hh" #include "nix/expr/attr-path.hh" -#include "nix/util/signals.hh" #include "nix/expr/value-to-xml.hh" #include "nix/expr/value-to-json.hh" #include "nix/store/store-open.hh" @@ -14,7 +13,6 @@ #include "nix/cmd/legacy.hh" #include "man-pages.hh" -#include #include using namespace nix; diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index e932690679..a534e19afb 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -35,7 +35,6 @@ #include #include -#include "nix/store/build-result.hh" #include "nix/util/exit.hh" #include "nix/store/serve-protocol-impl.hh" diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index e000026fcc..5edbabc3bd 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -2,8 +2,6 @@ #include "nix/main/shared.hh" #include "nix/store/store-api.hh" -#include - using namespace nix; struct CmdOptimiseStore : StoreCommand diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 3e36197b95..9abe38bfdd 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -5,7 +5,6 @@ #include "nix/store/nar-info.hh" #include -#include #include diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3a7e448cda..1ace94a8b7 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -3,7 +3,6 @@ #include "nix/main/shared.hh" #include "nix/store/store-open.hh" #include "nix/store/filetransfer.hh" -#include "nix/util/finally.hh" #include "nix/main/loggers.hh" #include "nix/util/tarfile.hh" #include "nix/expr/attr-path.hh" @@ -11,7 +10,6 @@ #include "nix/cmd/legacy.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/cmd/misc-store-flags.hh" -#include "nix/util/terminal.hh" #include "nix/util/environment-variables.hh" #include "nix/util/url.hh" #include "nix/store/path.hh" diff --git a/src/nix/registry.cc b/src/nix/registry.cc index c943e80f9e..5f77700d65 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -1,8 +1,6 @@ #include "nix/cmd/command.hh" -#include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/expr/eval.hh" -#include "nix/flake/flake.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/registry.hh" diff --git a/src/nix/repl.cc b/src/nix/repl.cc index ea1ecb305a..85192faa64 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,8 +1,6 @@ #include "nix/expr/eval.hh" #include "nix/expr/eval-settings.hh" #include "nix/util/config-global.hh" -#include "nix/store/globals.hh" -#include "nix/store/store-open.hh" #include "nix/cmd/command.hh" #include "nix/cmd/installable-value.hh" #include "nix/cmd/repl.hh" diff --git a/src/nix/run.cc b/src/nix/run.cc index 38dbc42b68..a87a7679d4 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,14 +1,12 @@ #include "nix/util/current-process.hh" #include "run.hh" #include "nix/cmd/command-installable-value.hh" -#include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/util/signals.hh" #include "nix/store/store-api.hh" #include "nix/store/derivations.hh" #include "nix/store/local-fs-store.hh" #include "nix/util/finally.hh" -#include "nix/util/source-accessor.hh" #include "nix/expr/eval.hh" #include "nix/util/util.hh" #include "nix/store/globals.hh" @@ -20,8 +18,6 @@ # include "nix/store/personality.hh" #endif -#include - extern char ** environ __attribute__((weak)); using namespace nix; diff --git a/src/nix/search.cc b/src/nix/search.cc index dac60ceba5..4f5ae8cafa 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -4,7 +4,6 @@ #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" #include "nix/store/names.hh" -#include "nix/expr/get-drvs.hh" #include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/expr/eval-cache.hh" @@ -13,7 +12,6 @@ #include "nix/util/strings-inline.hh" #include -#include #include #include "nix/util/strings.hh" diff --git a/src/nix/self-exe.cc b/src/nix/self-exe.cc index 36f6e17ec8..d799181001 100644 --- a/src/nix/self-exe.cc +++ b/src/nix/self-exe.cc @@ -1,6 +1,5 @@ #include "nix/util/current-process.hh" -#include "nix/util/file-system.hh" -#include "nix/store/globals.hh" +#include "nix/util/environment-variables.hh" #include "self-exe.hh" #include "cli-config-private.hh" diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index 6e442f3713..cbd4e04f6a 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -3,10 +3,6 @@ #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" #include "nix/store/log-store.hh" -#include "nix/util/sync.hh" -#include "nix/util/thread-pool.hh" - -#include using namespace nix; diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 3e2e0f2ba5..5d28979a49 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -1,5 +1,4 @@ #include "nix/cmd/command.hh" -#include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" #include "nix/store/store-cast.hh" diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index 971be26be5..f4624a40e8 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -5,7 +5,6 @@ #include "nix/store/store-cast.hh" #include "nix/store/gc-store.hh" #include "nix/util/error.hh" -#include "nix/util/logging.hh" using namespace nix; diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index a8cfb089fc..278eaef374 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -1,25 +1,22 @@ ///@file #include "nix/util/signals.hh" -#include "nix/util/unix-domain-socket.hh" #include "nix/cmd/command.hh" #include "nix/main/shared.hh" -#include "nix/store/local-fs-store.hh" #include "nix/store/local-store.hh" #include "nix/store/uds-remote-store.hh" #include "nix/store/remote-store.hh" #include "nix/store/remote-store-connection.hh" #include "nix/store/store-open.hh" #include "nix/util/serialise.hh" -#include "nix/util/archive.hh" #include "nix/store/globals.hh" #include "nix/util/config-global.hh" #include "nix/store/derivations.hh" -#include "nix/util/finally.hh" #include "nix/cmd/legacy.hh" #include "nix/cmd/unix-socket-server.hh" #include "nix/store/daemon.hh" #include "man-pages.hh" +#include "nix/util/socket.hh" #include #include From b7c97d2043471b0d447a809b4332e0d3396c5ee5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 9 Apr 2026 16:09:47 -0400 Subject: [PATCH 222/927] `linux::openat2` wrapper return `std::optional` not `std::optional` This is because it is returning a (newly opened) owned not borrowed descriptor. This is a follow up from 89dd96efbfe55d5e56f99b85cb4f67dbf3d548cc when we did the same thing, to the same reason, to other `open*` functions. --- src/libutil/include/nix/util/file-system-at.hh | 3 ++- src/libutil/unix/file-system-at.cc | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libutil/include/nix/util/file-system-at.hh b/src/libutil/include/nix/util/file-system-at.hh index c78cafe6a5..75f4e6b2cf 100644 --- a/src/libutil/include/nix/util/file-system-at.hh +++ b/src/libutil/include/nix/util/file-system-at.hh @@ -120,7 +120,8 @@ namespace linux { * * @return nullopt if openat2 is not supported by the kernel. */ -std::optional openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve); +std::optional +openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve); } // namespace linux #endif diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index d8be6fdeb7..01a4c664d3 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -29,7 +29,7 @@ namespace nix { namespace linux { -std::optional openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve) +std::optional openat2(Descriptor dirFd, const char * path, uint64_t flags, uint64_t mode, uint64_t resolve) { # if HAVE_OPENAT2 /* Cache the result of whether openat2 is not supported. */ @@ -47,7 +47,7 @@ std::optional openat2(Descriptor dirFd, const char * path, uint64_t return std::nullopt; } - return res; + return AutoCloseFD{static_cast(res)}; } # endif return std::nullopt; @@ -202,9 +202,9 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & auto maybeFd = linux::openat2( dirFd, path.rel_c_str(), flags, static_cast(mode), RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS); if (maybeFd) { - if (*maybeFd < 0 && errno == ELOOP) + if (!*maybeFd && errno == ELOOP) throw SymlinkNotAllowed(path); - return AutoCloseFD{*maybeFd}; + return std::move(*maybeFd); } #endif return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode); From 5f1f60ff2b200c1dd2c9f4c6a38ec37a8d29dcd2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 10 Apr 2026 11:33:21 -0400 Subject: [PATCH 223/927] Fix Windows Build This header was accidentally removed in 303fc45c71cb7ac6f366ac8901dab92e428c9355. --- src/libutil/windows/current-process.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/windows/current-process.cc b/src/libutil/windows/current-process.cc index b6805417be..3d252257e1 100644 --- a/src/libutil/windows/current-process.cc +++ b/src/libutil/windows/current-process.cc @@ -1,4 +1,5 @@ #include "nix/util/current-process.hh" +#include "nix/util/error.hh" #include #ifdef _WIN32 From f7b59de6217fdc63a784ee6d0f337d45f59ffedf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 10 Apr 2026 11:33:21 -0400 Subject: [PATCH 224/927] Femove a bunch of redundant `#ifdef _WIN32` in Windows-only files This was always redundant, but was formally needed to run `clang-tidy`. Now we have a better `clang-tidy` setup that doesn't need this hack. --- src/libstore/windows/pathlocks.cc | 8 +++----- src/libutil/windows/current-process.cc | 6 ++---- src/libutil/windows/environment-variables.cc | 4 +--- .../windows/include/nix/util/windows-async-pipe.hh | 2 -- src/libutil/windows/muxable-pipe.cc | 10 ++++------ src/libutil/windows/os-string.cc | 4 ---- src/libutil/windows/processes.cc | 8 ++------ src/libutil/windows/users.cc | 6 ++---- src/libutil/windows/windows-async-pipe.cc | 7 +------ src/libutil/windows/windows-error.cc | 11 ++++------- 10 files changed, 19 insertions(+), 47 deletions(-) diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 8021bb072a..f048986c75 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -5,10 +5,9 @@ #include "nix/util/util.hh" #include "nix/util/windows-environment.hh" -#ifdef _WIN32 -# include -# include -# include +#include +#include +#include namespace nix { @@ -167,4 +166,3 @@ FdLock::FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view w } } // namespace nix -#endif diff --git a/src/libutil/windows/current-process.cc b/src/libutil/windows/current-process.cc index 3d252257e1..4d1ecdc1d7 100644 --- a/src/libutil/windows/current-process.cc +++ b/src/libutil/windows/current-process.cc @@ -2,9 +2,8 @@ #include "nix/util/error.hh" #include -#ifdef _WIN32 -# define WIN32_LEAN_AND_MEAN -# include +#define WIN32_LEAN_AND_MEAN +#include namespace nix { @@ -31,4 +30,3 @@ std::chrono::microseconds getCpuUserTime() } } // namespace nix -#endif // ifdef _WIN32 diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 34825793e4..767c5cd40e 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -1,7 +1,6 @@ #include "nix/util/environment-variables.hh" -#ifdef _WIN32 -# include +#include namespace nix { @@ -85,4 +84,3 @@ int setEnvOs(const OsString & name, const OsString & value) } } // namespace nix -#endif diff --git a/src/libutil/windows/include/nix/util/windows-async-pipe.hh b/src/libutil/windows/include/nix/util/windows-async-pipe.hh index e700aa4f63..7fd94313a7 100644 --- a/src/libutil/windows/include/nix/util/windows-async-pipe.hh +++ b/src/libutil/windows/include/nix/util/windows-async-pipe.hh @@ -2,7 +2,6 @@ ///@file #include "nix/util/file-descriptor.hh" -#ifdef _WIN32 namespace nix::windows { @@ -26,4 +25,3 @@ public: }; } // namespace nix::windows -#endif diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc index 5c427d67ae..1e89ccc5e3 100644 --- a/src/libutil/windows/muxable-pipe.cc +++ b/src/libutil/windows/muxable-pipe.cc @@ -1,9 +1,8 @@ -#ifdef _WIN32 -# include +#include -# include "nix/util/logging.hh" -# include "nix/util/util.hh" -# include "nix/util/muxable-pipe.hh" +#include "nix/util/logging.hh" +#include "nix/util/util.hh" +#include "nix/util/muxable-pipe.hh" namespace nix { @@ -70,4 +69,3 @@ void MuxablePipePollState::iterate( } } // namespace nix -#endif diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc index 2fface4195..f0fdb0c02b 100644 --- a/src/libutil/windows/os-string.cc +++ b/src/libutil/windows/os-string.cc @@ -5,8 +5,6 @@ #include "nix/util/os-string.hh" -#ifdef _WIN32 - namespace nix { std::string os_string_to_string(OsStringView s) @@ -32,5 +30,3 @@ OsString string_to_os_string(std::string s) } } // namespace nix - -#endif diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 9c7fa5d35f..c0b0fa6c10 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -25,10 +25,8 @@ #include #include -#ifdef _WIN32 - -# define WIN32_LEAN_AND_MEAN -# include +#define WIN32_LEAN_AND_MEAN +#include namespace nix { @@ -404,5 +402,3 @@ int execvpe(const wchar_t * file0, const wchar_t * const argv[], const wchar_t * } } // namespace nix - -#endif diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index caab6745d4..e5acd8cbdc 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -3,9 +3,8 @@ #include "nix/util/environment-variables.hh" #include "nix/util/file-system.hh" -#ifdef _WIN32 -# define WIN32_LEAN_AND_MEAN -# include +#define WIN32_LEAN_AND_MEAN +#include namespace nix { @@ -50,4 +49,3 @@ bool isRootUser() } } // namespace nix -#endif diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc index 09b72277a9..960dd63f80 100644 --- a/src/libutil/windows/windows-async-pipe.cc +++ b/src/libutil/windows/windows-async-pipe.cc @@ -1,7 +1,4 @@ - - -#ifdef _WIN32 -# include "nix/util/windows-async-pipe.hh" +#include "nix/util/windows-async-pipe.hh" namespace nix::windows { @@ -49,5 +46,3 @@ void AsyncPipe::close() } } // namespace nix::windows - -#endif diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index f04b70a216..85fcc1c1e5 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -1,10 +1,8 @@ -#ifdef _WIN32 +#include "nix/util/error.hh" -# include "nix/util/error.hh" - -# include -# define WIN32_LEAN_AND_MEAN -# include +#include +#define WIN32_LEAN_AND_MEAN +#include namespace nix::windows { @@ -33,4 +31,3 @@ std::string WinError::renderError(DWORD lastError) } } // namespace nix::windows -#endif From 00298039678721b4d11fb0796bf7c62fcb859389 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 10 Apr 2026 13:06:58 -0400 Subject: [PATCH 225/927] Replace top-level `using namespace nix...;` Instead of pulling in the `nix` namespace at file scope with `using namespace nix;`, wrap the code in `namespace nix { }` blocks. This is cleaner and avoids polluting the global namespace. Sub-namespace `using namespace nix::*;` declarations are either replaced with explicit qualification (e.g. `linux::getCgroupFS()`) or moved into the functions that need them. All remaining `using namespace` directives now use the fully qualified form (e.g. `using namespace nix::fetchers;` rather than `using namespace fetchers;`). Also remove `using std::cin;` and `using std::cout;` in `nix-store.cc`, qualifying them explicitly as `std::cin`/`std::cout` instead. Some POSIX functions (`read`, `write`) that could collide with `nix::` members now use `::` global scope qualification. (`pread` won't collide, but it doesn't hurt to use `::pread` too for symmetry.) --- src/libexpr/lexer.l | 2 + src/libflake/flake.cc | 2 - src/libflake/flakeref.cc | 2 +- .../unix/build/linux-derivation-builder.cc | 10 ++--- src/libstore/windows/pathlocks.cc | 3 +- src/libutil-test-support/hash.cc | 1 + .../include/nix/util/tests/hash.hh | 1 + src/libutil-tests/checked-arithmetic.cc | 1 + src/libutil-tests/git.cc | 37 ++++++++++------ src/libutil-tests/unix/file-system-at.cc | 6 ++- src/libutil-tests/unix/unix-domain-socket.cc | 6 ++- src/libutil/unix/file-descriptor.cc | 2 +- .../unix/include/nix/util/signals-impl.hh | 2 +- src/libutil/unix/signals.cc | 3 +- src/libutil/windows/file-descriptor.cc | 17 ++++--- src/libutil/windows/file-system-at.cc | 22 +++++----- src/libutil/windows/file-system.cc | 10 ++--- src/libutil/windows/known-folders.cc | 2 - src/libutil/windows/muxable-pipe.cc | 6 +-- src/libutil/windows/processes.cc | 20 ++++----- src/libutil/windows/users.cc | 6 +-- src/nix/add-to-store.cc | 4 +- src/nix/build-remote/build-remote.cc | 6 ++- src/nix/build.cc | 4 +- src/nix/bundle.cc | 4 +- src/nix/cat.cc | 4 +- src/nix/config-check.cc | 4 +- src/nix/config.cc | 4 +- src/nix/copy.cc | 4 +- src/nix/derivation-add.cc | 5 ++- src/nix/derivation-show.cc | 5 ++- src/nix/derivation.cc | 4 +- src/nix/develop.cc | 4 +- src/nix/diff-closures.cc | 6 +-- src/nix/dump-path.cc | 4 +- src/nix/edit.cc | 4 +- src/nix/env.cc | 4 +- src/nix/eval.cc | 4 +- src/nix/flake-command.hh | 4 +- src/nix/flake-prefetch-inputs.cc | 6 ++- src/nix/flake.cc | 26 +++++------ src/nix/formatter.cc | 4 +- src/nix/hash.cc | 4 +- src/nix/log.cc | 4 +- src/nix/ls.cc | 4 +- src/nix/main.cc | 8 ++-- src/nix/make-content-addressed.cc | 6 ++- src/nix/nar.cc | 4 +- src/nix/nix-build/nix-build.cc | 5 ++- src/nix/nix-channel/nix-channel.cc | 4 +- .../nix-collect-garbage.cc | 4 +- src/nix/nix-copy-closure/nix-copy-closure.cc | 4 +- src/nix/nix-env/nix-env.cc | 11 ++--- src/nix/nix-instantiate/nix-instantiate.cc | 4 +- src/nix/nix-store/nix-store.cc | 44 +++++++++---------- src/nix/optimise-store.cc | 4 +- src/nix/path-from-hash-part.cc | 4 +- src/nix/path-info.cc | 5 ++- src/nix/prefetch.cc | 4 +- src/nix/profile.cc | 4 +- src/nix/realisation.cc | 4 +- src/nix/registry.cc | 5 ++- src/nix/run.cc | 8 ++-- src/nix/search.cc | 5 ++- src/nix/sigs.cc | 4 +- src/nix/store-copy-log.cc | 4 +- src/nix/store-delete.cc | 4 +- src/nix/store-gc.cc | 4 +- src/nix/store-info.cc | 4 +- src/nix/store-repair.cc | 4 +- src/nix/store.cc | 4 +- src/nix/unix/daemon.cc | 13 +++--- src/nix/unix/store-roots-daemon.cc | 4 +- src/nix/upgrade-nix.cc | 4 +- src/nix/verify.cc | 4 +- src/nix/why-depends.cc | 4 +- tests/functional/plugins/plugintest.cc | 4 +- .../functional/test-libstoreconsumer/main.cc | 4 +- 78 files changed, 293 insertions(+), 196 deletions(-) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 5bdb5335b8..477eee1df9 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -33,6 +33,8 @@ namespace nix { struct LexerState; } +// OK because we keep this in a separate compilation unit even for unity +// builds. using namespace nix; using namespace nix::lexer::internal; diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 277adc092f..3d2351be08 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -63,8 +63,6 @@ namespace nix { struct SourceAccessor; -using namespace flake; - namespace flake { static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index 2960a158e3..01b5be1a82 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -270,7 +270,7 @@ std::pair parseFlakeRefWithFragment( bool isFlake, bool preserveRelativePaths) { - using namespace fetchers; + using namespace nix::fetchers; if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) { return *res; diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index c71d23e15e..0bc32a91e6 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -37,8 +37,6 @@ namespace nix { -using namespace nix::linux; - static void setupSeccomp(const LocalSettings & localSettings) { if (!localSettings.filterSyscalls) @@ -337,10 +335,10 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu /* If we're running from the daemon, then this will return the root cgroup of the service. Otherwise, it will return the current cgroup. */ - auto cgroupFS = getCgroupFS(); + auto cgroupFS = linux::getCgroupFS(); if (!cgroupFS) throw Error("cannot determine the cgroups file system"); - auto rootCgroupPath = *cgroupFS / getRootCgroup().rel(); + auto rootCgroupPath = *cgroupFS / linux::getRootCgroup().rel(); if (!pathExists(rootCgroupPath)) throw Error("expected cgroup directory %s", PathFmt(rootCgroupPath)); @@ -363,7 +361,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu if (pathExists(cgroupFile)) { auto prevCgroup = readFile(cgroupFile); - destroyCgroup(prevCgroup); + linux::destroyCgroup(prevCgroup); } writeFile(cgroupFile, cgroup->native()); @@ -825,7 +823,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu void killSandbox(bool getStats) override { if (cgroup) { - auto stats = destroyCgroup(*cgroup); + auto stats = linux::destroyCgroup(*cgroup); if (getStats) { buildResult.cpuUser = stats.cpuUser; buildResult.cpuSystem = stats.cpuSystem; diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index f048986c75..fe0e056d7c 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -11,8 +11,6 @@ namespace nix { -using namespace nix::windows; - void deleteLockFile(const std::filesystem::path & path, Descriptor desc) { @@ -59,6 +57,7 @@ AutoCloseFD openLockFile(const std::filesystem::path & path, bool create) template static bool warnOrThrowWine(DWORD lastError, const std::string & fs, const Args &... args) { + using namespace nix::windows; if (isWine()) { warn(fs + ": %s (ignored under Wine)", args..., lastError); return true; diff --git a/src/libutil-test-support/hash.cc b/src/libutil-test-support/hash.cc index fdc95a6f81..d9c7a0f747 100644 --- a/src/libutil-test-support/hash.cc +++ b/src/libutil-test-support/hash.cc @@ -8,6 +8,7 @@ #include "nix/util/tests/hash.hh" namespace rc { + using namespace nix; Gen Arbitrary::arbitrary() diff --git a/src/libutil-test-support/include/nix/util/tests/hash.hh b/src/libutil-test-support/include/nix/util/tests/hash.hh index 633f7bbf76..ee676f3eb2 100644 --- a/src/libutil-test-support/include/nix/util/tests/hash.hh +++ b/src/libutil-test-support/include/nix/util/tests/hash.hh @@ -6,6 +6,7 @@ #include "nix/util/hash.hh" namespace rc { + using namespace nix; template<> diff --git a/src/libutil-tests/checked-arithmetic.cc b/src/libutil-tests/checked-arithmetic.cc index 2b5970fb64..cc88d35393 100644 --- a/src/libutil-tests/checked-arithmetic.cc +++ b/src/libutil-tests/checked-arithmetic.cc @@ -10,6 +10,7 @@ #include "nix/util/tests/gtest-with-params.hh" namespace rc { + using namespace nix; template diff --git a/src/libutil-tests/git.cc b/src/libutil-tests/git.cc index dc7bd8fed4..81e34a065c 100644 --- a/src/libutil-tests/git.cc +++ b/src/libutil-tests/git.cc @@ -7,8 +7,6 @@ namespace nix { -using namespace git; - class GitTest : public CharacterizationTest { std::filesystem::path unitTestData = getUnitTestData() / "git"; @@ -36,6 +34,7 @@ class GitTest : public CharacterizationTest TEST(GitMode, gitMode_directory) { + using namespace git; Mode m = Mode::Directory; RawMode r = 0040000; ASSERT_EQ(static_cast(m), r); @@ -44,6 +43,7 @@ TEST(GitMode, gitMode_directory) TEST(GitMode, gitMode_executable) { + using namespace git; Mode m = Mode::Executable; RawMode r = 0100755; ASSERT_EQ(static_cast(m), r); @@ -52,6 +52,7 @@ TEST(GitMode, gitMode_executable) TEST(GitMode, gitMode_regular) { + using namespace git; Mode m = Mode::Regular; RawMode r = 0100644; ASSERT_EQ(static_cast(m), r); @@ -60,6 +61,7 @@ TEST(GitMode, gitMode_regular) TEST(GitMode, gitMode_symlink) { + using namespace git; Mode m = Mode::Symlink; RawMode r = 0120000; ASSERT_EQ(static_cast(m), r); @@ -68,6 +70,7 @@ TEST(GitMode, gitMode_symlink) TEST_F(GitTest, blob_read) { + using namespace git; readTest("hello-world-blob.bin", [&](const auto & encoded) { StringSource in{encoded}; StringSink out; @@ -83,6 +86,7 @@ TEST_F(GitTest, blob_read) TEST_F(GitTest, blob_write) { + using namespace git; writeTest("hello-world-blob.bin", [&]() { auto decoded = readFile(goldenMaster("hello-world.bin")); StringSink s; @@ -97,11 +101,11 @@ TEST_F(GitTest, blob_write) * so that we can check our test data in a small shell script test test * (`src/libutil-tests/data/git/check-data.sh`). */ -const static Tree treeSha1 = { +const static git::Tree treeSha1 = { { "Foo", { - .mode = Mode::Regular, + .mode = git::Mode::Regular, // hello world with special chars from above .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), }, @@ -109,7 +113,7 @@ const static Tree treeSha1 = { { "bAr", { - .mode = Mode::Executable, + .mode = git::Mode::Executable, // ditto .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), }, @@ -117,7 +121,7 @@ const static Tree treeSha1 = { { "baZ/", { - .mode = Mode::Directory, + .mode = git::Mode::Directory, // Empty directory hash .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", HashAlgorithm::SHA1), }, @@ -125,7 +129,7 @@ const static Tree treeSha1 = { { "quuX", { - .mode = Mode::Symlink, + .mode = git::Mode::Symlink, // hello world with special chars from above (symlink target // can be anything) .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), @@ -137,11 +141,11 @@ const static Tree treeSha1 = { * Same conceptual object as `treeSha1`, just different hash algorithm. * See that one for details. */ -const static Tree treeSha256 = { +const static git::Tree treeSha256 = { { "Foo", { - .mode = Mode::Regular, + .mode = git::Mode::Regular, .hash = Hash::parseAny( "ce60f5ad78a08ac24872ef74d78b078f077be212e7a246893a1a5d957dfbc8b1", HashAlgorithm::SHA256), }, @@ -149,7 +153,7 @@ const static Tree treeSha256 = { { "bAr", { - .mode = Mode::Executable, + .mode = git::Mode::Executable, .hash = Hash::parseAny( "ce60f5ad78a08ac24872ef74d78b078f077be212e7a246893a1a5d957dfbc8b1", HashAlgorithm::SHA256), }, @@ -157,7 +161,7 @@ const static Tree treeSha256 = { { "baZ/", { - .mode = Mode::Directory, + .mode = git::Mode::Directory, .hash = Hash::parseAny( "6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321", HashAlgorithm::SHA256), }, @@ -165,15 +169,16 @@ const static Tree treeSha256 = { { "quuX", { - .mode = Mode::Symlink, + .mode = git::Mode::Symlink, .hash = Hash::parseAny( "ce60f5ad78a08ac24872ef74d78b078f077be212e7a246893a1a5d957dfbc8b1", HashAlgorithm::SHA256), }, }, }; -static auto mkTreeReadTest(HashAlgorithm hashAlgo, Tree tree, const ExperimentalFeatureSettings & mockXpSettings) +static auto mkTreeReadTest(HashAlgorithm hashAlgo, git::Tree tree, const ExperimentalFeatureSettings & mockXpSettings) { + using namespace git; return [hashAlgo, tree, mockXpSettings](const auto & encoded) { StringSource in{encoded}; NullFileSystemObjectSink out; @@ -208,6 +213,7 @@ TEST_F(GitTest, tree_sha256_read) TEST_F(GitTest, tree_sha1_write) { + using namespace git; writeTest("tree-sha1.bin", [&]() { StringSink s; dumpTree(treeSha1, s, mockXpSettings); @@ -217,6 +223,7 @@ TEST_F(GitTest, tree_sha1_write) TEST_F(GitTest, tree_sha256_write) { + using namespace git; writeTest("tree-sha256.bin", [&]() { StringSink s; dumpTree(treeSha256, s, mockXpSettings); @@ -232,6 +239,7 @@ extern ref exampleComplex(); TEST_F(GitTest, both_roundrip) { + using namespace git; auto files = memory_source_accessor::exampleComplex(); for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) { @@ -285,6 +293,7 @@ TEST_F(GitTest, both_roundrip) TEST(GitLsRemote, parseSymrefLineWithReference) { + using namespace git; auto line = "ref: refs/head/main HEAD"; auto res = parseLsRemoteLine(line); ASSERT_TRUE(res.has_value()); @@ -295,6 +304,7 @@ TEST(GitLsRemote, parseSymrefLineWithReference) TEST(GitLsRemote, parseSymrefLineWithNoReference) { + using namespace git; auto line = "ref: refs/head/main"; auto res = parseLsRemoteLine(line); ASSERT_TRUE(res.has_value()); @@ -305,6 +315,7 @@ TEST(GitLsRemote, parseSymrefLineWithNoReference) TEST(GitLsRemote, parseObjectRefLine) { + using namespace git; auto line = "abc123 refs/head/main"; auto res = parseLsRemoteLine(line); ASSERT_TRUE(res.has_value()); diff --git a/src/libutil-tests/unix/file-system-at.cc b/src/libutil-tests/unix/file-system-at.cc index 5fb0d26b9c..09e427a26f 100644 --- a/src/libutil-tests/unix/file-system-at.cc +++ b/src/libutil-tests/unix/file-system-at.cc @@ -15,14 +15,14 @@ namespace nix { -using namespace nix::unix; - /* ---------------------------------------------------------------------------- * fchmodatTryNoFollow * --------------------------------------------------------------------------*/ TEST(fchmodatTryNoFollow, works) { + using namespace nix::unix; + std::filesystem::path tmpDir = nix::createTempDir(); nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true); @@ -106,6 +106,8 @@ TEST(fchmodatTryNoFollow, works) TEST(fchmodatTryNoFollow, fallbackWithoutProc) { + using namespace nix::unix; + if (!userNamespacesSupported()) GTEST_SKIP() << "User namespaces not supported"; diff --git a/src/libutil-tests/unix/unix-domain-socket.cc b/src/libutil-tests/unix/unix-domain-socket.cc index 6c50eab9b2..439d3188ac 100644 --- a/src/libutil-tests/unix/unix-domain-socket.cc +++ b/src/libutil-tests/unix/unix-domain-socket.cc @@ -9,14 +9,14 @@ namespace nix { -using namespace nix::unix; - /* ---------------------------------------------------------------------------- * sendMessageWithFds / receiveMessageWithFds * --------------------------------------------------------------------------*/ TEST(MessageWithFds, streamWithData) { + using namespace nix::unix; + int sockets[2]; ASSERT_EQ(socketpair(AF_UNIX, SOCK_STREAM, 0, sockets), 0); AutoCloseFD sender(sockets[0]); @@ -77,6 +77,8 @@ TEST(MessageWithFds, streamWithData) TEST(MessageWithFds, datagramEmptyData) { + using namespace nix::unix; + int sockets[2]; ASSERT_EQ(socketpair(AF_UNIX, SOCK_DGRAM, 0, sockets), 0); AutoCloseFD sender(sockets[0]); diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index 317bc33fff..545f4f2b70 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -33,7 +33,7 @@ size_t readOffset(Descriptor fd, off_t offset, std::span buffer) ssize_t n; do { checkInterrupt(); - n = pread(fd, buffer.data(), buffer.size(), offset); + n = ::pread(fd, buffer.data(), buffer.size(), offset); } while (n == -1 && errno == EINTR); if (n == -1) throw SysError("pread of %1% bytes at offset %2%", buffer.size(), offset); diff --git a/src/libutil/unix/include/nix/util/signals-impl.hh b/src/libutil/unix/include/nix/util/signals-impl.hh index dab0b09ed2..e222682144 100644 --- a/src/libutil/unix/include/nix/util/signals-impl.hh +++ b/src/libutil/unix/include/nix/util/signals-impl.hh @@ -77,7 +77,7 @@ static inline bool getInterrupted() static inline bool isInterrupted() { - using namespace unix; + using namespace nix::unix; return _isInterrupted || (interruptCheck && interruptCheck()); } diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index e236a3cc2a..7e425ab090 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -9,8 +9,6 @@ namespace nix { -using namespace unix; - std::atomic unix::_isInterrupted = false; thread_local std::function unix::interruptCheck; @@ -56,6 +54,7 @@ static Sync & getInterruptCallbacks() static void signalHandlerThread(sigset_t set) { + using namespace nix::unix; while (true) { int signal = 0; sigwait(&set, &signal); diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 0020623e31..c571d0a3e9 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -14,13 +14,11 @@ namespace nix { -using namespace nix::windows; - std::make_unsigned_t getFileSize(Descriptor fd) { LARGE_INTEGER li; if (!GetFileSizeEx(fd, &li)) { - throw WinError([&] { return HintFmt("getting size of file %s", PathFmt(descriptorToPath(fd))); }); + throw windows::WinError([&] { return HintFmt("getting size of file %s", PathFmt(descriptorToPath(fd))); }); } return li.QuadPart; } @@ -34,7 +32,8 @@ size_t read(Descriptor fd, std::span buffer) if (lastError == ERROR_BROKEN_PIPE) n = 0; // Treat as EOF else - throw WinError(lastError, "reading %1% bytes from %2%", buffer.size(), PathFmt(descriptorToPath(fd))); + throw windows::WinError( + lastError, "reading %1% bytes from %2%", buffer.size(), PathFmt(descriptorToPath(fd))); } return static_cast(n); } @@ -48,7 +47,7 @@ size_t readOffset(Descriptor fd, off_t offset, std::span buffer) ov.OffsetHigh = static_cast(offset >> 32); DWORD n; if (!ReadFile(fd, buffer.data(), static_cast(buffer.size()), &n, &ov)) { - throw WinError([&] { + throw windows::WinError([&] { return HintFmt( "reading %1% bytes at offset %2% from %3%", buffer.size(), offset, PathFmt(descriptorToPath(fd))); }); @@ -62,7 +61,7 @@ size_t write(Descriptor fd, std::span buffer, bool allowInterru checkInterrupt(); // For consistency with unix DWORD n; if (!WriteFile(fd, buffer.data(), static_cast(buffer.size()), &n, NULL)) { - throw WinError( + throw windows::WinError( [&] { return HintFmt("writing %1% bytes to %2%", buffer.size(), PathFmt(descriptorToPath(fd))); }); } return static_cast(n); @@ -72,7 +71,7 @@ AutoCloseFD dupDescriptor(Descriptor fd) { HANDLE newHandle; if (!DuplicateHandle(GetCurrentProcess(), fd, GetCurrentProcess(), &newHandle, 0, FALSE, DUPLICATE_SAME_ACCESS)) { - throw WinError("duplicating handle"); + throw windows::WinError("duplicating handle"); } return AutoCloseFD{newHandle}; } @@ -88,7 +87,7 @@ void Pipe::create() HANDLE hReadPipe, hWritePipe; if (!CreatePipe(&hReadPipe, &hWritePipe, &saAttr, 0)) - throw WinError("CreatePipe"); + throw windows::WinError("CreatePipe"); readSide = hReadPipe; writeSide = hWritePipe; @@ -130,7 +129,7 @@ off_t lseek(HANDLE h, off_t offset, int whence) void syncDescriptor(Descriptor fd) { if (!::FlushFileBuffers(fd)) { - throw WinError([&] { return HintFmt("flushing file %s", PathFmt(descriptorToPath(fd))); }); + throw windows::WinError([&] { return HintFmt("flushing file %s", PathFmt(descriptorToPath(fd))); }); } } diff --git a/src/libutil/windows/file-system-at.cc b/src/libutil/windows/file-system-at.cc index a10dedbb88..668bd7eb40 100644 --- a/src/libutil/windows/file-system-at.cc +++ b/src/libutil/windows/file-system-at.cc @@ -13,8 +13,6 @@ namespace nix { -using namespace nix::windows; - namespace windows { namespace { @@ -214,7 +212,7 @@ PosixStat fstat(Descriptor fd) { BY_HANDLE_FILE_INFORMATION info; if (!GetFileInformationByHandle(fd, &info)) - throw WinError("getting file information for %s", PathFmt(descriptorToPath(fd))); + throw windows::WinError("getting file information for %s", PathFmt(descriptorToPath(fd))); PosixStat st; windows::statFromFileInfo( @@ -253,9 +251,9 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( /* Helper to check if a component is a symlink and throw SymlinkNotAllowed if so */ auto throwIfSymlink = [&](std::wstring_view component, const CanonPath & pathForError) { try { - auto testHandle = - ntOpenAt(getParentFd(), component, FILE_READ_ATTRIBUTES | SYNCHRONIZE, FILE_OPEN_REPARSE_POINT); - if (isReparsePoint(testHandle.get())) + auto testHandle = windows::ntOpenAt( + getParentFd(), component, FILE_READ_ATTRIBUTES | SYNCHRONIZE, FILE_OPEN_REPARSE_POINT); + if (windows::isReparsePoint(testHandle.get())) throw SymlinkNotAllowed(pathForError); } catch (SymlinkNotAllowed &) { throw; @@ -272,13 +270,13 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( /* Open directory without following symlinks */ AutoCloseFD parentFd2; try { - parentFd2 = ntOpenAt( + parentFd2 = windows::ntOpenAt( getParentFd(), wcomponent, FILE_TRAVERSE | SYNCHRONIZE, // Just need traversal rights FILE_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT // Open directory, don't follow symlinks ); - } catch (WinError & e) { + } catch (windows::WinError & e) { /* Check if this is because it's a symlink */ if (e.lastError == ERROR_CANT_ACCESS_FILE || e.lastError == ERROR_ACCESS_DENIED) { throwIfSymlink(wcomponent, pathUpTo(std::next(it))); @@ -287,7 +285,7 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( } /* Check if what we opened is actually a symlink */ - if (isReparsePoint(parentFd2.get())) { + if (windows::isReparsePoint(parentFd2.get())) { throw SymlinkNotAllowed(pathUpTo(std::next(it))); } @@ -299,13 +297,13 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( AutoCloseFD finalHandle; try { - finalHandle = ntOpenAt( + finalHandle = windows::ntOpenAt( getParentFd(), finalComponent, desiredAccess, createOptions | FILE_OPEN_REPARSE_POINT, // Don't follow symlinks on final component either createDisposition); - } catch (WinError & e) { + } catch (windows::WinError & e) { /* Check if final component is a symlink when we requested to not follow it */ if (e.lastError == ERROR_CANT_ACCESS_FILE) { throwIfSymlink(finalComponent, path); @@ -314,7 +312,7 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( } /* Final check: did we accidentally open a symlink? */ - if (isReparsePoint(finalHandle.get())) + if (windows::isReparsePoint(finalHandle.get())) throw SymlinkNotAllowed(path); return finalHandle; diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 9dab6f2b7a..e644f52580 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -16,8 +16,6 @@ static_assert(S_IFLNK != S_IFCHR, "S_IFLNK must not equal S_IFCHR"); namespace nix { -using namespace nix::windows; - void setWriteTime( const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) { @@ -72,7 +70,7 @@ std::filesystem::path defaultTempDir() wchar_t buf[MAX_PATH + 1]; DWORD len = GetTempPathW(MAX_PATH + 1, buf); if (len == 0 || len > MAX_PATH) - throw WinError("getting default temporary directory"); + throw windows::WinError("getting default temporary directory"); return std::filesystem::path(buf); } @@ -106,7 +104,7 @@ std::filesystem::path descriptorToPath(Descriptor handle) if (dw > buf.size()) { buf.resize(dw); if (GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED) != dw - 1) - throw WinError("GetFinalPathNameByHandleW"); + throw windows::WinError("GetFinalPathNameByHandleW"); dw -= 1; } return std::filesystem::path{std::wstring{buf.data(), dw}}; @@ -179,7 +177,7 @@ PosixStat lstat(const std::filesystem::path & path) { WIN32_FILE_ATTRIBUTE_DATA attrData; if (!GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &attrData)) - throw WinError("getting status of %s", PathFmt(path)); + throw windows::WinError("getting status of %s", PathFmt(path)); return statFromFileInfo(attrData); } @@ -190,7 +188,7 @@ std::optional maybeLstat(const std::filesystem::path & path) auto lastError = GetLastError(); if (lastError == ERROR_FILE_NOT_FOUND || lastError == ERROR_PATH_NOT_FOUND) return std::nullopt; - throw WinError(lastError, "getting status of %s", PathFmt(path)); + throw windows::WinError(lastError, "getting status of %s", PathFmt(path)); } return statFromFileInfo(attrData); } diff --git a/src/libutil/windows/known-folders.cc b/src/libutil/windows/known-folders.cc index 1d28674284..5e0966c849 100644 --- a/src/libutil/windows/known-folders.cc +++ b/src/libutil/windows/known-folders.cc @@ -8,8 +8,6 @@ namespace nix::windows::known_folders { -using namespace nix::windows; - static std::filesystem::path getKnownFolder(REFKNOWNFOLDERID rfid) { PWSTR str = nullptr; diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc index 1e89ccc5e3..b1f98b63af 100644 --- a/src/libutil/windows/muxable-pipe.cc +++ b/src/libutil/windows/muxable-pipe.cc @@ -6,8 +6,6 @@ namespace nix { -using namespace nix::windows; - void MuxablePipePollState::poll(HANDLE ioport, std::optional timeout) { /* We are on at least Windows Vista / Server 2008 and can get many @@ -16,7 +14,7 @@ void MuxablePipePollState::poll(HANDLE ioport, std::optional timeo ioport, oentries, sizeof(oentries) / sizeof(*oentries), &removed, timeout ? *timeout : INFINITE, false)) { auto lastError = GetLastError(); if (lastError != WAIT_TIMEOUT) - throw WinError(lastError, "GetQueuedCompletionStatusEx"); + throw windows::WinError(lastError, "GetQueuedCompletionStatusEx"); assert(removed == 0); } else { assert(0 < removed && removed <= sizeof(oentries) / sizeof(*oentries)); @@ -58,7 +56,7 @@ void MuxablePipePollState::iterate( handleEOF((*p)->readSide.get()); nextp = channels.erase(p); // no need to maintain `channels` ? } else if (lastError != ERROR_IO_PENDING) - throw WinError(lastError, "ReadFile(%s, ..)", (*p)->readSide.get()); + throw windows::WinError(lastError, "ReadFile(%s, ..)", (*p)->readSide.get()); } } break; diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index c0b0fa6c10..46a149603f 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -30,8 +30,6 @@ namespace nix { -using namespace nix::windows; - Pid::Pid() {} Pid::Pid(Pid && other) noexcept @@ -64,7 +62,7 @@ int Pid::kill(bool allowInterrupts) debug("killing process %1%", pid.get()); if (!TerminateProcess(pid.get(), 1)) - logError(WinError("terminating process %1%", pid.get()).info()); + logError(windows::WinError("terminating process %1%", pid.get()).info()); return wait(allowInterrupts); } @@ -76,11 +74,11 @@ int Pid::wait(bool allowInterrupts) assert(pid.get() != INVALID_DESCRIPTOR); DWORD status = WaitForSingleObject(pid.get(), INFINITE); if (status != WAIT_OBJECT_0) - throw WinError("waiting for process %1%", pid.get()); + throw windows::WinError("waiting for process %1%", pid.get()); DWORD exitCode = 0; if (GetExitCodeProcess(pid.get(), &exitCode) == FALSE) - throw WinError("getting exit code of process %1%", pid.get()); + throw windows::WinError("getting exit code of process %1%", pid.get()); pid.close(); return exitCode; @@ -126,7 +124,7 @@ void setFDInheritable(AutoCloseFD & fd, bool inherit) { if (fd.get() != INVALID_DESCRIPTOR) { if (!SetHandleInformation(fd.get(), HANDLE_FLAG_INHERIT, inherit ? HANDLE_FLAG_INHERIT : 0)) { - throw WinError("Couldn't disable inheriting of handle"); + throw windows::WinError("Couldn't disable inheriting of handle"); } } } @@ -146,7 +144,7 @@ AutoCloseFD nullFD() 0, NULL); if (!nul.get()) { - throw WinError("Couldn't open NUL device"); + throw windows::WinError("Couldn't open NUL device"); } // Let this handle be inheritable by child processes setFDInheritable(nul, true); @@ -258,7 +256,7 @@ Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & o &startInfo, &procInfo) == 0) { - throw WinError("CreateProcessW failed (%1%)", os_string_to_string(cmdline)); + throw windows::WinError("CreateProcessW failed (%1%)", os_string_to_string(cmdline)); } // Convert these to use RAII @@ -271,15 +269,15 @@ Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & o Descriptor job = CreateJobObjectW(NULL, NULL); if (job == NULL) { TerminateProcess(procInfo.hProcess, 0); - throw WinError("Couldn't create job object for child process"); + throw windows::WinError("Couldn't create job object for child process"); } if (AssignProcessToJobObject(job, procInfo.hProcess) == FALSE) { TerminateProcess(procInfo.hProcess, 0); - throw WinError("Couldn't assign child process to job object"); + throw windows::WinError("Couldn't assign child process to job object"); } if (ResumeThread(procInfo.hThread) == (DWORD) -1) { TerminateProcess(procInfo.hProcess, 0); - throw WinError("Couldn't resume child process thread"); + throw windows::WinError("Couldn't resume child process thread"); } return process; diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index e5acd8cbdc..eb82a827d6 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -8,8 +8,6 @@ namespace nix { -using namespace nix::windows; - std::string getUserName() { // Get the required buffer size @@ -17,7 +15,7 @@ std::string getUserName() if (!GetUserNameA(nullptr, &size)) { auto lastError = GetLastError(); if (lastError != ERROR_INSUFFICIENT_BUFFER) - throw WinError(lastError, "cannot figure out size of user name"); + throw windows::WinError(lastError, "cannot figure out size of user name"); } std::string name; @@ -28,7 +26,7 @@ std::string getUserName() // Retrieve the username if (!GetUserNameA(&name[0], &size)) - throw WinError("cannot figure out user name"); + throw windows::WinError("cannot figure out user name"); return name; } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 2a24c4199f..4f0fe722f1 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -4,7 +4,7 @@ #include "nix/util/posix-source-accessor.hh" #include "nix/cmd/misc-store-flags.hh" -using namespace nix; +namespace nix { struct CmdAddToStore : MixDryRun, StoreCommand { @@ -84,3 +84,5 @@ struct CmdAddPath : CmdAddToStore static auto rCmdAddFile = registerCommand2({"store", "add-file"}); static auto rCmdAddPath = registerCommand2({"store", "add-path"}); static auto rCmdAdd = registerCommand2({"store", "add"}); + +} // namespace nix diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index 6436466374..e5ebf948d7 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -5,6 +5,7 @@ #include #include #include + #ifdef __APPLE__ # include #endif @@ -24,8 +25,7 @@ #include "nix/util/experimental-features.hh" #include "nix/store/globals.hh" -using namespace nix; -using std::cin; +namespace nix { static void handleAlarm(int sig) {} @@ -410,3 +410,5 @@ static int main_build_remote(int argc, char ** argv) } static RegisterLegacyCommand r_build_remote("build-remote", main_build_remote); + +} // namespace nix diff --git a/src/nix/build.cc b/src/nix/build.cc index 2de7fdf423..4677f325fb 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -5,7 +5,7 @@ #include -using namespace nix; +namespace nix { /* This serialization code is diferent from the canonical (single) derived path serialization because: @@ -186,3 +186,5 @@ struct CmdBuild : InstallablesCommand, MixOutLinkByDefault, MixDryRun, MixJSON, }; static auto rCmdBuild = registerCommand("build"); + +} // namespace nix diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 85c8cdb587..4808a2fc71 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -6,7 +6,7 @@ #include "nix/expr/eval-inline.hh" #include "nix/store/globals.hh" -using namespace nix; +namespace nix { struct CmdBundle : InstallableValueCommand { @@ -131,3 +131,5 @@ struct CmdBundle : InstallableValueCommand }; static auto r2 = registerCommand("bundle"); + +} // namespace nix diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 8a816f5288..a68285efba 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -6,7 +6,7 @@ #include -using namespace nix; +namespace nix { struct MixCat : virtual Args { @@ -120,3 +120,5 @@ struct CmdCatNar : StoreCommand, MixCat static auto rCmdCatStore = registerCommand2({"store", "cat"}); static auto rCmdCatNar = registerCommand2({"nar", "cat"}); + +} // namespace nix diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 76c2eaa74e..8bd923aabf 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -11,7 +11,7 @@ #include "nix/util/executable-path.hh" #include "nix/store/globals.hh" -using namespace nix; +namespace nix { namespace { @@ -178,3 +178,5 @@ struct CmdConfigCheck : StoreCommand }; static auto rCmdConfigCheck = registerCommand2({"config", "check"}); + +} // namespace nix diff --git a/src/nix/config.cc b/src/nix/config.cc index 3fd0cde808..6230fd1036 100644 --- a/src/nix/config.cc +++ b/src/nix/config.cc @@ -4,7 +4,7 @@ #include -using namespace nix; +namespace nix { struct CmdConfig : NixMultiCommand { @@ -79,3 +79,5 @@ struct CmdConfigShow : Command, MixJSON static auto rCmdConfig = registerCommand("config"); static auto rShowConfig = registerCommand2({"config", "show"}); + +} // namespace nix diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 86306e7fdb..11a0cbf623 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -3,7 +3,7 @@ #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" -using namespace nix; +namespace nix { struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile, MixNoCheckSigs { @@ -75,3 +75,5 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile, Mix }; static auto rCmdCopy = registerCommand("copy"); + +} // namespace nix diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc index 2046ecca38..def1d7158c 100644 --- a/src/nix/derivation-add.cc +++ b/src/nix/derivation-add.cc @@ -7,9 +7,10 @@ #include "nix/store/globals.hh" #include -using namespace nix; using json = nlohmann::json; +namespace nix { + struct CmdAddDerivation : MixDryRun, StoreCommand { std::string description() override @@ -43,3 +44,5 @@ struct CmdAddDerivation : MixDryRun, StoreCommand }; static auto rCmdAddDerivation = registerCommand2({"derivation", "add"}); + +} // namespace nix diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc index 1a6f00c8cd..5880c6c8f1 100644 --- a/src/nix/derivation-show.cc +++ b/src/nix/derivation-show.cc @@ -7,9 +7,10 @@ #include "nix/store/derivations.hh" #include -using namespace nix; using json = nlohmann::json; +namespace nix { + struct CmdShowDerivation : InstallablesCommand, MixPrintJSON { bool recursive = false; @@ -68,3 +69,5 @@ struct CmdShowDerivation : InstallablesCommand, MixPrintJSON }; static auto rCmdShowDerivation = registerCommand2({"derivation", "show"}); + +} // namespace nix diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc index 2634048ac2..98f41d7736 100644 --- a/src/nix/derivation.cc +++ b/src/nix/derivation.cc @@ -1,6 +1,6 @@ #include "nix/cmd/command.hh" -using namespace nix; +namespace nix { struct CmdDerivation : NixMultiCommand { @@ -21,3 +21,5 @@ struct CmdDerivation : NixMultiCommand }; static auto rCmdDerivation = registerCommand("derivation"); + +} // namespace nix diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 301e35b55f..50b248bcb8 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -21,7 +21,7 @@ #include "nix/util/strings.hh" -using namespace nix; +namespace nix { struct DevelopSettings : Config { @@ -752,3 +752,5 @@ struct CmdPrintDevEnv : Common, MixJSON static auto rCmdPrintDevEnv = registerCommand("print-dev-env"); static auto rCmdDevelop = registerCommand("develop"); + +} // namespace nix diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index ff37100651..72108399d1 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -112,10 +112,6 @@ void printClosureDiff( } } -} // namespace nix - -using namespace nix; - struct CmdDiffClosures : SourceExprCommand, MixOperateOnOptions { std::string _before, _after; @@ -149,3 +145,5 @@ struct CmdDiffClosures : SourceExprCommand, MixOperateOnOptions }; static auto rCmdDiffClosures = registerCommand2({"store", "diff-closures"}); + +} // namespace nix diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 2a6253b026..2512948778 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -3,7 +3,7 @@ #include "nix/util/archive.hh" #include "nix/util/terminal.hh" -using namespace nix; +namespace nix { static FdSink getNarSink() { @@ -77,3 +77,5 @@ struct CmdNarDumpPath : CmdDumpPath2 static auto rCmdNarPack = registerCommand2({"nar", "pack"}); static auto rCmdNarDumpPath = registerCommand2({"nar", "dump-path"}); + +} // namespace nix diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 0657301f36..aeb26915ba 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -7,7 +7,7 @@ #include -using namespace nix; +namespace nix { struct CmdEdit : InstallableValueCommand { @@ -58,3 +58,5 @@ struct CmdEdit : InstallableValueCommand }; static auto rCmdEdit = registerCommand("edit"); + +} // namespace nix diff --git a/src/nix/env.cc b/src/nix/env.cc index 79d187d85e..4022c3effa 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -10,7 +10,7 @@ #include "nix/util/environment-variables.hh" #include "nix/util/mounted-source-accessor.hh" -using namespace nix; +namespace nix { struct CmdEnv : NixMultiCommand { @@ -121,3 +121,5 @@ struct CmdShell : InstallablesCommand, MixEnvironment }; static auto rCmdShell = registerCommand2({"env", "shell"}); + +} // namespace nix diff --git a/src/nix/eval.cc b/src/nix/eval.cc index f41d98f1a8..ed6fe41be6 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -8,7 +8,7 @@ #include -using namespace nix; +namespace nix { struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption { @@ -126,3 +126,5 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption }; static auto rCmdEval = registerCommand("eval"); + +} // namespace nix diff --git a/src/nix/flake-command.hh b/src/nix/flake-command.hh index 3636bd5251..ae52bcabb9 100644 --- a/src/nix/flake-command.hh +++ b/src/nix/flake-command.hh @@ -6,8 +6,6 @@ namespace nix { -using namespace nix::flake; - class FlakeCommand : virtual Args, public MixFlakeOptions { protected: @@ -19,7 +17,7 @@ public: FlakeRef getFlakeRef(); - LockedFlake lockFlake(); + flake::LockedFlake lockFlake(); std::vector getFlakeRefsForCompletion() override; }; diff --git a/src/nix/flake-prefetch-inputs.cc b/src/nix/flake-prefetch-inputs.cc index 4ea6342c36..eb73c6c915 100644 --- a/src/nix/flake-prefetch-inputs.cc +++ b/src/nix/flake-prefetch-inputs.cc @@ -6,8 +6,7 @@ #include -using namespace nix; -using namespace nix::flake; +namespace nix { struct CmdFlakePrefetchInputs : FlakeCommand { @@ -25,6 +24,7 @@ struct CmdFlakePrefetchInputs : FlakeCommand void run(nix::ref store) override { + using namespace nix::flake; auto flake = lockFlake(); ThreadPool pool{fileTransferSettings.httpConnections}; @@ -69,3 +69,5 @@ struct CmdFlakePrefetchInputs : FlakeCommand }; static auto rCmdFlakePrefetchInputs = registerCommand2({"flake", "prefetch-inputs"}); + +} // namespace nix diff --git a/src/nix/flake.cc b/src/nix/flake.cc index df2a3a9114..42de784534 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -30,9 +30,7 @@ // FIXME is this supposed to be private or not? #include "flake-command.hh" -using namespace nix; -using namespace nix::flake; -using json = nlohmann::json; +namespace nix { struct CmdFlakeUpdate; @@ -52,7 +50,7 @@ FlakeRef FlakeCommand::getFlakeRef() return parseFlakeRef(fetchSettings, flakeUrl, std::filesystem::current_path().string()); // FIXME } -LockedFlake FlakeCommand::lockFlake() +flake::LockedFlake FlakeCommand::lockFlake() { return flake::lockFlake(flakeSettings, *getEvalState(), getFlakeRef(), lockFlags); } @@ -89,7 +87,7 @@ struct CmdFlakeUpdate : FlakeCommand .optional = true, .handler = {[&](std::vector inputsToUpdate) { for (const auto & inputToUpdate : inputsToUpdate) { - std::optional inputAttrPath; + std::optional inputAttrPath; try { inputAttrPath = flake::NonEmptyInputAttrPath::parse(inputToUpdate); if (!inputAttrPath) @@ -269,9 +267,9 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON if (!lockedFlake.lockFile.root->inputs.empty()) logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); - std::set> visited{lockedFlake.lockFile.root}; + std::set> visited{lockedFlake.lockFile.root}; - [&](this const auto & recurse, const Node & node, const std::string & prefix) -> void { + [&](this const auto & recurse, const flake::Node & node, const std::string & prefix) -> void { for (const auto & [i, input] : enumerate(node.inputs)) { bool last = i + 1 == node.inputs.size(); @@ -295,7 +293,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON "%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'", prefix + (last ? treeLast : treeConn), input.first, - printInputAttrPath(*follows)); + flake::printInputAttrPath(*follows)); } } }(*lockedFlake.lockFile.root, ""); @@ -856,7 +854,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand std::string templateUrl = "templates"; std::filesystem::path destDir; - const LockFlags lockFlags{.writeLockFile = false}; + const flake::LockFlags lockFlags{.writeLockFile = false}; CmdFlakeInitCommon() { @@ -1097,9 +1095,9 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs sources.insert(storePath); // FIXME: use graph output, handle cycles. - std::function traverse; - traverse = [&](const Node & node) { - nlohmann::json jsonObj2 = json ? json::object() : nlohmann::json(nullptr); + std::function traverse; + traverse = [&](const flake::Node & node) { + nlohmann::json jsonObj2 = json ? nlohmann::json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { std::optional storePath; @@ -1174,7 +1172,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON evalSettings.enableImportFromDerivation.setDefault(false); auto state = getEvalState(); - auto flake = make_ref(lockFlake()); + auto flake = make_ref(lockFlake()); auto localSystem = std::string(settings.thisSystem.get()); std::function @@ -1567,3 +1565,5 @@ static auto rCmdFlakeNew = registerCommand2({"flake", "new"}); static auto rCmdFlakePrefetch = registerCommand2({"flake", "prefetch"}); static auto rCmdFlakeShow = registerCommand2({"flake", "show"}); static auto rCmdFlakeUpdate = registerCommand2({"flake", "update"}); + +} // namespace nix diff --git a/src/nix/formatter.cc b/src/nix/formatter.cc index 619dcf4fad..19d4738635 100644 --- a/src/nix/formatter.cc +++ b/src/nix/formatter.cc @@ -7,7 +7,7 @@ #include "run.hh" -using namespace nix; +namespace nix { struct CmdFormatter : NixMultiCommand { @@ -158,3 +158,5 @@ struct CmdFmt : CmdFormatterRun }; static auto rFmt = registerCommand("fmt"); + +} // namespace nix diff --git a/src/nix/hash.cc b/src/nix/hash.cc index bc1ce313b0..90f44fdecd 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -9,7 +9,7 @@ #include "man-pages.hh" #include "nix/util/fun.hh" -using namespace nix; +namespace nix { /** * Base for `nix hash path`, `nix hash file` (deprecated), and `nix-hash` (legacy). @@ -366,3 +366,5 @@ static int compatNixHash(int argc, char ** argv) } static RegisterLegacyCommand r_nix_hash("nix-hash", compatNixHash); + +} // namespace nix diff --git a/src/nix/log.cc b/src/nix/log.cc index acc03cbb67..1ee3af6699 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -3,7 +3,7 @@ #include "nix/main/shared.hh" #include "nix/store/globals.hh" -using namespace nix; +namespace nix { struct CmdLog : InstallableCommand { @@ -47,3 +47,5 @@ struct CmdLog : InstallableCommand }; static auto rCmdLog = registerCommand("log"); + +} // namespace nix diff --git a/src/nix/ls.cc b/src/nix/ls.cc index a9d082fb37..410f4f420e 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -4,7 +4,7 @@ #include "nix/main/common-args.hh" #include -using namespace nix; +namespace nix { struct MixLs : virtual Args, MixJSON { @@ -160,3 +160,5 @@ struct CmdLsNar : Command, MixLs static auto rCmdLsStore = registerCommand2({"store", "ls"}); static auto rCmdLsNar = registerCommand2({"nar", "ls"}); + +} // namespace nix diff --git a/src/nix/main.cc b/src/nix/main.cc index c19aa26925..98423b3b10 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -41,16 +41,16 @@ # include "nix/util/linux-namespaces.hh" #endif +#include "nix/util/strings.hh" + +namespace nix { + #ifndef _WIN32 extern std::string chrootHelperName; void chrootHelper(int argc, char ** argv); #endif -#include "nix/util/strings.hh" - -namespace nix { - /* Check if we have a non-loopback/link-local network interface. */ static bool haveInternet() { diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index cd0f3f59f6..1268163967 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -5,10 +5,10 @@ #include -using namespace nix; - using nlohmann::json; +namespace nix { + struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON { CmdMakeContentAddressed() @@ -56,3 +56,5 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, }; static auto rCmdMakeContentAddressed = registerCommand2({"store", "make-content-addressed"}); + +} // namespace nix diff --git a/src/nix/nar.cc b/src/nix/nar.cc index bae77b6cc1..fe6c89ec12 100644 --- a/src/nix/nar.cc +++ b/src/nix/nar.cc @@ -1,6 +1,6 @@ #include "nix/cmd/command.hh" -using namespace nix; +namespace nix { struct CmdNar : NixMultiCommand { @@ -28,3 +28,5 @@ struct CmdNar : NixMultiCommand }; static auto rCmdNar = registerCommand("nar"); + +} // namespace nix diff --git a/src/nix/nix-build/nix-build.cc b/src/nix/nix-build/nix-build.cc index d9e2f2dc8d..e68e775e02 100644 --- a/src/nix/nix-build/nix-build.cc +++ b/src/nix/nix-build/nix-build.cc @@ -31,11 +31,12 @@ #include "nix/util/fun.hh" #include "man-pages.hh" -using namespace nix; using namespace std::string_literals; extern char ** environ __attribute__((weak)); +namespace nix { + /* Recreate the effect of the perl shellwords function, breaking up a * string into arguments like a shell word, including escapes */ @@ -747,3 +748,5 @@ static void main_nix_build(int argc, char ** argv) static RegisterLegacyCommand r_nix_build("nix-build", main_nix_build); static RegisterLegacyCommand r_nix_shell("nix-shell", main_nix_build); + +} // namespace nix diff --git a/src/nix/nix-channel/nix-channel.cc b/src/nix/nix-channel/nix-channel.cc index 911aabd438..aa58beb243 100644 --- a/src/nix/nix-channel/nix-channel.cc +++ b/src/nix/nix-channel/nix-channel.cc @@ -16,7 +16,7 @@ #include #include -using namespace nix; +namespace nix { typedef StringMap Channels; @@ -306,3 +306,5 @@ static int main_nix_channel(int argc, char ** argv) } static RegisterLegacyCommand r_nix_channel("nix-channel", main_nix_channel); + +} // namespace nix diff --git a/src/nix/nix-collect-garbage/nix-collect-garbage.cc b/src/nix/nix-collect-garbage/nix-collect-garbage.cc index a9f8b7fb6b..c267e95c9f 100644 --- a/src/nix/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix/nix-collect-garbage/nix-collect-garbage.cc @@ -12,7 +12,7 @@ #include -using namespace nix; +namespace nix { std::string deleteOlderThan; bool dryRun = false; @@ -109,3 +109,5 @@ static int main_nix_collect_garbage(int argc, char ** argv) } static RegisterLegacyCommand r_nix_collect_garbage("nix-collect-garbage", main_nix_collect_garbage); + +} // namespace nix diff --git a/src/nix/nix-copy-closure/nix-copy-closure.cc b/src/nix/nix-copy-closure/nix-copy-closure.cc index 33aec6c3e6..0c939df659 100644 --- a/src/nix/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix/nix-copy-closure/nix-copy-closure.cc @@ -5,7 +5,7 @@ #include "nix/cmd/legacy.hh" #include "man-pages.hh" -using namespace nix; +namespace nix { static int main_nix_copy_closure(int argc, char ** argv) { @@ -69,3 +69,5 @@ static int main_nix_copy_closure(int argc, char ** argv) } static RegisterLegacyCommand r_nix_copy_closure("nix-copy-closure", main_nix_copy_closure); + +} // namespace nix diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index b93a51ac53..abe04d9634 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -33,8 +33,7 @@ #include #include -using namespace nix; -using std::cout; +namespace nix { /** * Settings related to Nix user environments. @@ -1073,7 +1072,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) /* Print the desired columns, or XML output. */ if (jsonOutput) { queryJSON(globals, elems, printOutPath, printDrvPath, printMeta); - cout << '\n'; + std::cout << '\n'; return; } @@ -1082,7 +1081,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) Table table; std::ostringstream dummy; - XMLWriter xml(true, *(xmlOutput ? &cout : &dummy)); + XMLWriter xml(true, *(xmlOutput ? &std::cout : &dummy)); XMLOpenElement xmlRoot(xml, "items"); for (auto & i : elems) { @@ -1274,7 +1273,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } else table.push_back(columns); - cout.flush(); + std::cout.flush(); } catch (AssertionError & e) { printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName()); @@ -1536,3 +1535,5 @@ static int main_nix_env(int argc, char ** argv) } static RegisterLegacyCommand r_nix_env("nix-env", main_nix_env); + +} // namespace nix diff --git a/src/nix/nix-instantiate/nix-instantiate.cc b/src/nix/nix-instantiate/nix-instantiate.cc index b368346eea..27a11767b0 100644 --- a/src/nix/nix-instantiate/nix-instantiate.cc +++ b/src/nix/nix-instantiate/nix-instantiate.cc @@ -15,7 +15,7 @@ #include -using namespace nix; +namespace nix { std::filesystem::path gcRoot; static int rootNr = 0; @@ -207,3 +207,5 @@ static int main_nix_instantiate(int argc, char ** argv) } static RegisterLegacyCommand r_nix_instantiate("nix-instantiate", main_nix_instantiate); + +} // namespace nix diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index a534e19afb..2c344d4cd9 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -41,8 +41,6 @@ namespace nix_store { using namespace nix; -using std::cin; -using std::cout; typedef void (*Operation)(Strings opFlags, Strings opArgs); @@ -182,7 +180,7 @@ static void opRealise(Strings opFlags, Strings opArgs) auto paths2 = realisePath(i, false); if (!noOutput) for (auto & j : paths2) - cout << fmt("%s\n", j.string()); + std::cout << fmt("%s\n", j.string()); } } @@ -194,7 +192,7 @@ static void opAdd(Strings opFlags, Strings opArgs) for (auto & i : opArgs) { auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), sourcePath))); + std::cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), sourcePath))); } } @@ -242,7 +240,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string hash = *i++; std::string name = *i++; - cout << fmt( + std::cout << fmt( "%s\n", store->printStorePath(store->makeFixedOutputPath( name, @@ -280,11 +278,11 @@ static void printTree(const StorePath & path, const std::string & firstPad, const std::string & tailPad, StorePathSet & done) { if (!done.insert(path).second) { - cout << fmt("%s%s [...]\n", firstPad, store->printStorePath(path)); + std::cout << fmt("%s%s [...]\n", firstPad, store->printStorePath(path)); return; } - cout << fmt("%s%s\n", firstPad, store->printStorePath(path)); + std::cout << fmt("%s%s\n", firstPad, store->printStorePath(path)); auto info = store->queryPathInfo(path); @@ -387,7 +385,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & i : opArgs) { auto outputs = maybeUseOutputs(store->followLinksToStorePath(i), true, forceRealise); for (auto & outputPath : outputs) - cout << fmt("%1%\n", store->printStorePath(outputPath)); + std::cout << fmt("%1%\n", store->printStorePath(outputPath)); } break; } @@ -416,14 +414,14 @@ static void opQuery(Strings opFlags, Strings opArgs) } auto sorted = store->topoSortPaths(paths); for (StorePaths::reverse_iterator i = sorted.rbegin(); i != sorted.rend(); ++i) - cout << fmt("%s\n", store->printStorePath(*i)); + std::cout << fmt("%s\n", store->printStorePath(*i)); break; } case qDeriver: for (auto & i : opArgs) { auto info = store->queryPathInfo(store->followLinksToStorePath(i)); - cout << fmt("%s\n", info->deriver ? store->printStorePath(*info->deriver) : "unknown-deriver"); + std::cout << fmt("%s\n", info->deriver ? store->printStorePath(*info->deriver) : "unknown-deriver"); } break; @@ -437,7 +435,7 @@ static void opQuery(Strings opFlags, Strings opArgs) } auto sorted = store->topoSortPaths(result); for (StorePaths::reverse_iterator i = sorted.rbegin(); i != sorted.rend(); ++i) - cout << fmt("%s\n", store->printStorePath(*i)); + std::cout << fmt("%s\n", store->printStorePath(*i)); break; } @@ -449,7 +447,7 @@ static void opQuery(Strings opFlags, Strings opArgs) if (j == drv.env.end()) throw Error( "derivation '%s' has no environment binding named '%s'", store->printStorePath(path), bindingName); - cout << fmt("%s\n", j->second); + std::cout << fmt("%s\n", j->second); } break; @@ -460,9 +458,9 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.algo == HashAlgorithm::SHA256); - cout << fmt("%s\n", info->narHash.to_string(HashFormat::Nix32, true)); + std::cout << fmt("%s\n", info->narHash.to_string(HashFormat::Nix32, true)); } else if (query == qSize) - cout << fmt("%d\n", info->narSize); + std::cout << fmt("%d\n", info->narSize); } } break; @@ -494,7 +492,7 @@ static void opQuery(Strings opFlags, Strings opArgs) case qResolve: { for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->followLinksToStorePath(i))); + std::cout << fmt("%s\n", store->printStorePath(store->followLinksToStorePath(i))); break; } @@ -513,7 +511,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & [target, links] : roots) if (referrers.find(target) != referrers.end()) for (auto & link : links) - cout << fmt("%1% -> %2%\n", link, gcStore.printStorePath(target)); + std::cout << fmt("%1% -> %2%\n", link, gcStore.printStorePath(target)); break; } @@ -539,7 +537,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) /* Also output the arguments. */ std::string argsStr = concatStringsSep(" ", drv.args); - cout << "export _args; _args=" << escapeShellArgAlways(argsStr) << "\n"; + std::cout << "export _args; _args=" << escapeShellArgAlways(argsStr) << "\n"; } static void opReadLog(Strings opFlags, Strings opArgs) @@ -566,10 +564,10 @@ static void opDumpDB(Strings opFlags, Strings opArgs) throw UsageError("unknown flag"); if (!opArgs.empty()) { for (auto & i : opArgs) - cout << store->makeValidityRegistration({store->followLinksToStorePath(i)}, true, true); + std::cout << store->makeValidityRegistration({store->followLinksToStorePath(i)}, true, true); } else { for (auto & i : store->queryAllValidPaths()) - cout << store->makeValidityRegistration({i}, true, true); + std::cout << store->makeValidityRegistration({i}, true, true); } } @@ -585,7 +583,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) std::numeric_limits::max(), }} : std::nullopt; - auto info = decodeValidPathInfo(*store, cin, hashResultOpt); + auto info = decodeValidPathInfo(*store, std::cin, hashResultOpt); if (!info) break; if (!store->isValidPath(info->path) || reregister) { @@ -651,7 +649,7 @@ static void opCheckValidity(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); if (!store->isValidPath(path)) { if (printInvalid) - cout << fmt("%s\n", store->printStorePath(path)); + std::cout << fmt("%s\n", store->printStorePath(path)); else throw Error("path '%s' is not valid", store->printStorePath(path)); } @@ -703,7 +701,7 @@ static void opGC(Strings opFlags, Strings opArgs) Finally printer([&] { if (options.action != GCOptions::gcDeleteDead) for (auto & i : results.paths) - cout << i << std::endl; + std::cout << i << std::endl; else printFreed(false, results); }); @@ -788,7 +786,7 @@ static void opImport(Strings opFlags, Strings opArgs) auto paths = importPaths(*store, source, NoCheckSigs); for (auto & i : paths) - cout << fmt("%s\n", store->printStorePath(i)) << std::flush; + std::cout << fmt("%s\n", store->printStorePath(i)) << std::flush; } /* Initialise the Nix databases. */ diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index 5edbabc3bd..2b2f3be48f 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -2,7 +2,7 @@ #include "nix/main/shared.hh" #include "nix/store/store-api.hh" -using namespace nix; +namespace nix { struct CmdOptimiseStore : StoreCommand { @@ -25,3 +25,5 @@ struct CmdOptimiseStore : StoreCommand }; static auto rCmdOptimiseStore = registerCommand2({"store", "optimise"}); + +} // namespace nix diff --git a/src/nix/path-from-hash-part.cc b/src/nix/path-from-hash-part.cc index 7e6c6ec280..a2f16393be 100644 --- a/src/nix/path-from-hash-part.cc +++ b/src/nix/path-from-hash-part.cc @@ -1,7 +1,7 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" -using namespace nix; +namespace nix { struct CmdPathFromHashPart : StoreCommand { @@ -37,3 +37,5 @@ struct CmdPathFromHashPart : StoreCommand }; static auto rCmdPathFromHashPart = registerCommand2({"store", "path-from-hash-part"}); + +} // namespace nix diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 9abe38bfdd..5be20f66da 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -10,7 +10,8 @@ #include "nix/util/strings.hh" -using namespace nix; +namespace nix { + using nlohmann::json; /** @@ -237,3 +238,5 @@ struct CmdPathInfo : StorePathsCommand, MixJSON }; static auto rCmdPathInfo = registerCommand("path-info"); + +} // namespace nix diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 1ace94a8b7..7457931d6f 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -18,7 +18,7 @@ #include -using namespace nix; +namespace nix { /* If ‘url’ starts with ‘mirror://’, then resolve it using the list of mirrors defined in Nixpkgs. */ @@ -349,3 +349,5 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON }; static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); + +} // namespace nix diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 9819004c14..c85c406aa5 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -20,7 +20,7 @@ #include "nix/util/strings.hh" -using namespace nix; +namespace nix { struct ProfileElementSource { @@ -1029,3 +1029,5 @@ struct CmdProfile : NixMultiCommand }; static auto rCmdProfile = registerCommand("profile"); + +} // namespace nix diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index 8dd608d23b..129f06f6c5 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -3,7 +3,7 @@ #include -using namespace nix; +namespace nix { struct CmdRealisation : NixMultiCommand { @@ -78,3 +78,5 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON }; static auto rCmdRealisationInfo = registerCommand2({"realisation", "info"}); + +} // namespace nix diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 5f77700d65..713c0d5d17 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -5,8 +5,7 @@ #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/registry.hh" -using namespace nix; -using namespace nix::flake; +namespace nix { class RegistryCommand : virtual Args { @@ -268,3 +267,5 @@ struct CmdRegistry : NixMultiCommand }; static auto rCmdRegistry = registerCommand("registry"); + +} // namespace nix diff --git a/src/nix/run.cc b/src/nix/run.cc index a87a7679d4..d9056ba91d 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -20,12 +20,10 @@ extern char ** environ __attribute__((weak)); -using namespace nix; +namespace nix { std::string chrootHelperName = "__run_in_chroot"; -namespace nix { - /* Convert `env` to a list of strings suitable for `execve`'s `envp` argument. */ Strings toEnvp(StringMap env) { @@ -108,8 +106,6 @@ void execProgramInStore( throw SysError("unable to execute '%s'", program); } -} // namespace nix - struct CmdRun : InstallableValueCommand, MixEnvironment { using InstallableCommand::run; @@ -264,3 +260,5 @@ void chrootHelper(int argc, char ** argv) throw Error("mounting the Nix store on '%s' is not supported on this platform", storeDir); #endif } + +} // namespace nix diff --git a/src/nix/search.cc b/src/nix/search.cc index 4f5ae8cafa..c4b37b8a71 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -16,9 +16,10 @@ #include "nix/util/strings.hh" -using namespace nix; using json = nlohmann::json; +namespace nix { + std::string wrap(std::string prefix, std::string s) { return concatStrings(prefix, s, ANSI_NORMAL); @@ -196,3 +197,5 @@ struct CmdSearch : InstallableValueCommand, MixJSON }; static auto rCmdSearch = registerCommand("search"); + +} // namespace nix diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index c72204cea3..bcc5773179 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -7,7 +7,7 @@ #include -using namespace nix; +namespace nix { struct CmdCopySigs : StorePathsCommand { @@ -226,3 +226,5 @@ struct CmdKey : NixMultiCommand }; static auto rCmdKey = registerCommand("key"); + +} // namespace nix diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index cbd4e04f6a..bcdbee75f2 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -4,7 +4,7 @@ #include "nix/store/store-cast.hh" #include "nix/store/log-store.hh" -using namespace nix; +namespace nix { struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand { @@ -37,3 +37,5 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand }; static auto rCmdCopyLog = registerCommand2({"store", "copy-log"}); + +} // namespace nix diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 5d28979a49..c18b518776 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -4,7 +4,7 @@ #include "nix/store/store-cast.hh" #include "nix/store/gc-store.hh" -using namespace nix; +namespace nix { struct CmdStoreDelete : StorePathsCommand { @@ -45,3 +45,5 @@ struct CmdStoreDelete : StorePathsCommand }; static auto rCmdStoreDelete = registerCommand2({"store", "delete"}); + +} // namespace nix diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index f4624a40e8..c2923fdc56 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -6,7 +6,7 @@ #include "nix/store/gc-store.hh" #include "nix/util/error.hh" -using namespace nix; +namespace nix { struct CmdStoreGC : StoreCommand, MixDryRun { @@ -49,3 +49,5 @@ struct CmdStoreGC : StoreCommand, MixDryRun }; static auto rCmdStoreGC = registerCommand2({"store", "gc"}); + +} // namespace nix diff --git a/src/nix/store-info.cc b/src/nix/store-info.cc index 4526d9cda1..ade9230c98 100644 --- a/src/nix/store-info.cc +++ b/src/nix/store-info.cc @@ -5,7 +5,7 @@ #include -using namespace nix; +namespace nix { struct CmdInfoStore : StoreCommand, MixJSON { @@ -45,3 +45,5 @@ struct CmdInfoStore : StoreCommand, MixJSON }; static auto rCmdInfoStore = registerCommand2({"store", "info"}); + +} // namespace nix diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc index cd243691c5..a6eee6dfbf 100644 --- a/src/nix/store-repair.cc +++ b/src/nix/store-repair.cc @@ -1,7 +1,7 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" -using namespace nix; +namespace nix { struct CmdStoreRepair : StorePathsCommand { @@ -25,3 +25,5 @@ struct CmdStoreRepair : StorePathsCommand }; static auto rStoreRepair = registerCommand2({"store", "repair"}); + +} // namespace nix diff --git a/src/nix/store.cc b/src/nix/store.cc index 45e505d069..ee3c748752 100644 --- a/src/nix/store.cc +++ b/src/nix/store.cc @@ -1,6 +1,6 @@ #include "nix/cmd/command.hh" -using namespace nix; +namespace nix { struct CmdStore : NixMultiCommand { @@ -24,3 +24,5 @@ struct CmdStore : NixMultiCommand }; static auto rCmdStore = registerCommand("store"); + +} // namespace nix diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 278eaef374..05e47f79c3 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -38,8 +38,7 @@ # include "nix/util/cgroup.hh" #endif -using namespace nix; -using namespace nix::daemon; +namespace nix { /** * Settings related to authenticating clients for the Nix daemon. @@ -106,12 +105,12 @@ static ssize_t splice(int fd_in, void * off_in, int fd_out, void * off_out, size { // We ignore most parameters, we just have them for conformance with the linux syscall std::vector buf(8192); - auto read_count = read(fd_in, buf.data(), buf.size()); + auto read_count = ::read(fd_in, buf.data(), buf.size()); if (read_count == -1) return read_count; auto write_count = decltype(read_count)(0); while (write_count < read_count) { - auto res = write(fd_out, buf.data() + write_count, read_count - write_count); + auto res = ::write(fd_out, buf.data() + write_count, read_count - write_count); if (res == -1) return res; write_count += res; @@ -250,6 +249,8 @@ static void daemonLoop( std::optional forceTrustClientOpt, std::filesystem::path socketPath) { + using namespace nix::daemon; + if (chdir("/") == -1) throw SysError("cannot change current directory"); @@ -404,7 +405,7 @@ static void forwardStdioConnection(RemoteStore & store) */ static void processStdioConnection(ref store, TrustedFlag trustClient) { - processConnection(store, FdSource(STDIN_FILENO), FdSink(STDOUT_FILENO), trustClient, NotRecursive); + processConnection(store, FdSource(STDIN_FILENO), FdSink(STDOUT_FILENO), trustClient, daemon::NotRecursive); } /** @@ -622,3 +623,5 @@ struct CmdDaemon : StoreConfigCommand }; static auto rCmdDaemon = registerCommand2({"daemon"}); + +} // namespace nix diff --git a/src/nix/unix/store-roots-daemon.cc b/src/nix/unix/store-roots-daemon.cc index c9728a2efe..85b0a67a91 100644 --- a/src/nix/unix/store-roots-daemon.cc +++ b/src/nix/unix/store-roots-daemon.cc @@ -7,7 +7,7 @@ #include -using namespace nix; +namespace nix { struct CmdRootsDaemon : StoreConfigCommand { @@ -64,3 +64,5 @@ struct CmdRootsDaemon : StoreConfigCommand }; static auto rCmdStoreRootsDaemon = registerCommand2({"store", "roots-daemon"}); + +} // namespace nix diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 17b410b325..3136f4d5fd 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -13,7 +13,7 @@ #include "nix/util/config-global.hh" #include "self-exe.hh" -using namespace nix; +namespace nix { /** * Check whether a path has a "profiles" component. @@ -210,3 +210,5 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand }; static auto rCmdUpgradeNix = registerCommand("upgrade-nix"); + +} // namespace nix diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 6fb10bf3e8..27a349062e 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -9,7 +9,7 @@ #include "nix/util/exit.hh" -using namespace nix; +namespace nix { struct CmdVerify : StorePathsCommand { @@ -186,3 +186,5 @@ struct CmdVerify : StorePathsCommand }; static auto rCmdVerify = registerCommand2({"store", "verify"}); + +} // namespace nix diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index e7224f6cc9..d953b022f3 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -7,7 +7,7 @@ #include -using namespace nix; +namespace nix { static std::string hilite(const std::string & s, size_t pos, size_t len, const std::string & colour = ANSI_RED) { @@ -295,3 +295,5 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions }; static auto rCmdWhyDepends = registerCommand("why-depends"); + +} // namespace nix diff --git a/tests/functional/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc index f562bab56a..dae7a34c95 100644 --- a/tests/functional/plugins/plugintest.cc +++ b/tests/functional/plugins/plugintest.cc @@ -1,7 +1,7 @@ #include "nix/util/config-global.hh" #include "nix/expr/primops.hh" -using namespace nix; +namespace nix { struct MySettings : Config { @@ -25,3 +25,5 @@ static RegisterPrimOp rp({ .arity = 0, .impl = prim_anotherNull, }); + +} // namespace nix diff --git a/tests/functional/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc index 6cfe50047a..cab02d799d 100644 --- a/tests/functional/test-libstoreconsumer/main.cc +++ b/tests/functional/test-libstoreconsumer/main.cc @@ -3,10 +3,10 @@ #include "nix/store/build-result.hh" #include -using namespace nix; - int main(int argc, char ** argv) { + using namespace nix; + try { if (argc != 2) { std::cerr << "Usage: " << argv[0] << " store/path/to/something.drv\n"; From 693a14190aabf379ef1ed67d219fa665bfc788a8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 10 Apr 2026 17:15:48 -0400 Subject: [PATCH 226/927] libutil-tests: Run tests under `enosys` to exercise syscall fallbacks Add a second test run that uses `enosys --syscall openat2 --syscall fchmodat2` to make both syscalls return `ENOSYS` via seccomp. This forces the iterative `openat`-based fallback in `openFileEnsureBeneathNoSymlinks` and the `/proc/self/fd` fallback in `fchmodatTryNoFollow`, giving CI coverage for code paths that are otherwise only reachable on older kernels, or other Unix OSes. - `meson.build`: new `nix-util-tests-without-new-syscalls` test target, gated on Linux and `enosys` being available. - `package.nix`: new `run-without-new-syscalls` passthru test, gated on `hostPlatform.isLinux && buildPlatform.canExecute hostPlatform` (enosys uses seccomp and can't work through an emulator). - `everything.nix`: include the new test in `checkInputs` under the same platform guard so Hydra builds it on Linux. Idea-by: Sergei Zimmerman --- packaging/everything.nix | 5 +++++ src/libutil-tests/meson.build | 24 ++++++++++++++++++++++++ src/libutil-tests/package.nix | 28 +++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/packaging/everything.nix b/packaging/everything.nix index 4d8f94f4b9..751d861c9e 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -140,6 +140,11 @@ stdenv.mkDerivation (finalAttrs: { # Make sure the functional tests have passed nix-functional-tests ] + ++ + lib.optionals (stdenv.hostPlatform.isLinux && stdenv.buildPlatform.canExecute stdenv.hostPlatform) + [ + nix-util-tests.tests.run-without-new-syscalls + ] ++ lib.optionals (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index a76d8d9709..ff6e539c8a 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -126,3 +126,27 @@ test( }, protocol : 'gtest', ) + +# Run the same tests again under `enosys -d openat2` to exercise the +# iterative (non-openat2) fallback path on Linux. `enosys` uses +# seccomp to make `openat2` return `ENOSYS`, which the wrapper in +# file-system-at.cc caches and falls back to the iterative impl. +if host_machine.system() == 'linux' + enosys = find_program('enosys', required : false) + if enosys.found() + test( + meson.project_name() + '-without-new-syscalls', + enosys, + args : [ + '--syscall', 'openat2', + '--syscall', 'fchmodat2', + '--', + this_exe, + ], + env : { + '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + }, + protocol : 'gtest', + ) + endif +endif diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index f24d69243b..03c1c2d0ef 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -11,6 +11,7 @@ rapidcheck, gtest, runCommand, + util-linux, # Configuration Options @@ -44,6 +45,9 @@ mkMesonExecutable (finalAttrs: { nix-util-test-support rapidcheck gtest + ] + ++ lib.optionals stdenv.hostPlatform.isLinux [ + util-linux ]; mesonFlags = [ @@ -67,7 +71,29 @@ mkMesonExecutable (finalAttrs: { touch $out '' ); - }; + } + // + lib.optionalAttrs + (stdenv.hostPlatform.isLinux && stdenv.buildPlatform.canExecute stdenv.hostPlatform) + { + # Run the same tests with newer syscalls disabled via seccomp, + # to exercise fallback paths (iterative openat for openat2, + # /proc/self/fd for fchmodat2). + run-without-new-syscalls = + runCommand "${finalAttrs.pname}-run-without-new-syscalls" + { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + nativeBuildInputs = [ util-linux ]; + } + '' + export _NIX_TEST_UNIT_DATA=${./data} + enosys \ + --syscall openat2 \ + --syscall fchmodat2 \ + -- ${lib.getExe finalAttrs.finalPackage} + touch $out + ''; + }; }; meta = { From 4d470c30091375501a2a794683a5b73b562cee91 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Apr 2026 18:08:50 -0400 Subject: [PATCH 227/927] libutil: Ensure `openFileEnsureBeneathNoSymlinks` throws enough, simplify interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We had some issues where `openFileEnsureBeneathNoSymlinks` was not always throwing `SymlinkNotAllowed` when expected. This wasn't necessarily a problem, but opened the door to a bad situation where callers might handle different errors differently in unexpected and unintentional ways. When I was investigating this, I realized the presence or absence of `O_NOFOLLOW` in the caller-provided flags was doubling the number of cases that we needed to handle, and this was becoming quite burdensome. The function doesn't need to work with arbitrary, e.g. user-provided flags. All its callers (at least eventually, if you trace back far enough) in Nix should always have statically-known flags. So we can use our own invariants to cut down that state space. Specifically, callers of `openFileEnsureBeneathNoSymlinks` should never pass `O_NOFOLLOW` — the function owns symlink policy (it's in the name). Add an assert enforcing this and drop `O_NOFOLLOW` from the one caller that had it (`fs-sink.cc`'s directory open). Each implementation then handles trailing-symlink detection internally: - The `openat2` path uses `RESOLVE_NO_SYMLINKS`, which gives a uniform `ELOOP` for every trailing-symlink case. - The iterative fallback adds its own defensive `O_NOFOLLOW` on the final `::openat`, with post-checks for two kernel oddities where `O_NOFOLLOW` produces unexpected results (`O_DIRECTORY` gives `ENOTDIR` instead of `ELOOP`; `O_PATH` silently succeeds with a symlink fd). With `O_PATH` (without `O_DIRECTORY`), a trailing symlink is intentionally allowed — the caller gets a path fd to the symlink itself. Interior symlinks are always rejected regardless of flags. Also add comprehensive tests covering every flag combination (`O_RDONLY`, `O_DIRECTORY`, `O_PATH | O_DIRECTORY`, `O_PATH`), including symlink rejection, trailing-symlink-fd success for `O_PATH`, non-symlink `ENOTDIR` propagation, happy paths, and death tests for the `O_NOFOLLOW` assert. Reviewed-by: Sergei Zimmerman --- src/libstore/unix/build/derivation-builder.cc | 2 +- src/libutil-tests/file-system-at.cc | 173 ++++++++++++++---- src/libutil/fs-sink.cc | 2 +- .../include/nix/util/file-system-at.hh | 48 +++-- src/libutil/unix/file-system-at.cc | 70 ++++++- 5 files changed, 237 insertions(+), 58 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 8288a4a318..85aa98c7ae 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1273,7 +1273,7 @@ void DerivationBuilderImpl::writeBuilderFile(const std::string & name, std::stri a single path component without any `..`, `.` components. */ auto relPath = CanonPath::fromFilename(name); AutoCloseFD fd = openFileEnsureBeneathNoSymlinks( - tmpDirFd.get(), relPath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC | O_EXCL | O_NOFOLLOW, 0666); + tmpDirFd.get(), relPath, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC | O_EXCL, 0666); auto path = tmpDir / relPath.rel(); /* This is used only for error messages. */ if (!fd) throw SysError("creating file %s", PathFmt(path)); diff --git a/src/libutil-tests/file-system-at.cc b/src/libutil-tests/file-system-at.cc index fa646820d7..82b73d7af0 100644 --- a/src/libutil-tests/file-system-at.cc +++ b/src/libutil-tests/file-system-at.cc @@ -4,6 +4,7 @@ #include "nix/util/file-system-at.hh" #include "nix/util/file-system.hh" #include "nix/util/fs-sink.hh" +#include "nix/util/tests/gmock-matchers.hh" namespace nix { @@ -120,51 +121,80 @@ TEST(openFileEnsureBeneathNoSymlinks, works) auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); - // Helper to open files with platform-specific arguments + /* Helpers that wrap `openFileEnsureBeneathNoSymlinks` to throw + `SysError` on null-fd failure. This way every failure is an + exception and tests can use plain `EXPECT_THROW`/`EXPECT_TRUE` + without juggling errno-vs-throw state. Callers that want to + check a specific errno can catch `SysError` and inspect + `.errNo`. */ + auto check = [](std::string_view what, AutoCloseFD fd) { + if (!fd) + throw SysError("openFileEnsureBeneathNoSymlinks: %s", what); + return fd; + }; + auto openRead = [&](std::string_view path) -> AutoCloseFD { - return openFileEnsureBeneathNoSymlinks( - dirFd.get(), - CanonPath(path), + return check( + path, + openFileEnsureBeneathNoSymlinks( + dirFd.get(), + CanonPath(path), #ifdef _WIN32 - FILE_READ_DATA | FILE_READ_ATTRIBUTES | SYNCHRONIZE, - 0 + FILE_READ_DATA | FILE_READ_ATTRIBUTES | SYNCHRONIZE, + 0 #else - O_RDONLY, - 0 + O_RDONLY, + 0 #endif - ); + )); }; auto openReadDir = [&](std::string_view path) -> AutoCloseFD { - return openFileEnsureBeneathNoSymlinks( - dirFd.get(), - CanonPath(path), + return check( + path, + openFileEnsureBeneathNoSymlinks( + dirFd.get(), + CanonPath(path), #ifdef _WIN32 - FILE_READ_ATTRIBUTES | SYNCHRONIZE, - FILE_DIRECTORY_FILE + FILE_READ_ATTRIBUTES | SYNCHRONIZE, + FILE_DIRECTORY_FILE #else - O_RDONLY | O_DIRECTORY, - 0 + O_RDONLY | O_DIRECTORY, + 0 #endif - ); + )); + }; + +#if defined(__linux__) + auto openReadDirPath = [&](std::string_view path) -> AutoCloseFD { + return check( + path, openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath(path), O_PATH | O_DIRECTORY | O_CLOEXEC, 0)); }; + auto openPath = [&](std::string_view path) -> AutoCloseFD { + return check(path, openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath(path), O_PATH | O_CLOEXEC, 0)); + }; +#endif auto openCreateExclusive = [&](std::string_view path) -> AutoCloseFD { - return openFileEnsureBeneathNoSymlinks( - dirFd.get(), - CanonPath(path), + return check( + path, + openFileEnsureBeneathNoSymlinks( + dirFd.get(), + CanonPath(path), #ifdef _WIN32 - FILE_WRITE_DATA | SYNCHRONIZE, - 0, - FILE_CREATE // Create new file, fail if exists (equivalent to O_CREAT | O_EXCL) + FILE_WRITE_DATA | SYNCHRONIZE, + 0, + FILE_CREATE // Create new file, fail if exists (equivalent to O_CREAT | O_EXCL) #else - O_CREAT | O_WRONLY | O_EXCL, - 0666 + O_CREAT | O_WRONLY | O_EXCL, + 0666 #endif - ); + )); }; - // Test that symlinks are detected and rejected + using nix::testing::ThrowsSysError; + + // Symlinks in the path (any position) are rejected. EXPECT_THROW(openRead("a/absolute_symlink"), SymlinkNotAllowed); EXPECT_THROW(openRead("a/relative_symlink"), SymlinkNotAllowed); EXPECT_THROW(openRead("a/absolute_symlink/a"), SymlinkNotAllowed); @@ -173,21 +203,94 @@ TEST(openFileEnsureBeneathNoSymlinks, works) EXPECT_THROW(openRead("a/b/c/d"), SymlinkNotAllowed); EXPECT_THROW(openRead("a/broken_symlink"), SymlinkNotAllowed); + /* Trailing symlink with `O_DIRECTORY` (no `O_NOFOLLOW`): the + syscall reports `ELOOP`, handled by the direct + `SymlinkNotAllowed` throw in `openFileEnsureBeneathNoSymlinks`. */ + EXPECT_THROW(openReadDir("a/absolute_symlink"), SymlinkNotAllowed); + EXPECT_THROW(openReadDir("a/relative_symlink"), SymlinkNotAllowed); + EXPECT_THROW(openReadDir("a/b/c"), SymlinkNotAllowed); + EXPECT_THROW(openReadDir("a/broken_symlink"), SymlinkNotAllowed); + +#if defined(__linux__) + // Same thing for `O_PATH | O_DIRECTORY` (Linux-only flag). + EXPECT_THROW(openReadDirPath("a/absolute_symlink"), SymlinkNotAllowed); + EXPECT_THROW(openReadDirPath("a/relative_symlink"), SymlinkNotAllowed); + EXPECT_THROW(openReadDirPath("a/b/c"), SymlinkNotAllowed); + EXPECT_THROW(openReadDirPath("a/broken_symlink"), SymlinkNotAllowed); + + /* `O_PATH` on a trailing symlink is allowed: the caller gets a + path fd referring to the symlink itself. Interior symlinks are + still rejected. */ + EXPECT_TRUE(openPath("a/absolute_symlink")); + EXPECT_TRUE(openPath("a/relative_symlink")); + EXPECT_TRUE(openPath("a/b/c")); + EXPECT_TRUE(openPath("a/broken_symlink")); + EXPECT_THROW(openPath("a/absolute_symlink/a"), SymlinkNotAllowed); + EXPECT_THROW(openPath("a/absolute_symlink/c/d"), SymlinkNotAllowed); + EXPECT_THROW(openPath("a/relative_symlink/c"), SymlinkNotAllowed); + EXPECT_THROW(openPath("a/b/c/d"), SymlinkNotAllowed); +#endif + #if !defined(_WIN32) && !defined(__CYGWIN__) - // This returns ELOOP on cygwin when O_NOFOLLOW is used - EXPECT_FALSE(openCreateExclusive("a/broken_symlink")); - /* Sanity check, no symlink shenanigans and behaves the same as regular openat with O_EXCL | O_CREAT. */ - EXPECT_EQ(errno, EEXIST); + /* Sanity check: behaves the same as plain openat with + `O_EXCL | O_CREAT` on an existing broken-symlink path — + the symlink counts as "already there". */ + EXPECT_THAT([&] { openCreateExclusive("a/broken_symlink"); }, ThrowsSysError(EEXIST)); #endif EXPECT_THROW(openCreateExclusive("a/absolute_symlink/broken_symlink"), SymlinkNotAllowed); - // Test invalid paths - EXPECT_FALSE(openRead("c/d/regular/a")); - EXPECT_FALSE(openReadDir("c/d/regular")); + // Non-symlink failure modes: errno must survive. + EXPECT_THAT([&] { openRead("c/d/regular/a"); }, ThrowsSysError(ENOTDIR)); + EXPECT_THAT([&] { openReadDir("c/d/regular"); }, ThrowsSysError(ENOTDIR)); + EXPECT_THAT([&] { openRead("c/d/nonexistent"); }, ThrowsSysError(ENOENT)); - // Test valid paths work + // Happy paths. EXPECT_TRUE(openRead("c/d/regular")); + EXPECT_TRUE(openReadDir("c")); + EXPECT_TRUE(openReadDir("c/d")); EXPECT_TRUE(openCreateExclusive("a/regular")); +#if defined(__linux__) + EXPECT_TRUE(openReadDirPath("c")); + EXPECT_TRUE(openReadDirPath("c/d")); + EXPECT_TRUE(openPath("c/d/regular")); + EXPECT_TRUE(openPath("c")); +#endif +} + +/* ---------------------------------------------------------------------------- + * openFileEnsureBeneathNoSymlinks — O_NOFOLLOW is forbidden + * --------------------------------------------------------------------------*/ + +#if !defined(_WIN32) +TEST(openFileEnsureBeneathNoSymlinksDeathTest, rejectsONofollow) +{ + std::filesystem::path tmpDir = nix::createTempDir(); + nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true); + + { + RestoreSink sink(/*startFsync=*/false); + sink.dstPath = tmpDir; + sink.dirFd = openDirectory(tmpDir, FinalSymlink::Follow); + sink.createRegularFile(CanonPath("file"), [](CreateRegularFileSink & crf) {}); + } + + auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); + + /* The function asserts that callers don't pass `O_NOFOLLOW` — it + owns symlink policy. Verify the assert fires for every flag + combination that includes `O_NOFOLLOW`. */ + EXPECT_DEATH( + openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath("file"), O_RDONLY | O_NOFOLLOW, 0), "O_NOFOLLOW"); + EXPECT_DEATH( + openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath("file"), O_RDONLY | O_DIRECTORY | O_NOFOLLOW, 0), + "O_NOFOLLOW"); +# if defined(__linux__) + EXPECT_DEATH(openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath("file"), O_PATH | O_NOFOLLOW, 0), "O_NOFOLLOW"); + EXPECT_DEATH( + openFileEnsureBeneathNoSymlinks(dirFd.get(), CanonPath("file"), O_PATH | O_DIRECTORY | O_NOFOLLOW, 0), + "O_NOFOLLOW"); +# endif } +#endif } // namespace nix diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 6aebb3f5ea..b0bb20d99f 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -90,7 +90,7 @@ void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallba FILE_READ_ATTRIBUTES | SYNCHRONIZE, FILE_DIRECTORY_FILE #else - O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC, + O_RDONLY | O_DIRECTORY | O_CLOEXEC, 0 #endif ); diff --git a/src/libutil/include/nix/util/file-system-at.hh b/src/libutil/include/nix/util/file-system-at.hh index 75f4e6b2cf..0f4678461c 100644 --- a/src/libutil/include/nix/util/file-system-at.hh +++ b/src/libutil/include/nix/util/file-system-at.hh @@ -72,25 +72,45 @@ PosixStat fstatat(Descriptor dirFd, const std::filesystem::path & path); OsString readLinkAt(Descriptor dirFd, const CanonPath & path); /** - * Safe(r) function to open a file relative to dirFd, while - * disallowing escaping from a directory and any symlinks in the process. + * Open a file relative to @p dirFd, ensuring the path stays beneath + * @p dirFd and that no path component is a symlink (with the + * exception that `O_PATH` (without `O_DIRECTORY`) on Unix permits a + * trailing symlink). * - * @note On Windows, implemented via NtCreateFile single path component traversal - * with FILE_OPEN_REPARSE_POINT. On Unix, uses RESOLVE_BENEATH with openat2 when - * available, or falls back to openat single path component traversal. + * Callers must not pass `O_NOFOLLOW` on Unix (enforced by assert); + * this function owns symlink policy and handles the flag internally. * * @param dirFd Directory handle to open relative to - * @param path Relative path (no .. or . components) - * @param desiredAccess (Windows) Windows ACCESS_MASK (e.g., GENERIC_READ, FILE_WRITE_DATA) - * @param createOptions (Windows) Windows create options (e.g., FILE_NON_DIRECTORY_FILE) - * @param createDisposition (Windows) FILE_OPEN, FILE_CREATE, etc. - * @param flags (Unix) O_* flags - * @param mode (Unix) Mode for O_{CREAT,TMPFILE} + * @param path Relative path (with no `..` or `.` components) * - * @pre path.isRoot() is false + * @param desiredAccess (Windows) Windows `ACCESS_MASK` + * @param createOptions (Windows) Windows create options + * @param createDisposition (Windows) `FILE_OPEN`, `FILE_CREATE`, etc. + * + * @param flags (Unix) `O_*` flags (must not include `O_NOFOLLOW`) + * @param mode (Unix) Mode for `O_{CREAT,TMPFILE}` + * + * @pre `path.isRoot()` is false + * + * @throws SymlinkNotAllowed if an interior path component is a + * symlink, or if the final component is a symlink and `O_PATH` + * (without `O_DIRECTORY`) was *not* passed. With `O_PATH` + * (without `O_DIRECTORY`) on Unix, a trailing symlink is + * permitted and the caller receives a "path fd" to the symlink + * itself. + * + * @note With `O_CREAT | O_EXCL`, a pre-existing symlink at the + * final component causes the OS to return `EEXIST` rather + * than `ELOOP`, so `SymlinkNotAllowed` is *not* thrown — the + * caller sees a failed descriptor with `errno == EEXIST`. + * + * @return A valid descriptor on success, or an invalid descriptor + * on non-symlink errors (Unix: `errno` set, e.g. `ENOENT`, + * `ENOTDIR`, `EACCES`; Windows: last error set). The caller is + * responsible for checking the return value. * - * @throws SymlinkNotAllowed if any path components are symlinks - * @throws SystemError on other errors + * `errno` will never be `ELOOP` because that case is translated + * to a `SymlinkNotAllowed` throw instead. */ AutoCloseFD openFileEnsureBeneathNoSymlinks( Descriptor dirFd, diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index 01a4c664d3..762a245ded 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -175,6 +175,9 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat }); if (errno == ENOTDIR) /* Path component might be a symlink. */ { + /* Does not follow final symlink. We know `component` is a + single component so we don't have to worry about intermediate + symlinks either. */ if (auto st = maybeFstatat(getParentFd(), component); st && S_ISLNK(st->st_mode)) throw SymlinkNotAllowed(path2); errno = ENOTDIR; /* Restore the errno. */ @@ -188,25 +191,78 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat parentFd = std::move(parentFd2); } - AutoCloseFD res = ::openat(getParentFd(), std::string(path.baseName().value()).c_str(), flags | O_NOFOLLOW, mode); - if (!res && errno == ELOOP) - throw SymlinkNotAllowed(path); + auto lastComponent = std::string(path.baseName().value()); + AutoCloseFD res = ::openat(getParentFd(), lastComponent.c_str(), flags | O_NOFOLLOW, mode); + + if (!res) { + if (errno == ELOOP) + throw SymlinkNotAllowed(path); + /* `O_DIRECTORY | O_NOFOLLOW` on a trailing symlink returns + `ENOTDIR` rather than `ELOOP`. Post-check via `fstatat` to + disambiguate — only on the error path, so the common + successful-directory-open case pays no extra syscall. */ + if (errno == ENOTDIR) { + if (auto st = maybeFstatat(getParentFd(), lastComponent); st && S_ISLNK(st->st_mode)) + throw SymlinkNotAllowed(path); + /* Put back errno so the caller will get the original + error. */ + errno = ENOTDIR; + } + return res; + } + + /* For `O_PATH`, the defensive `| O_NOFOLLOW` we added above + means a trailing symlink silently succeeds with an fd to the + symlink itself (`O_PATH | O_NOFOLLOW` is the idiomatic way to + obtain a symlink fd). This is intentional — `O_PATH` callers + are asking for a path reference, and interior symlinks are + already guarded by the component-by-component walk above. */ + return res; } AutoCloseFD openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode) { - assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */ + /* Just in case the invariant is somehow broken. */ + assert(!path.rel().starts_with('/')); assert(!path.isRoot()); + + /* We don't want callers of this function to think about the presence or + absence of `O_NOFOLLOW`. "ensure beneath no symlinks" is in the name, so + we want them to trust us to handle it instead. */ + assert(!(flags & O_NOFOLLOW)); + + /* See doxygen in `file-system-at.hh` for why we reject this. */ + #if HAVE_OPENAT2 - auto maybeFd = linux::openat2( - dirFd, path.rel_c_str(), flags, static_cast(mode), RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS); - if (maybeFd) { + /* Two things are being fixed here: + + 1. For `O_PATH` (without `O_DIRECTORY`), add `O_NOFOLLOW` so + that a trailing symlink returns an fd to the symlink itself + rather than `ELOOP`. `O_PATH | O_NOFOLLOW` is the idiomatic + way to obtain a symlink fd, and `RESOLVE_NO_SYMLINKS` does + not refuse it. + + 2. We must not add `O_NOFOLLOW` when `O_DIRECTORY` is set, + because `O_DIRECTORY | O_NOFOLLOW` on a trailing symlink + returns `ENOTDIR` instead of `ELOOP`. Interior symlinks are + still caught by `RESOLVE_NO_SYMLINKS` regardless, but the + non-inclusion of `O_NOFOLLOW` is needed for + `RESOLVE_NO_SYMLINKS` to make the final symlink an `ELOOP` + rather than `O_DIRECTORY` making it an `ENOTDIR`. + + For other cases, `O_NOFOLLOW` doesn't really matter, but we + default to not including it. */ + auto flagsAdj = (flags & O_PATH) && !(flags & O_DIRECTORY) ? flags | O_NOFOLLOW : flags; + + if (auto maybeFd = linux::openat2( + dirFd, path.rel_c_str(), flagsAdj, static_cast(mode), RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS)) { if (!*maybeFd && errno == ELOOP) throw SymlinkNotAllowed(path); return std::move(*maybeFd); } #endif + return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode); } From dc83e83e73cbbcd07bd7fb45d85474bb798648cc Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 18 Mar 2026 15:43:50 -0400 Subject: [PATCH 228/927] Add option `--skip-alive` to `nix store delete` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows garbage collecting a closure. `nix store delete --recursive --skip-alive` is morally equivalent to ```bash for validPath in $(nix path-info --recursive foo); do nix store delete "$validPath" || true done ``` Resolves NixOS#7239 Co-authored-by: Théophane Hufschmitt Co-authored-by: Alexander Bantyev Signed-off-by: Lisanna Dettwyler --- doc/manual/rl-next/closure-gc.md | 9 ++ src/libstore/daemon.cc | 17 ++- src/libstore/gc.cc | 115 +++++++++++------- src/libstore/include/nix/store/gc-store.hh | 8 +- .../include/nix/store/worker-protocol.hh | 8 ++ src/libstore/remote-store.cc | 26 +++- src/libstore/worker-protocol.cc | 32 +++++ .../nix-collect-garbage.cc | 1 + src/nix/nix-store/nix-store.cc | 5 +- src/nix/store-delete.cc | 11 +- src/nix/store-gc.cc | 1 + tests/functional/gc-closure.sh | 29 +++++ tests/functional/meson.build | 1 + 13 files changed, 213 insertions(+), 50 deletions(-) create mode 100644 doc/manual/rl-next/closure-gc.md create mode 100755 tests/functional/gc-closure.sh diff --git a/doc/manual/rl-next/closure-gc.md b/doc/manual/rl-next/closure-gc.md new file mode 100644 index 0000000000..fc24d5e08b --- /dev/null +++ b/doc/manual/rl-next/closure-gc.md @@ -0,0 +1,9 @@ +--- +synopsis: "Added `--skip-alive` option to `nix store delete` for collecting garbage within a closure" +issues: 7239 +prs: 15236 +--- + +`nix store delete --recursive --skip-alive` can be used to collect garbage +within a closure, in which case it will only collect the dead paths that are +part of the closure of its arguments. diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 31d30097fc..52e1c121c3 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -17,6 +17,7 @@ #include "nix/util/args.hh" #include "nix/util/logging.hh" #include "nix/store/globals.hh" +#include #ifndef _WIN32 // TODO need graceful async exit support on Windows? # include "nix/util/monitor-fd.hh" @@ -745,13 +746,27 @@ static void performOp( case WorkerProto::Op::CollectGarbage: { GCOptions options; options.action = WorkerProto::Serialise::read(*store, rconn); - options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); + if (rconn.version.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); + } else { + auto paths = WorkerProto::Serialise::read(*store, rconn); + if (options.action != GCAction::gcDeleteSpecific && paths.empty()) + options.pathsToDelete = GCOptions::WholeStore{}; + else + options.pathsToDelete = paths; + } conn.from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(conn.from); readInt(conn.from); readInt(conn.from); + if (options.action == GCAction::gcDeleteDead && std::holds_alternative(options.pathsToDelete) + && !conn.protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + throw Error( + "Garbage collecting specific paths requested but it is not supported by the negotiated protocol"); + } + GCResults results; logger->startWork(); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f3bded97e4..835d068646 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,3 +1,4 @@ +#include "nix/store/gc-store.hh" #include "nix/store/local-gc.hh" #include "nix/store/local-settings.hh" #include "nix/store/local-store.hh" @@ -22,6 +23,7 @@ #include #include #include +#include #if HAVE_STATVFS # include #endif @@ -360,6 +362,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) boost::unordered_flat_set> roots, dead, alive; + /* Return early if nothing to delete */ + if (std::holds_alternative(options.pathsToDelete) + && std::get(options.pathsToDelete).empty()) + return; + struct Shared { // The temp roots only store the hash part to make it easier to @@ -577,7 +584,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) via the referrers edges and optionally derivers and derivation output edges. If none of those paths are roots, then all visited paths are garbage and are deleted. */ - auto deleteReferrersClosure = [&](const StorePath & start) { + auto maybeDeleteReferrersClosure = [&](const StorePath & start) { StorePathSet visited; std::queue todo; @@ -634,7 +641,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) return markAlive(); } - if (options.action == GCOptions::gcDeleteSpecific && !options.pathsToDelete.count(*path)) + if (std::holds_alternative(options.pathsToDelete) + && !std::get(options.pathsToDelete).contains(*path)) return; { @@ -694,50 +702,73 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - /* Either delete all garbage paths, or just the specified - paths (for gcDeleteSpecific). */ - if (options.action == GCOptions::gcDeleteSpecific) { - - for (auto & i : options.pathsToDelete) { - deleteReferrersClosure(i); - if (!dead.count(i)) - throw Error( - "Cannot delete path '%1%' since it is still alive. " - "To find out why, use: " - "nix-store --query --roots and nix-store --query --referrers", - printStorePath(i)); - } + try { + /* Either delete all garbage paths, or just the specified paths. */ + std::visit( + overloaded{ + [&](const StorePathSet & paths) { + for (auto & i : paths) { + switch (options.action) { + case GCOptions::gcDeleteDead: + printInfo("deleting garbage within specified paths..."); + break; + case GCOptions::gcDeleteSpecific: + printInfo("deleting specified paths..."); + break; + case GCOptions::gcReturnDead: + case GCOptions::gcReturnLive: + printInfo("determining live/dead paths..."); + } - } else if (options.maxFreed > 0) { + maybeDeleteReferrersClosure(i); - if (shouldDelete) - printInfo("deleting garbage..."); - else - printInfo("determining live/dead paths..."); + if (options.action == GCOptions::gcDeleteSpecific && !dead.count(i)) + throw Error( + "Cannot delete path '%1%' since it is still alive. " + "To find out why, use: " + "nix-store --query --roots and nix-store --query --referrers", + printStorePath(i)); + } + }, + [&](const GCOptions::WholeStore & _) { + if (options.maxFreed == 0) + return; - try { - AutoCloseDir dir(opendir(config->realStoreDir.get().string().c_str())); - if (!dir) - throw SysError("opening directory %1%", PathFmt(config->realStoreDir.get())); - - /* Read the store and delete all paths that are invalid or - unreachable. We don't use readDirectory() here so that - GCing can start faster. */ - auto linksName = linksDir.filename(); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == ".." || name == linksName) - continue; + switch (options.action) { + case GCOptions::gcDeleteDead: + printInfo("deleting garbage..."); + break; + case GCOptions::gcDeleteSpecific: + throw Error("Cannot delete the entire store"); + case GCOptions::gcReturnDead: + case GCOptions::gcReturnLive: + printInfo("determining live/dead paths..."); + } - if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) - deleteReferrersClosure(*storePath); - else - deleteFromStore(name, false); - } - } catch (GCLimitReached & e) { - } + AutoCloseDir dir(opendir(config->realStoreDir.get().string().c_str())); + if (!dir) + throw SysError("opening directory %1%", PathFmt(config->realStoreDir.get())); + + /* Read the store and delete all paths that are invalid or + unreachable. We don't use readDirectory() here so that + GCing can start faster. */ + auto linksName = linksDir.filename(); + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == ".." || name == linksName) + continue; + + if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) + maybeDeleteReferrersClosure(*storePath); + else + deleteFromStore(name, false); + } + }, + }, + options.pathsToDelete); + } catch (GCLimitReached & e) { } if (options.action == GCOptions::gcReturnLive) { diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index de016e241d..5e23f20524 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -39,6 +39,9 @@ struct GCOptions using GCAction = nix::GCAction; using enum GCAction; + struct WholeStore + {}; + GCAction action{gcDeleteDead}; /** @@ -50,9 +53,10 @@ struct GCOptions bool ignoreLiveness{false}; /** - * For `gcDeleteSpecific`, the paths to delete. + * The paths from which to delete. */ - StorePathSet pathsToDelete; + using GCPaths = std::variant; + GCPaths pathsToDelete; /** * Stop after at least `maxFreed` bytes have been freed. diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 8e20d3c992..ea1cbb502d 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -6,6 +6,7 @@ #include #include "nix/store/common-protocol.hh" +#include "nix/store/gc-store.hh" namespace nix { @@ -124,6 +125,11 @@ struct WorkerProto */ static constexpr std::string_view featureRealisationWithPath = "realisation-with-path-not-hash"; + /** + * Feature for garbage collecting a specific set of paths. + */ + static constexpr std::string_view featureDeleteDeadSpecific = "delete-dead-specific"; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. @@ -339,6 +345,8 @@ template<> DECLARE_WORKER_SERIALISER(std::optional); template<> DECLARE_WORKER_SERIALISER(WorkerProto::ClientHandshakeInfo); +template<> +DECLARE_WORKER_SERIALISER(GCOptions::GCPaths); template DECLARE_WORKER_SERIALISER(std::vector); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 3bb5b07557..2def1bf87a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,3 +1,5 @@ +#include "nix/store/path.hh" +#include "nix/store/store-api.hh" #include "nix/util/serialise.hh" #include "nix/util/util.hh" #include "nix/store/path-with-outputs.hh" @@ -18,6 +20,7 @@ #include "nix/store/filetransfer.hh" #include "nix/util/signals.hh" #include "nix/util/socket.hh" +#include #ifndef _WIN32 # include @@ -679,9 +682,26 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { auto conn(getConnection()); - conn->to << WorkerProto::Op::CollectGarbage; - WorkerProto::write(*this, *conn, options.action); - WorkerProto::write(*this, *conn, options.pathsToDelete); + if (conn->protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + conn->to << WorkerProto::Op::CollectGarbage; + WorkerProto::write(*this, *conn, options.action); + WorkerProto::write(*this, *conn, options.pathsToDelete); + } else { + auto paths = std::visit( + overloaded{ + [&](const StorePathSet & paths) { + if (options.action != GCOptions::gcDeleteSpecific) + throw Error( + "Your daemon version is too old to support garbage collecting a specific set of paths"); + return paths; + }, + [](const GCOptions::WholeStore & _) { return StorePathSet{}; }, + }, + options.pathsToDelete); + conn->to << WorkerProto::Op::CollectGarbage; + WorkerProto::write(*this, *conn, options.action); + WorkerProto::write(*this, *conn, paths); + } conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 7f9bd163d9..31373a128f 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -1,3 +1,4 @@ +#include "nix/store/store-dir-config.hh" #include "nix/util/serialise.hh" #include "nix/store/path-with-outputs.hh" #include "nix/store/store-api.hh" @@ -8,8 +9,10 @@ #include "nix/store/worker-protocol-impl.hh" #include "nix/store/path-info.hh" #include "nix/util/json-utils.hh" +#include "nix/util/util.hh" #include +#include #include namespace nix { @@ -25,6 +28,7 @@ const WorkerProto::Version WorkerProto::latest = { std::string{ WorkerProto::featureRealisationWithPath, }, + std::string{WorkerProto::featureDeleteDeadSpecific}, }, }; @@ -529,4 +533,32 @@ void WorkerProto::Serialise::write(const StoreDirConfig & store, Wr WorkerProto::write(store, conn, static_cast(info)); } +GCOptions::GCPaths WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + uint8_t wholeStore; + conn.from >> wholeStore; + switch (wholeStore) { + case 0: + return WorkerProto::Serialise::read(store, conn); + case 1: + return GCOptions::WholeStore{}; + default: + throw Error("Invalid whole store indicator from remote"); + } +} + +void WorkerProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const GCOptions::GCPaths & gcPaths) +{ + std::visit( + overloaded{ + [&](const StorePathSet paths) { + conn.to << uint8_t{0}; + WorkerProto::write(store, conn, paths); + }, + [&](const GCOptions::WholeStore & _) { conn.to << uint8_t{1}; }, + }, + gcPaths); +} + } // namespace nix diff --git a/src/nix/nix-collect-garbage/nix-collect-garbage.cc b/src/nix/nix-collect-garbage/nix-collect-garbage.cc index a9f8b7fb6b..abab2ce4d2 100644 --- a/src/nix/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix/nix-collect-garbage/nix-collect-garbage.cc @@ -100,6 +100,7 @@ static int main_nix_collect_garbage(int argc, char ** argv) auto store = openStore(); auto & gcStore = require(*store); options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + options.pathsToDelete = GCOptions::WholeStore{}; GCResults results; Finally printer([&] { printFreed(dryRun, results); }); gcStore.collectGarbage(options, results); diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index a534e19afb..9daf15aa94 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -663,6 +663,7 @@ static void opGC(Strings opFlags, Strings opArgs) bool printRoots = false; GCOptions options; options.action = GCOptions::gcDeleteDead; + options.pathsToDelete = GCOptions::WholeStore{}; GCResults results; @@ -725,8 +726,10 @@ static void opDelete(Strings opFlags, Strings opArgs) else throw UsageError("unknown flag '%1%'", i); + StorePathSet paths; for (auto & i : opArgs) - options.pathsToDelete.insert(store->followLinksToStorePath(i)); + paths.insert(store->followLinksToStorePath(i)); + options.pathsToDelete = std::move(paths); auto & gcStore = require(*store); diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 5d28979a49..12b03f870a 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -17,6 +17,13 @@ struct CmdStoreDelete : StorePathsCommand .description = "Do not check whether the paths are reachable from a root.", .handler = {&options.ignoreLiveness, true}, }); + + addFlag({ + .longName = "skip-alive", + .description = + "Do not emit errors when attempting to delete something that is still alive, useful with --recursive.", + .handler = {&options.action, GCOptions::gcDeleteDead}, + }); } std::string description() override @@ -35,8 +42,10 @@ struct CmdStoreDelete : StorePathsCommand { auto & gcStore = require(*store); + StorePathSet paths; for (auto & path : storePaths) - options.pathsToDelete.insert(path); + paths.insert(path); + options.pathsToDelete = std::move(paths); GCResults results; Finally printer([&] { printFreed(false, results); }); diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index f4624a40e8..02df4ec060 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -42,6 +42,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun auto & gcStore = require(*store); options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + options.pathsToDelete = GCOptions::WholeStore{}; GCResults results; Finally printer([&] { printFreed(dryRun, results); }); gcStore.collectGarbage(options, results); diff --git a/tests/functional/gc-closure.sh b/tests/functional/gc-closure.sh new file mode 100755 index 0000000000..03e2a0fc25 --- /dev/null +++ b/tests/functional/gc-closure.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +nix_gc_closure() { + clearStore + nix build -f dependencies.nix input0_drv --out-link "$TEST_ROOT/gc-root" + input0=$(realpath "$TEST_ROOT/gc-root") + input1=$(nix build -f dependencies.nix input1_drv --no-link --print-out-paths) + input2=$(nix build -f dependencies.nix input2_drv --no-link --print-out-paths) + top=$(nix build -f dependencies.nix --no-link --print-out-paths) + somthing_else=$(nix store add-path ./dependencies.nix) + + if isDaemonNewer "2.35pre"; then + # Check that nix store delete --recursive --skip-alive is best-effort (doesn't fail when some paths in the closure are alive) + nix store delete --recursive --skip-alive "$top" + [[ ! -e "$top" ]] || fail "top should have been deleted" + [[ -e "$input0" ]] || fail "input0 is a gc root, shouldn't have been deleted" + [[ ! -e "$input2" ]] || fail "input2 is not a gc root and is part of top's closure, it should have been deleted" + [[ -e "$input1" ]] || fail "input1 is not in the closure of top, it shouldn't have been deleted" + [[ -e "$somthing_else" ]] || fail "somthing_else is not in the closure of top, it shouldn't have been deleted" + else + expectStderr 1 nix store delete --recursive --skip-alive "$top" | grepQuiet "Your daemon version is too old to support garbage collecting a specific set of paths" + fi +} + +nix_gc_closure diff --git a/tests/functional/meson.build b/tests/functional/meson.build index bc4d2643e2..e2668c716e 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -89,6 +89,7 @@ suites = [ 'hash-convert.sh', 'hash-path.sh', 'gc-non-blocking.sh', + 'gc-closure.sh', 'check.sh', 'nix-shell.sh', 'check-refs.sh', From 26bb1f12c8030454f324d71769f509a6f23bf4d4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 11 Apr 2026 17:13:33 +0300 Subject: [PATCH 229/927] libmain: Don't raise the RLIMIT_NOFILE to RLIM_INFINITY, cap at 1048576 used by the daemon See the comment. Should fix https://github.com/NixOS/nix/issues/15619. See: https://cgit.git.savannah.gnu.org/cgit/patch.git/commit/?id=61d7788b83b302207a67b82786f4fd79e3538f30 --- src/libmain/shared.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index f3293060f1..6ddabf12c3 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -133,7 +133,14 @@ void bumpFileLimit() return; if (limit.rlim_cur < limit.rlim_max) { - limit.rlim_cur = limit.rlim_max; + // Some software misbehaves really bad when we try to raise the + // limit to RLIM_INFINITY, so cap the limit at the 1048576 limit used + // by the daemon. + // + // GNU patch < 2.8 crashes with **** out of memory, which breaks in nixpkgs darwin bootstrap tools. + // This was fixed in: + // https://cgit.git.savannah.gnu.org/cgit/patch.git/commit/?id=61d7788b83b302207a67b82786f4fd79e3538f30 + limit.rlim_cur = std::min(limit.rlim_max, rlim_t(1048576)); // Ignore errors, this is best effort. setrlimit(RLIMIT_NOFILE, &limit); } From 9418535d877b043327c1c16ac45c0be71e6d49cd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 5 Apr 2026 12:19:58 +0200 Subject: [PATCH 230/927] Remove shallow.lock before running git fetch An interrupted `git fetch` can leave behind a `shallow.lock` file which causes subsequent fetches to fail. Since Nix already has a PathLock on the repo, it should be safe to just delete this file. --- src/libfetchers/git-utils.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index fcffdcfa8a..0bf984652e 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -638,6 +638,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this // then use code that was removed in this commit (see blame) auto dir = this->path; + + // Remove shallow.lock left behind by a previously interrupted `git fetch`, as it would prevent `git fetch` + // from running. Note that we already have a repository-wide `PathLock` (see git.cc), so this is safe. + tryUnlink(dir / "shallow.lock"); + OsStrings gitArgs = { OS_STR("-C"), dir.native(), From 729819d21f5d098706fc4195d3eb624b3abcd08f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 11 Apr 2026 12:38:12 -0400 Subject: [PATCH 231/927] tests/functional/nars: test edge-case restore destinations Add expected-failure tests for `nix-store --restore` with destinations that can't work: - Trailing `/.` and `/..` require the named directory to already exist, which conflicts with `--restore` creating something new. Tested for file, symlink, and directory NARs. - Empty string destination. All currently fail with "No such file or directory". These establish a baseline so we notice if the behaviour changes during refactoring. Also replace `(! ...)` with `expectStderr 1` for the case-hack collision dump test, matching the rest of the file's convention. Also add a comment on a symlink test I found interesting. --- tests/functional/nars.sh | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index ecae66c141..9b8fe67716 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -35,6 +35,11 @@ rm -rf "$TEST_ROOT/out" mkdir -p "$TEST_ROOT/out" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" +# Note that the target of the symlink doesn't exist, but the parent does. +# +# A more lenient implementation could allow this, but it would hard/impossible +# to implement without TOCTOU (to the extent avoiding TOCTOU is even +# well-defined with symlinks, though). rm -rf "$TEST_ROOT/out" ln -s "$TEST_ROOT/out2" "$TEST_ROOT/out" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" @@ -47,6 +52,19 @@ rm -rf "$TEST_ROOT/out" (cd "$TEST_ROOT" && nix-store --restore out < tmp.nar) [[ -f "$TEST_ROOT/out" ]] +# Trailing `/.` and `/..` refer to entries within a directory that +# must already exist, which conflicts with --restore creating +# something new. +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out/." < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" + +# Destination with trailing `/..` should fail. +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out/.." < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" + +# Empty destination should fail. +expectStderr 1 nix-store --restore "" < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" + # The same, but for a symlink. ln -sfn foo "$TEST_ROOT/symlink" nix-store --dump "$TEST_ROOT/symlink" > "$TEST_ROOT/tmp.nar" @@ -73,6 +91,10 @@ rm -rf "$TEST_ROOT/out" [[ -L "$TEST_ROOT/out" ]] [[ $(readlink "$TEST_ROOT/out") = foo ]] +# Trailing `/.` — same baseline as above (currently fails). +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out/." < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" + # Check whether restoring and dumping a NAR that contains case # collisions is round-tripping, even on a case-insensitive system. rm -rf "$TEST_ROOT/case" @@ -96,7 +118,11 @@ rm -rf "$TEST_ROOT/case" # Check whether we detect true collisions (e.g. those remaining after # removal of the suffix). touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" -(! nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > /dev/null) +expectStderr 1 nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > /dev/null + +# Trailing `/.` — same baseline as above (currently fails). +rm -rf "$TEST_ROOT/case" +expectStderr 1 nix-store "${opts[@]}" --restore "$TEST_ROOT/case/." < case.nar | grepQuiet "No such file or directory" # Detect NARs that have a directory entry that after case-hacking # collides with another entry (e.g. a directory containing 'Test', From 95e4446b4eaf5b44dccc320aa76246bfdfb71cdf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Apr 2026 16:53:50 -0400 Subject: [PATCH 232/927] libutil: Use `*at` operations for root file/symlink in `RestoreSink` Previously, when the root entry was a file or symlink (rather than a directory), `RestoreSink` fell back to path-based operations without the race-free symlink safety of `*at` calls. Now, all three methods (`createDirectory`, `createRegularFile`, `createSymlink`) use a shared static `getParentFdAndName` helper that always returns a valid parent fd and single-component name, ensuring consistent use of `openFileEnsureBeneathNoSymlinks`, `mkdirat`, and `symlinkat` across all cases. Multi-component paths also now go through `openFileEnsureBeneathNoSymlinks`, fixing a case where `createDirectory` would follow symlinks in intermediate path components. Update the `openFileEnsureBeneathNoSymlinks` unit test to expect `SymlinkNotAllowed` on Unix too (previously a FIXME that noted `createDirectory` incorrectly succeeded). This is now caught because `getParentFdAndName` routes multi-component paths through `openFileEnsureBeneathNoSymlinks` rather than passing them directly to `mkdirat`. This doesn't help the caller so much because they cannot provide the parent FD, but that will be fixed next. Co-authored-by: Sergei Zimmerman --- src/libutil-tests/file-system-at.cc | 5 -- src/libutil/fs-sink.cc | 111 +++++++++++++++++++--------- tests/functional/nars.sh | 20 ++--- 3 files changed, 87 insertions(+), 49 deletions(-) diff --git a/src/libutil-tests/file-system-at.cc b/src/libutil-tests/file-system-at.cc index 82b73d7af0..527e6fc261 100644 --- a/src/libutil-tests/file-system-at.cc +++ b/src/libutil-tests/file-system-at.cc @@ -102,12 +102,7 @@ TEST(openFileEnsureBeneathNoSymlinks, works) dirSink.createDirectory(CanonPath("d")); dirSink.createSymlink(CanonPath("c"), "./d"); }); -#ifdef _WIN32 EXPECT_THROW(sink.createDirectory(CanonPath("a/b/c/e")), SymlinkNotAllowed); -#else - // FIXME: This still follows symlinks on Unix (incorrectly succeeds) - sink.createDirectory(CanonPath("a/b/c/e")); -#endif // Test that symlinks in intermediate path are detected during nested operations EXPECT_THROW( sink.createDirectory( diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index b0bb20d99f..e41d153e92 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -70,6 +70,61 @@ static std::filesystem::path append(const std::filesystem::path & src, const Can return dst; } +#ifndef _WIN32 +/** + * Return a descriptor and single-component name suitable for + * `*at` operations. The returned `CanonPath` is always a pure + * name (no slashes). The `Descriptor` is the fd to use. The + * `AutoCloseFD` keeps it alive when it was opened temporarily + * (for multi-component paths); otherwise it is empty and the + * `Descriptor` borrows from `dirFd`. When `dirFd` is not set, + * temporarily opens the parent of `dstPath`. + */ +static std::tuple +getParentFdAndName(Descriptor dirFd, const std::filesystem::path & dstPath, const CanonPath & path) +{ + if (dirFd != INVALID_DESCRIPTOR) { + /* dirFd is the root of the restore tree, which means we already created + a root directory, which means that path must be relative (i.e. not + root) within it. */ + assert(!path.isRoot()); + auto parent = path.parent(); + if (parent->isRoot()) + return {AutoCloseFD{}, dirFd, CanonPath::fromFilename(*path.baseName())}; + auto parentFd = openFileEnsureBeneathNoSymlinks(dirFd, *parent, O_RDONLY | O_DIRECTORY | O_CLOEXEC, 0); + if (!parentFd) + throw SysError("opening parent directory of %s", PathFmt(append(dstPath, path))); + auto fd = parentFd.get(); + return {std::move(parentFd), fd, CanonPath::fromFilename(*path.baseName())}; + } + + /* Without dirFd, we're creating the root entry itself, so path + must be root. If it's not, someone forgot to create the root + directory first. */ + auto p = append(dstPath, path); + if (!path.isRoot()) + throw Error("cannot create non-root path %s without a root directory", PathFmt(p)); + if (p.empty()) + throw Error("restore destination path is empty"); + auto filename = p.filename(); + if (filename == "." || filename == "..") + throw Error( + "restore destination '%s' ends in '%s', which is not a valid filename", p.native(), filename.native()); + auto parentPath = p.parent_path(); + /* Relative path with no directory component (e.g. "out") — + the parent is the current working directory. Open it so we + hold a stable reference in case something else in the process + changes the working directory mid-unpack. */ + if (parentPath.empty()) + parentPath = "."; + AutoCloseFD parentFd{::open(parentPath.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC)}; + if (!parentFd) + throw SysError("opening parent directory of %s", PathFmt(p)); + auto fd = parentFd.get(); + return {std::move(parentFd), fd, CanonPath::fromFilename(p.filename().native())}; +} +#endif + void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallback callback) { if (path.isRoot()) { @@ -103,34 +158,27 @@ void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallba void RestoreSink::createDirectory(const CanonPath & path) { - auto p = append(dstPath, path); - #ifndef _WIN32 - if (dirFd) { - if (path.isRoot()) - /* Trying to create a directory that we already have a file descriptor for. */ - throw Error("path %s already exists", PathFmt(p)); + if (dirFd && path.isRoot()) + /* Trying to create a directory that we already have a file descriptor for. */ + throw Error("path %s already exists", PathFmt(append(dstPath, path))); - if (::mkdirat(dirFd.get(), path.rel_c_str(), 0777) == -1) - throw SysError("creating directory %s", PathFmt(p)); + auto [_parentFd, fd, name] = getParentFdAndName(dirFd.get(), dstPath, path); - return; - } -#endif - - if (!std::filesystem::create_directory(p)) - throw Error("path '%s' already exists", p.string()); - -#ifndef _WIN32 - if (path.isRoot()) { - assert(!dirFd); // Handled above + if (::mkdirat(fd, name.rel_c_str(), 0777) == -1) + throw SysError("creating directory %s", PathFmt(append(dstPath, path))); + if (!dirFd) { /* Open directory for further *at operations relative to the sink root directory. */ - dirFd = open(p.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + dirFd = openFileEnsureBeneathNoSymlinks(fd, name, O_RDONLY | O_DIRECTORY | O_CLOEXEC, 0); if (!dirFd) - throw SysError("creating directory %1%", PathFmt(p)); + throw SysError("opening directory %s", PathFmt(append(dstPath, path))); } +#else + auto p = append(dstPath, path); + if (!std::filesystem::create_directory(p)) + throw Error("path '%s' already exists", p.string()); #endif }; @@ -172,13 +220,11 @@ struct RestoreRegularFile : CreateRegularFileSink, FdSink void RestoreSink::createRegularFile(const CanonPath & path, fun func) { - auto p = append(dstPath, path); - auto crf = RestoreRegularFile( startFsync, #ifdef _WIN32 CreateFileW( - p.c_str(), + append(dstPath, path).c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, @@ -190,14 +236,13 @@ void RestoreSink::createRegularFile(const CanonPath & path, fun '%2%'", PathFmt(p), target); - return; - } + auto [_parentFd, fd, name] = getParentFdAndName(dirFd.get(), dstPath, path); + if (::symlinkat(requireCString(target), fd, name.rel_c_str()) == -1) + throw SysError("creating symlink from %1% -> '%2%'", PathFmt(append(dstPath, path)), target); +#else + nix::createSymlink(target, append(dstPath, path).string()); #endif - nix::createSymlink(target, p.string()); } void RegularFileSink::createRegularFile(const CanonPath & path, fun func) diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index 9b8fe67716..02ee66a14d 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -11,18 +11,18 @@ rm -rf "$TEST_ROOT/out" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "NAR directory is not sorted" # Check that nix-store --restore fails if the output already exists. -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "path '.*/out' already exists" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet 'creating directory ".*/out": File exists' rm -rf "$TEST_ROOT/out" echo foo > "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "File exists" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet 'creating directory ".*/out": File exists' rm -rf "$TEST_ROOT/out" ln -s "$TEST_ROOT/out2" "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "File exists" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet 'creating directory ".*/out": File exists' mkdir -p "$TEST_ROOT/out2" -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "path '.*/out' already exists" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet 'creating directory ".*/out": File exists' # The same, but for a regular file. nix-store --dump ./nars.sh > "$TEST_ROOT/tmp.nar" @@ -56,14 +56,14 @@ rm -rf "$TEST_ROOT/out" # must already exist, which conflicts with --restore creating # something new. rm -rf "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out/." < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" +expectStderr 1 nix-store --restore "$TEST_ROOT/out/." < "$TEST_ROOT/tmp.nar" | grepQuiet "ends in '\.'.*not a valid filename" # Destination with trailing `/..` should fail. rm -rf "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out/.." < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" +expectStderr 1 nix-store --restore "$TEST_ROOT/out/.." < "$TEST_ROOT/tmp.nar" | grepQuiet "ends in '\.\.'.*not a valid filename" # Empty destination should fail. -expectStderr 1 nix-store --restore "" < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" +expectStderr 1 nix-store --restore "" < "$TEST_ROOT/tmp.nar" | grepQuiet "destination path is empty" # The same, but for a symlink. ln -sfn foo "$TEST_ROOT/symlink" @@ -93,7 +93,7 @@ rm -rf "$TEST_ROOT/out" # Trailing `/.` — same baseline as above (currently fails). rm -rf "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out/." < "$TEST_ROOT/tmp.nar" | grepQuiet "No such file or directory" +expectStderr 1 nix-store --restore "$TEST_ROOT/out/." < "$TEST_ROOT/tmp.nar" | grepQuiet "ends in '\.'.*not a valid filename" # Check whether restoring and dumping a NAR that contains case # collisions is round-tripping, even on a case-insensitive system. @@ -122,7 +122,7 @@ expectStderr 1 nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > /dev/null # Trailing `/.` — same baseline as above (currently fails). rm -rf "$TEST_ROOT/case" -expectStderr 1 nix-store "${opts[@]}" --restore "$TEST_ROOT/case/." < case.nar | grepQuiet "No such file or directory" +expectStderr 1 nix-store "${opts[@]}" --restore "$TEST_ROOT/case/." < case.nar | grepQuiet "ends in '\.'.*not a valid filename" # Detect NARs that have a directory entry that after case-hacking # collides with another entry (e.g. a directory containing 'Test', @@ -156,7 +156,7 @@ if (( unicodeTestCode == 1 )); then # If the command failed (MacOS or ZFS + normalization), checks that it failed # with the expected "already exists" error, and that this is the same # behavior as `touch` - echo "$unicodeTestOut" | grepQuiet "creating directory \".*/out/â\": File exists" + echo "$unicodeTestOut" | grepQuiet 'creating directory ".*/out/â": File exists' (( touchFilesCount == 1 )) elif (( unicodeTestCode == 0 )); then From a4064a93f779c939450dbdcb4a145fde4ab34708 Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Sun, 12 Apr 2026 14:52:18 +1000 Subject: [PATCH 233/927] flake: remove x86_64-unknown-freebsd13 seems this was a no op, freebsd13 was changed to freebsd a while ago --- flake.nix | 3 --- 1 file changed, 3 deletions(-) diff --git a/flake.nix b/flake.nix index 97db5eef93..89cbe93cff 100644 --- a/flake.nix +++ b/flake.nix @@ -113,9 +113,6 @@ { config = crossSystem; } - // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") { - useLLVM = true; - } // lib.optionalAttrs (crossSystem == "x86_64-w64-mingw32") { emulator = pkgs: "${pkgs.buildPackages.wineWow64Packages.stable_11}/bin/wine"; }; From 9fae7a90f1699fcc823c6d2853d8cfc023c60852 Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Mon, 13 Apr 2026 12:49:41 +1000 Subject: [PATCH 234/927] libstore: enable sandbox by default on freebsd --- src/libstore/include/nix/store/local-settings.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index 962e831a3e..4fe28818d2 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -342,7 +342,7 @@ struct LocalSettings : public virtual Config, public GCSettings, public AutoAllo Setting sandboxMode{ this, -#ifdef __linux__ +#if defined(__linux__) || defined(__FreeBSD__) smEnabled #else smDisabled From 72fb36eeff3428d3742bb7630039672f8b6f0991 Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Mon, 13 Apr 2026 12:50:03 +1000 Subject: [PATCH 235/927] packaging: add x86_64-freebsd to installer script --- packaging/hydra.nix | 1 + packaging/installer/install.in | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index fa41b070cc..8f7e13c1f2 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -286,6 +286,7 @@ rec { self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + self.hydraJobs.binaryTarballCross."x86_64-linux"."x86_64-unknown-freebsd" ]; installerScriptForGHA = forAllSystems ( diff --git a/packaging/installer/install.in b/packaging/installer/install.in index b4e808d8e9..af61a433dd 100755 --- a/packaging/installer/install.in +++ b/packaging/installer/install.in @@ -65,6 +65,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_aarch64-darwin@ system=aarch64-darwin ;; + FreeBSD.amd64|FreeBSD.x86_64) + hash=@tarballHash_x86_64-freebsd@ + path=@tarballPath_x86_64-freebsd@ + system=x86_64-freebsd + ;; *) oops "sorry, there is no binary distribution of Nix for your platform";; esac From 9edc035b7ee3a112d8531ef7430dcac05114f458 Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Mon, 13 Apr 2026 12:50:34 +1000 Subject: [PATCH 236/927] maintainers/upload-release.pl: add x86_64-freebsd to nix-fallback-paths --- maintainers/upload-release.pl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index f7678b7d1c..06678553e7 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -279,6 +279,10 @@ sub downloadFile { downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); }; warn "$@" if $@; + eval { + downloadFile("binaryTarballCross.x86_64-linux.x86_64-unknown-freebsd", "1"); + }; + warn "$@" if $@; downloadFile("installerScript", "1"); # Upload nix-fallback-paths.nix. @@ -290,6 +294,7 @@ sub downloadFile { " riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . " x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" . " aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" . + " x86_64-freebsd = \"" . getStorePath("buildCross.nix-everything.x86_64-unknown-freebsd.x86_64-linux") . "\";\n" . "}\n"); for my $fn (glob "$tmpDir/*") { From 65ff1e6ebf294684d6f6c4ec24b45897b6bd6f60 Mon Sep 17 00:00:00 2001 From: espes Date: Wed, 25 Mar 2026 10:44:34 +0000 Subject: [PATCH 237/927] libutil: emit multi-frame zstd for parallel-decodable output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Libarchive's zstd filter always produces a single frame, so decompression of large NARs is stuck on one core. Replace the libarchive zstd compression path with a direct-libzstd sink that cuts a new frame every 16 MiB of uncompressed input, each with an exact pledged size so Frame_Content_Size lands in the header. Frame concatenation is mandatory in RFC 8878 §3.1, so existing nix binaries, libarchive and the zstd CLI decode the result unchanged. Nix still decompresses serially today, but the independent sized frames let a future parallel decoder exploit data already on disk. Per-frame compression uses up to 4 zstd workers (bounded by getMaxCPU()/hardware_concurrency). parallel-compression now defaults to true for zstd; xz keeps its false default. As a side effect peak RSS during compression drops substantially (~600 -> ~100 MiB for a 1 GiB store path) with effectively unchanged ratio. --- doc/manual/rl-next/zstd-multiframe.md | 18 +++ src/libstore/binary-cache-store.cc | 6 +- .../include/nix/store/binary-cache-store.hh | 6 +- src/libutil-tests/compression.cc | 58 ++++++++ src/libutil-tests/meson.build | 3 + src/libutil-tests/package.nix | 2 + src/libutil/compression.cc | 134 +++++++++++++++++- src/libutil/meson.build | 4 + src/libutil/package.nix | 2 + 9 files changed, 227 insertions(+), 6 deletions(-) create mode 100644 doc/manual/rl-next/zstd-multiframe.md diff --git a/doc/manual/rl-next/zstd-multiframe.md b/doc/manual/rl-next/zstd-multiframe.md new file mode 100644 index 0000000000..6f5d9875df --- /dev/null +++ b/doc/manual/rl-next/zstd-multiframe.md @@ -0,0 +1,18 @@ +--- +synopsis: zstd compression now emits multi-frame output and uses less memory +prs: [15550] +--- + +zstd-compressed NARs are now written as a sequence of independent 16 MiB +frames instead of a single large frame. This lays the groundwork for +parallel decompression in a future release without requiring caches to be +repopulated, and significantly lowers peak memory use during compression +(e.g. from ~600 MiB to ~100 MiB for a 1 GiB store path). + +The output remains standard zstd and is decoded unchanged by existing Nix +binaries and the `zstd` CLI; compression ratio is effectively unchanged. + +Per-frame compression now uses up to 4 worker threads. For zstd this is the +new default: the `parallel-compression` store setting defaults to `true` when +`compression=zstd` (it remains `false` for `xz`). Set +`?parallel-compression=false` to opt out. diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 624bbd6496..64fe33536b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -150,8 +150,10 @@ ref BinaryCacheStore::addToStoreCommon( { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed{fileSink, fileHashSink}; - auto compressionSink = makeCompressionSink( - config.compression, teeSinkCompressed, config.parallelCompression, config.compressionLevel); + bool parallel = config.parallelCompression.overridden ? config.parallelCompression.get() + : config.compression.get() == CompressionAlgo::zstd; + auto compressionSink = + makeCompressionSink(config.compression, teeSinkCompressed, parallel, config.compressionLevel); TeeSink teeSinkUncompressed{*compressionSink, narHashSink}; TeeSource teeSource{narSource, teeSinkUncompressed}; narAccessor = makeNarAccessor(parseNarListing(teeSource)); diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index ad38ddaa5b..7871ad03c8 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -58,7 +58,11 @@ struct BinaryCacheStoreConfig : virtual StoreConfig this, false, "parallel-compression", - "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + R"( + Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`. + + If not set explicitly, defaults to `true` when `compression` is `zstd` and `false` otherwise. + )"}; Setting compressionLevel{ this, diff --git a/src/libutil-tests/compression.cc b/src/libutil-tests/compression.cc index 53d476fa85..f1575a67c5 100644 --- a/src/libutil-tests/compression.cc +++ b/src/libutil-tests/compression.cc @@ -1,5 +1,6 @@ #include "nix/util/compression.hh" #include +#include namespace nix { @@ -61,6 +62,63 @@ TEST(decompress, decompressBrCompressed) ASSERT_EQ(o, str); } +TEST(decompress, decompressZstdCompressed) +{ + auto method = "zstd"; + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + auto o = decompress(method, compress(CompressionAlgo::zstd, str)); + + ASSERT_EQ(o, str); +} + +TEST(decompress, decompressZstdCompressedParallel) +{ + auto method = "zstd"; + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + auto o = decompress(method, compress(CompressionAlgo::zstd, str, true)); + + ASSERT_EQ(o, str); +} + +TEST(decompress, decompressZstdMultiFrameLargeInput) +{ + // Create input larger than the 16 MiB frame boundary to exercise + // multi-frame emission. + std::string str(20 * 1024 * 1024, 'x'); + for (size_t i = 0; i < str.size(); i += 997) + str[i] = 'y'; // add some variation + auto compressed = compress(CompressionAlgo::zstd, str); + auto o = decompress("zstd", compressed); + + ASSERT_EQ(o, str); +} + +TEST(compress, zstdEmptyInput) +{ + auto compressed = compress(CompressionAlgo::zstd, ""); + // Empty input should still emit a valid (empty-content) zstd + // frame so it round-trips through the decompressor. + ASSERT_FALSE(compressed.empty()); + auto o = decompress("zstd", compressed); + ASSERT_EQ(o, ""); +} + +TEST(compress, zstdExactFrameBoundary) +{ + // Input exactly equal to the 16 MiB frame boundary should not + // emit a trailing zero-content frame. + std::string str(16 * 1024 * 1024, 'z'); + auto compressed = compress(CompressionAlgo::zstd, str); + auto o = decompress("zstd", compressed); + ASSERT_EQ(o, str); + + // Verify there is exactly one frame by checking that + // the first frame's compressed size equals the total size. + size_t frameSize = ZSTD_findFrameCompressedSize(compressed.data(), compressed.size()); + ASSERT_FALSE(ZSTD_isError(frameSize)); + ASSERT_EQ(frameSize, compressed.size()); +} + TEST(decompress, decompressInvalidInputThrowsCompressionError) { auto method = "bzip2"; diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index ff6e539c8a..6a86504ded 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -36,6 +36,9 @@ deps_private += gtest gmock = dependency('gmock') deps_private += gmock +zstd = dependency('libzstd', version : '>= 1.4.0') +deps_private += zstd + if host_machine.system() == 'windows' # Boost.ASIO needs this unconditionally. socket = cxx.find_library('ws2_32') diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index 03c1c2d0ef..265e58e312 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -10,6 +10,7 @@ rapidcheck, gtest, + zstd, runCommand, util-linux, @@ -45,6 +46,7 @@ mkMesonExecutable (finalAttrs: { nix-util-test-support rapidcheck gtest + zstd ] ++ lib.optionals stdenv.hostPlatform.isLinux [ util-linux diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index f07b22dcab..24539a4633 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -2,6 +2,7 @@ #include "nix/util/signals.hh" #include "nix/util/tarfile.hh" #include "nix/util/logging.hh" +#include "nix/util/current-process.hh" #include #include @@ -11,6 +12,9 @@ #include #include +#include +#include + namespace nix { static const int COMPRESSION_LEVEL_DEFAULT = -1; @@ -68,7 +72,11 @@ struct ArchiveDecompressionSource : Source } }; -/* Happens to match enum names. */ +/* Algorithms whose *compression* is handled by libarchive. zstd is + intentionally absent: ZstdMultiFrameCompressionSink compresses it + directly so the output is split into independent frames; zstd + *decompression* is still handled by libarchive via + ArchiveDecompressionSource. */ #define NIX_FOR_EACH_LA_ALGO(MACRO) \ MACRO(bzip2) \ MACRO(compress) \ @@ -79,8 +87,7 @@ struct ArchiveDecompressionSource : Source MACRO(lzip) \ MACRO(lzma) \ MACRO(lzop) \ - MACRO(xz) \ - MACRO(zstd) + MACRO(xz) struct ArchiveCompressionSink : CompressionSink { @@ -99,6 +106,7 @@ struct ArchiveCompressionSink : CompressionSink switch (method) { case CompressionAlgo::none: case CompressionAlgo::brotli: + case CompressionAlgo::zstd: unreachable(); #define NIX_DEF_LA_ALGO_CASE(algo) \ case CompressionAlgo::algo: \ @@ -317,6 +325,124 @@ struct BrotliCompressionSink : ChunkedCompressionSink } }; +/** + * Zstd compression that cuts a new frame every `bytesPerFrame` of + * uncompressed input. The result is a concatenation of independent + * frames, which any conformant zstd decoder (RFC 8878 §3.1) handles + * transparently — including libarchive's, which is what the nix + * substituter path uses for decompression. Because each frame is + * independent and carries its decompressed size, a parallel decoder + * can split work across them. + * + * Frame size is fixed at 16 MiB of input. zstd's window size is + * level-dependent (~2 MiB at the default level 3, up to 8 MiB at + * higher levels), so the ratio loss from not being able to reference + * across a frame boundary is small. 16 MiB gives ~700 frames for the + * biggest NARs, which is ample parallelism and lets a decoder start + * work before the whole blob is downloaded. + */ +struct ZstdMultiFrameCompressionSink : CompressionSink +{ + Sink & nextSink; + std::unique_ptr cctx{nullptr, ZSTD_freeCCtx}; + std::vector outbuf; + /** + * Input buffer for the current frame. We accumulate a full + * frame's worth before compressing so we can set an exact + * `ZSTD_CCtx_setPledgedSrcSize` — that writes `Frame_Content_Size` + * into the frame header, allowing a parallel decoder to compute + * each frame's output offset up front. + */ + std::vector inbuf; + bool emittedAnyFrame = false; + static constexpr uint64_t bytesPerFrame = 16 * 1024 * 1024; + + ZstdMultiFrameCompressionSink(Sink & nextSink, bool parallel, int level) + : nextSink(nextSink) + , outbuf(ZSTD_CStreamOutSize()) + { + inbuf.reserve(bytesPerFrame); + cctx.reset(ZSTD_createCCtx()); + if (!cctx) + throw CompressionError("unable to initialise zstd encoder"); + if (level != COMPRESSION_LEVEL_DEFAULT) + checkZstd(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, level)); + if (parallel) { + unsigned ncpu = getMaxCPU(); + if (ncpu == 0) + ncpu = std::thread::hardware_concurrency(); + /* Cap nbWorkers: zstd's MT engine splits each frame into + per-worker jobs. With 16 MiB frames, more than ~4 + workers yields diminishing returns (< 4 MiB per worker) + and the thread synchronisation overhead can make + compression slower than single-threaded. */ + if (ncpu > 4) + ncpu = 4; + if (ncpu > 1) + /* Don't checkZstd(): if libzstd was built without + ZSTD_MULTITHREAD this returns an error, but per the + zstd docs the parameter is simply ignored and + compression falls back to single-threaded. */ + ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_nbWorkers, ncpu); + } + } + + void checkZstd(size_t ret) + { + if (ZSTD_isError(ret)) + throw CompressionError("zstd error: %s", ZSTD_getErrorName(ret)); + } + + /** + * Compress all of `inbuf` as one complete frame, pledged at its + * exact size so `Frame_Content_Size` lands in the header. + */ + void emitFrame() + { + checkZstd(ZSTD_CCtx_reset(cctx.get(), ZSTD_reset_session_only)); + checkZstd(ZSTD_CCtx_setPledgedSrcSize(cctx.get(), inbuf.size())); + + ZSTD_inBuffer in = {inbuf.data(), inbuf.size(), 0}; + for (;;) { + checkInterrupt(); + ZSTD_outBuffer out = {outbuf.data(), outbuf.size(), 0}; + size_t remaining = ZSTD_compressStream2(cctx.get(), &out, &in, ZSTD_e_end); + checkZstd(remaining); + if (out.pos > 0) + nextSink({outbuf.data(), out.pos}); + if (remaining == 0) + break; + } + inbuf.clear(); + emittedAnyFrame = true; + } + + void writeUnbuffered(std::string_view data) override + { + while (!data.empty()) { + uint64_t room = bytesPerFrame - inbuf.size(); + size_t n = (room < data.size()) ? room : data.size(); + + inbuf.insert(inbuf.end(), data.data(), data.data() + n); + data.remove_prefix(n); + + if (inbuf.size() >= bytesPerFrame) + emitFrame(); + } + } + + void finish() override + { + flush(); + /* Emit the trailing partial frame, or an empty frame if we + never wrote anything — the output must contain at least one + frame header to be valid zstd (otherwise the libarchive + decoder chokes on round-tripped empty input). */ + if (!inbuf.empty() || !emittedAnyFrame) + emitFrame(); + } +}; + ref makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel, int level) { switch (method) { @@ -324,6 +450,8 @@ ref makeCompressionSink(CompressionAlgo method, Sink & nextSink return make_ref(nextSink); case CompressionAlgo::brotli: return make_ref(nextSink); + case CompressionAlgo::zstd: + return make_ref(nextSink, parallel, level); /* Everything else is supported via libarchive. */ #define NIX_DEF_LA_ALGO_CASE(algo) case CompressionAlgo::algo: NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE) diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 7e996b8c3d..50ade66887 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -101,6 +101,10 @@ brotli = [ ] deps_private += brotli +# Direct libzstd linkage for ZstdMultiFrameCompressionSink. +zstd = dependency('libzstd', version : '>= 1.4.0') +deps_private += zstd + cpuid_required = get_option('cpuid') if host_machine.cpu_family() != 'x86_64' and cpuid_required.enabled() warning('Force-enabling seccomp on non-x86_64 does not make sense') diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 3deb7ba3ae..049c75c580 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -11,6 +11,7 @@ libsodium, nlohmann_json, openssl, + zstd, # Configuration Options @@ -52,6 +53,7 @@ mkMesonLibrary (finalAttrs: { libblake3 libsodium openssl + zstd ] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; From 0edf06155d96ba8d98aa422607f239ac05d72445 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 13 Apr 2026 20:25:24 +0300 Subject: [PATCH 238/927] Only bump the RLIMIT_STACK limit in EvalState constructor For regular commands like `nix store add` we have no reason to bump the stack size. I've seen pthread_create fail in nix-ninja with ENOMEM. This is a cross-port of https://gerrit.lix.systems/c/lix/+/5053. --- src/libexpr/eval.cc | 12 ++++++++++++ src/libutil/current-process.cc | 2 +- src/libutil/include/nix/util/current-process.hh | 5 +++-- src/nix/main.cc | 8 -------- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 42436f5ea9..1d66b8ead1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -314,6 +315,17 @@ EvalState::EvalState( #endif , staticBaseEnv{std::make_shared(nullptr, nullptr)} { +#ifndef _WIN32 + static std::once_flag stackSizeBumped; + std::call_once(stackSizeBumped, []() { + // Increase the default stack size for the evaluator and for + // libstdc++'s std::regex. + // This used to be 64 MiB, but macOS as deployed on GitHub Actions has a + // hard limit slightly under that, so we round it down a bit. + nix::ensureStackSizeAtLeast(60 * 1024 * 1024); + }); +#endif + corepkgsFS->setPathDisplay(""); internalFS->setPathDisplay("«nix-internal»", ""); diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 177728a633..acb6e52dd7 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -58,7 +58,7 @@ unsigned int getMaxCPU() #ifndef _WIN32 size_t savedStackSize = 0; -void setStackSize(size_t stackSize) +void ensureStackSizeAtLeast(size_t stackSize) { struct rlimit limit; if (getrlimit(RLIMIT_STACK, &limit) == 0 && static_cast(limit.rlim_cur) < stackSize) { diff --git a/src/libutil/include/nix/util/current-process.hh b/src/libutil/include/nix/util/current-process.hh index 657ff0b44e..cdd5a8689c 100644 --- a/src/libutil/include/nix/util/current-process.hh +++ b/src/libutil/include/nix/util/current-process.hh @@ -27,9 +27,10 @@ unsigned int getMaxCPU(); // It does not seem possible to dynamically change stack size on Windows. #ifndef _WIN32 /** - * Change the stack size. + * Increase the RLIMIT_STACK rlimit if it is currently smaller than `stackSize`. + * @note Not thread safe. Calls to this should be wrapped in a std::call_once. */ -void setStackSize(size_t stackSize); +void ensureStackSizeAtLeast(size_t stackSize); #endif /** diff --git a/src/nix/main.cc b/src/nix/main.cc index 98423b3b10..541f79622e 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -597,13 +597,5 @@ int main(int argc, char ** argv) { // The CLI has a more detailed version than the libraries; see nixVersion. nix::nixVersion = NIX_CLI_VERSION; -#ifndef _WIN32 - // Increase the default stack size for the evaluator and for - // libstdc++'s std::regex. - // This used to be 64 MiB, but macOS as deployed on GitHub Actions has a - // hard limit slightly under that, so we round it down a bit. - nix::setStackSize(60 * 1024 * 1024); -#endif - return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv); }); } From e4c4f920d32d9b6fd48ef1151f705e9d87d01182 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 11 Mar 2026 20:57:37 -0400 Subject: [PATCH 239/927] Make post-build-hook asynchronous Register the post-build-hook as a child to allow waiting on child events to service the hook (output and PID waiting) *without* blocking the main event loop. Because the servicing is handled as part of the goal, the current behavior is maintained wherein waiting goals will not start until the `post-build-hook` for their dependencies have completed. However, multiple `post-build-hook`s can now run in parallel rather than a single one blocking the entire event loop. Resolves #15406 Signed-off-by: Lisanna Dettwyler --- doc/manual/rl-next/async-post-build-hook.md | 9 + .../build/derivation-building-goal.cc | 179 +++++++++++++----- 2 files changed, 136 insertions(+), 52 deletions(-) create mode 100644 doc/manual/rl-next/async-post-build-hook.md diff --git a/doc/manual/rl-next/async-post-build-hook.md b/doc/manual/rl-next/async-post-build-hook.md new file mode 100644 index 0000000000..ea061f14ae --- /dev/null +++ b/doc/manual/rl-next/async-post-build-hook.md @@ -0,0 +1,9 @@ +--- +synopsis: Make post-build-hook asynchronous +prs: [15451] +issues: [15406] +--- + +This change makes the `post-build-hook` run asynchronously but still as part of the goal. +This retains the current behavior that a waiting goal will not start until the `post-build-hook` of the goal it is waiting on completes. +However, multiple `post-build-hook`s can now run concurrently just as multiple goals can run concurrently. diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 2235cf872b..4b012c8a0b 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -16,6 +16,7 @@ #include "nix/store/local-store.hh" // TODO remove, along with remaining downcasts #include "nix/store/outputs-query.hh" #include "nix/store/globals.hh" +#include "nix/util/current-process.hh" #include #include @@ -66,7 +67,72 @@ std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & dr return msg; } -static void runPostBuildHook( +struct LogSink : Sink +{ + Activity & act; + std::string currentLine; + + LogSink(Activity & act) + : act(act) + { + } + + void operator()(std::string_view data) override + { + for (auto c : data) { + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() + { + act.result(resPostBuildLogLine, currentLine); + currentLine.clear(); + } + + ~LogSink() + { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } +}; + +struct PostBuildHookState +{ + const std::string hook; + Activity act; + std::unique_ptr sink; + std::unique_ptr out; + Pid pid; + + PostBuildHookState(Logger & logger, const std::string hook, const std::string drvPath) + : hook(hook) + , act(logger, + lvlTalkative, + actPostBuildHook, + fmt("running post-build-hook '%s'", hook), + Logger::Fields{drvPath}) + , out(std::make_unique()) + { + out->create(); + sink = std::make_unique(act); + } + + void complete() + { + if (int ret = pid.wait()) { + throw Error("program \"%s\" %s", hook, statusToString(ret)); + } + } +}; + +static std::unique_ptr runPostBuildHook( const WorkerSettings & workerSettings, const StoreDirConfig & store, Logger & logger, @@ -762,7 +828,21 @@ Goal::Co DerivationBuildingGoal::buildWithHook( StorePathSet outputPaths; for (auto & [_, output] : builtOutputs) outputPaths.insert(output.outPath); - runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + + if (worker.settings.postBuildHook.get() != "") { + auto hookState = runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + worker.childStarted(shared_from_this(), {hookState->out->readSide.get()}, false, false); + while (true) { + auto event = co_await WaitForChildEvent{}; + if (auto * output = std::get_if(&event)) { + (*hookState->sink)(output->data); + } else if (std::get_if(&event)) { + hookState->complete(); + worker.childTerminated(this); + break; + } + } + } /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will @@ -992,7 +1072,21 @@ Goal::Co DerivationBuildingGoal::buildLocally( worker.markContentsGood(output.outPath); outputPaths.insert(output.outPath); } - runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + + if (worker.settings.postBuildHook.get() != "") { + auto hookState = runPostBuildHook(worker.settings, worker.store, *logger, drvPath, outputPaths); + worker.childStarted(shared_from_this(), {hookState->out->readSide.get()}, false, false); + while (true) { + auto event = co_await WaitForChildEvent{}; + if (auto * output = std::get_if(&event)) { + (*hookState->sink)(output->data); + } else if (std::get_if(&event)) { + hookState->complete(); + worker.childTerminated(this); + break; + } + } + } /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will @@ -1005,24 +1099,21 @@ Goal::Co DerivationBuildingGoal::buildLocally( #endif } -static void runPostBuildHook( +static std::unique_ptr runPostBuildHook( const WorkerSettings & workerSettings, const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths) { - auto hook = workerSettings.postBuildHook; - if (hook == "") - return; +#ifdef _WIN32 + throw UnimplementedError("post-build-hook is not implemented on Windows"); +#else + auto state = + std::make_unique(logger, workerSettings.postBuildHook.get(), store.printStorePath(drvPath)); + + auto hook = workerSettings.postBuildHook.get(); - Activity act( - logger, - lvlTalkative, - actPostBuildHook, - fmt("running post-build-hook '%s'", workerSettings.postBuildHook), - Logger::Fields{store.printStorePath(drvPath)}); - PushActivity pact(act.id); OsStringMap hookEnvironment = getEnvOs(); hookEnvironment.emplace(OS_STR("DRV_PATH"), string_to_os_string(store.printStorePath(drvPath))); @@ -1030,50 +1121,34 @@ static void runPostBuildHook( OS_STR("OUT_PATHS"), string_to_os_string(chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))))); hookEnvironment.emplace(OS_STR("NIX_CONFIG"), string_to_os_string(globalConfig.toKeyValue())); - struct LogSink : Sink - { - Activity & act; - std::string currentLine; + ProcessOptions processOptions; + processOptions.allowVfork = false; - LogSink(Activity & act) - : act(act) - { - } + state->pid = startProcess( + [&] { + replaceEnv(hookEnvironment); + if (dup2(state->out->writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); - void operator()(std::string_view data) override - { - for (auto c : data) { - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } + Strings args_; + args_.push_front(hook); - void flushLine() - { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } + unix::closeExtraFDs(); - ~LogSink() - { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; + restoreProcessContext(); - LogSink sink(act); + execvp(hook.c_str(), stringsToCharPtrs(args_).data()); - runProgram2({ - .program = workerSettings.postBuildHook.get(), - .environment = hookEnvironment, - .standardOut = &sink, - .mergeStderrToStdout = true, - }); + throw SysError("executing %s", PathFmt(hook)); + }, + processOptions); + + state->out->writeSide.close(); + + return state; +#endif } BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailureError e, BuildLog & buildLog) From 892d870d9640fe5a51b0e5e38107b27ab276c2f7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 13 Apr 2026 22:11:52 +0300 Subject: [PATCH 240/927] libstore: Plug thread safety issues in recursive-nix --- .../include/nix/store/restricted-store.hh | 52 +++++++++++++++---- src/libstore/restricted-store.cc | 15 ++++-- .../unix/build/chroot-derivation-builder.cc | 2 - src/libstore/unix/build/derivation-builder.cc | 19 ++++--- 4 files changed, 65 insertions(+), 23 deletions(-) diff --git a/src/libstore/include/nix/store/restricted-store.hh b/src/libstore/include/nix/store/restricted-store.hh index ca4e0b8536..79167a3d08 100644 --- a/src/libstore/include/nix/store/restricted-store.hh +++ b/src/libstore/include/nix/store/restricted-store.hh @@ -2,6 +2,9 @@ ///@file #include "nix/store/store-api.hh" +#include "nix/util/sync.hh" + +#include namespace nix { @@ -28,15 +31,20 @@ struct RestrictionContext */ virtual const StorePathSet & originalPaths() = 0; - /** - * Paths that were added via recursive Nix calls. - */ - StorePathSet addedPaths; + struct State + { + /** + * Paths that were added via recursive Nix calls. + */ + std::map> addedPaths; - /** - * Realisations that were added via recursive Nix calls. - */ - std::set addedDrvOutputs; + /** + * Realisations that were added via recursive Nix calls. + */ + std::set addedDrvOutputs; + }; + + Sync state_; /** * Recursive Nix calls are only allowed to build or realize paths @@ -56,7 +64,33 @@ struct RestrictionContext { if (isAllowed(path)) return; - addDependencyImpl(path); + + std::promise promise; + + auto [future, shouldAdd] = [&]() -> std::pair, bool> { + auto state(state_.lock()); + if (auto iter = state->addedPaths.find(path); iter != state->addedPaths.end()) { + return {iter->second, false}; + } + auto [iter2, _] = state->addedPaths.emplace(path, promise.get_future().share()); + return {iter2->second, true}; + }(); + + /* Another daemon worker thread already started adding the dependency. Just wait for it + to complete. */ + if (!shouldAdd) { + future.get(); + return; + } + + try { + addDependencyImpl(path); + promise.set_value(); + } catch (...) { + /* Notify all other waiters that we are done. */ + promise.set_exception(std::current_exception()); + throw; + } } virtual ~RestrictionContext() = default; diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 8001d43ec8..03c130da96 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -167,8 +167,11 @@ StorePathSet RestrictedStore::queryAllValidPaths() StorePathSet paths; for (auto & p : goal.originalPaths()) paths.insert(p); - for (auto & p : goal.addedPaths) - paths.insert(p); + for (auto & [p, future] : goal.state_.lock()->addedPaths) { + /* Only report the paths that have finished materialising in the sandbox. */ + if (future.wait_for(std::chrono::seconds(0)) == std::future_status::ready) + paths.insert(p); + } return paths; } @@ -300,8 +303,12 @@ std::vector RestrictedStore::buildPathsWithResults( next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); - for (auto & real : newRealisations) - goal.addedDrvOutputs.insert(real.id); + + { + auto state(goal.state_.lock()); + for (auto & real : newRealisations) + state->addedDrvOutputs.insert(real.id); + } return results; } diff --git a/src/libstore/unix/build/chroot-derivation-builder.cc b/src/libstore/unix/build/chroot-derivation-builder.cc index d10d98245a..157d9e3191 100644 --- a/src/libstore/unix/build/chroot-derivation-builder.cc +++ b/src/libstore/unix/build/chroot-derivation-builder.cc @@ -131,8 +131,6 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl std::pair addDependencyPrep(const StorePath & path) { - DerivationBuilderImpl::addDependencyImpl(path); - debug("materialising '%s' in the sandbox", store.printStorePath(path)); std::filesystem::path source = store.toRealPath(path); diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 85aa98c7ae..3cf7977c50 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -238,12 +238,18 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder bool isAllowed(const StorePath & path) override { - return inputPaths.count(path) || addedPaths.count(path); + if (inputPaths.count(path)) + return true; + auto state(state_.lock()); + auto iter = state->addedPaths.find(path); + if (iter == state->addedPaths.end()) + return false; + return iter->second.wait_for(std::chrono::seconds(0)) == std::future_status::ready; } bool isAllowed(const DrvOutput & id) override { - return addedDrvOutputs.count(id); + return state_.lock()->addedDrvOutputs.count(id); } bool isAllowed(const DerivedPath & req); @@ -1165,7 +1171,7 @@ void DerivationBuilderImpl::startDaemon() ref(std::dynamic_pointer_cast(this->store.shared_from_this())), *this); - addedPaths.clear(); + state_.lock()->addedPaths.clear(); auto socketName = ".nix-socket"; std::filesystem::path socketPath = tmpDir / socketName; @@ -1246,10 +1252,7 @@ void DerivationBuilderImpl::stopDaemon() daemonSocket.close(); } -void DerivationBuilderImpl::addDependencyImpl(const StorePath & path) -{ - addedPaths.insert(path); -} +void DerivationBuilderImpl::addDependencyImpl(const StorePath & path) {} void DerivationBuilderImpl::chownToBuilder(const std::filesystem::path & path) { @@ -1434,7 +1437,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() referenceablePaths.insert(p); for (auto & i : scratchOutputs) referenceablePaths.insert(i.second); - for (auto & p : addedPaths) + for (auto & [p, _] : state_.lock()->addedPaths) referenceablePaths.insert(p); /* Check whether the output paths were created, and make all From 83a0d7e3923648e38918e81582ea9888d4a63597 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 22:39:41 +0000 Subject: [PATCH 241/927] build(deps): bump cachix/install-nix-action from 31.10.3 to 31.10.4 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.10.3 to 31.10.4. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/96951a368ba55167b55f1c916f7d416bac6505fe...616559265b40713947b9c190a8ff4b507b5df49b) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.10.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51cd9b48c7..be8cb0f0c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,7 +192,7 @@ jobs: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@96951a368ba55167b55f1c916f7d416bac6505fe # v31.10.3 + - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 if: ${{ !matrix.experimental-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} @@ -258,7 +258,7 @@ jobs: id: installer-tarball-url run: | echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@96951a368ba55167b55f1c916f7d416bac6505fe # v31.10.3 + - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} From 493acdfdd143e15863c948326f8df154c9618222 Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Tue, 7 Apr 2026 16:03:05 +0200 Subject: [PATCH 242/927] Use limit from macOS kernel instead of hardcoded --- src/libmain/shared.cc | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 6ddabf12c3..c57f71cc26 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -21,6 +21,9 @@ #ifndef _WIN32 # include #endif +#ifdef __APPLE__ +# include +#endif #ifdef __linux__ # include #endif @@ -132,15 +135,26 @@ void bumpFileLimit() if (getrlimit(RLIMIT_NOFILE, &limit) != 0) return; - if (limit.rlim_cur < limit.rlim_max) { - // Some software misbehaves really bad when we try to raise the - // limit to RLIM_INFINITY, so cap the limit at the 1048576 limit used - // by the daemon. - // - // GNU patch < 2.8 crashes with **** out of memory, which breaks in nixpkgs darwin bootstrap tools. - // This was fixed in: - // https://cgit.git.savannah.gnu.org/cgit/patch.git/commit/?id=61d7788b83b302207a67b82786f4fd79e3538f30 - limit.rlim_cur = std::min(limit.rlim_max, rlim_t(1048576)); + rlim_t target = limit.rlim_max; + +# ifdef __APPLE__ + // On macOS the hard limit is typically RLIM_INFINITY, but + // setting rlim_cur to that causes problems: child processes + // (e.g. GNU patch in the Nix sandbox) may allocate memory + // proportional to the fd limit and OOM. Use the kernel's + // per-process file limit instead, which is the effective cap. + // + // GNU patch < 2.8 crashes with **** out of memory, which breaks in nixpkgs darwin bootstrap tools. + // This was fixed in: + // https://cgit.git.savannah.gnu.org/cgit/patch.git/commit/?id=61d7788b83b302207a67b82786f4fd79e3538f30 + int maxfiles; + size_t len = sizeof(maxfiles); + if (sysctlbyname("kern.maxfilesperproc", &maxfiles, &len, nullptr, 0) == 0) + target = maxfiles; +# endif + + if (limit.rlim_cur < target) { + limit.rlim_cur = target; // Ignore errors, this is best effort. setrlimit(RLIMIT_NOFILE, &limit); } From de276c66d0b5a2dd7bc25822e3fbb7fd78d42dc1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Apr 2026 14:09:43 +0200 Subject: [PATCH 243/927] NarInfo::to_string(): Loosen fileInfo/fileSize requirements These fields are in fact optional, and various binary cache implementations do not return them (e.g. when NARs are compressed on the fly). --- src/libstore/nar-info.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index d470569ea2..028302963e 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -110,9 +110,12 @@ std::string NarInfo::to_string(const StoreDirConfig & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash && fileHash->algo == HashAlgorithm::SHA256); - res += "FileHash: " + fileHash->to_string(HashFormat::Nix32, true) + "\n"; - res += "FileSize: " + std::to_string(fileSize) + "\n"; + if (fileHash) { + assert(fileHash->algo == HashAlgorithm::SHA256); + res += "FileHash: " + fileHash->to_string(HashFormat::Nix32, true) + "\n"; + } + if (fileSize) + res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.algo == HashAlgorithm::SHA256); res += "NarHash: " + narHash.to_string(HashFormat::Nix32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; From 779b983a4dadb3674b6e817acf3bf55c73cebb3c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Apr 2026 17:41:03 +0200 Subject: [PATCH 244/927] LocalStore::addToStore(): Show hash mismatches in SRI format This is more consistent with e.g. hash mismatches in fixed-output derivations and the output of `nix path-info`. --- src/libstore/local-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d5e4577312..d52daaae1f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1064,8 +1064,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF throw Error( "hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), - info.narHash.to_string(HashFormat::Nix32, true), - hashResult.hash.to_string(HashFormat::Nix32, true)); + info.narHash.to_string(HashFormat::SRI, true), + hashResult.hash.to_string(HashFormat::SRI, true)); if (hashResult.numBytesDigested != info.narSize) throw Error( From 2a71fbf41017b4b9ad87f7c6fa5506b80743e1a7 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 13 Nov 2025 12:45:01 +0700 Subject: [PATCH 245/927] docs: add time complexity to relevant primops --- src/libexpr/primops.cc | 192 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 191 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 98ed1b4509..5d28cd0dca 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2999,6 +2999,12 @@ static RegisterPrimOp primop_attrNames({ Return the names of the attributes in the set *set* in an alphabetically sorted list. For instance, `builtins.attrNames { y = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`. + + # Time Complexity + + - O(n log n), where: + + n = number of attributes in the set )", .fun = prim_attrNames, }); @@ -3031,6 +3037,12 @@ static RegisterPrimOp primop_attrValues({ .doc = R"( Return the values of the attributes in the set *set* in the order corresponding to the sorted attribute names. + + # Time Complexity + + - O(n log n), where: + + n = number of attributes in the set )", .fun = prim_attrValues, }); @@ -3056,6 +3068,10 @@ static RegisterPrimOp primop_getAttr({ aborts if the attribute doesn’t exist. This is a dynamic version of the `.` operator, since *s* is an expression rather than an identifier. + + # Time Complexity + + O(log n) where n = number of attributes in the set )", .fun = prim_getAttr, }); @@ -3144,6 +3160,10 @@ static RegisterPrimOp primop_hasAttr({ `hasAttr` returns `true` if *set* has an attribute named *s*, and `false` otherwise. This is a dynamic version of the `?` operator, since *s* is an expression rather than an identifier. + + # Time Complexity + + O(log n) where n = number of attributes in the set )", .fun = prim_hasAttr, }); @@ -3203,6 +3223,13 @@ static RegisterPrimOp primop_removeAttrs({ ``` evaluates to `{ y = 2; }`. + + # Time Complexity + + O(n + k log k) where: + + n = number of attributes in input set + k = number of attribute names to remove )", .fun = prim_removeAttrs, }); @@ -3290,6 +3317,10 @@ static RegisterPrimOp primop_listToAttrs({ ```nix { foo = 123; bar = 456; } ``` + + # Time Complexity + + O(n log n) where n = number of list elements )", .fun = prim_listToAttrs, }); @@ -3366,7 +3397,12 @@ static RegisterPrimOp primop_intersectAttrs({ Return a set consisting of the attributes in the set *e2* which have the same name as some attribute in *e1*. - Performs in O(*n* log *m*) where *n* is the size of the smaller set and *m* the larger set's size. + # Time Complexity + + O(n * log m) where: + + n = number of attributes in the smaller set + m = number of attributes in the larger set )", .fun = prim_intersectAttrs, }); @@ -3406,6 +3442,13 @@ static RegisterPrimOp primop_catAttrs({ ``` evaluates to `[1 2]`. + + # Time Complexity + + O(n * log m) where: + + n = number of sets in input list + m = number of attributes per set )", .fun = prim_catAttrs, }); @@ -3449,6 +3492,10 @@ static RegisterPrimOp primop_functionArgs({ "Formal argument" here refers to the attributes pattern-matched by the function. Plain lambdas are not included, e.g. `functionArgs (x: ...) = { }`. + + # Time Complexity + + O(n) where n = number of formal arguments )", .fun = prim_functionArgs, }); @@ -3481,6 +3528,14 @@ static RegisterPrimOp primop_mapAttrs({ ``` evaluates to `{ a = 10; b = 20; }`. + + # Time Complexity + + O(n) where: + + n = number of attributes + + Calls to `f` are performed afterwards, when needed )", .fun = prim_mapAttrs, }); @@ -3568,6 +3623,15 @@ static RegisterPrimOp primop_zipAttrsWith({ b = { name = "b"; values = [ "z" ]; }; } ``` + + # Time Complexity + + O(n * k * log k) worst case, where: + + n = number of attribute sets in input list + k = number of unique keys across all sets + + More precisely: O(n * m * log k) where m ≤ k is average number of attributes per set )", .fun = prim_zipAttrsWith, }); @@ -3633,6 +3697,10 @@ static RegisterPrimOp primop_head({ Return the first element of a list; abort evaluation if the argument isn’t a list or is an empty list. You can test whether a list is empty by comparing it with `[]`. + + # Time Complexity + + O(1) )", .fun = prim_head, }); @@ -3664,6 +3732,10 @@ static RegisterPrimOp primop_tail({ > This function should generally be avoided since it's inefficient: > unlike Haskell's `tail`, it takes O(n) time, so recursing over a > list by repeatedly calling `tail` takes O(n^2) time. + + # Time Complexity + + O(n) where n = list length (copies n-1 elements) )", .fun = prim_tail, }); @@ -3698,6 +3770,14 @@ static RegisterPrimOp primop_map({ ``` evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + # Time Complexity + + O(n) where: + + n = list length + + Calls to `f` are performed afterwards when needed. )", .fun = prim_map, }); @@ -3747,6 +3827,13 @@ static RegisterPrimOp primop_filter({ .doc = R"( Return a list consisting of the elements of *list* for which the function *f* returns `true`. + + # Time Complexity + + O(n * T_f) where: + + n = list length + T_f = predicate evaluation time )", .fun = prim_filter, }); @@ -3770,6 +3857,15 @@ static RegisterPrimOp primop_elem({ .doc = R"( Return `true` if a value equal to *x* occurs in the list *xs*, and `false` otherwise. + + # Time Complexity + + O(n * T) (worst case) where: + + n = list length + T = time to compare two elements + + returns early if the elements is found )", .fun = prim_elem, }); @@ -3792,6 +3888,12 @@ static RegisterPrimOp primop_concatLists({ .args = {"lists"}, .doc = R"( Concatenate a list of lists into a single list. + + # Time Complexity + + O(N) where: + + N = total number of elements across all lists )", .fun = prim_concatLists, }); @@ -3808,6 +3910,10 @@ static RegisterPrimOp primop_length({ .args = {"e"}, .doc = R"( Return the length of the list *e*. + + # Time Complexity + + O(1) )", .fun = prim_length, }); @@ -3851,6 +3957,13 @@ static RegisterPrimOp primop_foldlStrict({ argument is the current element being processed. The return value of each application of `op` is evaluated immediately, even for intermediate values. + + # Time Complexity + + O(n * T_op) where: + + n = list length + T_op = `op` call evaluation time )", .fun = prim_foldlStrict, }); @@ -3889,6 +4002,15 @@ static RegisterPrimOp primop_any({ .doc = R"( Return `true` if the function *pred* returns `true` for at least one element of *list*, and `false` otherwise. + + # Time Complexity + + O(n * T_pred) where: + + - n = `list` length + - T_pred = `pred` call evaluation time + + returns early when `pred` returns `true` )", .fun = prim_any, }); @@ -3904,6 +4026,15 @@ static RegisterPrimOp primop_all({ .doc = R"( Return `true` if the function *pred* returns `true` for all elements of *list*, and `false` otherwise. + + # Time Complexity + + O(n * T_f) where: + + - n = list length + - T_f = predicate evaluation time + + returns early when `pred` returns `false` )", .fun = prim_all, }); @@ -3942,6 +4073,17 @@ static RegisterPrimOp primop_genList({ ``` returns the list `[ 0 1 4 9 16 ]`. + + # Time Complexity + + Complexity of `genList generator n`: O(n) + + Complexity of `deepSeq (genList generator n)`: O(n * T_f) + + where: + + n = requested length + T_f = `generator` call evaluation time )", .fun = prim_genList, }); @@ -4036,6 +4178,16 @@ static RegisterPrimOp primop_sort({ If the *comparator* violates any of these properties, then `builtins.sort` reorders elements in an unspecified manner. + + # Time Complexity + + O(n log n * T_cmp), where: + + n = `list` length + T_cmp = `comparator` call evaluation time + + Uses an adaptive sort that exploits existing sorted runs in the input, + down to O(n * T_cmp) when the list is already sorted. )", .fun = prim_sort, }); @@ -4097,6 +4249,13 @@ static RegisterPrimOp primop_partition({ ```nix { right = [ 23 42 ]; wrong = [ 1 9 3 ]; } ``` + + # Time Complexity + + O(n * T_pred) where: + + n = list length + T_pred = `pred` call evaluation time )", .fun = prim_partition, }); @@ -4150,6 +4309,14 @@ static RegisterPrimOp primop_groupBy({ ```nix { b = [ "bar" "baz" ]; f = [ "foo" ]; } ``` + + # Time Complexity + + O(N * T_f + N * log k) where: + + N = number of `list` elements + T_f = `f` call evaluation time + k = number of unique groups )", .fun = prim_groupBy, }); @@ -4192,6 +4359,14 @@ static RegisterPrimOp primop_concatMap({ .doc = R"( This function is equivalent to `builtins.concatLists (map f list)` but is more efficient. + + # Time Complexity + + O(k * T_f + N) where: + + k = length of input list + T_f = time to call `f` on an element + N = total number of elements returned by `f` calls )", .fun = prim_concatMap, }); @@ -4888,6 +5063,13 @@ static RegisterPrimOp primop_concatStringsSep({ Concatenate a list of strings with a separator between each element, e.g. `concatStringsSep "/" ["usr" "local" "bin"] == "usr/local/bin"`. + + # Time Complexity + + O(n + m) where: + + n = number of list elements + m = total length of output string )", .fun = prim_concatStringsSep, }); @@ -4972,6 +5154,14 @@ static RegisterPrimOp primop_replaceStrings({ ``` evaluates to `"fabir"`. + + # Time Complexity + + O(n * k * c) where: + + n = length of input string + k = number of replacement patterns + c = average length of patterns in 'from' list )", .fun = prim_replaceStrings, }); From 3872910b2a545816688ea027e42cc632597617a1 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 14 Apr 2026 19:38:27 +0200 Subject: [PATCH 246/927] docs: big-o fix minor inconsistencies --- src/libexpr/primops.cc | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5d28cd0dca..d5a0c24280 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3447,7 +3447,7 @@ static RegisterPrimOp primop_catAttrs({ O(n * log m) where: - n = number of sets in input list + n = list length m = number of attributes per set )", .fun = prim_catAttrs, @@ -3535,7 +3535,7 @@ static RegisterPrimOp primop_mapAttrs({ n = number of attributes - Calls to `f` are performed afterwards, when needed + Calls to `f` are performed afterwards, when needed. )", .fun = prim_mapAttrs, }); @@ -3626,12 +3626,10 @@ static RegisterPrimOp primop_zipAttrsWith({ # Time Complexity - O(n * k * log k) worst case, where: + O(N * log k) where: - n = number of attribute sets in input list + N = total attributes across all sets k = number of unique keys across all sets - - More precisely: O(n * m * log k) where m ≤ k is average number of attributes per set )", .fun = prim_zipAttrsWith, }); @@ -3830,7 +3828,7 @@ static RegisterPrimOp primop_filter({ # Time Complexity - O(n * T_f) where: + O(n * T_f) (eager; predicate is forced) where: n = list length T_f = predicate evaluation time @@ -5066,7 +5064,7 @@ static RegisterPrimOp primop_concatStringsSep({ # Time Complexity - O(n + m) where: + O(n + m) (amortized) where: n = number of list elements m = total length of output string @@ -5157,7 +5155,7 @@ static RegisterPrimOp primop_replaceStrings({ # Time Complexity - O(n * k * c) where: + O(n * k * c) (worst case) where: n = length of input string k = number of replacement patterns From 58aeb0683d05a00b3152b617b012cba1ee5e5e0a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Apr 2026 23:55:40 +0300 Subject: [PATCH 247/927] edit: Allow opening paths that don't exist on disk in EDITOR Since https://github.com/NixOS/nix/pull/14050 we now mount over the store paths in the rootFS accessor in the evaluator. This broke :e and nix edit on something like `:e "${outPath}/src/nix-manual/utils.nix"` when having the `github:nixos/nix` flake loaded in a `--pure-eval` repl. `nix edit` was also broken similarly with: `because it has no physical path`. For regular files, we can copy the outputs into a temporary read-only file and open that instead. For directories, we could open a directory listing in the editor, but I haven't implemented that here. Copying the whole directory tree seems a bit wasteful to me, so we can't truthfully mimic the old behavior of `:e` on directories (that editors usually allow the user to traverse when opened). This also implements some infrastructure for getting a writable file descriptor for the file that would get edited - I intend to use it for https://github.com/NixOS/nix/issues/15633. --- src/libcmd/editor-for.cc | 54 +++++++++++++++++---- src/libcmd/include/nix/cmd/editor-for.hh | 13 ++++- src/libcmd/repl.cc | 15 +++--- src/libutil/include/nix/util/file-system.hh | 4 ++ src/libutil/include/nix/util/os-string.hh | 10 ++++ src/libutil/unix/file-system.cc | 2 +- src/libutil/windows/file-system.cc | 2 +- src/libutil/windows/os-string.cc | 14 ++++++ src/nix/edit.cc | 21 ++++---- tests/functional/flakes/edit.sh | 4 ++ tests/functional/repl.sh | 3 ++ 11 files changed, 112 insertions(+), 30 deletions(-) diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index 95fdf95ad0..83aba78d8f 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,22 +1,56 @@ #include "nix/cmd/editor-for.hh" #include "nix/util/environment-variables.hh" #include "nix/util/source-path.hh" +#include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" namespace nix { -Strings editorFor(const SourcePath & file, uint32_t line) +std::tuple editorFor(const SourcePath & file, uint32_t line, bool readOnly) { auto path = file.getPhysicalPath(); - if (!path) - throw Error("cannot open '%s' in an editor because it has no physical path", file); - auto editor = getEnv("EDITOR").value_or("cat"); - auto args = tokenizeString(editor); + OsString editor = getEnvOsNonEmpty(OS_STR("EDITOR")).value_or(OS_STR("cat")); + auto args = tokenizeString(editor); if (line > 0 - && (editor.find("emacs") != std::string::npos || editor.find("nano") != std::string::npos - || editor.find("vim") != std::string::npos || editor.find("kak") != std::string::npos)) - args.push_back(fmt("+%d", line)); - args.push_back(path->string()); - return args; + && (editor.contains(OS_STR("emacs")) || editor.contains(OS_STR("nano")) || editor.contains(OS_STR("vim")) + || editor.contains(OS_STR("kak")))) + args.push_back(string_to_os_string(fmt("+%d", line))); + + if (path) { + args.push_back(path->native()); + return {std::move(args), AutoCloseFD{}, AutoDelete{}}; + } + + /* Resolve symlinks when creating a temporary. That's how a regular editor + would behave. Also GitSourceAccessor behaves poorly with symlinks in the + path and fails with "«...» does not exist". */ + auto file2 = file.resolveSymlinks(); + auto stat = file2.lstat(); + /* TODO: Maybe we should print a directory listing and open that instead? */ + if (stat.type != SourceAccessor::tRegular) + throw Error("can't open a file %s of type '%s'", file2.to_string(), stat.typeString()); + + auto tempDir = createTempDir(defaultTempDir(), "nix-edit", 0700); + AutoDelete autoDel(tempDir, /*recursive=*/true); + auto tempPath = tempDir / file2.path.baseName().value_or("nix-edit"); + + /* Create the file with the same name, so editors that recognise file + extensions spin up syntax highlighting and LSPs. */ + auto tempFd = openNewFileForWrite( + tempPath, + readOnly ? 0400 : 0600, + {.truncateExisting = false, .followSymlinksOnTruncate = false, .writeOnly = false}); + + if (!tempFd) + throw NativeSysError("failed to create temporary file %s", PathFmt(tempPath)); + + /* Copy the contents into the created copy. */ + FdSink fileSink(tempFd.get()); + file2.readFile(fileSink); + fileSink.flush(); + args.push_back(tempPath); + + return {std::move(args), std::move(tempFd), std::move(autoDel)}; } } // namespace nix diff --git a/src/libcmd/include/nix/cmd/editor-for.hh b/src/libcmd/include/nix/cmd/editor-for.hh index 3fb8a072e7..998b60f4ee 100644 --- a/src/libcmd/include/nix/cmd/editor-for.hh +++ b/src/libcmd/include/nix/cmd/editor-for.hh @@ -1,15 +1,24 @@ #pragma once ///@file -#include "nix/util/types.hh" #include "nix/util/source-path.hh" +#include "nix/util/os-string.hh" +#include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" namespace nix { /** * Helper function to generate args that invoke $EDITOR on * filename:lineno. + * + * When file doesn't have a physical path, the contents get copied into a + * temporary file, and a file descriptor and RAII cleanup guard for it are + * returned. + * + * @param readOnly make the temporary file readonly if the file has no physical + * path. Ignored otherwise. */ -Strings editorFor(const SourcePath & file, uint32_t line); +std::tuple editorFor(const SourcePath & file, uint32_t line, bool readOnly = true); } // namespace nix diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 9968cd2983..2eabde9d6b 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -484,8 +484,8 @@ ProcessLineResult NixRepl::processLine(std::string line) } }(); - // Open in EDITOR - auto args = editorFor(path, line); + /* Open file in EDITOR, or edit a read-only copy if the file doesn't have a physical path. */ + auto [args, fd, autoDel] = editorFor(path, line, /*readOnly=*/true); auto editor = args.front(); args.pop_front(); @@ -494,13 +494,16 @@ ProcessLineResult NixRepl::processLine(std::string line) runProgram2({ .program = editor, .lookupPath = true, - .args = toOsStrings(std::move(args)), + .args = std::move(args), .isInteractive = true, }); - // Reload right after exiting the editor - state->resetFileCache(); - reloadFilesAndFlakes(); + /* If we had to open a temporary read-only file, there's no need to + reload (no files could have changed anyway). */ + if (!fd) { + state->resetFileCache(); + reloadFilesAndFlakes(); + } } else if (command == ":t") { diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index ff225eae0f..30301d97da 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -232,6 +232,10 @@ struct OpenNewFileForWriteParams * Whether to follow symlinks if @ref truncateExisting is true. */ bool followSymlinksOnTruncate:1 = false; + /** + * Whether to open the newly created file as write-only. + */ + bool writeOnly:1 = true; }; /** diff --git a/src/libutil/include/nix/util/os-string.hh b/src/libutil/include/nix/util/os-string.hh index 7e3bf47d38..af686a62cb 100644 --- a/src/libutil/include/nix/util/os-string.hh +++ b/src/libutil/include/nix/util/os-string.hh @@ -104,4 +104,14 @@ inline OsStrings toOsStrings(std::list ss) # define OS_STR(s) L##s #endif +#ifdef _WIN32 + +template +C tokenizeString(OsStringView s, OsStringView separators = OS_STR(" \t\n\r")); + +extern template std::list tokenizeString(OsStringView s, OsStringView separators); +extern template std::vector tokenizeString(OsStringView s, OsStringView separators); + +#endif + } // namespace nix diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index b154ee2152..9181f4466e 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -37,7 +37,7 @@ AutoCloseFD openFileReadonly(const std::filesystem::path & path, FinalSymlink fi AutoCloseFD openNewFileForWrite(const std::filesystem::path & path, mode_t mode, OpenNewFileForWriteParams params) { - auto flags = O_WRONLY | O_CREAT | O_CLOEXEC; + auto flags = (params.writeOnly ? O_WRONLY : O_RDWR) | O_CREAT | O_CLOEXEC; if (params.truncateExisting) { flags |= O_TRUNC; if (!params.followSymlinksOnTruncate) diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index e644f52580..98d41a9caa 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -57,7 +57,7 @@ openNewFileForWrite(const std::filesystem::path & path, [[maybe_unused]] mode_t { return AutoCloseFD{CreateFileW( path.c_str(), - GENERIC_WRITE, + GENERIC_WRITE | (params.writeOnly ? 0 : GENERIC_READ), FILE_SHARE_READ | FILE_SHARE_DELETE, /*lpSecurityAttributes=*/nullptr, params.truncateExisting ? CREATE_ALWAYS : CREATE_NEW, /* TODO: Reparse points. */ diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc index f0fdb0c02b..9d3cf583c3 100644 --- a/src/libutil/windows/os-string.cc +++ b/src/libutil/windows/os-string.cc @@ -4,6 +4,7 @@ #include #include "nix/util/os-string.hh" +#include "nix/util/strings-inline.hh" namespace nix { @@ -29,4 +30,17 @@ OsString string_to_os_string(std::string s) return string_to_os_string(std::string_view{s}); } +#ifdef _WIN32 + +template +C tokenizeString(OsStringView s, OsStringView separators) +{ + return basicTokenizeString(s, separators); +} + +template std::list tokenizeString(OsStringView s, OsStringView separators); +template std::vector tokenizeString(OsStringView s, OsStringView separators); + +#endif + } // namespace nix diff --git a/src/nix/edit.cc b/src/nix/edit.cc index aeb26915ba..a57371bea5 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -44,16 +44,17 @@ struct CmdEdit : InstallableValueCommand logger->stop(); - auto args = editorFor(file, line); - - restoreProcessContext(); - - execvp(args.front().c_str(), stringsToCharPtrs(args).data()); - - std::string command; - for (const auto & arg : args) - command += " '" + arg + "'"; - throw SysError("cannot run command%s", command); + auto [args, tempFd, delTemp] = editorFor(file, line, /*readOnly=*/true); + auto program = args.front(); + args.pop_front(); + + runProgram2( + RunOptions{ + .program = program, + .lookupPath = true, + .args = std::move(args), + .isInteractive = true, + }); } }; diff --git a/tests/functional/flakes/edit.sh b/tests/functional/flakes/edit.sh index fda5c29fd0..0d926c2874 100755 --- a/tests/functional/flakes/edit.sh +++ b/tests/functional/flakes/edit.sh @@ -6,3 +6,7 @@ createFlake1 export EDITOR=cat nix edit "$flake1Dir#" | grepQuiet simple.builder.sh +tar --exclude=".git*" -czf "$TEST_ROOT"/flake1Dir.tar.gz -C "$(dirname "$flake1Dir")" "$(basename "$flake1Dir")" +# Test that editing a file from a tarball flake works and the file is readonly. +nix edit "file://$TEST_ROOT/flake1Dir.tar.gz" | grepQuiet simple.builder.sh +EDITOR='test ! -w' nix edit "file://$TEST_ROOT/flake1Dir.tar.gz" diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 31f2fbdd88..9fc803d092 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -381,6 +381,9 @@ import $testDir/lang/parse-fail-eof-pos.nix " \ '.*error: syntax error, unexpected end of file.*' +EDITOR='cat' nix repl <<< ':e derivation' 2>&1 | grepQuiet 'derivationStrict' +EDITOR='cat' nix repl <<< ':e ' 2>&1 | grepQuiet 'builtin:fetchurl' + # TODO: move init to characterisation/framework.sh badDiff=0 badExitCode=0 From 291e2662ad0b5a302bb50e2258e50c000f3fd2f9 Mon Sep 17 00:00:00 2001 From: steveoliphant <8796107+steveoliphant@users.noreply.github.com> Date: Wed, 15 Apr 2026 08:47:54 -0600 Subject: [PATCH 248/927] Fix typo in content-address documentation --- doc/manual/source/store/derivation/outputs/content-address.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/store/derivation/outputs/content-address.md b/doc/manual/source/store/derivation/outputs/content-address.md index aa65fbe493..9fadcff3c6 100644 --- a/doc/manual/source/store/derivation/outputs/content-address.md +++ b/doc/manual/source/store/derivation/outputs/content-address.md @@ -13,7 +13,7 @@ Given the method, the output's name (computed from the derivation name and outpu ## Fixed-output content-addressing {#fixed} In this case the content address of the *fixed* in advanced by the derivation itself. -In other words, when the derivation has finished [building](@docroot@/store/building.md), and the provisional output' content-address is computed as part of the process to turn it into a *bona fide* store object, the calculated content address must much that given in the derivation, or the build of that derivation will be deemed a failure. +In other words, when the derivation has finished [building](@docroot@/store/building.md), and the provisional output' content-address is computed as part of the process to turn it into a *bona fide* store object, the calculated content address must match that given in the derivation, or the build of that derivation will be deemed a failure. The output spec for an output with a fixed content addresses additionally contains: From e89a8dbde63121f684b4de7896069cf8c4db2aec Mon Sep 17 00:00:00 2001 From: drorspei Date: Wed, 15 Apr 2026 18:44:56 +0200 Subject: [PATCH 249/927] fix concurrent localstore migrations --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d52daaae1f..b0c41c35ce 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -584,7 +584,7 @@ void LocalStore::upgradeDBSchema(State & state) debug("executing Nix database schema migration '%s'...", migrationName); SQLiteTxn txn(state.db); - state.db.exec(stmt + fmt(";\ninsert into SchemaMigrations values('%s')", migrationName)); + state.db.exec(stmt + fmt(";\ninsert or ignore into SchemaMigrations values('%s')", migrationName)); txn.commit(); schemaMigrations.insert(migrationName); From 099fd303c0b94e717fe70c0eedfc3ea37b3d038a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Apr 2026 21:50:20 +0300 Subject: [PATCH 250/927] libstore: Reduce memory usage of the Worker even more Several things. Derivation is the primary memory hog in the build loop, with the total memory usage getting into gigabytes just to hold the derivations in memory. Previously I've optimised most goals to share a single ref, but turns out I missed DerivationResolutionGoal. Also fixes a bug in the move assignment of Goal::Co that didn't destroy the coroutine frame of the Goal being moved into. Just this single commit lowers the memory usage of the build loop by 25% in my tests (from 4.2G -> 3.2G) on a 70k derivation closure. --- src/libstore/build/derivation-goal.cc | 17 +++++++---------- .../build/derivation-resolution-goal.cc | 4 ++-- src/libstore/build/goal.cc | 9 +++++++-- src/libstore/build/worker.cc | 2 +- .../store/build/derivation-resolution-goal.hh | 5 +++-- src/libstore/include/nix/store/build/goal.hh | 4 +++- src/libstore/include/nix/store/build/worker.hh | 2 +- 7 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6e2d3223b1..6d76cd9d27 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -144,11 +144,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) worker.store.printStorePath(drvPath)); } - auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, *drv, buildMode); - { - Goals waitees{resolutionGoal}; - co_await await(std::move(waitees)); - } + auto resolutionGoal = worker.makeDerivationResolutionGoal(drvPath, drv, buildMode); + co_await await({resolutionGoal}); + if (nrFailed != 0) { co_return doneFailure({BuildResult::Failure::DependencyFailed, "Build failed due to failed dependency"}); } @@ -219,6 +217,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) assert(false); } + /* We don't need it any more and don't want to hold on to it while suspended. */ + resolutionGoal.reset(); + /* Give up on substitution for the output we want, actually build this derivation */ auto g = worker.makeDerivationBuildingGoal(drvPath, drv, buildMode, storeDerivation); @@ -226,11 +227,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) /* We will finish with it ourselves, as if we were the derivational goal. */ g->preserveFailure = true; - { - Goals waitees; - waitees.insert(g); - co_await await(std::move(waitees)); - } + co_await await({g}); trace("outer build done"); diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index 81c698e185..1343267eff 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -7,10 +7,10 @@ namespace nix { DerivationResolutionGoal::DerivationResolutionGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode) + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode) : Goal(worker, resolveDerivation()) , drvPath(drvPath) - , drv{std::make_unique(drv)} + , drv(std::move(drv)) , buildMode{buildMode} { name = fmt("resolving derivation '%s'", worker.store.printStorePath(drvPath)); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 946314afe1..1bfa563d15 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -65,10 +65,15 @@ Co::Co(Co && rhs) rhs.handle = nullptr; } -void Co::operator=(Co && rhs) +Co & Co::operator=(Co && rhs) { - this->handle = rhs.handle; + if (handle) { + handle.promise().alive = false; + handle.destroy(); + } + handle = rhs.handle; rhs.handle = nullptr; + return *this; } Co::~Co() diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 0c63beb4dd..002bb1a30d 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -104,7 +104,7 @@ std::shared_ptr Worker::makeDerivationGoal( } std::shared_ptr -Worker::makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode) +Worker::makeDerivationResolutionGoal(const StorePath & drvPath, ref drv, BuildMode buildMode) { return initGoalIfNeeded(derivationResolutionGoals[drvPath], drvPath, drv, *this, buildMode); } diff --git a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh index 843e4031aa..972558e970 100644 --- a/src/libstore/include/nix/store/build/derivation-resolution-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-resolution-goal.hh @@ -37,7 +37,8 @@ struct DerivationResolutionGoal : public Goal { friend class Worker; - DerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode); + DerivationResolutionGoal( + const StorePath & drvPath, ref drv, Worker & worker, BuildMode buildMode); /** * If the derivation needed to be resolved, this is resulting @@ -55,7 +56,7 @@ private: /** * The derivation stored at drvPath. */ - std::unique_ptr drv; + ref drv; /** * The remainder is state held during the build. diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 0b7367ff7e..f8293609e1 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -233,8 +233,10 @@ public: explicit Co(handle_type handle) : handle(handle) {}; - void operator=(Co &&); + Co & operator=(Co &&); Co(Co && rhs); + Co & operator=(const Co &) = delete; + Co(const Co & rhs) = delete; ~Co(); bool await_ready() diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 4c83698681..12e6c0122e 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -265,7 +265,7 @@ public: * @ref DerivationResolutionGoal "derivation resolution goal" */ std::shared_ptr - makeDerivationResolutionGoal(const StorePath & drvPath, const Derivation & drv, BuildMode buildMode); + makeDerivationResolutionGoal(const StorePath & drvPath, ref drv, BuildMode buildMode); /** * @ref DerivationBuildingGoal "derivation building goal" From 88c137708fc581dd64c67a9ae6fa3b452af6eb3f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Apr 2026 21:50:32 +0300 Subject: [PATCH 251/927] libstore: Move childEvents from the promise into Goal itself ChildEvents has no business being in the coroutines promise type - it bloats it unnecessarily. The whole promise type was like 500 bytes just because it stored TimedOut inline - now it's behind a unique_ptr as it should have been. Shaves off like 300MB of memory usage on my stress-testing benchmark. --- .../build/derivation-building-goal.cc | 8 +- src/libstore/build/goal.cc | 24 +++--- src/libstore/include/nix/store/build/goal.hh | 82 +++++++++---------- 3 files changed, 55 insertions(+), 59 deletions(-) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 57a4017e8e..248ef7507c 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -764,9 +764,9 @@ Goal::Co DerivationBuildingGoal::buildWithHook( } else if (std::get_if(&event)) { buildLog->flush(); break; - } else if (auto * timeout = std::get_if(&event)) { + } else if (auto * timeout = std::get_if>(&event)) { hook.reset(); - co_return doneFailure(std::move(*timeout)); + co_return doneFailure(std::move(**timeout)); } } @@ -1035,9 +1035,9 @@ Goal::Co DerivationBuildingGoal::buildLocally( } else if (std::get_if(&event)) { buildLog->flush(); break; - } else if (auto * timeout = std::get_if(&event)) { + } else if (auto * timeout = std::get_if>(&event)) { builder->killChild(); - co_return doneFailure(std::move(*timeout)); + co_return doneFailure(std::move(**timeout)); } } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 1bfa563d15..7651282e3e 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -12,16 +12,15 @@ TimedOut::TimedOut(time_t maxDuration) using Co = nix::Goal::Co; using promise_type = nix::Goal::promise_type; -using ChildEvents = decltype(promise_type::childEvents); -void ChildEvents::pushChildEvent(ChildOutput event) +void Goal::ChildEvents::pushChildEvent(ChildOutput event) { if (childTimeout) return; // Already timed out, ignore childOutputs.push(std::move(event)); } -void ChildEvents::pushChildEvent(ChildEOF event) +void Goal::ChildEvents::pushChildEvent(ChildEOF event) { if (childTimeout) return; // Already timed out, ignore @@ -29,20 +28,20 @@ void ChildEvents::pushChildEvent(ChildEOF event) childEOF = std::move(event); } -void ChildEvents::pushChildEvent(TimedOut event) +void Goal::ChildEvents::pushChildEvent(TimedOut event) { // Timeout is immediate - flush pending events childOutputs = {}; childEOF.reset(); - childTimeout = std::move(event); + childTimeout = std::make_unique(std::move(event)); } -bool ChildEvents::hasChildEvent() const +bool Goal::ChildEvents::hasChildEvent() const { return !childOutputs.empty() || childEOF || childTimeout; } -Goal::ChildEvent ChildEvents::popChildEvent() +Goal::ChildEvent Goal::ChildEvents::popChildEvent() { if (!childOutputs.empty()) { auto event = std::move(childOutputs.front()); @@ -52,7 +51,7 @@ Goal::ChildEvent ChildEvents::popChildEvent() if (childEOF) return *std::exchange(childEOF, std::nullopt); if (childTimeout) - return *std::exchange(childTimeout, std::nullopt); + return std::exchange(childTimeout, nullptr); unreachable(); } @@ -278,22 +277,19 @@ void Goal::work() void Goal::handleChildOutput(Descriptor fd, std::string_view data) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); + childEvents.pushChildEvent(ChildOutput{fd, std::string{data}}); worker.wakeUp(shared_from_this()); } void Goal::handleEOF(Descriptor fd) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(ChildEOF{fd}); + childEvents.pushChildEvent(ChildEOF{fd}); worker.wakeUp(shared_from_this()); } void Goal::timedOut(TimedOut && ex) { - assert(top_co); - top_co->handle.promise().childEvents.pushChildEvent(std::move(ex)); + childEvents.pushChildEvent(std::move(ex)); worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index f8293609e1..6ddc73250d 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -74,7 +74,43 @@ enum struct JobCategory { struct Goal : public std::enable_shared_from_this { + /** + * Event types for child process communication, delivered via coroutines. + */ + struct ChildOutput + { + Descriptor fd; + std::string data; + }; + + struct ChildEOF + { + Descriptor fd; + }; + + using ChildEvent = std::variant>; + private: + class ChildEvents + { + /** + * Structured queue of child events: + * - outputs: stream of data from child + * - eof: optional end-of-stream marker + * - timeout: optional timeout that flushes/overrides other events + */ + std::queue childOutputs; + std::optional childEOF; + std::unique_ptr childTimeout; + + public: + void pushChildEvent(ChildOutput event); + void pushChildEvent(ChildEOF event); + void pushChildEvent(TimedOut event); + bool hasChildEvent() const; + ChildEvent popChildEvent(); + }; + /** * Goals that this goal is waiting for. */ @@ -85,6 +121,8 @@ private: */ std::optional cachedKey; + ChildEvents childEvents; + public: typedef enum { ecBusy, ecSuccess, ecFailed, ecNoSubstituters } ExitCode; @@ -152,22 +190,6 @@ public: friend Goal; }; - /** - * Event types for child process communication, delivered via coroutines. - */ - struct ChildOutput - { - Descriptor fd; - std::string data; - }; - - struct ChildEOF - { - Descriptor fd; - }; - - using ChildEvent = std::variant; - /** * Tag type for `co_await`-ing child events. * Returns a `ChildEvent` when resumed. @@ -321,28 +343,6 @@ public: */ bool alive = true; - class - { - /** - * Structured queue of child events: - * - outputs: stream of data from child - * - eof: optional end-of-stream marker - * - timeout: optional timeout that flushes/overrides other events - */ - std::queue childOutputs; - std::optional childEOF; - std::optional childTimeout; - - public: - - void pushChildEvent(ChildOutput event); - void pushChildEvent(ChildEOF event); - void pushChildEvent(TimedOut event); - bool hasChildEvent() const; - ChildEvent popChildEvent(); - - } childEvents; - /** * The awaiter used by @ref final_suspend. */ @@ -450,7 +450,7 @@ public: bool await_ready() { - assert(!promise.childEvents.hasChildEvent()); + assert(!promise.goal->childEvents.hasChildEvent()); return false; } @@ -478,7 +478,7 @@ public: bool await_ready() { - return handle && handle.promise().childEvents.hasChildEvent(); + return handle && handle.promise().goal->childEvents.hasChildEvent(); } void await_suspend(handle_type h) @@ -489,7 +489,7 @@ public: ChildEvent await_resume() { assert(handle); - return handle.promise().childEvents.popChildEvent(); + return handle.promise().goal->childEvents.popChildEvent(); } }; From 1cded5e512927a79a6e60b5a68c97aca4c1ef0a4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Apr 2026 21:50:39 +0300 Subject: [PATCH 252/927] derivation-resolution-goal: Make use of deducing this for a recursive lambda, get rid of allocationsA std::function allocates for the captures, but with C++23 we have a much better way of writing recursive lambdas - so use it. --- .../build/derivation-resolution-goal.cc | 66 +++++++++---------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index 1343267eff..f457c5614a 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -40,42 +40,38 @@ Goal::Co DerivationResolutionGoal::resolveDerivation() std::map, GoalPtr, value_comparison> inputGoals; - { - std::function, const DerivedPathMap::ChildNode &)> - addWaiteeDerivedPath; - - addWaiteeDerivedPath = [&](ref inputDrv, - const DerivedPathMap::ChildNode & inputNode) { - if (!inputNode.value.empty()) { - auto g = worker.makeGoal( - DerivedPath::Built{ - .drvPath = inputDrv, - .outputs = inputNode.value, - }, - buildMode == bmRepair ? bmRepair : bmNormal); - inputGoals.insert_or_assign(inputDrv, g); - waitees.insert(std::move(g)); - } - for (const auto & [outputName, childNode] : inputNode.childMap) - addWaiteeDerivedPath( - make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); - }; - - for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) { - /* Ensure that pure, non-fixed-output derivations don't - depend on impure derivations. */ - if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() - && !drv->type().isFixed()) { - auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); - if (inputDrv.type().isImpure()) - throw Error( - "pure derivation '%s' depends on impure derivation '%s'", - worker.store.printStorePath(drvPath), - worker.store.printStorePath(inputDrvPath)); - } - - addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); + auto addWaiteeDerivedPath = [&worker = worker, buildMode = buildMode, &waitees, &inputGoals]( + this const auto & self, + ref inputDrv, + const DerivedPathMap::ChildNode & inputNode) -> void { + if (!inputNode.value.empty()) { + auto g = worker.makeGoal( + DerivedPath::Built{ + .drvPath = inputDrv, + .outputs = inputNode.value, + }, + buildMode == bmRepair ? bmRepair : bmNormal); + inputGoals.insert_or_assign(inputDrv, g); + waitees.insert(std::move(g)); } + for (const auto & [outputName, childNode] : inputNode.childMap) + self(make_ref(SingleDerivedPath::Built{inputDrv, outputName}), childNode); + }; + + for (const auto & [inputDrvPath, inputNode] : drv->inputDrvs.map) { + /* Ensure that pure, non-fixed-output derivations don't + depend on impure derivations. */ + if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() + && !drv->type().isFixed()) { + auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); + if (inputDrv.type().isImpure()) + throw Error( + "pure derivation '%s' depends on impure derivation '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(inputDrvPath)); + } + + addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); } co_await await(std::move(waitees)); From 50923476b9dca028051263922e607a1a1887abd8 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 15 Apr 2026 21:50:47 +0300 Subject: [PATCH 253/927] derivation-resolution-goal: Replace value_comparison with a lambda Having such helpers in the namespace scope in a translation unit is a bad time when using unity builds. At some point this was duplicated between several files and caused name collisions with unity builds. A lambda is much cleaner and doesn't have the same footgun. Lambdas can have template parameters too now. --- .../build/derivation-resolution-goal.cc | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/libstore/build/derivation-resolution-goal.cc b/src/libstore/build/derivation-resolution-goal.cc index f457c5614a..6f55ea29d9 100644 --- a/src/libstore/build/derivation-resolution-goal.cc +++ b/src/libstore/build/derivation-resolution-goal.cc @@ -22,23 +22,16 @@ std::string DerivationResolutionGoal::key() return "dc$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } -/** - * Used for `inputGoals` local variable below - */ -struct value_comparison -{ - template - bool operator()(const ref & lhs, const ref & rhs) const - { - return *lhs < *rhs; - } -}; - Goal::Co DerivationResolutionGoal::resolveDerivation() { Goals waitees; - std::map, GoalPtr, value_comparison> inputGoals; + using ValueComparison = decltype([](const ref & lhs, const ref & rhs) { + /* Compare the values, not the pointers themselves. */ + return *lhs < *rhs; + }); + + std::map, GoalPtr, ValueComparison> inputGoals; auto addWaiteeDerivedPath = [&worker = worker, buildMode = buildMode, &waitees, &inputGoals]( this const auto & self, From d3d6e7cdb15406b832da0d8f5a94ec0c765e6ef9 Mon Sep 17 00:00:00 2001 From: zowoq <59103226+zowoq@users.noreply.github.com> Date: Thu, 16 Apr 2026 12:53:07 +1000 Subject: [PATCH 254/927] packaging/dependencies: bump mimalloc to 3.3.0 --- packaging/dependencies.nix | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 9903a93ba9..b44f6ae46c 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -16,15 +16,19 @@ in scope: { inherit stdenv; - mimalloc = pkgs.mimalloc.overrideAttrs rec { - version = "3.1.6"; - src = pkgs.fetchFromGitHub { - owner = "microsoft"; - repo = "mimalloc"; - tag = "v${version}"; - hash = "sha256-7zG0Sqloanz/b+fkJ4wzO86uBmtf9fdYNAT9ixLouyY="; - }; - }; + mimalloc = + if lib.versionAtLeast pkgs.mimalloc.version "3.3.0" then + pkgs.mimalloc + else + pkgs.mimalloc.overrideAttrs rec { + version = "3.3.0"; + src = pkgs.fetchFromGitHub { + owner = "microsoft"; + repo = "mimalloc"; + tag = "v${version}"; + hash = "sha256-xy9gPihw3xvhnd6BrCYfMnnRp5dPSodynKRToYwxuzg="; + }; + }; boehmgc = (pkgs.boehmgc.override { From c485da7fdcf18d79b7ffc278c04e539057db92d8 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 16 Apr 2026 23:26:36 +0300 Subject: [PATCH 255/927] libfethers: Removed unused input parameter to runHg --- src/libfetchers/mercurial.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index db0333e908..af5c94c1b4 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -28,10 +28,9 @@ static RunOptions hgOptions(OsStrings args) } // runProgram wrapper that uses hgOptions instead of stock RunOptions. -static std::string runHg(OsStrings args, const std::optional & input = {}) +static std::string runHg(OsStrings args) { RunOptions opts = hgOptions(std::move(args)); - opts.input = input; auto res = runProgram(std::move(opts)); From 355e3c030aa5a4077b4cdb4b95b1d598ff4c7bce Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 16 Apr 2026 23:36:16 +0300 Subject: [PATCH 256/927] libfetchers: Pass commit message in a temp file This is much easier and will allow us to get rid of a bunch of code. This is a cross port of https://gerrit.lix.systems/c/lix/+/1509. Lix patch was done by eldritch horrors - I just reimplemented this. This is not a cherry-pick because it would to too painful with how far the codebases have diverged. --- src/libfetchers/git.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 3c191d32f5..8b1d4dffec 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -515,6 +515,10 @@ struct GitInputScheme : InputScheme }); if (commitMsg) { + auto [tempFd, tempPath] = createTempFile("nix-msg"); + AutoDelete delTemp(tempPath, /*recursive=*/false); + writeFull(tempFd.get(), *commitMsg); + // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` auto suspension = logger->suspend(); runProgram( @@ -528,9 +532,10 @@ struct GitInputScheme : InputScheme OS_STR("commit"), string_to_os_string(std::string(path.rel())), OS_STR("-F"), - OS_STR("-"), - }, - *commitMsg); + tempPath.native(), + }); + + delTemp.deletePath(); } } } From 28a799548c387a647175eefd118bbd5262ab01e1 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 16 Apr 2026 23:52:32 +0300 Subject: [PATCH 257/927] Remove all input redirection code from runCommand This was only used for one thing (see previous commit), which is much more cleanly handled by just a temporary file. That allows us to get rid of all the threading entirely. This is the same stuff as in https://gerrit.lix.systems/c/lix/+/1510. --- src/libcmd/include/nix/cmd/repl.hh | 2 +- src/libcmd/repl.cc | 6 +- src/libfetchers/git.cc | 2 +- src/libutil/include/nix/util/processes.hh | 3 - src/libutil/unix/processes.cc | 63 ++----------------- src/libutil/windows/processes.cc | 75 +++-------------------- src/nix/repl.cc | 3 +- 7 files changed, 19 insertions(+), 135 deletions(-) diff --git a/src/libcmd/include/nix/cmd/repl.hh b/src/libcmd/include/nix/cmd/repl.hh index d46aa94b6b..81c7b8df5a 100644 --- a/src/libcmd/include/nix/cmd/repl.hh +++ b/src/libcmd/include/nix/cmd/repl.hh @@ -28,7 +28,7 @@ struct AbstractNixRepl * @param programName Name of the command, e.g. `nix` or `nix-env`. * @param args arguments to the command. */ - using RunNix = void(const std::string & programName, OsStrings args, const std::optional & input); + using RunNix = void(const std::string & programName, OsStrings args); /** * @param runNix Function to run the nix CLI to support various diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 9968cd2983..7109eafce0 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -75,7 +75,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc RunNix * runNixPtr; - void runNix(const std::string & program, OsStrings args, const std::optional & input = {}); + void runNix(const std::string & program, OsStrings args); std::unique_ptr interacter; @@ -889,10 +889,10 @@ void NixRepl::evalString(std::string s, Value & v) state->forceValue(v, v.determinePos(noPos)); } -void NixRepl::runNix(const std::string & program, OsStrings args, const std::optional & input) +void NixRepl::runNix(const std::string & program, OsStrings args) { if (runNixPtr) - (*runNixPtr)(program, std::move(args), input); + (*runNixPtr)(program, std::move(args)); else throw Error( "Cannot run '%s' because no method of calling the Nix CLI was provided. This is a configuration problem pertaining to how this program was built. See Nix 2.25 release notes", diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8b1d4dffec..3941c34256 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -455,7 +455,7 @@ struct GitInputScheme : InputScheme args.push_back(destDir.native()); - runProgram("git", true, args, {}, true); + runProgram("git", true, args, true); } std::optional getSourcePath(const Input & input) const override diff --git a/src/libutil/include/nix/util/processes.hh b/src/libutil/include/nix/util/processes.hh index 6a9f75c859..d2cb7adb7d 100644 --- a/src/libutil/include/nix/util/processes.hh +++ b/src/libutil/include/nix/util/processes.hh @@ -116,7 +116,6 @@ std::string runProgram( std::filesystem::path program, bool lookupPath = false, const OsStrings & args = OsStrings(), - const std::optional & input = {}, bool isInteractive = false); struct RunOptions @@ -130,8 +129,6 @@ struct RunOptions #endif std::optional chdir; std::optional environment; - std::optional input; - Source * standardIn = nullptr; Sink * standardOut = nullptr; bool mergeStderrToStdout = false; bool isInteractive = false; diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 9f23da876a..cb17499aa7 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -291,20 +291,15 @@ pid_t startProcess(fun processMain, const ProcessOptions & options) return pid; } -std::string runProgram( - std::filesystem::path program, - bool lookupPath, - const OsStrings & args, - const std::optional & input, - bool isInteractive) +std::string runProgram(std::filesystem::path program, bool lookupPath, const OsStrings & args, bool isInteractive) { auto res = runProgram( RunOptions{ .program = program, .lookupPath = lookupPath, .args = args, - .input = input, - .isInteractive = isInteractive}); + .isInteractive = isInteractive, + }); if (!statusOk(res.first)) throw ExecError(res.first, "program %s %s", PathFmt(program), statusToString(res.first)); @@ -333,22 +328,10 @@ void runProgram2(const RunOptions & options) { checkInterrupt(); - assert(!(options.standardIn && options.input)); - - std::unique_ptr source_; - Source * source = options.standardIn; - - if (options.input) { - source_ = std::make_unique(*options.input); - source = source_.get(); - } - /* Create a pipe. */ - Pipe out, in; + Pipe out; if (options.standardOut) out.create(); - if (source) - in.create(); ProcessOptions processOptions; // vfork implies that the environment of the main process and the fork will @@ -368,8 +351,6 @@ void runProgram2(const RunOptions & options) if (options.mergeStderrToStdout) if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) throw SysError("cannot dup stdout into stderr"); - if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); if (options.chdir && chdir((*options.chdir).c_str()) == -1) throw SysError("chdir failed"); @@ -399,47 +380,11 @@ void runProgram2(const RunOptions & options) out.writeSide.close(); - std::thread writerThread; - - std::promise promise; - - Finally doJoin([&] { - if (writerThread.joinable()) - writerThread.join(); - }); - - if (source) { - in.readSide.close(); - writerThread = std::thread([&] { - try { - std::vector buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), {buf.data(), n}); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide.close(); - }); - } - if (options.standardOut) drainFD(out.readSide.get(), *options.standardOut); /* Wait for the child to finish. */ int status = pid.wait(); - - /* Wait for the writer thread to finish. */ - if (source) - promise.get_future().get(); - if (status) throw ExecError(status, "program %1% %2%", PathFmt(options.program), statusToString(status)); } diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 46a149603f..b2e7b8af08 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -85,20 +85,15 @@ int Pid::wait(bool allowInterrupts) } // TODO: Merge this with Unix's runProgram since it's identical logic. -std::string runProgram( - std::filesystem::path program, - bool lookupPath, - const OsStrings & args, - const std::optional & input, - bool isInteractive) +std::string runProgram(std::filesystem::path program, bool lookupPath, const OsStrings & args, bool isInteractive) { auto res = runProgram( RunOptions{ .program = program, .lookupPath = lookupPath, .args = args, - .input = input, - .isInteractive = isInteractive}); + .isInteractive = isInteractive, + }); if (!statusOk(res.first)) throw ExecError(res.first, "program %s %s", PathFmt(program), statusToString(res.first)); @@ -197,7 +192,7 @@ OsString windowsEscape(const OsString & str, bool cmd) return buffer + L'"'; } -Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & options, Pipe & out, Pipe & in) +Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & options, Pipe & out) { // Setup pipes. if (options.standardOut) { @@ -206,17 +201,13 @@ Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & o } else { out.writeSide = nullFD(); } - if (options.standardIn) { - // Don't inherit the write end of the input pipe - setFDInheritable(in.writeSide, false); - } else { - in.readSide = nullFD(); - } + + AutoCloseFD in = nullFD(); STARTUPINFOW startInfo = {0}; startInfo.cb = sizeof(startInfo); startInfo.dwFlags = STARTF_USESTDHANDLES; - startInfo.hStdInput = in.readSide.get(); + startInfo.hStdInput = in.get(); startInfo.hStdOutput = out.writeSide.get(); startInfo.hStdError = out.writeSide.get(); @@ -305,24 +296,12 @@ void runProgram2(const RunOptions & options) { checkInterrupt(); - assert(!(options.standardIn && options.input)); - - std::unique_ptr source_; - Source * source = options.standardIn; - - if (options.input) { - source_ = std::make_unique(*options.input); - source = source_.get(); - } - /* Create a pipe. */ - Pipe out, in; + Pipe out; // TODO: I copied this from unix but this is handled again in spawnProcess, so might be weird to split it up like // this if (options.standardOut) out.create(); - if (source) - in.create(); std::filesystem::path realProgram = options.program; // TODO: Implement shebang / program interpreter lookup on Windows @@ -330,52 +309,16 @@ void runProgram2(const RunOptions & options) auto suspension = logger->suspendIf(options.isInteractive); - Pid pid = spawnProcess(interpreter.has_value() ? *interpreter : realProgram, options, out, in); + Pid pid = spawnProcess(interpreter.has_value() ? *interpreter : realProgram, options, out); // TODO: This is identical to unix, deduplicate? out.writeSide.close(); - std::thread writerThread; - - std::promise promise; - - Finally doJoin([&] { - if (writerThread.joinable()) - writerThread.join(); - }); - - if (source) { - in.readSide.close(); - writerThread = std::thread([&] { - try { - std::vector buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), {buf.data(), n}); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide.close(); - }); - } - if (options.standardOut) drainFD(out.readSide.get(), *options.standardOut); /* Wait for the child to finish. */ int status = pid.wait(); - - /* Wait for the writer thread to finish. */ - if (source) - promise.get_future().get(); - if (status) throw ExecError(status, "program %1% %2%", PathFmt(options.program), statusToString(status)); } diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 85192faa64..bc7cdedfdc 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -11,7 +11,7 @@ namespace nix { -void runNix(const std::string & program, OsStrings args, const std::optional & input = {}) +void runNix(const std::string & program, OsStrings args) { auto subprocessEnv = getEnvOs(); subprocessEnv[OS_STR("NIX_CONFIG")] = string_to_os_string(globalConfig.toKeyValue()); @@ -21,7 +21,6 @@ void runNix(const std::string & program, OsStrings args, const std::optional Date: Fri, 17 Apr 2026 00:00:25 +0300 Subject: [PATCH 258/927] Deduplicate runProgram between unix/windows The code was identical already, so we can safely dedup it. --- src/libutil/include/nix/util/processes.hh | 1 + src/libutil/processes.cc | 17 +++++++++++++++++ src/libutil/unix/processes.cc | 17 ----------------- src/libutil/windows/processes.cc | 18 ------------------ 4 files changed, 18 insertions(+), 35 deletions(-) diff --git a/src/libutil/include/nix/util/processes.hh b/src/libutil/include/nix/util/processes.hh index d2cb7adb7d..4db35f7e8a 100644 --- a/src/libutil/include/nix/util/processes.hh +++ b/src/libutil/include/nix/util/processes.hh @@ -134,6 +134,7 @@ struct RunOptions bool isInteractive = false; }; +// Output = error code + "standard out" output stream std::pair runProgram(RunOptions && options); void runProgram2(const RunOptions & options); diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc index a9ce2b5212..d36feb19a5 100644 --- a/src/libutil/processes.cc +++ b/src/libutil/processes.cc @@ -1,4 +1,5 @@ #include "nix/util/processes.hh" +#include "nix/util/serialise.hh" namespace nix { @@ -8,4 +9,20 @@ Pid & Pid::operator=(Pid && other) noexcept return *this; } +std::pair runProgram(RunOptions && options) +{ + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} + } // namespace nix diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index cb17499aa7..ef37aee226 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -307,23 +307,6 @@ std::string runProgram(std::filesystem::path program, bool lookupPath, const OsS return res.second; } -// Output = error code + "standard out" output stream -std::pair runProgram(RunOptions && options) -{ - StringSink sink; - options.standardOut = &sink; - - int status = 0; - - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } - - return {status, std::move(sink.s)}; -} - void runProgram2(const RunOptions & options) { checkInterrupt(); diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index b2e7b8af08..3a8703462b 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -274,24 +274,6 @@ Pid spawnProcess(const std::filesystem::path & realProgram, const RunOptions & o return process; } -// TODO: Merge this with Unix's runProgram since it's identical logic. -// Output = error code + "standard out" output stream -std::pair runProgram(RunOptions && options) -{ - StringSink sink; - options.standardOut = &sink; - - int status = 0; - - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } - - return {status, std::move(sink.s)}; -} - void runProgram2(const RunOptions & options) { checkInterrupt(); From fbc6f35dff0bf5710e8697f52a41d1dc0c8c4d87 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Apr 2026 16:04:02 +0200 Subject: [PATCH 259/927] Move Deleter into a separate header --- src/libfetchers/git-lfs-fetch.cc | 1 + src/libfetchers/git-utils.cc | 1 + .../include/nix/fetchers/git-utils.hh | 11 ----------- src/libutil/include/nix/util/deleter.hh | 19 +++++++++++++++++++ src/libutil/include/nix/util/file-system.hh | 11 ++--------- src/libutil/include/nix/util/meson.build | 1 + 6 files changed, 24 insertions(+), 20 deletions(-) create mode 100644 src/libutil/include/nix/util/deleter.hh diff --git a/src/libfetchers/git-lfs-fetch.cc b/src/libfetchers/git-lfs-fetch.cc index 4585e68e58..9d2fb928d6 100644 --- a/src/libfetchers/git-lfs-fetch.cc +++ b/src/libfetchers/git-lfs-fetch.cc @@ -8,6 +8,7 @@ #include "nix/util/util.hh" #include "nix/util/hash.hh" #include "nix/store/ssh.hh" +#include "nix/util/deleter.hh" #include #include diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index abc257fb9e..6bc474395a 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -13,6 +13,7 @@ #include "nix/util/util.hh" #include "nix/util/thread-pool.hh" #include "nix/util/pool.hh" +#include "nix/util/deleter.hh" #include #include diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 24a7b80087..725fbf3984 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -133,17 +133,6 @@ struct GitRepo virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; -// A helper to ensure that the `git_*_free` functions get called. -template -struct Deleter -{ - template - void operator()(T * p) const - { - del(p); - }; -}; - // A helper to ensure that we don't leak objects returned by libgit2. template struct Setter diff --git a/src/libutil/include/nix/util/deleter.hh b/src/libutil/include/nix/util/deleter.hh new file mode 100644 index 0000000000..7f349b10ee --- /dev/null +++ b/src/libutil/include/nix/util/deleter.hh @@ -0,0 +1,19 @@ +#pragma once + +namespace nix { + +/** + * A helper for `std::unique_ptr` that ensures that C APIs that require manual memory management get properly freed. The + * template parameter `del` is a function that takes a pointer and frees it. + */ +template +struct Deleter +{ + template + void operator()(T * p) const + { + del(p); + }; +}; + +} // namespace nix \ No newline at end of file diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 30301d97da..f977e089bf 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -12,6 +12,7 @@ #include "nix/util/types.hh" #include "nix/util/file-descriptor.hh" #include "nix/util/file-path.hh" +#include "nix/util/deleter.hh" #include #include @@ -436,15 +437,7 @@ public: } }; -struct DIRDeleter -{ - void operator()(DIR * dir) const - { - closedir(dir); - } -}; - -typedef std::unique_ptr AutoCloseDir; +typedef std::unique_ptr> AutoCloseDir; /** * Create a temporary directory. diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index e8f6bc345a..0f7a40df7a 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -33,6 +33,7 @@ headers = [ config_pub_h ] + files( 'config-impl.hh', 'configuration.hh', 'current-process.hh', + 'deleter.hh', 'demangle.hh', 'english.hh', 'environment-variables.hh', From 136e1a2c93e153eb05518f4175aa453071a443b6 Mon Sep 17 00:00:00 2001 From: dramforever Date: Tue, 7 Apr 2026 15:53:57 +0800 Subject: [PATCH 260/927] fetchClosure: Add temproot before checking path This avoids the fetched path from getting garbage collected if it is already valid. --- src/libexpr/primops/fetchClosure.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index db182ab499..fe3873c101 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -23,6 +23,8 @@ static void runFetchClosureWithRewrite( const std::optional & toPathMaybe, Value & v) { + if (toPathMaybe) + state.store->addTempRoot(*toPathMaybe); // establish toPath or throw @@ -74,6 +76,7 @@ static void runFetchClosureWithRewrite( static void runFetchClosureWithContentAddressedPath( EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + state.store->addTempRoot(fromPath); if (!state.store->isValidPath(fromPath)) copyClosure(fromStore, *state.store, RealisedPath::Set{fromPath}); @@ -103,6 +106,7 @@ static void runFetchClosureWithContentAddressedPath( static void runFetchClosureWithInputAddressedPath( EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + state.store->addTempRoot(fromPath); if (!state.store->isValidPath(fromPath)) copyClosure(fromStore, *state.store, RealisedPath::Set{fromPath}); From 7fe57da4b294f8e28f6155194293dde1f59a33ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 20 Apr 2026 13:43:08 +0000 Subject: [PATCH 261/927] libstore: don't print URL userinfo in FileTransfer diagnostics When a URL with embedded credentials (scheme://user:pass@host/...) is passed to builtins.fetchurl or any other downloadFile caller, the password is printed verbatim in progress, retry-warning and error messages because they all format request.uri directly: warning: unable to download 'http://user:SECRET@host/file': ...; retrying in 256 ms Add FileTransferRequest::displayUri() which returns uri with the userinfo component stripped, and use it at every diagnostic call site. request.uri itself is unchanged so CURLOPT_URL, result.urls and fetcher-cache keys are unaffected. --- src/libstore-tests/filetransfer-request.cc | 19 +++++++++ src/libstore-tests/meson.build | 1 + src/libstore/filetransfer.cc | 42 +++++++++++++------ .../include/nix/store/filetransfer.hh | 8 ++++ tests/nixos/fetchurl.nix | 20 +++++++++ 5 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 src/libstore-tests/filetransfer-request.cc diff --git a/src/libstore-tests/filetransfer-request.cc b/src/libstore-tests/filetransfer-request.cc new file mode 100644 index 0000000000..b89bb7ed0b --- /dev/null +++ b/src/libstore-tests/filetransfer-request.cc @@ -0,0 +1,19 @@ +#include + +#include "nix/store/filetransfer.hh" + +namespace nix { + +TEST(FileTransferRequest, displayUriStripsUserinfo) +{ + FileTransferRequest req(VerbatimURL{std::string{"https://alice:s3cr3t@example.org:8443/path/file.toml?x=1"}}); + // uri itself is untouched (used for CURLOPT_URL, result.urls, cache keys). + EXPECT_EQ(req.uri.to_string(), "https://alice:s3cr3t@example.org:8443/path/file.toml?x=1"); + // displayUri() drops the userinfo for diagnostics. + EXPECT_EQ(req.displayUri(), "https://example.org:8443/path/file.toml?x=1"); + + FileTransferRequest plain(VerbatimURL{std::string{"https://example.org/file"}}); + EXPECT_EQ(plain.displayUri(), "https://example.org/file"); +} + +} // namespace nix diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index a126a87aca..78a1cc4a12 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -64,6 +64,7 @@ sources = files( 'derived-path.cc', 'downstream-placeholder.cc', 'dummy-store.cc', + 'filetransfer-request.cc', 'filetransfer-retry.cc', 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 85bc650dc8..6d97fb4e3f 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -281,7 +281,11 @@ struct curlFileTransfer : public FileTransfer try { if (!done && enqueued) fail(FileTransferError( - Interrupted, {}, "%s of '%s' was interrupted", Uncolored(request.noun()), request.uri)); + Interrupted, + {}, + "%s of '%s' was interrupted", + Uncolored(request.noun()), + request.displayUri())); } catch (...) { ignoreExceptionInDestructor(); } @@ -297,7 +301,7 @@ struct curlFileTransfer : public FileTransfer /* Already descriptive enough. */ } catch (nix::Error & e) { /* Add more context to the error message. */ - e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.uri.to_string()); + e.addTrace({}, "during %s of '%s'", Uncolored(request.noun()), request.displayUri()); } catch (...) { /* Can't add more context to the error. */ } @@ -361,7 +365,7 @@ struct curlFileTransfer : public FileTransfer try { size_t realSize = size * nmemb; std::string line((char *) contents, realSize); - printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); + printMsg(lvlVomit, "got header for '%s': %s", request.displayUri(), trim(line)); static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); if (std::smatch match; std::regex_match(line, match, statusLine)) { @@ -450,8 +454,8 @@ struct curlFileTransfer : public FileTransfer *logger, lvlTalkative, actFileTransfer, - fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri), - Logger::Fields{request.uri.to_string()}, + fmt("%s '%s'", request.verb(/*continuous=*/true), request.displayUri()), + Logger::Fields{request.displayUri()}, request.parentAct); // Reset the start time to when we actually started the download. startTime = std::chrono::steady_clock::now(); @@ -716,7 +720,7 @@ struct curlFileTransfer : public FileTransfer debug( "finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", Uncolored(request.noun()), - request.uri, + request.displayUri(), code, httpStatus, result.bodySize, @@ -815,14 +819,14 @@ struct curlFileTransfer : public FileTransfer std::move(response), "%s of '%s' was interrupted", Uncolored(request.noun()), - request.uri) + request.displayUri()) : httpStatus != 0 ? FileTransferError( err, std::move(response), "unable to %s '%s': HTTP error %d%s", Uncolored(request.verb()), - request.uri, + request.displayUri(), httpStatus, code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) : FileTransferError( @@ -830,7 +834,7 @@ struct curlFileTransfer : public FileTransfer std::move(response), "unable to %s '%s': %s (%d) %s", Uncolored(request.verb()), - request.uri, + request.displayUri(), curl_easy_strerror(code), code, errbuf); @@ -1081,7 +1085,7 @@ struct curlFileTransfer : public FileTransfer } for (auto & item : incoming) { - debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.uri); + debug("starting %s of '%s'", Uncolored(item->request.noun()), item->request.displayUri()); item->init(); curl_multi_add_handle(curlm.get(), item->req); item->active = true; @@ -1133,7 +1137,7 @@ struct curlFileTransfer : public FileTransfer { if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https" && item->request.uri.scheme() != "s3") - throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string()); + throw nix::Error("uploading to '%s' is not supported", item->request.displayUri()); { auto state(state_.lock()); @@ -1194,6 +1198,20 @@ ref makeFileTransfer(const FileTransferSettings & settings) return makeCurlFileTransfer(settings); } +std::string FileTransferRequest::displayUri() const +{ + try { + auto parsed = uri.parsed(); + if (parsed.authority && parsed.authority->user) { + parsed.authority->user.reset(); + parsed.authority->password.reset(); + return parsed.to_string(); + } + } catch (BadURL &) { + } + return uri.to_string(); +} + void FileTransferRequest::setupForS3() { auto parsedS3 = ParsedS3URL::parse(uri.parsed()); @@ -1283,7 +1301,7 @@ void FileTransfer::download( state->request.notify_one(); }); - request.dataCallback = [_state, uri = request.uri.to_string()](std::string_view data) -> PauseTransfer { + request.dataCallback = [_state, uri = request.displayUri()](std::string_view data) -> PauseTransfer { auto state(_state->lock()); if (state->quit) diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index a423249e66..2bd9ce201d 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -307,6 +307,14 @@ struct FileTransferRequest { } + /** + * `uri` with any userinfo (`user:password@`) stripped, for use in + * progress, warning and error messages so credentials embedded in + * the URL don't leak into logs. Returns `uri` verbatim if it can't + * be parsed. + */ + std::string displayUri() const; + /** * Returns the method description for logging purposes. */ diff --git a/tests/nixos/fetchurl.nix b/tests/nixos/fetchurl.nix index e8663debbc..b3e744424e 100644 --- a/tests/nixos/fetchurl.nix +++ b/tests/nixos/fetchurl.nix @@ -54,6 +54,14 @@ in echo 'foobar' > "$out/index.html" ''; }; + + virtualHosts."auth" = { + basicAuth.alice = "s3cr3t"; + root = pkgs.runCommand "nginx-root" { } '' + mkdir "$out" + echo 'authed' > "$out/index.html" + ''; + }; }; security.pki.certificateFiles = [ "${goodCert}/cert.pem" ]; @@ -61,6 +69,7 @@ in networking.hosts."127.0.0.1" = [ "good" "bad" + "auth" ]; virtualisation.writableStore = true; @@ -89,5 +98,16 @@ in # Fetching from a server with a trusted cert should work via environment variable override. machine.succeed("NIX_SSL_CERT_FILE=/tmp/cafile.pem nix build --no-substitute --expr 'import { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }'") + + # builtins.fetchurl should authenticate using userinfo from the URL. + out = machine.succeed("nix eval --raw --impure --no-substitute --expr 'builtins.readFile (builtins.fetchurl { url = \"http://alice:s3cr3t@auth/index.html\"; sha256 = \"sha256-67y3HalfTt2zfgMt7BwU5vkaGWcelFlR4jryIkA1dTo=\"; })'") + assert out == "authed\n", out + + # On failure the transport-layer diagnostic must show the stripped URL, + # not the userinfo. (The eval trace still echoes the source expression; + # that is the user's own input, not a leak.) + err = machine.fail("nix eval --raw --impure --no-substitute --option tarball-ttl 0 --expr 'builtins.fetchurl { url = \"http://alice:wrong-secret@auth/index.html\"; sha256 = \"sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\"; }' 2>&1") + print(err) + assert "unable to download 'http://auth/index.html'" in err, err ''; } From f0d9109fa355d59ffdd5f07220e5b3d396256360 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 03:10:54 +0300 Subject: [PATCH 262/927] libexpr: Fix error message in InvalidPathError It was only printing the base name, which isn't how we usually print store paths. --- src/libexpr/eval-error.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 45cb1e409d..67e8d5ea1d 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -1,11 +1,12 @@ #include "nix/expr/eval-error.hh" #include "nix/expr/eval.hh" #include "nix/expr/value.hh" +#include "nix/store/store-api.hh" namespace nix { InvalidPathError::InvalidPathError(EvalState & state, const StorePath & path) - : CloneableError(state, "path '%s' is not valid", path.to_string()) + : CloneableError(state, "path '%s' is not valid", state.store->printStorePath(path)) , path{path} { } From 2949729038395be70f99173388e992244e4678f0 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Apr 2026 21:38:23 +0300 Subject: [PATCH 263/927] libflake: Fix argument order evaluation footgun mountInput modifies lockedRef.input to stuff narHash into it. --- src/libflake/flake.cc | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index deb1c16b71..45871cff87 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -391,14 +391,9 @@ static Flake getFlake( lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); } + auto rootDir = state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)); // Re-parse flake.nix from the store. - return readFlake( - state, - originalRef, - resolvedRef, - lockedRef, - state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor)), - lockRootAttrPath); + return readFlake(state, originalRef, resolvedRef, lockedRef, rootDir, lockRootAttrPath); } Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries) From 49b2680eaf80bda3759139df6094dd979bc2b1b0 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 03:52:59 +0300 Subject: [PATCH 264/927] nix flake archive: Use 'deducing this' recursive lambda instead of std::function One slight blemish I noticed while touching this code. With C++23 we can simplify things. --- src/nix/flake.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 42de784534..15e9700d9d 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1095,8 +1095,8 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs sources.insert(storePath); // FIXME: use graph output, handle cycles. - std::function traverse; - traverse = [&](const flake::Node & node) { + auto traverse = [&store, json = json, dryRun = dryRun, &sources]( + this const auto & self, const flake::Node & node) -> nlohmann::json { nlohmann::json jsonObj2 = json ? nlohmann::json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { @@ -1110,9 +1110,9 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs auto & jsonObj3 = jsonObj2[inputName]; if (storePath) jsonObj3["path"] = store->printStorePath(*storePath); - jsonObj3["inputs"] = traverse(**inputNode); + jsonObj3["inputs"] = self(**inputNode); } else - traverse(**inputNode); + self(**inputNode); } } return jsonObj2; From d5f162d1166f71a8eb1e343727a51fcfb8a88235 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 15:12:45 +0300 Subject: [PATCH 265/927] libexpr-c: Fix UAF on readOnlyMode Turns out the readOnlyMode was always dangling in the C API and nobody noticed... Previous commits just started accessing it, which was caught by ASan. This is the minimal fix I can think of. --- src/libexpr-c/nix_api_expr.cc | 21 ++++++++----------- src/libexpr-c/nix_api_expr_internal.h | 3 +-- src/libexpr/eval-settings.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/include/nix/expr/eval-settings.hh | 9 +++++++- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 97680ac6bf..99c8e42cc2 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -129,23 +129,20 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto if (context) context->last_err_code = NIX_OK; try { - return unsafe_new_with_self([&](auto * self) { - return nix_eval_state_builder{ - .store = nix::ref(store->ptr), - .settings = nix::EvalSettings{/* &bool */ self->readOnlyMode}, - .fetchSettings = nix::fetchers::Settings{}, - .readOnlyMode = true, - }; - }); + auto readOnly = nix::make_ref(true); + return new nix_eval_state_builder{ + .store = nix::ref(store->ptr), + .settings = nix::EvalSettings{/* &bool */ *readOnly}, + .fetchSettings = nix::fetchers::Settings{}, + .readOnlyMode = readOnly, + }; } NIXC_CATCH_ERRS_NULL } void nix_eval_state_builder_free(nix_eval_state_builder * builder) { - if (builder) - builder->~nix_eval_state_builder(); - operator delete(builder, static_cast(alignof(nix_eval_state_builder))); + delete builder; } nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder) @@ -154,7 +151,7 @@ nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_buil context->last_err_code = NIX_OK; try { // TODO: load in one go? - builder->settings.readOnlyMode = nix::settings.readOnlyMode; + builder->settings.readOnlyMode = &nix::settings.readOnlyMode; loadConfFile(builder->settings); loadConfFile(builder->fetchSettings); } diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index b38aeaf7b4..3f7e4bf1df 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -18,8 +18,7 @@ struct nix_eval_state_builder nix::EvalSettings settings; nix::fetchers::Settings fetchSettings; nix::LookupPath lookupPath; - // TODO: make an EvalSettings setting own this instead? - bool readOnlyMode; + nix::ref readOnlyMode; }; struct EvalState diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 5cf0ae0430..a6a3e829f2 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -70,7 +70,7 @@ Strings EvalSettings::parseNixPath(const std::string & s) } EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks) - : readOnlyMode{readOnlyMode} + : readOnlyMode{&readOnlyMode} , lookupPathHooks{lookupPathHooks} { auto var = getEnv("NIX_ABORT_ON_WARN"); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1d66b8ead1..e4fe3ba6a5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2583,7 +2583,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat fetchSettings, *store, path.resolveSymlinks(SymlinkResolution::Ancestors), - settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + settings.isReadOnly() ? FetchMode::DryRun : FetchMode::Copy, path.baseName(), ContentAddressMethod::Raw::NixArchive, nullptr, diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index d9dba95370..a03a8dadbe 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -71,7 +71,14 @@ struct EvalSettings : Config EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {}); - bool & readOnlyMode; + /* FIXME: This really shouldn't be public. The C API should have non-global settings instead. */ + bool * readOnlyMode = nullptr; + + bool isReadOnly() const + { + assert(readOnlyMode); + return *readOnlyMode; + } static Strings getDefaultNixPath(); From d34b442634de5d27a221e4c5885724f271228fcd Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 20 Apr 2026 21:40:25 +0300 Subject: [PATCH 266/927] flake archive: Factor out storePath computation/fetching into a lambda This is necessary for making flake store paths lazier and also slightly more concise anyway. --- src/nix/flake.cc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 15e9700d9d..2a5239549e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1090,20 +1090,25 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs StorePathSet sources; - auto storePath = store->toStorePath(flake.flake.path.path.abs()).first; + auto getStorePath = [&](const FlakeRef & lockedRef) { + return dryRun ? lockedRef.input.computeStorePath(*store) + : std::get(lockedRef.input.fetchToStore(fetchSettings, *store)); + }; + + auto storePath = getStorePath(flake.flake.lockedRef); sources.insert(storePath); // FIXME: use graph output, handle cycles. - auto traverse = [&store, json = json, dryRun = dryRun, &sources]( + auto traverse = [&store, json = json, &sources, &getStorePath]( this const auto & self, const flake::Node & node) -> nlohmann::json { nlohmann::json jsonObj2 = json ? nlohmann::json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { std::optional storePath; - if (!(*inputNode)->lockedRef.input.isRelative()) { - storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetchToStore(fetchSettings, *store).first; + const auto & lockedRef = (*inputNode)->lockedRef; + if (!lockedRef.input.isRelative()) { + storePath = getStorePath(lockedRef); sources.insert(*storePath); } if (json) { From 9fcb58f388611e0660db51fed4195594eb21a985 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 20 Apr 2026 21:59:22 +0300 Subject: [PATCH 267/927] libexpr-c: Remove unsafe_new_with_self This is thankfully not used by anything else anymore. Good riddance, since all usages of it had bugs in them. --- src/libexpr-c/nix_api_expr.cc | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 99c8e42cc2..2387ee8c38 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -19,27 +19,6 @@ # include #endif -/** - * @brief Allocate and initialize using self-reference - * - * This allows a brace initializer to reference the object being constructed. - * - * @warning Use with care, as the pointer points to an object that is not fully constructed yet. - * - * @tparam T Type to allocate - * @tparam F A function type for `init`, taking a T* and returning the initializer for T - * @param init Function that takes a T* and returns the initializer for T - * @return Pointer to allocated and initialized object - */ -template -static T * unsafe_new_with_self(F && init) -{ - // Allocate - void * p = ::operator new(sizeof(T), static_cast(alignof(T))); - // Initialize with placement new - return new (p) T(init(static_cast(p))); -} - extern "C" { nix_err nix_libexpr_init(nix_c_context * context) From 7d12269604077f6f302f75449c20c50d38b03f7f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:11:38 +0300 Subject: [PATCH 268/927] libfetchers: Use source accessor if the mercurial fetcher, use makeFSSourceAccessor This significantly simplifies path filtering and gets rid of raw file system accesses. --- src/libfetchers/mercurial.cc | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index af5c94c1b4..ab1d31ed33 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -240,26 +240,25 @@ struct MercurialInputScheme : InputScheme }), "\0"s); - auto actualPath = absPath(localPath); + /* FIXME: Check that the access to this path is allowed. */ + auto accessor = makeFSSourceAccessor(absPath(localPath)); PathFilter filter = [&](const std::string & p) -> bool { - assert(hasPrefix(p, actualPath.string())); - std::string file(p, actualPath.string().size() + 1); + auto cp = CanonPath(p); + auto st = accessor->lstat(cp); - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; + if (st.type == SourceAccessor::tDirectory) { + auto prefix = cp.rel() + "/"; auto i = files.lower_bound(prefix); return i != files.end() && hasPrefix(*i, prefix); } - return files.count(file); + return files.count(cp.rel()); }; return store.addToStore( input.getName(), - {getFSSourceAccessor(), CanonPath(actualPath.string())}, + {accessor, CanonPath::root}, ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {}, @@ -381,7 +380,7 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir / ".hg_archival.txt"); - auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir.string())}); + auto storePath = store.addToStore(name, {makeFSSourceAccessor(tmpDir), CanonPath::root}); Attrs infoAttrs({ {"revCount", (uint64_t) revCount}, From 48100ab18c99a63c195d1d17f249fe9fc685b60f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 00:14:41 +0300 Subject: [PATCH 269/927] libutil: Add a dirFd callback for iterative openFileEnsureBeneathNoSymlinks This would be useful for caching parent directory file descriptors in the source accessor. I don't think the windows code works too well now, so I left the callback there as a FIXME to be called. It's just a perf improvement in general and shouldn't break if it never gets called. --- .../include/nix/util/file-system-at.hh | 8 ++++--- src/libutil/unix/file-system-at.cc | 23 +++++++++++++++---- src/libutil/windows/file-system-at.cc | 8 ++++++- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/libutil/include/nix/util/file-system-at.hh b/src/libutil/include/nix/util/file-system-at.hh index 0f4678461c..17cd8b4668 100644 --- a/src/libutil/include/nix/util/file-system-at.hh +++ b/src/libutil/include/nix/util/file-system-at.hh @@ -90,6 +90,8 @@ OsString readLinkAt(Descriptor dirFd, const CanonPath & path); * @param flags (Unix) `O_*` flags (must not include `O_NOFOLLOW`) * @param mode (Unix) Mode for `O_{CREAT,TMPFILE}` * + * @param dirFdCallback Callback invoked that gets the ownership of an intermediate directory fd. + * * @pre `path.isRoot()` is false * * @throws SymlinkNotAllowed if an interior path component is a @@ -118,12 +120,12 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks( #ifdef _WIN32 ACCESS_MASK desiredAccess, ULONG createOptions, - ULONG createDisposition = FILE_OPEN + ULONG createDisposition = FILE_OPEN, #else int flags, - mode_t mode = 0 + mode_t mode = 0, #endif -); + std::function dirFdCallback = nullptr); #ifdef __linux__ namespace linux { diff --git a/src/libutil/unix/file-system-at.cc b/src/libutil/unix/file-system-at.cc index 762a245ded..11c7bcd064 100644 --- a/src/libutil/unix/file-system-at.cc +++ b/src/libutil/unix/file-system-at.cc @@ -140,20 +140,27 @@ void unix::fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t } } -static AutoCloseFD -openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode) +static AutoCloseFD openFileEnsureBeneathNoSymlinksIterative( + Descriptor dirFd, + const CanonPath & path, + int flags, + mode_t mode, + std::function dirFdCallback) { AutoCloseFD parentFd; auto nrComponents = std::ranges::distance(path); assert(nrComponents >= 1); auto components = std::views::take(path, nrComponents - 1); /* Everything but last component */ auto getParentFd = [&]() { return parentFd ? parentFd.get() : dirFd; }; + auto currentRelPath = CanonPath::root; /* This rather convoluted loop is necessary to avoid TOCTOU when validating that no inner path component is a symlink. */ for (auto it = components.begin(); it != components.end(); ++it) { auto component = std::string(*it); /* Copy into a string to make NUL terminated. */ assert(component != ".." && !component.starts_with('/')); /* In case invariant is broken somehow.. */ + auto prevRelPath = currentRelPath; + currentRelPath = currentRelPath / *it; AutoCloseFD parentFd2 = ::openat( getParentFd(), /* First iteration uses dirFd. */ @@ -188,6 +195,9 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat return AutoCloseFD{}; } + if (dirFdCallback && parentFd) + dirFdCallback(std::move(parentFd), std::move(prevRelPath)); + parentFd = std::move(parentFd2); } @@ -221,7 +231,12 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat return res; } -AutoCloseFD openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode) +AutoCloseFD openFileEnsureBeneathNoSymlinks( + Descriptor dirFd, + const CanonPath & path, + int flags, + mode_t mode, + std::function dirFdCallback) { /* Just in case the invariant is somehow broken. */ assert(!path.rel().starts_with('/')); @@ -263,7 +278,7 @@ AutoCloseFD openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & } #endif - return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode); + return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode, std::move(dirFdCallback)); } OsString readLinkAt(Descriptor dirFd, const CanonPath & path) diff --git a/src/libutil/windows/file-system-at.cc b/src/libutil/windows/file-system-at.cc index 668bd7eb40..54248f2a36 100644 --- a/src/libutil/windows/file-system-at.cc +++ b/src/libutil/windows/file-system-at.cc @@ -229,7 +229,13 @@ PosixStat fstat(Descriptor fd) } AutoCloseFD openFileEnsureBeneathNoSymlinks( - Descriptor dirFd, const CanonPath & path, ACCESS_MASK desiredAccess, ULONG createOptions, ULONG createDisposition) + Descriptor dirFd, + const CanonPath & path, + ACCESS_MASK desiredAccess, + ULONG createOptions, + ULONG createDisposition, + /* FIXME: Actually call this callback. */ + [[maybe_unused]] std::function dirFdCallback) { assert(!path.isRoot()); assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */ From 293aa8ded34dd79d04217505897b678a9d7bc522 Mon Sep 17 00:00:00 2001 From: Krish Jaiswal Date: Tue, 21 Apr 2026 02:52:45 +0530 Subject: [PATCH 270/927] Add redirect for language/values.html --- doc/manual/source/_redirects | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/source/_redirects b/doc/manual/source/_redirects index 07b3130f9c..7e4557f7d5 100644 --- a/doc/manual/source/_redirects +++ b/doc/manual/source/_redirects @@ -36,6 +36,7 @@ /expressions/language-values /language/values 301! /expressions/* /language/:splat 301! /language/values /language/types 301! +/language/values.html /language/types 301! /language/constructs /language/syntax 301! /language/builtin-constants /language/builtins 301! From e069dae4ef9f7e97cf5869f9c0c124979b54364a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 21 Apr 2026 00:01:32 +0200 Subject: [PATCH 271/927] release-jobs: include all buildCross.nix-everything targets Avoids drifting from upload-release.pl when new cross targets such as x86_64-unknown-freebsd are added to fallback-paths. Also drop the reference to a not-yet-existing Python rewrite of the upload script. Addresses review comments on #15640. --- packaging/release-jobs.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packaging/release-jobs.nix b/packaging/release-jobs.nix index 3da8c80de3..23add3905e 100644 --- a/packaging/release-jobs.nix +++ b/packaging/release-jobs.nix @@ -1,5 +1,5 @@ # Hydra jobset containing only the artifacts consumed by -# `maintainers/upload-release.{pl,py}`, so a release can be cut without +# `maintainers/upload-release.pl`, so a release can be cut without # waiting on the full `hydraJobs` CI matrix. # # Evaluated as a legacy (non-flake) jobset because Hydra hard-codes flake @@ -36,8 +36,7 @@ let # `fallback-paths.nix` and (on x86_64-linux) the rendered manual via # its `doc` output. build.nix-everything = hydraJobs.build.nix-everything; - buildCross.nix-everything.riscv64-unknown-linux-gnu = - hydraJobs.buildCross.nix-everything.riscv64-unknown-linux-gnu; + buildCross.nix-everything = hydraJobs.buildCross.nix-everything; inherit (hydraJobs) manual From 7981f28017b7af317546749e25bddb412b3f4c52 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 01:54:04 +0300 Subject: [PATCH 272/927] libutil: Implement unix source accessors that work with file descriptors --- src/libutil/posix-source-accessor.cc | 351 +++++++++++++++++- tests/functional/fetchGit.sh | 2 +- ...val-fail-readDir-not-a-directory-1.err.exp | 2 +- ...val-fail-readDir-not-a-directory-2.err.exp | 2 +- 4 files changed, 352 insertions(+), 5 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 6098552813..bd28f81fa5 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -1,11 +1,19 @@ #include "nix/util/posix-source-accessor.hh" #include "nix/util/file-system-at.hh" +#include "nix/util/lru-cache.hh" +#include "nix/util/sync.hh" #include "nix/util/memory-source-accessor.hh" #include "nix/util/source-path.hh" #include "nix/util/signals.hh" #include +#include + +#ifndef _WIN32 +# include +#endif + namespace nix { static SourceAccessor::Stat sourceAccessorStatFromPosixStat(const PosixStat & st) @@ -31,8 +39,14 @@ static SourceAccessor::Stat sourceAccessorStatFromPosixStat(const PosixStat & st namespace { +#ifndef _WIN32 + +class PosixDirectorySourceAccessor; + class PosixFileSourceAccessor : public detail::PosixSourceAccessorBase { + friend class PosixDirectorySourceAccessor; + AutoCloseFD fd; std::filesystem::path fsPath; /** @@ -122,6 +136,331 @@ std::string PosixFileSourceAccessor::readLink(const CanonPath & path) throw NotASymlink("path '%1%' is not a symlink", showPath(path)); } +static unsigned getGlobalDirFdCacheLimit() +{ + ::rlimit lim{}; + if (::getrlimit(RLIMIT_NOFILE, &lim) == -1) + throw SysError("querying RLIMIT_NOFILE"); + /* Some sane upper bound in case we have a huge rlimit. */ + return std::min(4096, lim.rlim_cur / 8); +} + +class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase +{ +public: + static unsigned getGlobalFdLimit() + { + static auto res = getGlobalDirFdCacheLimit(); + return res; + } + + static void registerAccessor(ref accessor) + { + auto reg = globalDirFdCacheRegistry.lock(); + std::erase_if(*reg, [](auto & maybeAccessor) { return maybeAccessor.expired(); }); + reg->push_back(accessor.get_ptr()); + } + +private: + AutoCloseFD dirFd; + std::filesystem::path fsPath; + + std::shared_ptr>>> dirFdCache; + + static inline std::atomic globalDirFdCount = 0; + + static inline Sync>> globalDirFdCacheRegistry; + + static void maybeEvictFromGlobalCaches() + { + if (globalDirFdCount.load(std::memory_order_relaxed) < getGlobalFdLimit()) + return; + + auto registry(globalDirFdCacheRegistry.lock()); + for (auto it = registry->begin(); it != registry->end();) { + if (globalDirFdCount.load(std::memory_order_relaxed) < getGlobalFdLimit()) + break; + + auto accessor = it->lock(); + if (!accessor) { + it = registry->erase(it); + continue; + } + + /* TODO: Would be nicer if we could evict a portion of the utilised + cache to avoid cold-start issues. Should be fine for now. */ + if (accessor->dirFdCache) { + auto cache = accessor->dirFdCache->lock(); + globalDirFdCount.fetch_sub(cache->size(), std::memory_order_relaxed); + cache->clear(); + } + + ++it; + } + } + + void insertIntoDirFdCache(const CanonPath & key, ref fd) + { + assert(dirFdCache); + auto cache = dirFdCache->lock(); + auto before = cache->size(); + cache->upsert(key, std::move(fd)); + globalDirFdCount.fetch_add(cache->size() - before, std::memory_order_relaxed); + } + + /** + * Get the parent directory of path. The second pair element might be an owning file descriptor + * if path.parent().isRoot() is false. + */ + std::pair> openParent(const CanonPath & path); + + std::function makeDirFdCallback(); + +public: + PosixDirectorySourceAccessor( + AutoCloseFD fd, std::filesystem::path path, bool trackLastModified, unsigned dirFdCacheSize) + : PosixSourceAccessorBase(trackLastModified) + , dirFd(std::move(fd)) + , fsPath(std::move(path)) + { + assert(fsPath.is_absolute()); /* Only used for error messages, but still nice to enforce this invariant. */ + setPathDisplay(fsPath.generic_string()); + + if (dirFdCacheSize) + dirFdCache = std::make_shared>>>(dirFdCacheSize); + } + + PosixDirectorySourceAccessor(PosixDirectorySourceAccessor &&) = delete; + PosixDirectorySourceAccessor(const PosixDirectorySourceAccessor &) = delete; + PosixDirectorySourceAccessor & operator=(PosixDirectorySourceAccessor &&) = delete; + PosixDirectorySourceAccessor & operator=(const PosixDirectorySourceAccessor &) = delete; + + ~PosixDirectorySourceAccessor() + { + if (dirFdCache) { + auto cache = dirFdCache->lock(); + globalDirFdCount.fetch_sub(cache->size(), std::memory_order_relaxed); + } + } + + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override + { + if (path.isRoot()) + return fsPath; + return std::filesystem::path(fsPath) / path.rel(); /* RHS *must* be a relative path. */ + } + + std::string showPath(const CanonPath & path) override + { + if (path.isRoot()) + return displayPrefix; /* No trailing slash. */ + if (displayPrefix.ends_with('/')) + return displayPrefix + path.rel(); + return displayPrefix + path.abs(); + } +}; + +std::function PosixDirectorySourceAccessor::makeDirFdCallback() +{ + if (!dirFdCache) + return nullptr; + + return [this](AutoCloseFD fd, CanonPath key) { + assert(fd); + insertIntoDirFdCache(std::move(key), make_ref(std::move(fd))); + }; +} + +std::pair> PosixDirectorySourceAccessor::openParent(const CanonPath & path) +{ + assert(!path.isRoot()); + auto parent = path.parent().value(); + if (parent.isRoot()) + return {dirFd.get(), nullptr}; + + maybeEvictFromGlobalCaches(); + + if (dirFdCache) { + if (auto cachedFd = dirFdCache->lock()->get(parent)) { + assert((*cachedFd)->get()); + return {(*cachedFd)->get(), *cachedFd}; + } + } + + AutoCloseFD parentFdOwning = openFileEnsureBeneathNoSymlinks( + dirFd.get(), parent, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, makeDirFdCallback()); + + return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; +} + +std::optional PosixDirectorySourceAccessor::maybeLstat(const CanonPath & path) +try { + PosixStat st; + + if (path.isRoot()) { + /* Must never fail - we already have the file descriptor for the directory. */ + st = nix::fstat(dirFd.get()); + } else { + auto [parentFd, parentFdOwning] = openParent(path); + if (parentFd == INVALID_DESCRIPTOR) { + if (errno == ENOENT || errno == ENOTDIR) + return std::nullopt; + throw SysError("opening directory '%1%'", showPath(path.parent().value())); + } + + if (dirFdCache && parentFdOwning) { + assert(*parentFdOwning); + insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); + } + + /* We know that CanonPath returns a NUL-terminated string_view, so the use of ->data() here is safe. */ + if (::fstatat(parentFd, path.baseName()->data(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) + return std::nullopt; + throw SysError("getting status of '%1%'", showPath(path)); + } + } + + maybeUpdateMtime(st.st_mtime); + return sourceAccessorStatFromPosixStat(st); +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +void PosixDirectorySourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) +try { + if (path.isRoot()) + throw NotARegularFile("'%s' is not a regular file", showPath(path)); + + AutoCloseFD fileFd = + openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_CLOEXEC, /*mode=*/0, makeDirFdCallback()); + + if (!fileFd) { + if (errno == ENOENT || errno == ENOTDIR) /* Intermediate component might not exist. */ + throw FileNotFound("file '%s' does not exist", showPath(path)); + throw SysError("opening '%s'", showPath(path)); + } + + auto st = nix::fstat(fileFd.get()); + if (!S_ISREG(st.st_mode)) + throw Error("file '%s' has an unsupported type", showPath(path)); + PosixFileSourceAccessor fileAccessor(std::move(fileFd), fsPath / path.rel(), trackLastModified, st); + maybeUpdateMtime(st.st_mtime); + fileAccessor.readFile(CanonPath::root, sink, sizeCallback); +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +SourceAccessor::DirEntries PosixDirectorySourceAccessor::readDirectory(const CanonPath & path) +try { + AutoCloseFD dirFdOwning; + + if (path.isRoot()) { + /* Get a fresh file descriptor for thread-safety. */ + dirFdOwning = ::openat(dirFd.get(), ".", O_DIRECTORY | O_RDONLY | O_CLOEXEC); + if (!dirFdOwning) + throw SysError("opening directory '%s'", showPath(path)); + } else { + dirFdOwning = openFileEnsureBeneathNoSymlinks( + dirFd.get(), path, O_DIRECTORY | O_RDONLY | O_CLOEXEC, /*mode=*/0, makeDirFdCallback()); + + if (!dirFdOwning) { + if (errno == ENOTDIR) + throw NotADirectory("'%s' is not a directory", showPath(path)); + throw SysError("opening directory '%s'", showPath(path)); + } + } + + AutoCloseDir dir(::fdopendir(dirFdOwning.get())); + if (!dir) + throw SysError("reading directory '%s'", showPath(path)); + dirFdOwning.release(); + + DirEntries entries; + const ::dirent * dirent = nullptr; + + while (errno = 0, dirent = ::readdir(dir.get())) { + checkInterrupt(); + std::string_view name(dirent->d_name); + if (name == "." || name == "..") + continue; + + std::optional type; + switch (dirent->d_type) { + case DT_REG: + type = tRegular; + break; + case DT_DIR: + type = tDirectory; + break; + case DT_LNK: + type = tSymlink; + break; + case DT_CHR: + type = tChar; + break; + case DT_BLK: + type = tBlock; + break; + case DT_FIFO: + type = tFifo; + break; + case DT_SOCK: + type = tSocket; + break; + default: + type = std::nullopt; + break; + } + entries.emplace(name, type); + } + + if (errno) + throw SysError("reading directory '%1%'", showPath(path)); + + return entries; +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +std::string PosixDirectorySourceAccessor::readLink(const CanonPath & path) +try { + if (path.isRoot()) + throw NotASymlink("file '%s' is not a symlink", showPath(path)); + + auto [parentFd, parentFdOwning] = openParent(path); + if (parentFd == INVALID_DESCRIPTOR) { + if (errno == ENOENT || errno == ENOTDIR) + throw FileNotFound("path '%s' does not exist", showPath(path)); + throw SysError("opening directory '%1%'", showPath(path.parent().value())); + } + + if (dirFdCache && parentFdOwning) { + assert(*parentFdOwning); + insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); + } + + try { + return readLinkAt(parentFd, CanonPath(path.baseName().value())); + } catch (SysError & e) { + if (e.errNo == EINVAL) + throw NotASymlink("file '%s' is not a symlink", showPath(path)); + throw; + } +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +#endif + } // namespace PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot, bool trackLastModified) @@ -303,7 +642,7 @@ void PosixSourceAccessor::assertNoSymlinks(CanonPath path) ref getFSSourceAccessor() { - static auto rootFS = make_ref(); + static auto rootFS = makeFSSourceAccessor("/", /*trackLastModified=*/false); return rootFS; } @@ -383,7 +722,15 @@ ref makeFSSourceAccessor(std::filesystem::path root, bool trackL if (S_ISREG(st.st_mode)) return make_ref(std::move(fd), std::move(root), trackLastModified, st); - /* TODO: Use the file descriptor for fd-relative operations on the directory. */ + else if (S_ISDIR(st.st_mode)) { + auto res = make_ref( + std::move(fd), std::move(root), trackLastModified, PosixDirectorySourceAccessor::getGlobalFdLimit() / 8); + PosixDirectorySourceAccessor::registerAccessor(res); + return res; + } + + else + throw Error("file %1% has an unsupported type", PathFmt(root)); #endif return make_ref(std::move(root), trackLastModified); diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index 2992020f16..ef57e47196 100755 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -35,7 +35,7 @@ nix-instantiate --eval -E "builtins.readFile ((builtins.fetchGit \"file://$TEST_ # Fetch a worktree. unset _NIX_FORCE_HTTP -expectStderr 0 nix eval -vvvv --impure --raw --expr "(builtins.fetchGit \"file://$TEST_ROOT/worktree\").outPath" | grepQuiet "copying '$TEST_ROOT/worktree/' to the store" +expectStderr 0 nix eval -vvvv --impure --raw --expr "(builtins.fetchGit \"file://$TEST_ROOT/worktree\").outPath" | grepQuiet "copying '$TEST_ROOT/worktree' to the store" path0=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$TEST_ROOT/worktree\").outPath") path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/worktree\"; }).outPath") [[ $path0 = "$path0_" ]] diff --git a/tests/functional/lang/eval-fail-readDir-not-a-directory-1.err.exp b/tests/functional/lang/eval-fail-readDir-not-a-directory-1.err.exp index f94a7ed745..7db863b03d 100644 --- a/tests/functional/lang/eval-fail-readDir-not-a-directory-1.err.exp +++ b/tests/functional/lang/eval-fail-readDir-not-a-directory-1.err.exp @@ -13,4 +13,4 @@ error: | ^ 3| } - error: cannot read directory "/pwd/lang/readDir/bar": Not a directory + error: '/pwd/lang/readDir/bar' is not a directory diff --git a/tests/functional/lang/eval-fail-readDir-not-a-directory-2.err.exp b/tests/functional/lang/eval-fail-readDir-not-a-directory-2.err.exp index f5e6775545..20bbc3072e 100644 --- a/tests/functional/lang/eval-fail-readDir-not-a-directory-2.err.exp +++ b/tests/functional/lang/eval-fail-readDir-not-a-directory-2.err.exp @@ -13,4 +13,4 @@ error: | ^ 3| } - error: cannot read directory "/pwd/lang/readDir/foo/git-hates-directories": Not a directory + error: '/pwd/lang/readDir/foo/git-hates-directories' is not a directory From 732f9c1cc06baf13f5b992e6ed6967da932b44dc Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:47:10 +0300 Subject: [PATCH 273/927] libstore: LocalStoreAccessor uses makeFSSourceAccessor This more honestly wraps the underlying FS accessor (we want to use the unix-specific dirfd-based one). --- src/libstore/local-fs-store.cc | 52 ++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 9f5864ee44..12818f32a4 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,4 +1,3 @@ -#include "nix/util/posix-source-accessor.hh" #include "nix/store/store-api.hh" #include "nix/store/local-fs-store.hh" #include "nix/util/compression.hh" @@ -27,13 +26,14 @@ LocalFSStore::LocalFSStore(const Config & config) { } -struct LocalStoreAccessor : PosixSourceAccessor +struct LocalStoreAccessor : SourceAccessor { + ref accessor; ref store; bool requireValidPath; LocalStoreAccessor(ref store, bool requireValidPath) - : PosixSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()}) + : accessor(makeFSSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()})) , store(store) , requireValidPath(requireValidPath) { @@ -54,25 +54,61 @@ struct LocalStoreAccessor : PosixSourceAccessor return Stat{.type = tDirectory}; requireStoreObject(path); - return PosixSourceAccessor::maybeLstat(path); + return accessor->maybeLstat(path); + } + + Stat lstat(const CanonPath & path) override + { + /* Also allow `path` to point to the entire store, which is + needed for resolving symlinks. */ + if (path.isRoot()) + return Stat{.type = tDirectory}; + + requireStoreObject(path); + return accessor->lstat(path); } DirEntries readDirectory(const CanonPath & path) override { requireStoreObject(path); - return PosixSourceAccessor::readDirectory(path); + return accessor->readDirectory(path); } void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { requireStoreObject(path); - return PosixSourceAccessor::readFile(path, sink, sizeCallback); + return accessor->readFile(path, sink, sizeCallback); } std::string readLink(const CanonPath & path) override { requireStoreObject(path); - return PosixSourceAccessor::readLink(path); + return accessor->readLink(path); + } + + std::string showPath(const CanonPath & path) override + { + return accessor->showPath(path); + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return accessor->getPhysicalPath(path); + } + + std::pair> getFingerprint(const CanonPath & path) override + { + return accessor->getFingerprint(path); + } + + std::optional getLastModified() override + { + return accessor->getLastModified(); + } + + bool pathExists(const CanonPath & path) override + { + return accessor->pathExists(path); } }; @@ -96,7 +132,7 @@ std::shared_ptr LocalFSStore::getFSAccessor(const StorePath & pa if (!pathExists(absPath)) return nullptr; } - return std::make_shared(std::move(absPath)); + return makeFSSourceAccessor(std::move(absPath)); } const std::filesystem::path LocalFSStore::drvsLogDir = "drvs"; From 7847c5136fce2c5f7fed343b82eaebe493e5fdc6 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:54:36 +0300 Subject: [PATCH 274/927] libstore: Use requireStoreObjectAccessor in addToStore --- src/libstore/local-store.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b0c41c35ce..b41856871b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1077,8 +1077,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF if (info.ca) { auto & specified = *info.ca; auto actualHash = ({ - auto accessor = getFSAccessor(false); - CanonPath path{info.path.to_string()}; + SourcePath sourcePath = requireStoreObjectAccessor(info.path, /*requireValidPath=*/false); Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++ auto fim = specified.method.getFileIngestionMethod(); switch (fim) { @@ -1088,12 +1087,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF specified.hash.algo, std::string{info.path.hashPart()}, }; - dumpPath({accessor, path}, caSink, (FileSerialisationMethod) fim); + dumpPath(sourcePath, caSink, (FileSerialisationMethod) fim); h = caSink.finish().hash; break; } case FileIngestionMethod::Git: - h = git::dumpHash(specified.hash.algo, {accessor, path}).hash; + h = git::dumpHash(specified.hash.algo, sourcePath).hash; break; } ContentAddress{ From 344ab0a5ee5b6fe2147047d7c449db20095d0297 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:09:10 +0300 Subject: [PATCH 275/927] libstore: Use makeFSStoreAccessor in derivation builder With the addition of the file descriptor based source accessor (that does caching) we now create a fresh instance of the accessor and the last path component is not followed, so the transformation is safe (actually tests fail without this because some directories get unlinked and result in ENOENT if the file descriptor gets cached). --- src/libstore/unix/build/derivation-builder.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 85aa98c7ae..2df2b74bb1 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1713,12 +1713,11 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() HashModuloSink caSink{outputHash.hashAlgo, oldHashPart}; auto fim = outputHash.method.getFileIngestionMethod(); dumpPath( - {getFSSourceAccessor(), CanonPath(actualPath.native())}, caSink, (FileSerialisationMethod) fim); + {makeFSSourceAccessor(actualPath), CanonPath::root}, caSink, (FileSerialisationMethod) fim); return caSink.finish().hash; } case FileIngestionMethod::Git: { - return git::dumpHash(outputHash.hashAlgo, {getFSSourceAccessor(), CanonPath(actualPath.native())}) - .hash; + return git::dumpHash(outputHash.hashAlgo, {makeFSSourceAccessor(actualPath), CanonPath::root}).hash; } } assert(false); @@ -1740,7 +1739,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() { HashResult narHashAndSize = hashPath( - {getFSSourceAccessor(), CanonPath(actualPath.native())}, + {makeFSSourceAccessor(actualPath), CanonPath::root}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); newInfo0.narHash = narHashAndSize.hash; @@ -1783,7 +1782,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() std::string{scratchPath->hashPart()}, std::string{requiredFinalPath.hashPart()}); rewriteOutput(outputRewrites); HashResult narHashAndSize = hashPath( - {getFSSourceAccessor(), CanonPath(actualPath.native())}, + {makeFSSourceAccessor(actualPath), CanonPath::root}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); ValidPathInfo newInfo0{requiredFinalPath, {store, narHashAndSize.hash}}; From b16a5c365c45fce2f3a77632bff883a100c2c5dc Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:27:31 +0300 Subject: [PATCH 276/927] nix: Use makeFSSourceAccessor in place of createAtRoot The API is the same and uses the unix-specific implementation. We can also drop all makeParentCanonical calls, because the function follows symlinks in the parents already when opening the dirFd/file. One slight wrinkle it that we first have to do absPath - this is fine. --- src/nix/add-to-store.cc | 2 +- src/nix/hash.cc | 4 +--- src/nix/nix-store/nix-store.cc | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 4f0fe722f1..09972a2890 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -36,7 +36,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand if (!namePart) namePart = path.filename().string(); - auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); + auto sourcePath = makeFSSourceAccessor(absPath(path)); auto storePath = dryRun ? store->computeStorePath(*namePart, sourcePath, caMethod, hashAlgo, {}).first : store->addToStoreSlow(*namePart, sourcePath, caMethod, hashAlgo, {}).path; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 90f44fdecd..9cd0592073 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -84,9 +84,7 @@ struct CmdHashBase : Command return std::make_unique(hashAlgo); }; - auto makeSourcePath = [&]() -> SourcePath { - return PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); - }; + auto makeSourcePath = [&]() -> SourcePath { return makeFSSourceAccessor(absPath(path)); }; Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++ switch (mode) { diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index c21625e040..15a4e878f5 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -191,7 +191,7 @@ static void opAdd(Strings opFlags, Strings opArgs) throw UsageError("unknown flag"); for (auto & i : opArgs) { - auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); + auto sourcePath = makeFSSourceAccessor(absPath(i)); std::cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), sourcePath))); } } @@ -215,7 +215,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) { - auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); + auto sourcePath = makeFSSourceAccessor(absPath(i)); std::cout << fmt( "%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), sourcePath, method, hashAlgo).path)); } From a8bcc083a270f86334b5ef0268ddc56a23235713 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:34:47 +0300 Subject: [PATCH 277/927] libutil: Use makeFSSourceAccessor in dumpPath Unlike previously, we now follow symlinks in parents (not the last path component), but this is secure and what versions like 2.18 did. --- src/libutil/archive.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index f704acdf78..6188860ce1 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -101,14 +101,15 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & time_t dumpPathAndGetMtime(const std::filesystem::path & path, Sink & sink, PathFilter & filter) { - auto path2 = PosixSourceAccessor::createAtRoot(path, /*trackLastModified=*/true); + SourcePath path2 = makeFSSourceAccessor(absPath(path), /*trackLastModified=*/true); path2.dumpPath(sink, filter); return path2.accessor->getLastModified().value(); } void dumpPath(const std::filesystem::path & path, Sink & sink, PathFilter & filter) { - dumpPathAndGetMtime(path, sink, filter); + SourcePath path2 = makeFSSourceAccessor(absPath(path), /*trackLastModified=*/false); + path2.dumpPath(sink, filter); } void dumpString(std::string_view s, Sink & sink) From c349fbf4c835b2eea7ab8c266f1c02d829a83029 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 02:26:06 +0300 Subject: [PATCH 278/927] libutil: Add a new overload of readDirectory (for fd-relative operations) and use in recursive traversal This ensures race-free traversal throughout. --- src/libutil/archive.cc | 34 +++++++++++-------- src/libutil/fs-sink.cc | 10 +++--- .../include/nix/util/source-accessor.hh | 16 +++++++++ src/libutil/posix-source-accessor.cc | 27 ++++++++++++++- 4 files changed, 68 insertions(+), 19 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6188860ce1..91af4b9e7f 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -34,10 +34,10 @@ PathFilter defaultPathFilter = [](const std::string &) { return true; }; void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter) { - auto dumpContents = [&](const CanonPath & path) { + auto dumpContents = [&sink](SourceAccessor & accessor, const CanonPath & path) { sink << "contents"; std::optional size; - readFile(path, sink, [&](uint64_t _size) { + accessor.readFile(path, sink, [&](uint64_t _size) { size = _size; sink << _size; }); @@ -47,10 +47,14 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & sink << narVersionMagic1; - [&, &this_(*this)](this const auto & dump, const CanonPath & path) -> void { + [&sink, &filter, &dumpContents]( + this const auto & dump, + SourceAccessor & accessor, + const CanonPath & path, + const CanonPath & filterPath) -> void { checkInterrupt(); - auto st = this_.lstat(path); + auto st = accessor.lstat(path); sink << "("; @@ -58,7 +62,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & sink << "type" << "regular"; if (st.isExecutable) sink << "executable" << ""; - dumpContents(path); + dumpContents(accessor, path); } else if (st.type == tDirectory) { @@ -67,7 +71,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & /* If we're on a case-insensitive system like macOS, undo the case hack applied by restorePath(). */ StringMap unhacked; - for (auto & i : this_.readDirectory(path)) + for (auto & i : accessor.readDirectory(path)) if (archiveSettings.useCaseHack) { std::string name(i.first); size_t pos = i.first.find(caseHackSuffix); @@ -81,22 +85,24 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & } else unhacked.emplace(i.first, i.first); - for (auto & i : unhacked) - if (filter((path / i.first).abs())) { - sink << "entry" << "(" << "name" << i.first << "node"; - dump(path / i.second); - sink << ")"; - } + accessor.readDirectory(path, [&](SourceAccessor & subdirAccessor, const CanonPath & subdirRelPath) { + for (auto & i : unhacked) + if (filter((filterPath / i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(subdirAccessor, subdirRelPath / i.second, filterPath / i.second); + sink << ")"; + } + }); } else if (st.type == tSymlink) - sink << "type" << "symlink" << "target" << this_.readLink(path); + sink << "type" << "symlink" << "target" << accessor.readLink(path); else throw Error("file '%s' has an unsupported type", path); sink << ")"; - }(path); + }(*this, path, path); } time_t dumpPathAndGetMtime(const std::filesystem::path & path, Sink & sink, PathFilter & filter) diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index e41d153e92..deb8ec3c8f 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -34,10 +34,12 @@ void copyRecursive(SourceAccessor & accessor, const CanonPath & from, FileSystem } case SourceAccessor::tDirectory: { - sink.createDirectory(to, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPath) { - for (auto & [name, _] : accessor.readDirectory(from)) { - copyRecursive(accessor, from / name, dirSink, relDirPath / name); - } + sink.createDirectory(to, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPathTo) { + accessor.readDirectory(from, [&](SourceAccessor & subdirAccessor, const CanonPath & relDirPathFrom) { + for (auto & [name, _] : subdirAccessor.readDirectory(relDirPathFrom)) { + copyRecursive(subdirAccessor, relDirPathFrom / name, dirSink, relDirPathTo / name); + } + }); }); break; } diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index c458cf8b5c..737383aba2 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -138,6 +138,22 @@ struct SourceAccessor : std::enable_shared_from_this */ virtual DirEntries readDirectory(const CanonPath & path) = 0; + /** + * Variation of readDirectory that receives a SourceAccessor possibly scoped to \ref dirPath. + * Primary meant for recursive traversal functions that would benefit from *at-style syscalls + * relative to a particular directory. + * + * @note Like `readFile`, this method should *not* follow symlinks. + * @param callback Caller-provided function invoked with a maximally deeply scoped SourceAccessor and the path that + * would have to be prepended to each path relative to dirPath to access a particular file with it. + */ + virtual void readDirectory( + const CanonPath & dirPath, + std::function callback) + { + callback(*this, dirPath); + } + virtual std::string readLink(const CanonPath & path) = 0; virtual void dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter = defaultPathFilter); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index bd28f81fa5..6a69a16128 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -216,6 +216,8 @@ class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase std::function makeDirFdCallback(); + AutoCloseFD openSubdirectory(const CanonPath & path); + public: PosixDirectorySourceAccessor( AutoCloseFD fd, std::filesystem::path path, bool trackLastModified, unsigned dirFdCacheSize) @@ -249,6 +251,10 @@ class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase DirEntries readDirectory(const CanonPath & path) override; + void readDirectory( + const CanonPath & dirPath, + std::function callback) override; + std::string readLink(const CanonPath & path) override; std::optional getPhysicalPath(const CanonPath & path) override @@ -359,7 +365,7 @@ try { throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } -SourceAccessor::DirEntries PosixDirectorySourceAccessor::readDirectory(const CanonPath & path) +AutoCloseFD PosixDirectorySourceAccessor::openSubdirectory(const CanonPath & path) try { AutoCloseFD dirFdOwning; @@ -379,6 +385,14 @@ try { } } + return dirFdOwning; +} catch (SymlinkNotAllowed & e) { + throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); +} + +SourceAccessor::DirEntries PosixDirectorySourceAccessor::readDirectory(const CanonPath & path) +try { + AutoCloseFD dirFdOwning = openSubdirectory(path); AutoCloseDir dir(::fdopendir(dirFdOwning.get())); if (!dir) throw SysError("reading directory '%s'", showPath(path)); @@ -431,6 +445,17 @@ try { throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } +void PosixDirectorySourceAccessor::readDirectory( + const CanonPath & dirPath, + std::function callback) +{ + auto fd = openSubdirectory(dirPath); + PosixDirectorySourceAccessor accessor{ + std::move(fd), fsPath / dirPath.rel(), trackLastModified, /*dirFdCacheSize=*/0}; + callback(accessor, CanonPath::root); + PosixSourceAccessorBase::maybeUpdateMtime(accessor.mtime); +} + std::string PosixDirectorySourceAccessor::readLink(const CanonPath & path) try { if (path.isRoot()) From ba7db4e05635e351201327db0a5bbbd85ca4a07a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:38:08 +0300 Subject: [PATCH 279/927] nix-perl: Get rid of the last occurence of createAtRoot --- src/perl/lib/Nix/Store.xs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index b0f70be3a9..8b28b0e539 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -259,7 +259,7 @@ hashPath(char * algo, int base32, char * path) PPCODE: try { Hash h = hashPath( - PosixSourceAccessor::createAtRoot(path), + makeFSSourceAccessor(absPath(path)), FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first; auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); @@ -339,7 +339,7 @@ StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat; auto path = THIS->store->addToStore( std::string(baseNameOf(srcPath)), - PosixSourceAccessor::createAtRoot(srcPath), + makeFSSourceAccessor(absPath(srcPath)), method, parseHashAlgo(algo)); XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); } catch (Error & e) { From 316e33a14834de36cd0792c021e00fddc260e3ba Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 10 Jan 2026 01:39:14 +0300 Subject: [PATCH 280/927] libutil: Get rid of PosixSourceAccessor::createAtRoot --- .../include/nix/util/posix-source-accessor.hh | 30 ------------------- src/libutil/posix-source-accessor.cc | 10 ------- 2 files changed, 40 deletions(-) diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index d93b68e0af..0da7dcdea8 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -76,36 +76,6 @@ public: std::optional getPhysicalPath(const CanonPath & path) override; - /** - * Create a `PosixSourceAccessor` and `SourcePath` corresponding to - * some native path. - * - * @param Whether the accessor should return a non-null getLastModified. - * When true the accessor must be used only by a single thread. - * - * The `PosixSourceAccessor` is rooted as far up the tree as - * possible, (e.g. on Windows it could scoped to a drive like - * `C:\`). This allows more `..` parent accessing to work. - * - * @note When `path` is trusted user input, canonicalize it using - * `std::filesystem::canonical`, `makeParentCanonical`, `std::filesystem::weakly_canonical`, etc, - * as appropriate for the use case. At least weak canonicalization is - * required for the `SourcePath` to do anything useful at the location it - * points to. - * - * @note A canonicalizing behavior is not built in `createAtRoot` so that - * callers do not accidentally introduce symlink-related security vulnerabilities. - * Furthermore, `createAtRoot` does not know whether the file pointed to by - * `path` should be resolved if it is itself a symlink. In other words, - * `createAtRoot` can not decide between aforementioned `canonical`, `makeParentCanonical`, etc. for its callers. - * - * See - * [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path) - * and - * [`std::filesystem::path::relative_path`](https://en.cppreference.com/w/cpp/filesystem/path/relative_path). - */ - static SourcePath createAtRoot(const std::filesystem::path & path, bool trackLastModified = false); - void invalidateCache(const CanonPath & path) override; private: diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 6a69a16128..affad4fbbe 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -3,7 +3,6 @@ #include "nix/util/lru-cache.hh" #include "nix/util/sync.hh" #include "nix/util/memory-source-accessor.hh" -#include "nix/util/source-path.hh" #include "nix/util/signals.hh" #include @@ -501,15 +500,6 @@ PosixSourceAccessor::PosixSourceAccessor() { } -SourcePath PosixSourceAccessor::createAtRoot(const std::filesystem::path & path, bool trackLastModified) -{ - std::filesystem::path path2 = absPath(path); - return { - make_ref(path2.root_path(), trackLastModified), - CanonPath{path2.relative_path().string()}, - }; -} - std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) { return root.empty() ? (std::filesystem::path{path.abs()}) From 52011de1b22206b080dc935b18a781a53417eeb0 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 02:49:31 +0300 Subject: [PATCH 281/927] libstore: Use copyRecursive when copying FOD outputs This completely bypasses coroutines and serialisation/deserialisation overhead and could start using reflinking in the future too. --- src/libstore/unix/build/derivation-builder.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 2df2b74bb1..73eff4dfd6 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1761,8 +1761,10 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() any stale writable file descriptors. Copy through the serialisation/deserialisation. TODO: Use copyRecursive here and make use of reflinking. */ - auto source = sinkToSource([&](Sink & nextSink) { dumpPath(actualPath, nextSink); }); - restorePath(tmpOutput, *source, store.config->getLocalSettings().fsyncStorePaths); + auto pathAccessor = makeFSSourceAccessor(actualPath); + RestoreSink restoreSink{store.config->getLocalSettings().fsyncStorePaths}; + restoreSink.dstPath = tmpOutput; + copyRecursive(*pathAccessor, CanonPath::root, restoreSink, CanonPath::root); /* This makes it slightly harder to make sense of the control flow. The rule of thumb is that actualPath points to the current location of the stuff that we'll end up registering. */ From 8e7702d7365ef5b88b0498949d8ee2a5109f44ea Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 03:02:49 +0300 Subject: [PATCH 282/927] makeFSSourceAccessor: add finalSymlink parameter Useful to avoid costly resolveSymlinks() when possible. --- src/libutil/include/nix/util/source-accessor.hh | 3 ++- src/libutil/posix-source-accessor.cc | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 737383aba2..92bd5d30fa 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -278,7 +278,8 @@ ref getFSSourceAccessor(); * * Symlinks in parents of `root` are resolved. Final symlink is not. */ -ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified = false); +ref makeFSSourceAccessor( + std::filesystem::path root, bool trackLastModified = false, FinalSymlink finalSymlink = FinalSymlink::DontFollow); /** * Construct an accessor that presents a "union" view of a vector of diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index affad4fbbe..a8046c5c17 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -661,14 +661,14 @@ ref getFSSourceAccessor() return rootFS; } -ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified) +ref makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified, FinalSymlink finalSymlink) { #ifndef _WIN32 assert(root.is_absolute()); - AutoCloseFD fd = openFileReadonly(root, FinalSymlink::DontFollow); + AutoCloseFD fd = openFileReadonly(root, finalSymlink); if (!fd) { - if (errno != ELOOP) + if (finalSymlink == FinalSymlink::Follow || errno != ELOOP) throw NativeSysError("opening file %1%", PathFmt(root)); /* A helper class that holds the symlink destination in memory. */ From c296e254b133c9c6e97c47d85fb43c5ef957b7f4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 03:39:39 +0300 Subject: [PATCH 283/927] PosixDirectorySourceAccessor: Improve dirFd caching --- src/libutil/posix-source-accessor.cc | 37 ++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index a8046c5c17..5dd4f56dd3 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -293,15 +293,42 @@ std::pair> PosixDirectorySourceAccessor maybeEvictFromGlobalCaches(); + std::shared_ptr intermediateParentFd; + CanonPath anchor = CanonPath::root; + if (dirFdCache) { - if (auto cachedFd = dirFdCache->lock()->get(parent)) { - assert((*cachedFd)->get()); - return {(*cachedFd)->get(), *cachedFd}; + auto cache = dirFdCache->lock(); + auto p = parent; + while (true) { + if (auto intermediateDirFdHit = cache->get(p)) { + if (p == parent) + return {(*intermediateDirFdHit)->get(), *intermediateDirFdHit}; + intermediateParentFd = intermediateDirFdHit->get_ptr(); + anchor = p; + break; + } + if (p.isRoot()) + break; + p.pop(); + } + } + + Descriptor startFd = intermediateParentFd ? intermediateParentFd->get() : dirFd.get(); + CanonPath relPath = intermediateParentFd ? parent.removePrefix(anchor) : parent; + + std::function cb; + if (auto base = makeDirFdCallback()) { + if (intermediateParentFd) { + cb = [base = std::move(base), prefix = anchor](AutoCloseFD fd, CanonPath relKey) { + base(std::move(fd), prefix / relKey); + }; + } else { + cb = std::move(base); } } - AutoCloseFD parentFdOwning = openFileEnsureBeneathNoSymlinks( - dirFd.get(), parent, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, makeDirFdCallback()); + AutoCloseFD parentFdOwning = + openFileEnsureBeneathNoSymlinks(startFd, relPath, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, std::move(cb)); return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; } From 043cafab68d9e87de0e35a35f3abd8426ec5094b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 03:50:36 +0300 Subject: [PATCH 284/927] SourceAccessor: remove invalidateCache, remove raw usage of PosixSourceAccessor PosixSourceAccessor is going to be renamed to WindowsSourceAccessor in the following commit and hopefully deleted soon. The lstat cache is finally removed from the unix case too. --- src/libfetchers/filtering-source-accessor.cc | 5 ----- .../include/nix/fetchers/filtering-source-accessor.hh | 2 -- src/libflake/flake.cc | 2 -- src/libstore/optimise-store.cc | 8 +------- src/libutil/include/nix/util/posix-source-accessor.hh | 2 -- src/libutil/include/nix/util/source-path.hh | 5 ----- src/libutil/mounted-source-accessor.cc | 6 ------ src/libutil/posix-source-accessor.cc | 5 ----- src/libutil/union-source-accessor.cc | 6 ------ 9 files changed, 1 insertion(+), 40 deletions(-) diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index 6fe7d2504e..b8455710fa 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -62,11 +62,6 @@ std::pair> FilteringSourceAccessor::getFin return next->getFingerprint(prefix / path); } -void FilteringSourceAccessor::invalidateCache(const CanonPath & path) -{ - next->invalidateCache(prefix / path); -} - void FilteringSourceAccessor::checkAccess(const CanonPath & path) { if (!isAllowed(path)) diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 13272719fe..b259e8ef8f 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -53,8 +53,6 @@ struct FilteringSourceAccessor : SourceAccessor std::pair> getFingerprint(const CanonPath & path) override; - void invalidateCache(const CanonPath & path) override; - /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` * exception if `isAllowed()` returns `false` for `path`. diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 45871cff87..e8dbf42f54 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -863,8 +863,6 @@ LockedFlake lockFlake( CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), newLockFileS, commitMessage); - - flake.lockFilePath().invalidateCache(); } /* Rewriting the lockfile changed the top-level diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 220e37fe9e..15f2b1f3d1 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -153,13 +153,7 @@ void LocalStore::optimisePath_( Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = ({ - hashPath( - {make_ref(), CanonPath(path.string())}, - FileSerialisationMethod::NixArchive, - HashAlgorithm::SHA256) - .hash; - }); + Hash hash = hashPath(makeFSSourceAccessor(path), FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).hash; debug("%s has hash '%s'", PathFmt(path), hash.to_string(HashFormat::Nix32, true)); /* Check if this is a known hash. */ diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index 0da7dcdea8..c8c54953eb 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -76,8 +76,6 @@ public: std::optional getPhysicalPath(const CanonPath & path) override; - void invalidateCache(const CanonPath & path) override; - private: /** diff --git a/src/libutil/include/nix/util/source-path.hh b/src/libutil/include/nix/util/source-path.hh index 24932e4cdd..f15b44fdd4 100644 --- a/src/libutil/include/nix/util/source-path.hh +++ b/src/libutil/include/nix/util/source-path.hh @@ -114,11 +114,6 @@ struct SourcePath return {accessor, accessor->resolveSymlinks(path, mode)}; } - void invalidateCache() const - { - accessor->invalidateCache(path); - } - friend class std::hash; }; diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index aab95f775b..84840352b3 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -99,12 +99,6 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor auto [accessor, subpath] = resolve(path); return accessor->getFingerprint(subpath); } - - void invalidateCache(const CanonPath & path) override - { - auto [accessor, subpath] = resolve(path); - accessor->invalidateCache(subpath); - } }; ref makeMountedSourceAccessor(std::map> mounts) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 5dd4f56dd3..e3850d1b6d 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -591,11 +591,6 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path return st; } -void PosixSourceAccessor::invalidateCache(const CanonPath & path) -{ - cache.erase(makeAbsPath(path).string()); -} - std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) { if (auto parent = path.parent()) diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index de50c75e37..da71903e6a 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -90,12 +90,6 @@ struct UnionSourceAccessor : SourceAccessor } return {path, std::nullopt}; } - - void invalidateCache(const CanonPath & path) override - { - for (auto & accessor : accessors) - accessor->invalidateCache(path); - } }; ref makeUnionSourceAccessor(std::vector> && accessors) From 5aa60ea0229ec093830c33b2c05064f6fb1a2be1 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 03:55:55 +0300 Subject: [PATCH 285/927] Rename PosixSourceAccessor to WindowsSourceAccessor This implementations sucks and really should just be removed. The assertNoSymlinks check is also useless on windows. We might as well just remove it. --- .../include/nix/util/posix-source-accessor.hh | 43 ---------- src/libutil/posix-source-accessor.cc | 86 ++++++++++++++----- 2 files changed, 63 insertions(+), 66 deletions(-) diff --git a/src/libutil/include/nix/util/posix-source-accessor.hh b/src/libutil/include/nix/util/posix-source-accessor.hh index c8c54953eb..686c66471c 100644 --- a/src/libutil/include/nix/util/posix-source-accessor.hh +++ b/src/libutil/include/nix/util/posix-source-accessor.hh @@ -45,47 +45,4 @@ protected: } // namespace detail -/** - * A source accessor that uses the Unix filesystem. - */ -class PosixSourceAccessor : public detail::PosixSourceAccessorBase -{ - /** - * Optional root path to prefix all operations into the native file - * system. This allows prepending funny things like `C:\` that - * `CanonPath` intentionally doesn't support. - */ - const std::filesystem::path root; - -public: - - PosixSourceAccessor(); - PosixSourceAccessor(std::filesystem::path && root, bool trackLastModified = false); - - void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; - - using SourceAccessor::readFile; - - bool pathExists(const CanonPath & path) override; - - std::optional maybeLstat(const CanonPath & path) override; - - DirEntries readDirectory(const CanonPath & path) override; - - std::string readLink(const CanonPath & path) override; - - std::optional getPhysicalPath(const CanonPath & path) override; - -private: - - /** - * Throw an error if `path` or any of its ancestors are symlinks. - */ - void assertNoSymlinks(CanonPath path); - - std::optional cachedLstat(const CanonPath & path); - - std::filesystem::path makeAbsPath(const CanonPath & path); -}; - } // namespace nix diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index e3850d1b6d..d5dc7be282 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -510,11 +510,53 @@ try { throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } -#endif +#else -} // namespace +/** + * A source accessor that uses the Windows filesystem. + * @todo Should be moved into a separate file. + */ +class WindowsSourceAccessor : public detail::PosixSourceAccessorBase +{ + /** + * Optional root path to prefix all operations into the native file + * system. This allows prepending funny things like `C:\` that + * `CanonPath` intentionally doesn't support. + */ + const std::filesystem::path root; + +public: + + WindowsSourceAccessor(); + WindowsSourceAccessor(std::filesystem::path && root, bool trackLastModified = false); + + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override; + + using SourceAccessor::readFile; -PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot, bool trackLastModified) + bool pathExists(const CanonPath & path) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override; + +private: + + /** + * Throw an error if `path` or any of its ancestors are symlinks. + */ + void assertNoSymlinks(CanonPath path); + + std::optional cachedLstat(const CanonPath & path); + + std::filesystem::path makeAbsPath(const CanonPath & path); +}; + +WindowsSourceAccessor::WindowsSourceAccessor(std::filesystem::path && argRoot, bool trackLastModified) : PosixSourceAccessorBase(trackLastModified) , root(std::move(argRoot)) { @@ -522,12 +564,12 @@ PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && argRoot, bool displayPrefix = root.string(); } -PosixSourceAccessor::PosixSourceAccessor() - : PosixSourceAccessor(std::filesystem::path{}) +WindowsSourceAccessor::WindowsSourceAccessor() + : WindowsSourceAccessor(std::filesystem::path{}) { } -std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) +std::filesystem::path WindowsSourceAccessor::makeAbsPath(const CanonPath & path) { return root.empty() ? (std::filesystem::path{path.abs()}) : path.isRoot() ? /* Don't append a slash for the root of the accessor, since @@ -537,19 +579,13 @@ std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) : root / path.rel(); } -void PosixSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) +void WindowsSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) { assertNoSymlinks(path); auto ap = makeAbsPath(path); - AutoCloseFD fd = toDescriptor(open( - ap.string().c_str(), - O_RDONLY -#ifndef _WIN32 - | O_NOFOLLOW | O_CLOEXEC -#endif - )); + AutoCloseFD fd = toDescriptor(open(ap.string().c_str(), O_RDONLY)); if (!fd) throw SysError("opening file '%1%'", ap.string()); @@ -563,7 +599,7 @@ void PosixSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun>; static Cache cache; -std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) +std::optional WindowsSourceAccessor::cachedLstat(const CanonPath & path) { // Note: we convert std::filesystem::path to std::string because the // former is not hashable on libc++. @@ -591,7 +627,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path return st; } -std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) +std::optional WindowsSourceAccessor::maybeLstat(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); @@ -603,7 +639,7 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP return sourceAccessorStatFromPosixStat(*st); } -SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) +SourceAccessor::DirEntries WindowsSourceAccessor::readDirectory(const CanonPath & path) { assertNoSymlinks(path); DirEntries res; @@ -655,19 +691,19 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & return res; } -std::string PosixSourceAccessor::readLink(const CanonPath & path) +std::string WindowsSourceAccessor::readLink(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); return nix::readLink(makeAbsPath(path)).string(); } -std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +std::optional WindowsSourceAccessor::getPhysicalPath(const CanonPath & path) { return makeAbsPath(path); } -void PosixSourceAccessor::assertNoSymlinks(CanonPath path) +void WindowsSourceAccessor::assertNoSymlinks(CanonPath path) { while (!path.isRoot()) { auto st = cachedLstat(path); @@ -677,6 +713,10 @@ void PosixSourceAccessor::assertNoSymlinks(CanonPath path) } } +#endif + +} // namespace + ref getFSSourceAccessor() { static auto rootFS = makeFSSourceAccessor("/", /*trackLastModified=*/false); @@ -768,9 +808,9 @@ ref makeFSSourceAccessor(std::filesystem::path root, bool trackL else throw Error("file %1% has an unsupported type", PathFmt(root)); +#else + return make_ref(std::move(root), trackLastModified); #endif - - return make_ref(std::move(root), trackLastModified); } } // namespace nix From 5450d99984d6f000964d4589f288ee6cb37a4551 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 04:44:31 +0300 Subject: [PATCH 286/927] tests/functional/multiple-output: Move invalid outtput name tests above nuking the store We now more robustly require the existence of the store directory. These tests have just been tacked on at the end of the test, even though it nukes the store right before them. --- tests/functional/multiple-outputs.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/functional/multiple-outputs.sh b/tests/functional/multiple-outputs.sh index f703fb02be..dec22ff817 100755 --- a/tests/functional/multiple-outputs.sh +++ b/tests/functional/multiple-outputs.sh @@ -96,6 +96,13 @@ if nix-build multiple-outputs.nix -A cyclic --no-out-link; then exit 1 fi +# TODO inspect why this doesn't work with floating content-addressing +# derivations. +if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + expect 1 nix build -f multiple-outputs.nix invalid-output-name-1 2>&1 | grep 'contains illegal character' + expect 1 nix build -f multiple-outputs.nix invalid-output-name-2 2>&1 | grep 'contains illegal character' +fi + # Do a GC. This should leave an empty store. echo "collecting garbage..." rm "$TEST_ROOT"/result* @@ -103,10 +110,3 @@ nix-store --gc --keep-derivations --keep-outputs nix-store --gc --print-roots rm -rf "$NIX_STORE_DIR"/.links rmdir "$NIX_STORE_DIR" - -# TODO inspect why this doesn't work with floating content-addressing -# derivations. -if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then - expect 1 nix build -f multiple-outputs.nix invalid-output-name-1 2>&1 | grep 'contains illegal character' - expect 1 nix build -f multiple-outputs.nix invalid-output-name-2 2>&1 | grep 'contains illegal character' -fi From 0214ee7abbf7d0bbadbd7de9d2f6aa887328370f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 21 Apr 2026 12:41:05 +0200 Subject: [PATCH 287/927] repl: Make :reload robust against load failures Previously a failing :l/:lf was recorded before evaluation, so a typo'd path or broken flake ref would be retried on every :reload. Worse, :reload cleared the file/flake lists up front and rebuilt them while iterating, so the first error aborted the loop and silently dropped all later entries (and skipped CLI installables and flakes entirely). Record loaded files/flakes only after they actually load, and during reload catch errors per entry so one broken file no longer wipes out the rest of the session. --- src/libcmd/repl.cc | 42 ++++++++++++++++++++++++++++------------ tests/functional/repl.sh | 24 +++++++++++++++++++++++ 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 777fc33256..f7833ea3a4 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -688,12 +688,13 @@ ProcessLineResult NixRepl::processLine(std::string line) void NixRepl::loadFile(const std::filesystem::path & path) { - loadedFiles.remove(path); - loadedFiles.push_back(path); Value v, v2; state->evalFile(lookupFileArg(*state, path.string()), v); state->autoCallFunction(*autoArgs, v, v2); addAttrsToScope(v2); + // Remember for :reload only on success. + loadedFiles.remove(path); + loadedFiles.push_back(path); } void NixRepl::loadFlake(const std::string & flakeRefS) @@ -701,9 +702,6 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - loadedFlakes.remove(flakeRefS); - loadedFlakes.push_back(flakeRefS); - std::filesystem::path cwd; try { cwd = std::filesystem::current_path(); @@ -730,6 +728,10 @@ void NixRepl::loadFlake(const std::string & flakeRefS) }), v); addAttrsToScope(v); + + // Remember for :reload only on success. + loadedFlakes.remove(flakeRefS); + loadedFlakes.push_back(flakeRefS); } void NixRepl::initEnv() @@ -772,28 +774,44 @@ void NixRepl::reloadFilesAndFlakes() void NixRepl::loadFiles() { - decltype(loadedFiles) old = loadedFiles; - loadedFiles.clear(); + // loadFile() rebuilds loadedFiles; keep failed entries and continue. + decltype(loadedFiles) old; + std::swap(old, loadedFiles); for (auto & i : old) { notice("Loading %1%...", PathFmt(i)); - loadFile(i); + try { + loadFile(i); + } catch (Error & e) { + loadedFiles.push_back(i); + printMsg(lvlError, e.msg()); + } } for (auto & [i, what] : getValues()) { notice("Loading installable '%1%'...", what); - addAttrsToScope(*i); + try { + addAttrsToScope(*i); + } catch (Error & e) { + printMsg(lvlError, e.msg()); + } } } void NixRepl::loadFlakes() { - Strings old = loadedFlakes; - loadedFlakes.clear(); + // See loadFiles(). + Strings old; + std::swap(old, loadedFlakes); for (auto & i : old) { notice("Loading flake '%1%'...", i); - loadFlake(i); + try { + loadFlake(i); + } catch (Error & e) { + loadedFlakes.push_back(i); + printMsg(lvlError, e.msg()); + } } } diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 9fc803d092..0dbc047ed8 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -281,6 +281,30 @@ exec 3>&- # Close fifo wait $repl_pid # Wait for process to finish grep -q "afterChange" repl_output +# Regression: a failed `:l` / `:lf` must not be remembered for `:reload`, +# and an error in one loaded file must not drop later ones from the reload list. +cat > reloadA.nix < reloadB.nix < Date: Tue, 21 Apr 2026 12:49:46 +0200 Subject: [PATCH 288/927] release: name artifact jobsets maintenance-X.Y-release Suffixing the existing maintenance-X.Y identifier keeps the full-CI and release-artifact jobsets adjacent in Hydra's alphabetical list and lets tooling derive one name from the other, whereas a parallel release-X.Y namespace scatters the pair and leaves no obvious slot for master. Teach upload-release.pl to read the git revision from the legacy `src` input so it works against evaluations of the new jobset, and trim buildCross in release-jobs.nix to the two targets fallback-paths.nix actually consumes so the artifact jobset stays minimal. --- .github/workflows/upload-release.yml | 2 +- maintainers/release-process.md | 28 ++++++++++++++++------------ maintainers/upload-release.pl | 8 +++++++- packaging/release-jobs.nix | 9 ++++++++- 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml index cd21336c89..f00dce4a5c 100644 --- a/.github/workflows/upload-release.yml +++ b/.github/workflows/upload-release.yml @@ -3,7 +3,7 @@ on: workflow_dispatch: inputs: eval_id: - description: "Hydra evaluation ID" + description: "Hydra evaluation ID (from the maintenance-X.Y-release jobset)" required: true type: number is_latest: diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 19e5e05053..aff0088a0d 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -84,24 +84,27 @@ release: * Create two jobsets for the release branch on Hydra: `maintenance-$VERSION` runs the full `hydraJobs` CI matrix. - `release-$VERSION` builds only the artifacts consumed by + `maintenance-$VERSION-release` builds only the artifacts consumed by `upload-release`, so a release can be cut without waiting on the full - matrix. + matrix. The `-release` suffix keeps the pair adjacent in Hydra's + alphabetical jobset list and lets scripts derive one name from the + other. * Clone the previous `maintenance-*` jobset, set identifier `maintenance-$VERSION`, description `$VERSION release branch`, flake URL `github:NixOS/nix/$VERSION-maintenance`. - * Clone the previous `release-*` jobset (or create a new **legacy** - jobset), set identifier `release-$VERSION`, description `$VERSION - release artifacts`, Nix expression `packaging/release-jobs.nix` in - input `src`, and add input `src` of type *Git checkout* pointing at + * Clone the previous `maintenance-*-release` jobset (or create a new + **legacy** jobset), set identifier `maintenance-$VERSION-release`, + description `$VERSION release artifacts`, Nix expression + `packaging/release-jobs.nix` in input `src`, and add input `src` of + type *Git checkout* pointing at `https://github.com/NixOS/nix $VERSION-maintenance`. -* Wait for the `release-$VERSION` jobset to evaluate and build. If - impatient, go to the evaluation and select `Actions -> Bump builds to - front of queue`. The aggregate job `release` turns green once every - required artifact is available. +* Wait for the `maintenance-$VERSION-release` jobset to evaluate and + build. If impatient, go to the evaluation and select `Actions -> Bump + builds to front of queue`. The aggregate job `release` turns green + once every required artifact is available. * When the release jobset evaluation has succeeded building, take note of the evaluation ID (e.g. `1780832` in @@ -177,8 +180,9 @@ release: $ git push ``` -* Wait for the desired evaluation of the `release-$VERSION` jobset to - finish building (the `release` aggregate job is the gating signal). +* Wait for the desired evaluation of the `maintenance-XX.YY-release` + jobset to finish building (the `release` aggregate job is the gating + signal). * Tag the release diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 06678553e7..b618bd900d 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -64,7 +64,13 @@ sub fetch { #print Dumper($evalInfo); my $flakeUrl = $evalInfo->{flake}; my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; -my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; +# Flake jobsets (`maintenance-X.Y`) expose the rev via the flake URL. +# The release-artifacts jobset (`maintenance-X.Y-release`) is a legacy +# jobset whose checkout is passed in as input `src`. +my $nixRev = ($flakeInfo + ? $flakeInfo->{revision} + : $evalInfo->{jobsetevalinputs}->{src}->{revision} + // $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix-everything.x86_64-linux", 'application/json')); #print Dumper($buildInfo); diff --git a/packaging/release-jobs.nix b/packaging/release-jobs.nix index 23add3905e..f82c23c9c1 100644 --- a/packaging/release-jobs.nix +++ b/packaging/release-jobs.nix @@ -8,6 +8,7 @@ # and share builds through the binary cache. # # Hydra jobset configuration: +# Identifier: maintenance--release # Type: Legacy # Nix expression: packaging/release-jobs.nix in input `src` # Inputs: @@ -36,7 +37,13 @@ let # `fallback-paths.nix` and (on x86_64-linux) the rendered manual via # its `doc` output. build.nix-everything = hydraJobs.build.nix-everything; - buildCross.nix-everything = hydraJobs.buildCross.nix-everything; + buildCross.nix-everything = { + # Only the cross targets that end up in `fallback-paths.nix`. + inherit (hydraJobs.buildCross.nix-everything) + riscv64-unknown-linux-gnu + x86_64-unknown-freebsd + ; + }; inherit (hydraJobs) manual From 50bce0b6de7cf1e6603616c8e1ea3b14b6ed05f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 21 Apr 2026 13:21:45 +0200 Subject: [PATCH 289/927] repl: Invalidate git workdir-info cache on :reload Since 7ba933e989b9, GitRepo::getCachedWorkdirInfo() memoises the workdir status (dirty flag, tracked files, head rev) in a process-global static to avoid repeated libgit2 status walks within a single command. In a long-lived `nix repl` session this cache outlives the underlying work tree: after the first `:load-flake` of a clean git checkout, every `:reload` keeps seeing the original head rev, so the source is served from the same fingerprinted store path even after the user edits files. Hook the cache invalidation into InputCache::clear(), which already expresses the "per evaluation" lifetime that resetFileCache() flushes on `:reload`, so reloading a git-backed flake (whether via `:lf` or as a CLI installable) now picks up uncommitted changes again. --- src/libfetchers/git-utils.cc | 9 +++++- .../include/nix/fetchers/git-utils.hh | 3 ++ src/libfetchers/input-cache.cc | 5 +++ tests/functional/repl.sh | 32 +++++++++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 6bc474395a..03a7783c18 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1502,9 +1502,11 @@ ref Settings::getTarballCache() const } // namespace fetchers +static Sync> workdirInfoCache_; + GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) { - static Sync> _cache; + auto & _cache = workdirInfoCache_; { auto cache(_cache.lock()); auto i = cache->find(path); @@ -1516,6 +1518,11 @@ GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & return workdirInfo; } +void GitRepo::invalidateWorkdirInfoCache() +{ + workdirInfoCache_.lock()->clear(); +} + bool isLegalRefName(const std::string & refName) { initLibGit2(); diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 725fbf3984..871bf5e43f 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -88,6 +88,9 @@ struct GitRepo static WorkdirInfo getCachedWorkdirInfo(const std::filesystem::path & path); + /* Drop all entries from the getCachedWorkdirInfo() cache. */ + static void invalidateWorkdirInfoCache(); + /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 85a611355e..3fe96d8503 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -1,4 +1,5 @@ #include "nix/fetchers/input-cache.hh" +#include "nix/fetchers/git-utils.hh" #include "nix/fetchers/registry.hh" #include "nix/util/sync.hh" @@ -65,6 +66,10 @@ struct InputCacheImpl : InputCache void clear() override { cache_.lock()->clear(); + /* The workdir info cache has the same "per evaluation" lifetime + as the input cache, so flush it here as well so that e.g. + `:reload` in `nix repl` picks up changes in git work trees. */ + GitRepo::invalidateWorkdirInfoCache(); } }; diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 0dbc047ed8..88d6e91cd9 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -281,6 +281,38 @@ exec 3>&- # Close fifo wait $repl_pid # Wait for process to finish grep -q "afterChange" repl_output +# Regression: `:reload` on a flake loaded from a *git* work tree must pick up +# uncommitted changes. Guards against the per-process workdir-info cache +# pinning the tree to the rev seen on first load. +if [[ $(type -p git) ]]; then + createGitRepo gitflake + cat > gitflake/flake.nix <> repl_output 2>&1 & + repl_pid=$! + exec 3>repl_fifo + echo "changingThing" >&3 + for _ in $(seq 1 1000); do + grep -q "beforeChange" repl_output && break + sleep 0.1 + done + grep -q "beforeChange" repl_output || fail "git flake didn't load" + sed -i 's/beforeChange/afterChange/' gitflake/flake.nix + echo ":reload" >&3 + echo "changingThing" >&3 + echo "exit" >&3 + exec 3>&- + wait $repl_pid + grep -q "afterChange" repl_output || fail ":reload didn't pick up git work tree change" +fi + # Regression: a failed `:l` / `:lf` must not be remembered for `:reload`, # and an error in one loaded file must not drop later ones from the reload list. cat > reloadA.nix < Date: Tue, 21 Apr 2026 12:36:37 +0200 Subject: [PATCH 290/927] PosixDirectorySourceAccessor: Drop dirfd cache on resetFileCache() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit openParent() caches O_DIRECTORY fds keyed by CanonPath. When a directory at a cached path is removed and recreated, the cached fd keeps referring to the now-orphaned inode and *at() calls through it report ENOENT for entries that exist in the replacement. This shows up in `nix repl` (and any LSP / tool that keeps an EvalState alive) via the long-lived rootFS accessor: $ cat proj/default.nix { x = builtins.readFile ./sub/f; } $ echo old > proj/sub/f $ nix repl -f proj/default.nix nix-repl> x "old\n" # in another shell: rm -rf proj/sub && mkdir proj/sub && echo new > proj/sub/f # (e.g. git checkout, git stash pop, regenerating a directory) nix-repl> :r nix-repl> builtins.pathExists ./sub/f false # file is right there nix-repl> x error: path '.../proj/sub/f' does not exist Pre-#15718 nix returned "new\n" here. The evaluator already assumes the filesystem is immutable for the duration of a single evaluation, so rather than recovering lazily inside the accessor, repurpose the (now-unused) `invalidateCache()` virtual to clear the dirfd LRU and call it through `rootFS` from `EvalState::resetFileCache()` — the existing "filesystem may have changed" hook used by `:l`/`:r`/`:e` and the C API. The wrapping accessors (union, mounted, filtering) forward the call. --- src/libexpr/eval.cc | 1 + .../nix/fetchers/filtering-source-accessor.hh | 5 ++++ src/libutil-tests/source-accessor.cc | 26 +++++++++++++++++++ .../include/nix/util/source-accessor.hh | 5 ++-- src/libutil/mounted-source-accessor.cc | 5 ++++ src/libutil/posix-source-accessor.cc | 6 +++++ src/libutil/union-source-accessor.cc | 6 +++++ 7 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e4fe3ba6a5..fb5d8d7a5e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1172,6 +1172,7 @@ void EvalState::resetFileCache() fileEvalCache->clear(); inputCache->clear(); positions.clear(); + rootFS->invalidateCache(); } void EvalState::eval(Expr * e, Value & v) diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index b259e8ef8f..c5c1ce282b 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -53,6 +53,11 @@ struct FilteringSourceAccessor : SourceAccessor std::pair> getFingerprint(const CanonPath & path) override; + void invalidateCache() override + { + next->invalidateCache(); + } + /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` * exception if `isAllowed()` returns `false` for `path`. diff --git a/src/libutil-tests/source-accessor.cc b/src/libutil-tests/source-accessor.cc index 1ee2c44cc6..836c5895a2 100644 --- a/src/libutil-tests/source-accessor.cc +++ b/src/libutil-tests/source-accessor.cc @@ -138,6 +138,32 @@ TEST_F(FSSourceAccessorTest, works) } } +TEST_F(FSSourceAccessorTest, invalidateCacheDropsStaleDirFds) +{ +#ifdef _WIN32 + GTEST_SKIP() << "fd-based accessor is Unix-only"; +#endif + auto accessor = makeFSSourceAccessor(tmpDir); + + createDirs(tmpDir / "a" / "b"); + writeFile(tmpDir / "a" / "b" / "f", "old"); + + EXPECT_TRUE(accessor->pathExists(CanonPath("a/b/f"))); + + deletePath(tmpDir / "a" / "b"); + createDirs(tmpDir / "a" / "b"); + writeFile(tmpDir / "a" / "b" / "g", "new"); + createSymlink("g", tmpDir / "a" / "b" / "l"); + + accessor->invalidateCache(); + + EXPECT_FALSE(accessor->pathExists(CanonPath("a/b/f"))); + EXPECT_TRUE(accessor->pathExists(CanonPath("a/b/g"))); + EXPECT_THAT(accessor, HasContents(CanonPath("a/b/g"), "new")); + EXPECT_THAT(accessor, HasDirectory(CanonPath("a/b"), (std::set{"g", "l"}))); + EXPECT_THAT(accessor, HasSymlink(CanonPath("a/b/l"), "g")); +} + /* ---------------------------------------------------------------------------- * RestoreSink non-directory at root (no dirFd) * --------------------------------------------------------------------------*/ diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 92bd5d30fa..a2fbaac349 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -231,9 +231,10 @@ struct SourceAccessor : std::enable_shared_from_this } /** - * Invalidate any cached value the accessor may have for the specified path. + * Drop any cached state that could go stale across external filesystem + * mutation (e.g. cached directory fds). */ - virtual void invalidateCache(const CanonPath & path) {} + virtual void invalidateCache() {} }; /** diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index 84840352b3..8c546bc535 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -73,6 +73,11 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor } } + void invalidateCache() override + { + mounts.visit_all([](auto & kv) { kv.second->invalidateCache(); }); + } + std::optional getPhysicalPath(const CanonPath & path) override { auto [accessor, subpath] = resolve(path); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index d5dc7be282..da18d398e8 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -237,10 +237,16 @@ class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase PosixDirectorySourceAccessor & operator=(const PosixDirectorySourceAccessor &) = delete; ~PosixDirectorySourceAccessor() + { + invalidateCache(); + } + + void invalidateCache() override { if (dirFdCache) { auto cache = dirFdCache->lock(); globalDirFdCount.fetch_sub(cache->size(), std::memory_order_relaxed); + cache->clear(); } } diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index da71903e6a..9cb004c8fd 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -69,6 +69,12 @@ struct UnionSourceAccessor : SourceAccessor return SourceAccessor::showPath(path); } + void invalidateCache() override + { + for (auto & accessor : accessors) + accessor->invalidateCache(); + } + std::optional getPhysicalPath(const CanonPath & path) override { for (auto & accessor : accessors) { From 387ae98fe5ad2ce8837bd56c57eecb2bc3c07aef Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 21:12:16 +0300 Subject: [PATCH 291/927] clang-tidy: Ban raw std::filesystem::create_directories, fix hydraJobs.clangTidy It doesn't wrap exceptions. Also clang-tidy seems to have been broken with unity builds. --- nix-meson-build-support/common/clang-tidy/.clang-tidy | 6 ++++-- packaging/hydra.nix | 2 ++ src/libfetchers/git.cc | 2 +- src/libstore-tests/register-valid-paths-bench.cc | 2 +- .../include/nix/util/tests/characterization.hh | 2 +- .../include/nix/util/tests/json-characterization.hh | 4 ++-- src/libutil/file-system.cc | 2 +- 7 files changed, 12 insertions(+), 8 deletions(-) diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy index f8394f4fab..87ce4d663b 100644 --- a/nix-meson-build-support/common/clang-tidy/.clang-tidy +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -41,8 +41,6 @@ Checks: - -bugprone-unused-local-non-trivial-variable # 2 warnings - returning const& from parameter - -bugprone-return-const-ref-from-parameter - # 1 warning - unsafe C functions (e.g., getenv) - - -bugprone-unsafe-functions # 1 warning - signed char misuse - -bugprone-signed-char-misuse # 1 warning - calling parent virtual instead of override @@ -90,3 +88,7 @@ CheckOptions: bugprone-reserved-identifier.AllowedIdentifiers: '__asan_default_options;__wrap___assert_fail;_SingleDerivedPathRaw;_DerivedPathRaw;_SingleBuiltPathRaw;_BuiltPathRaw' # Allow explicitly discarding return values with (void) cast bugprone-unused-return-value.AllowCastToVoid: true + bugprone-unsafe-functions.ReportDefaultFunctions: false + # Repurpose bugprone-unsafe-functions to lint functions that we'd want to wrap. + bugprone-unsafe-functions.CustomFunctions: > + ::std::filesystem::create_directories, nix::createDirs, "Use nix::createDirs (it wraps exceptions)"; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 8f7e13c1f2..5558b0309e 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -218,6 +218,8 @@ rec { tidyScope = pkgs.nixComponents2.overrideScope ( self: super: { withClangTidy = true; + # clang-tidy doesn't seem to like unity builds. + withUnityBuild = false; # nix-everything is built via callPackage (not the layer system), so # enableClangTidyLayer's doCheck=false doesn't reach it. Set it here # so checkInputs (the *-tests.tests.run derivations) aren't pulled in. diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 3941c34256..8e959764f9 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -815,7 +815,7 @@ struct GitInputScheme : InputScheme repoDir = cacheDir; repoInfo.gitDir = "."; - std::filesystem::create_directories(cacheDir.parent_path()); + createDirs(cacheDir.parent_path()); PathLocks cacheDirLock({cacheDir.string()}); auto repo = GitRepo::openRepo(cacheDir, {.create = true, .bare = true}); diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 51bcb29aa9..6417178245 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -22,7 +22,7 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) auto tmpRoot = createTempDir(); auto realStoreDir = tmpRoot / "nix/store"; - std::filesystem::create_directories(realStoreDir); + createDirs(realStoreDir); std::shared_ptr store = openStore(fmt("local?root=%s", tmpRoot.string())); auto localStore = std::dynamic_pointer_cast(store); diff --git a/src/libutil-test-support/include/nix/util/tests/characterization.hh b/src/libutil-test-support/include/nix/util/tests/characterization.hh index 6dd5f38866..154573b5e5 100644 --- a/src/libutil-test-support/include/nix/util/tests/characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/characterization.hh @@ -60,7 +60,7 @@ struct CharacterizationTest : virtual ::testing::Test auto got = test(); if (testAccept()) { - std::filesystem::create_directories(file.parent_path()); + createDirs(file.parent_path()); writeFile2(file, got); GTEST_SKIP() << "Updating golden master " << file; } else { diff --git a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh index 9bd4d7fbf1..75eba2b06a 100644 --- a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh +++ b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh @@ -73,7 +73,7 @@ void checkpointJson(CharacterizationTest & test, std::string_view testStem, cons json gotJson = static_cast(got); if (testAccept()) { - std::filesystem::create_directories(file.parent_path()); + createDirs(file.parent_path()); writeFile(file, gotJson.dump(2) + "\n"); ADD_FAILURE() << "Updating golden master " << file; } else { @@ -98,7 +98,7 @@ void checkpointJson(CharacterizationTest & test, std::string_view testStem, cons json gotJson = static_cast(*got); if (testAccept()) { - std::filesystem::create_directories(file.parent_path()); + createDirs(file.parent_path()); writeFile(file, gotJson.dump(2) + "\n"); ADD_FAILURE() << "Updating golden master " << file; } else { diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index b7e504e13a..0169b2728a 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -393,7 +393,7 @@ void createDir(const std::filesystem::path & path, mode_t mode) void createDirs(const std::filesystem::path & path) { try { - std::filesystem::create_directories(path); + std::filesystem::create_directories(path); // NOLINT(bugprone-unsafe-functions) } catch (std::filesystem::filesystem_error & e) { throw SystemError(e.code(), "creating directory %1%", PathFmt(path)); } From ad15006e36eabbb91c0fb7d28655e26168d60165 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 21:41:39 +0300 Subject: [PATCH 292/927] Clean up dead code, redundant .string() on std::filesystem::path makeParentCanonical is no longer needed with makeFSSourceAccessor achieving exactly the same behavior without the unnecessary syscalls. Also drops leftover .string() calls from the previous migrations to std::filesystem::path. --- src/libfetchers/git.cc | 2 +- src/libstore/local-store.cc | 6 +++--- src/libutil-tests/file-system.cc | 14 -------------- src/libutil/file-system.cc | 15 --------------- src/libutil/include/nix/util/file-system.hh | 17 ----------------- src/nix/dump-path.cc | 2 +- src/nix/profile.cc | 6 +++--- 7 files changed, 8 insertions(+), 54 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8e959764f9..447b4a3694 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1098,7 +1098,7 @@ struct GitInputScheme : InputScheme for (auto & file : repoInfo.workdirInfo.dirtyFiles) { writeString("modified:", hashSink); writeString(file.abs(), hashSink); - dumpPath((*repoPath / file.rel()).string(), hashSink); + dumpPath(*repoPath / file.rel(), hashSink); } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b41856871b..52fc0925f0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1207,7 +1207,7 @@ StorePath LocalStore::addToStoreFromDump( delTempDir = std::make_unique(tempDir); tempPath = tempDir / "x"; - restorePath(tempPath.string(), bothSource, dumpMethod, localSettings.fsyncStorePaths); + restorePath(tempPath, bothSource, dumpMethod, localSettings.fsyncStorePaths); dumpBuffer.reset(); dump = {}; @@ -1260,7 +1260,7 @@ StorePath LocalStore::addToStoreFromDump( } } else { /* Move the temporary path we restored above. */ - moveFile(tempPath.string(), realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1311,7 +1311,7 @@ std::pair LocalStore::createTempDirInStore() continue; } lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true); - } while (!pathExists(tmpDirFn.string()) || !lockedByUs); + } while (!pathExists(tmpDirFn) || !lockedByUs); return {tmpDirFn, std::move(tmpDirFd)}; } diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 995b75814f..f290e11780 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -256,20 +256,6 @@ TEST(pathExists, bogusPathDoesNotExist) ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); } -/* ---------------------------------------------------------------------------- - * makeParentCanonical - * --------------------------------------------------------------------------*/ - -TEST(makeParentCanonical, noParent) -{ - ASSERT_EQ(makeParentCanonical("file"), absPath(std::filesystem::path("file"))); -} - -TEST(makeParentCanonical, root) -{ - ASSERT_EQ(makeParentCanonical(FS_ROOT), FS_ROOT_NO_TRAILING_SLASH); -} - /* ---------------------------------------------------------------------------- * chmodIfNeeded * --------------------------------------------------------------------------*/ diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 0169b2728a..20dbf3ff3d 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -660,21 +660,6 @@ bool isExecutableFileAmbient(const std::filesystem::path & exe) == 0; } -std::filesystem::path makeParentCanonical(const std::filesystem::path & rawPath) -{ - std::filesystem::path path(absPath(rawPath)); - try { - auto parent = path.parent_path(); - if (parent == path) { - // `path` is a root directory => trivially canonical - return parent; - } - return std::filesystem::canonical(parent) / path.filename(); - } catch (std::filesystem::filesystem_error & e) { - throw SystemError(e.code(), "canonicalising parent path of %1%", PathFmt(path)); - } -} - void chmod(const std::filesystem::path & path, mode_t mode) { if ( diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index f977e089bf..2a13b311c9 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -159,23 +159,6 @@ std::optional maybeStat(const std::filesystem::path & path); */ bool pathExists(const std::filesystem::path & path); -/** - * Canonicalize a path except for the last component. - * - * This is useful for getting the canonical location of a symlink. - * - * Consider the case where `foo/l` is a symlink. `canonical("foo/l")` will - * resolve the symlink `l` to its target. - * `makeParentCanonical("foo/l")` will not resolve the symlink `l` to its target, - * but does ensure that the returned parent part of the path, `foo` is resolved - * to `canonical("foo")`, and can therefore be retrieved without traversing any - * symlinks. - * - * If a relative path is passed, it will be made absolute, so that the parent - * can always be canonicalized. - */ -std::filesystem::path makeParentCanonical(const std::filesystem::path & path); - /** * A version of pathExists that returns false on a permission error. * Useful for inferring default paths across directories that might not diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 2512948778..f21374c623 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -61,7 +61,7 @@ struct CmdDumpPath2 : Command void run() override { auto sink = getNarSink(); - dumpPath(path.string(), sink); + dumpPath(path, sink); sink.flush(); } }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index c85c406aa5..e63d0150c0 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -125,7 +125,7 @@ struct ProfileManifest auto manifestPath = profile / "manifest.json"; if (std::filesystem::exists(manifestPath)) { - auto json = nlohmann::json::parse(readFile(manifestPath.string())); + auto json = nlohmann::json::parse(readFile(manifestPath)); auto version = json.value("version", 0); std::string sUrl; @@ -248,13 +248,13 @@ struct ProfileManifest } } - buildProfile(tempDir.string(), std::move(pkgs)); + buildProfile(tempDir, std::move(pkgs)); writeFile(tempDir / "manifest.json", toJSON(*store).dump()); /* Add the symlink tree to the store. */ StringSink sink; - dumpPath(tempDir.string(), sink); + dumpPath(tempDir, sink); auto narHash = hashString(HashAlgorithm::SHA256, sink.s); From 3e56d682f8e20c635f435af4ec91fdc5e8f661e2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 21:52:49 +0300 Subject: [PATCH 293/927] Fix FreeBSD build, return includes, now with IWYU comments Got needlessly broken by trimming includes: https://hydra.nixos.org/build/326607395/nixlog/1 --- src/libexpr-test-support/tests/value/context.cc | 4 +--- src/libexpr-tests/derived-path.cc | 4 +--- src/libstore-test-support/derived-path.cc | 5 +---- .../include/nix/store/tests/outputs-spec.hh | 2 +- src/libstore-test-support/path.cc | 4 +--- src/libutil-test-support/hash.cc | 4 +--- 6 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/libexpr-test-support/tests/value/context.cc b/src/libexpr-test-support/tests/value/context.cc index ca7996acc1..22f8aa7cf0 100644 --- a/src/libexpr-test-support/tests/value/context.cc +++ b/src/libexpr-test-support/tests/value/context.cc @@ -1,6 +1,4 @@ -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/expr/tests/value/context.hh" diff --git a/src/libexpr-tests/derived-path.cc b/src/libexpr-tests/derived-path.cc index c685f6a094..a67f3df77b 100644 --- a/src/libexpr-tests/derived-path.cc +++ b/src/libexpr-tests/derived-path.cc @@ -1,8 +1,6 @@ #include #include -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/tests/derived-path.hh" diff --git a/src/libstore-test-support/derived-path.cc b/src/libstore-test-support/derived-path.cc index ee8018c326..c27edc95ef 100644 --- a/src/libstore-test-support/derived-path.cc +++ b/src/libstore-test-support/derived-path.cc @@ -1,7 +1,4 @@ - -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/tests/derived-path.hh" diff --git a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh index a30f837702..6cdb0a60ef 100644 --- a/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh +++ b/src/libstore-test-support/include/nix/store/tests/outputs-spec.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include // Needed by rapidcheck on Darwin +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/store/outputs-spec.hh" diff --git a/src/libstore-test-support/path.cc b/src/libstore-test-support/path.cc index 98a255ccc0..bca404cde4 100644 --- a/src/libstore-test-support/path.cc +++ b/src/libstore-test-support/path.cc @@ -1,6 +1,4 @@ -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include diff --git a/src/libutil-test-support/hash.cc b/src/libutil-test-support/hash.cc index d9c7a0f747..2dc5da5f2c 100644 --- a/src/libutil-test-support/hash.cc +++ b/src/libutil-test-support/hash.cc @@ -1,6 +1,4 @@ -#ifdef __APPLE__ -# include // Needed by rapidcheck on Darwin -#endif +#include // IWYU pragma: keep (Needed by rapidcheck on Darwin and FreeBSD) #include #include "nix/util/hash.hh" From 61e1be2c6d1fd8bef7186959b85ea950b8aacf61 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 21:54:52 +0300 Subject: [PATCH 294/927] Fix Darwin build Also needlessly broken by trimming includes... https://hydra.nixos.org/build/326734471/nixlog/1 --- src/libstore/optimise-store.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 15f2b1f3d1..eeac67ad27 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -7,6 +7,10 @@ #include #include +#ifdef __APPLE__ +# include +#endif + #include #include #include From b74401e8714151ef0e961ca7cd13bb7e2c9cc316 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 22:07:57 +0300 Subject: [PATCH 295/927] Fix functional_root tests This check isn't reliable with CAP_DAC_OVERRIDE. It's also not very critical anyway. --- tests/functional/flakes/edit.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/flakes/edit.sh b/tests/functional/flakes/edit.sh index 0d926c2874..2758c397ec 100755 --- a/tests/functional/flakes/edit.sh +++ b/tests/functional/flakes/edit.sh @@ -9,4 +9,3 @@ nix edit "$flake1Dir#" | grepQuiet simple.builder.sh tar --exclude=".git*" -czf "$TEST_ROOT"/flake1Dir.tar.gz -C "$(dirname "$flake1Dir")" "$(basename "$flake1Dir")" # Test that editing a file from a tarball flake works and the file is readonly. nix edit "file://$TEST_ROOT/flake1Dir.tar.gz" | grepQuiet simple.builder.sh -EDITOR='test ! -w' nix edit "file://$TEST_ROOT/flake1Dir.tar.gz" From d1b5ac384a22a8aeae29f54ea1898596c82fc172 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 23:35:07 +0300 Subject: [PATCH 296/927] Move std::filesystem cleanup Fix several issues: * Get rid of raw usage of std::filesystem::remove where it would only be removing links - not directories. std::filesystem::remove is equivalent to POSIX remove(3) and it there unlink is the better choice. * Get rid of more unnecessary .string() usage. * Replace exists(symlink_status()) with pathExists - it has the same semantics of not following symlinks. * Replace std::filesystem::remove_all usage with deletePath - mostly in tests to prepare to ban it via clang-tidy - C++ stdlib doesn't make any guarantees about symlink race safety for those. --- src/libfetchers/git-utils.cc | 4 ++-- .../include/nix/store/tests/nix_api_store.hh | 11 ++++------- src/libstore-tests/register-valid-paths-bench.cc | 2 +- src/libstore/local-overlay-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/optimise-store.cc | 6 +++--- src/libstore/profiles.cc | 11 +---------- src/libutil-tests/unix/file-system-at.cc | 14 ++++---------- src/libutil/file-system.cc | 8 ++++++++ src/libutil/include/nix/util/file-system.hh | 7 +++++++ 10 files changed, 32 insertions(+), 35 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 03a7783c18..216dcb741e 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -226,7 +226,7 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options options) { - if (pathExists(path.string())) + if (pathExists(path)) return; if (!options.create) @@ -570,7 +570,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this /* Get submodule info. */ auto modulesFile = path / ".gitmodules"; - if (pathExists(modulesFile.string())) + if (pathExists(modulesFile)) info.submodules = parseSubmodules(modulesFile); return info; diff --git a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh index 15df329cb2..df48a7469e 100644 --- a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh +++ b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh @@ -22,13 +22,10 @@ public: }; ~nix_api_store_test_base() override - { - if (exists(std::filesystem::path{nixDir})) { - for (auto & path : std::filesystem::recursive_directory_iterator(nixDir)) { - std::filesystem::permissions(path, std::filesystem::perms::owner_all); - } - std::filesystem::remove_all(nixDir); - } + try { + nix::deletePath(nixDir); + } catch (...) { + nix::ignoreExceptionInDestructor(); } std::string nixDir; diff --git a/src/libstore-tests/register-valid-paths-bench.cc b/src/libstore-tests/register-valid-paths-bench.cc index 6417178245..ecea1c8010 100644 --- a/src/libstore-tests/register-valid-paths-bench.cc +++ b/src/libstore-tests/register-valid-paths-bench.cc @@ -66,7 +66,7 @@ static void BM_RegisterValidPathsDerivations(benchmark::State & state) state.PauseTiming(); localStore.reset(); store.reset(); - std::filesystem::remove_all(tmpRoot); + deletePath(tmpRoot); state.ResumeTiming(); } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index afbd47b5bd..8d1a16f912 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -262,7 +262,7 @@ LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag StorePathSet done; auto existsInStoreDir = [&](const StorePath & storePath) { - return pathExists((config->realStoreDir.get() / storePath.to_string()).string()); + return pathExists(config->realStoreDir.get() / storePath.to_string()); }; bool errors = false; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 52fc0925f0..75f485c31f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1365,7 +1365,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printError( "link %s was modified! expected hash %s, got '%s'", PathFmt(link.path()), name.string(), hash); if (repair) { - std::filesystem::remove(link.path()); + unlinkIfExists(link.path()); printInfo("removed link %s", PathFmt(link.path())); } else { errors = true; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index eeac67ad27..0b85c6e3dc 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -164,7 +164,7 @@ void LocalStore::optimisePath_( std::filesystem::path linkPath = std::filesystem::path{linksDir} / hash.to_string(HashFormat::Nix32, false); /* Maybe delete the link, if it has been corrupted. */ - if (std::filesystem::exists(std::filesystem::symlink_status(linkPath))) { + if (pathExists(linkPath)) { auto stLink = lstat(linkPath); if (st.st_size != stLink.st_size || (repair && hash != ({ hashPath( @@ -178,11 +178,11 @@ void LocalStore::optimisePath_( warn( "There may be more corrupted paths." "\nYou should run `nix-store --verify --check-contents --repair` to fix them all"); - std::filesystem::remove(linkPath); + unlinkIfExists(linkPath); } } - if (!std::filesystem::exists(std::filesystem::symlink_status(linkPath))) { + if (!pathExists(linkPath)) { /* Nope, create a hard link in the links directory. */ try { std::filesystem::create_hard_link(path, linkPath); diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 519c2abc98..d015548e5e 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -95,19 +95,10 @@ std::filesystem::path createGeneration(LocalFSStore & store, std::filesystem::pa return generation; } -static void removeFile(const std::filesystem::path & path) -{ - try { - std::filesystem::remove(path); - } catch (std::filesystem::filesystem_error & e) { - throw SystemError(e.code(), "removing file %1%", PathFmt(path)); - } -} - void deleteGeneration(const std::filesystem::path & profile, GenerationNumber gen) { std::filesystem::path generation = makeName(profile, gen); - removeFile(generation); + unlinkIfExists(generation); } /** diff --git a/src/libutil-tests/unix/file-system-at.cc b/src/libutil-tests/unix/file-system-at.cc index 09e427a26f..e453a80d32 100644 --- a/src/libutil-tests/unix/file-system-at.cc +++ b/src/libutil-tests/unix/file-system-at.cc @@ -42,8 +42,6 @@ TEST(fchmodatTryNoFollow, works) auto dirFd = openDirectory(tmpDir, FinalSymlink::Follow); ASSERT_TRUE(dirFd); - struct ::stat st; - using nix::testing::ThrowsSysError; /* Check that symlinks are not followed and targets are not changed. @@ -82,22 +80,18 @@ TEST(fchmodatTryNoFollow, works) }; expectSymlinkChmod("filelink", 0777); - ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); - EXPECT_EQ(st.st_mode & 0777, 0644); + ASSERT_EQ(stat(tmpDir / "file").st_mode & 0777, 0644); expectSymlinkChmod("dirlink", 0777); - ASSERT_EQ(stat((tmpDir / "dir").c_str(), &st), 0); - EXPECT_EQ(st.st_mode & 0777, 0755); + ASSERT_EQ(stat(tmpDir / "dir").st_mode & 0777, 0755); /* Check fchmodatTryNoFollow works on regular files and directories. */ EXPECT_NO_THROW(fchmodatTryNoFollow(dirFd.get(), CanonPath("file"), 0600)); - ASSERT_EQ(stat((tmpDir / "file").c_str(), &st), 0); - EXPECT_EQ(st.st_mode & 0777, 0600); + ASSERT_EQ(stat(tmpDir / "file").st_mode & 0777, 0600); EXPECT_NO_THROW(fchmodatTryNoFollow(dirFd.get(), CanonPath("dir"), 0700)); - ASSERT_EQ(stat((tmpDir / "dir").c_str(), &st), 0); - EXPECT_EQ(st.st_mode & 0777, 0700); + ASSERT_EQ(stat(tmpDir / "dir").st_mode & 0777, 0700); EXPECT_THAT([&] { fchmodatTryNoFollow(dirFd.get(), CanonPath("nonexistent"), 0600); }, ThrowsSysError(ENOENT)); } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 20dbf3ff3d..b3087700d7 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -679,6 +679,14 @@ void chmod(const std::filesystem::path & path, mode_t mode) # define UNLINK_PROC ::unlink #endif +void unlinkIfExists(const std::filesystem::path & path) +{ + if (UNLINK_PROC(path.c_str()) == -1) { + if (errno != ENOENT) + throw SysError("removing %s", PathFmt(path)); + } +} + void unlink(const std::filesystem::path & path) { if (UNLINK_PROC(path.c_str()) == -1) diff --git a/src/libutil/include/nix/util/file-system.hh b/src/libutil/include/nix/util/file-system.hh index 2a13b311c9..fca79ceb8b 100644 --- a/src/libutil/include/nix/util/file-system.hh +++ b/src/libutil/include/nix/util/file-system.hh @@ -514,6 +514,13 @@ void chown(const std::filesystem::path & path, uid_t owner, gid_t group); */ void unlink(const std::filesystem::path & path); +/** + * Remove a file, throwing an exception on error. ENOENT is ignored. + * + * @param path Path to the file to remove. + */ +void unlinkIfExists(const std::filesystem::path & path); + /** * Try to remove a file, ignoring errors. * From b3e84b77679e727d4cba0b54c49ba0588b387093 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 21 Apr 2026 23:42:18 +0300 Subject: [PATCH 297/927] Ban std::filesystem::remove_all in the codebase All instances of those must use deletePath instead - standard library implementation is not guaranteed to be robust against symlink races. --- nix-meson-build-support/common/clang-tidy/.clang-tidy | 1 + src/libutil/windows/file-system.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy index 87ce4d663b..daf4c7f7c4 100644 --- a/nix-meson-build-support/common/clang-tidy/.clang-tidy +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -92,3 +92,4 @@ CheckOptions: # Repurpose bugprone-unsafe-functions to lint functions that we'd want to wrap. bugprone-unsafe-functions.CustomFunctions: > ::std::filesystem::create_directories, nix::createDirs, "Use nix::createDirs (it wraps exceptions)"; + ::std::filesystem::remove_all, nix::deletePath, "Use nix::deletePath (remove_all is not TOCTOU safe)"; diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 98d41a9caa..2ac2f74f80 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -77,7 +77,7 @@ std::filesystem::path defaultTempDir() void deletePath(const std::filesystem::path & path) { std::error_code ec; - std::filesystem::remove_all(path, ec); + std::filesystem::remove_all(path, ec); // NOLINT(bugprone-unsafe-functions) if (ec && ec != std::errc::no_such_file_or_directory) throw SysError(ec.default_error_condition().value(), "recursively deleting %1%", PathFmt(path)); } From 77db3d45cd36386ee222730419ad15c68adf4d4b Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 15 Apr 2026 12:25:24 +0000 Subject: [PATCH 298/927] libutil: Bound NAR directory depth and guard coroutine stacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parse()/dumpPath() recurse per directory level with no bound. On the main stack this overflows at a few thousand levels (DoS). Inside a sinkToSource/sourceToSink coroutine the stack is a 128 KiB malloc() chunk with no guard page, so overflow writes into adjacent heap — the v<1.25 wopAddToStore handler runs parseDump() on exactly such a stack and any client can downgrade to that protocol version. Reject NARs deeper than narMaxDepth (64) in both parse() and dumpPath(), and allocate coroutine stacks with boost::coroutines2::protected_fixedsize_stack so any remaining overflow hits a guard page instead of the heap. --- src/libutil/archive.cc | 25 +++++++++++++++++++------ src/libutil/serialise.cc | 5 +++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 91af4b9e7f..cfc0a51070 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -30,6 +30,12 @@ static ArchiveSettings archiveSettings; static GlobalConfig::Register rArchiveSettings(&archiveSettings); +/* Maximum directory nesting depth for dumpPath()/parseDump(). Bounds + stack usage so deep trees cannot overflow the (possibly coroutine) + stack these run on. Chosen to fit comfortably in the default 128 KiB + boost coroutine stack. */ +static constexpr size_t narMaxDepth = 64; + PathFilter defaultPathFilter = [](const std::string &) { return true; }; void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter) @@ -51,9 +57,13 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & this const auto & dump, SourceAccessor & accessor, const CanonPath & path, - const CanonPath & filterPath) -> void { + const CanonPath & filterPath, + size_t depth) -> void { checkInterrupt(); + if (depth >= narMaxDepth) + throw Error("path '%s' exceeds maximum NAR directory depth of %d", accessor.showPath(path), narMaxDepth); + auto st = accessor.lstat(path); sink << "("; @@ -89,7 +99,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & for (auto & i : unhacked) if (filter((filterPath / i.first).abs())) { sink << "entry" << "(" << "name" << i.first << "node"; - dump(subdirAccessor, subdirRelPath / i.second, filterPath / i.second); + dump(subdirAccessor, subdirRelPath / i.second, filterPath / i.second, depth + 1); sink << ")"; } }); @@ -102,7 +112,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & throw Error("file '%s' has an unsupported type", path); sink << ")"; - }(*this, path, path); + }(*this, path, path, 0); } time_t dumpPathAndGetMtime(const std::filesystem::path & path, Sink & sink, PathFilter & filter) @@ -153,8 +163,11 @@ struct CaseInsensitiveCompare } }; -static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath & path) +static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath & path, size_t depth) { + if (depth >= narMaxDepth) + throw badArchive("NAR directory nesting exceeds maximum depth of %d", narMaxDepth); + auto getString = [&]() { checkInterrupt(); return readString(source); @@ -237,7 +250,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath expectTag("node"); - parse(dirSink, source, relDirPath / name); + parse(dirSink, source, relDirPath / name, depth + 1); expectTag(")"); } @@ -268,7 +281,7 @@ void parseDump(FileSystemObjectSink & sink, Source & source) } if (version != narVersionMagic1) throw badArchive("input doesn't look like a Nix archive"); - parse(sink, source, CanonPath::root); + parse(sink, source, CanonPath::root, 0); } void restorePath(const std::filesystem::path & path, Source & source, bool startFsync) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 6c77c15fe5..542cee2b02 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -10,6 +10,7 @@ #include #include +#include #ifdef _WIN32 # include @@ -328,7 +329,7 @@ std::unique_ptr sourceToSink(fun reader) cur = in; if (!coro) { - coro = coro_t::push_type([&](coro_t::pull_type & yield) { + coro = coro_t::push_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::pull_type & yield) { LambdaSource source([&](char * out, size_t out_len) { if (cur.empty()) { yield(); @@ -385,7 +386,7 @@ std::unique_ptr sinkToSource(fun writer, fun eof) { bool hasCoro = coro.has_value(); if (!hasCoro) { - coro = coro_t::pull_type([&](coro_t::push_type & yield) { + coro = coro_t::pull_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::push_type & yield) { LambdaSink sink([&](std::string_view data) { if (!data.empty()) { yield(data); From 0d461dc28e9624685b3cc09ab877e2d3976c2b69 Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 15 Apr 2026 12:37:51 +0000 Subject: [PATCH 299/927] libutil: Bound string lengths in the NAR parser readString() defaults to no length limit, so every token, directory entry name, and symlink target in a NAR was read into a freshly allocated std::string of attacker-chosen size before any validation. A multi-GB length prefix where "(" is expected would be allocated and filled before expectTag() rejects it. Cap tags and keywords at 32 bytes, entry names at 255, and symlink targets at 4095. Overlong strings now throw SerialisationError("string is too long") without allocating. --- src/libutil/archive.cc | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index cfc0a51070..3ddc280f95 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -168,33 +168,42 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath if (depth >= narMaxDepth) throw badArchive("NAR directory nesting exceeds maximum depth of %d", narMaxDepth); - auto getString = [&]() { + /* NAR keywords are all <= 10 bytes; a little slack keeps error + messages useful for short garbage without allowing large + allocations. */ + constexpr size_t narMaxTag = 32; + /* Format-defined bounds, intentionally independent of host + NAME_MAX/PATH_MAX. */ + constexpr size_t narMaxName = 255; + constexpr size_t narMaxTarget = 4095; + + auto getString = [&](size_t max) { checkInterrupt(); - return readString(source); + return readString(source, max); }; auto expectTag = [&](std::string_view expected) { - auto tag = getString(); + auto tag = getString(narMaxTag); if (tag != expected) - throw badArchive("expected tag '%s', got '%s'", expected, tag.substr(0, 1024)); + throw badArchive("expected tag '%s', got '%s'", expected, tag); }; expectTag("("); expectTag("type"); - auto type = getString(); + auto type = getString(narMaxTag); if (type == "regular") { sink.createRegularFile(path, [&](auto & crf) { - auto tag = getString(); + auto tag = getString(narMaxTag); if (tag == "executable") { - auto s2 = getString(); + auto s2 = getString(0); if (s2 != "") throw badArchive("executable marker has non-empty value"); crf.isExecutable(); - tag = getString(); + tag = getString(narMaxTag); } if (tag != "contents") @@ -213,7 +222,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath std::string prevName; while (1) { - auto tag = getString(); + auto tag = getString(narMaxTag); if (tag == ")") break; @@ -225,7 +234,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath expectTag("name"); - auto name = getString(); + auto name = getString(narMaxName); if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos) throw badArchive("NAR contains invalid file name '%1%'", name); @@ -260,7 +269,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath else if (type == "symlink") { expectTag("target"); - auto target = getString(); + auto target = getString(narMaxTarget); sink.createSymlink(path, target); expectTag(")"); From ee68e870ece0332cf7886950ae9892296b36b821 Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 15 Apr 2026 12:37:51 +0000 Subject: [PATCH 300/927] libutil: Reject empty and NUL-containing symlink targets in NARs A NUL byte in a symlink target truncates at the symlink(2) boundary, so "foo\0junk" and "foo" restore to identical filesystem state but have different NAR hashes. LocalStore::addToStore() hashes the incoming stream, while `nix store verify` and re-export hash a fresh dump of the on-disk tree, so a non-canonical NAR would pass the ingest check but fail every later verification. Rejecting NUL (and empty, which symlink(2) refuses anyway) keeps parse() injective on accepted inputs, which the narHash model depends on. --- src/libutil/archive.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 3ddc280f95..cd7f58263c 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -270,6 +270,8 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath expectTag("target"); auto target = getString(narMaxTarget); + if (target.empty() || target.find((char) 0) != std::string::npos) + throw badArchive("NAR contains invalid symlink target"); sink.createSymlink(path, target); expectTag(")"); From 5d8406285b7d973477037135f3226fef0fb74db5 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Apr 2026 01:04:03 +0300 Subject: [PATCH 301/927] daemon: Limit the number of crashes before exiting to 64 In case someone is intentionally crashing the daemon and brute-force ASLR or get some other kind of info leak about the address-space layout, we'd like to to limit the blast radius by dying and (maybe) being restarted to get a fresh address space layout. --- .../include/nix/cmd/unix-socket-server.hh | 3 ++ src/libcmd/unix/unix-socket-server.cc | 3 ++ src/nix/unix/daemon.cc | 35 +++++++++++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/libcmd/include/nix/cmd/unix-socket-server.hh b/src/libcmd/include/nix/cmd/unix-socket-server.hh index 7a0d9fa793..48202cf182 100644 --- a/src/libcmd/include/nix/cmd/unix-socket-server.hh +++ b/src/libcmd/include/nix/cmd/unix-socket-server.hh @@ -68,6 +68,8 @@ struct ServeUnixSocketOptions #endif }; +MakeError(AbortServeSocket, BaseError); + /** * Run a server loop that accepts connections and calls the handler for each. * @@ -83,6 +85,7 @@ struct ServeUnixSocketOptions * * This function never returns normally. It runs until interrupted * (e.g., via SIGINT), at which point it throws `Interrupted`. + * Can be explicitly exited by throwing AbortServeSocket. * * @param options Configuration for the server. * @param handler Callback invoked for each accepted connection. diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index 5d1fba4622..c0348fa3b4 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -122,6 +122,9 @@ PeerInfo getPeerInfo(Descriptor remote) handler(std::move(remote), [&]() { listeningSockets.clear(); }); } + } catch (AbortServeSocket &) { + /* Explicitly aborted, bail out. */ + throw; } catch (Error & error) { auto ei = error.info(); // FIXME: add to trace? diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 05e47f79c3..4bc7a512d9 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -280,6 +280,21 @@ static void daemonLoop( } #endif + /* Check for anything that might be a crash. Too many crashes aren't + supposed to happen and we should limit the amount if someone is + intentionally triggering those as an ASLR bypass attempt (each forked + daemon worker has the same address space layout as we do). TODO: Ideally + we'd re-exec the daemon worker so that it gets a fresh address space + for each connection. Alternatively, we could make the daemon socket use + Accept=yes systemd.socket(5). */ + unsigned crashCount = 0; + + /* For now we are just limiting the number of crashes experienced by this + daemon instance. systemd (e.g.) would restart us, which would get us + a fresh address space layout - which is exactly what we want in case + someone is intentionally crashing the daemon to brute-force ASLR. */ + static constexpr unsigned crashLimit = 64; + try { unix::serveUnixSocket( { @@ -287,13 +302,29 @@ static void daemonLoop( .socketMode = 0666, .auxiliaryFd = sigChldPipe.pipe.readSide.get(), .onAuxiliaryFdPollin = - []() { + [&crashCount]() { sigChldPipe.drain(); /* Reap all dead children. */ pid_t pid = -1; int status; - while (pid = ::waitpid(/*pid (any child process)=*/-1, &status, WNOHANG), pid > 0) + while (pid = ::waitpid(/*pid (any child process)=*/-1, &status, WNOHANG), pid > 0) { printInfo("reaped child process %1%, status = %2%", pid, statusToString(status)); + + if (!WIFSIGNALED(status)) + continue; + + int sig = WTERMSIG(status); + for (auto i : {SIGILL, SIGSEGV, SIGBUS, SIGABRT, SIGSYS, SIGFPE}) { + if (sig == i) { + printInfo("daemon worker %1% crashed", pid); + ++crashCount; + break; + } + } + + if (crashCount >= crashLimit) + throw unix::AbortServeSocket("too many daemon worker crashes (%1%)", crashLimit); + } }, }, [&](AutoCloseFD remote, std::function closeListeners) { From 88b5bcd07751de3b5ca174d5192cf4d160434748 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Apr 2026 01:06:24 +0300 Subject: [PATCH 302/927] libutil: Make formatter happy --- src/libutil/serialise.cc | 42 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 542cee2b02..4e6daa84c9 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -329,20 +329,21 @@ std::unique_ptr sourceToSink(fun reader) cur = in; if (!coro) { - coro = coro_t::push_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::pull_type & yield) { - LambdaSource source([&](char * out, size_t out_len) { - if (cur.empty()) { - yield(); - if (yield.get()) - throw EndOfFile("coroutine has finished"); - } - - size_t n = cur.copy(out, out_len); - cur.remove_prefix(n); - return n; + coro = + coro_t::push_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::pull_type & yield) { + LambdaSource source([&](char * out, size_t out_len) { + if (cur.empty()) { + yield(); + if (yield.get()) + throw EndOfFile("coroutine has finished"); + } + + size_t n = cur.copy(out, out_len); + cur.remove_prefix(n); + return n; + }); + reader(source); }); - reader(source); - }); } if (!*coro) { @@ -386,14 +387,15 @@ std::unique_ptr sinkToSource(fun writer, fun eof) { bool hasCoro = coro.has_value(); if (!hasCoro) { - coro = coro_t::pull_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::push_type & yield) { - LambdaSink sink([&](std::string_view data) { - if (!data.empty()) { - yield(data); - } + coro = + coro_t::pull_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::push_type & yield) { + LambdaSink sink([&](std::string_view data) { + if (!data.empty()) { + yield(data); + } + }); + writer(sink); }); - writer(sink); - }); } if (cur.empty()) { From 7cdf3bad750b2af7c91e973f86ab8833312fa7ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 23 Apr 2026 00:23:28 +0200 Subject: [PATCH 303/927] packaging: build the Rust nix-installer with embedded Nix Builds NixOS/nix-installer via buildRustPackage with this revision's Nix closure baked in, exposed as hydraJobs.rustInstaller.. Replaces the dropped --nix-package-url knob (#15728). Source pinned via fetchFromGitHub to avoid a circular flake input. --- flake.nix | 3 ++ packaging/hydra.nix | 26 +++++++++++ packaging/rust-installer/default.nix | 64 ++++++++++++++++++++++++++++ packaging/rust-installer/tarball.nix | 50 ++++++++++++++++++++++ 4 files changed, 143 insertions(+) create mode 100644 packaging/rust-installer/default.nix create mode 100644 packaging/rust-installer/tarball.nix diff --git a/flake.nix b/flake.nix index 89cbe93cff..c9be5e1dd5 100644 --- a/flake.nix +++ b/flake.nix @@ -466,6 +466,9 @@ ) ) ) + // lib.optionalAttrs (self.hydraJobs.rustInstaller ? ${system}) { + rustInstaller = self.hydraJobs.rustInstaller.${system}; + } // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 5558b0309e..e3f8f1c1f1 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -298,6 +298,32 @@ rec { } ); + # `NixOS/nix-installer` with this revision's Nix closure embedded. + rustInstaller = + lib.genAttrs + ( + linux64BitSystems + ++ [ + "x86_64-darwin" + "aarch64-darwin" + ] + ) + ( + system: + let + pkgs = nixpkgsFor.${system}.native; + # Embed the native (glibc) Nix even though the Linux installer + # binary is static/musl. + tarball = pkgs.callPackage ./rust-installer/tarball.nix { + nix = pkgs.nixComponents2.nix-everything; + }; + builder = if pkgs.stdenv.hostPlatform.isLinux then pkgs.pkgsStatic else pkgs; + in + builder.callPackage ./rust-installer { + inherit tarball; + } + ); + # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); diff --git a/packaging/rust-installer/default.nix b/packaging/rust-installer/default.nix new file mode 100644 index 0000000000..dcfbf4badc --- /dev/null +++ b/packaging/rust-installer/default.nix @@ -0,0 +1,64 @@ +# `NixOS/nix-installer` built with *this* Nix closure embedded, so +# Hydra/CI can dogfood the Rust installer without the (removed) +# `--nix-package-url` knob. +{ + lib, + stdenv, + rustPlatform, + fetchFromGitHub, + tarball, +}: + +let + installerVersion = "2.34.5"; + src = fetchFromGitHub { + owner = "NixOS"; + repo = "nix-installer"; + tag = installerVersion; + hash = "sha256-+gM241qQOzQlOnP0a7d47z3iRf9+yNjbBJCLIWWNX+c="; + }; +in + +rustPlatform.buildRustPackage { + pname = "nix-installer"; + version = tarball.passthru.nixVersion; + + inherit src; + + cargoHash = "sha256-6pt2f7wznH672L5+SkbA5GA6Sxvk1KamAf3erGqZlLU="; + + doCheck = false; + + env = { + NIX_TARBALL_PATH = "${tarball}/nix.tar.zst"; + NIX_STORE_PATH = tarball.passthru.nixStorePath; + NSS_CACERT_STORE_PATH = tarball.passthru.cacertStorePath; + NIX_VERSION = tarball.passthru.nixVersion; + } + // lib.optionalAttrs stdenv.hostPlatform.isDarwin { + # Drop the unused libiconv dylib the darwin stdenv injects; the + # binary must run before `/nix/store` exists. + NIX_LDFLAGS = "-dead_strip_dylibs"; + }; + + postInstall = '' + install -m755 nix-installer.sh $out/bin/nix-installer.sh + + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix-installer" >> $out/nix-support/hydra-build-products + echo "file binary-dist $out/bin/nix-installer.sh" >> $out/nix-support/hydra-build-products + ''; + + # The binary embeds store-path strings (`NIX_STORE_PATH`, …) on + # purpose; don't let the reference scanner pull the whole Nix + # closure into this derivation's runtime closure. + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + + meta = { + description = "Rust-based Nix installer with an embedded Nix ${tarball.passthru.nixVersion}"; + homepage = "https://github.com/NixOS/nix-installer"; + license = lib.licenses.lgpl21Only; + mainProgram = "nix-installer"; + }; +} diff --git a/packaging/rust-installer/tarball.nix b/packaging/rust-installer/tarball.nix new file mode 100644 index 0000000000..4f236e0215 --- /dev/null +++ b/packaging/rust-installer/tarball.nix @@ -0,0 +1,50 @@ +# Zstd-compressed Nix closure in the layout expected by +# `NixOS/nix-installer` (`include_bytes!` at build time). +{ + lib, + stdenv, + runCommand, + buildPackages, + zstd, + nix, + cacert, +}: + +let + installerClosureInfo = buildPackages.closureInfo { + rootPaths = [ + nix + cacert + ]; + }; +in + +runCommand "nix-installer-tarball-${nix.version}" + { + nativeBuildInputs = [ zstd ]; + + passthru = { + nixStorePath = nix.outPath; + cacertStorePath = cacert.outPath; + nixVersion = nix.version; + }; + } + '' + mkdir -p $out + + dir=nix-${nix.version}-${stdenv.hostPlatform.system} + + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + + tar cf - \ + --sort=name \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --mtime='1970-01-01' \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) \ + | zstd -19 -T1 -o $out/nix.tar.zst + '' From b12be692c8c1f73ea13bde46c00100fbe4a851c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 23 Apr 2026 00:23:28 +0200 Subject: [PATCH 304/927] ci: run the Rust installer in installer_test Build .#rustInstaller alongside the bash installer, ship it in the same artifact and run the binary directly; drop the now-dead experimental-installer plumbing from install-nix-action. --- .../actions/install-nix-action/action.yaml | 70 ------------------- .github/workflows/ci.yml | 33 ++++----- .../prepare-installer-for-github-actions | 6 +- 3 files changed, 20 insertions(+), 89 deletions(-) diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml index 535ae9d08f..f889944b2d 100644 --- a/.github/actions/install-nix-action/action.yaml +++ b/.github/actions/install-nix-action/action.yaml @@ -4,22 +4,12 @@ inputs: dogfood: description: "Whether to use Nix installed from the latest artifact from master branch" required: true # Be explicit about the fact that we are using unreleased artifacts - experimental-installer: - description: "Whether to use the experimental installer to install Nix" - default: false - experimental-installer-version: - description: "Version of the experimental installer to use. If `latest`, the newest artifact from the default branch is used." - # TODO: This should probably be pinned to a release after https://github.com/NixOS/experimental-nix-installer/pull/49 lands in one - default: "latest" extra_nix_config: description: "Gets appended to `/etc/nix/nix.conf` if passed." install_url: description: "URL of the Nix installer" required: false default: "https://releases.nixos.org/nix/nix-2.32.1/install" - tarball_url: - description: "URL of the Nix tarball to use with the experimental installer" - required: false github_token: description: "Github token" required: true @@ -51,74 +41,14 @@ runs: gh run download "$RUN_ID" --repo "$DOGFOOD_REPO" -n "$INSTALLER_ARTIFACT" -D "$INSTALLER_DOWNLOAD_DIR" echo "installer-path=file://$INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" - TARBALL_PATH="$(find "$INSTALLER_DOWNLOAD_DIR" -name 'nix*.tar.xz' -print | head -n 1)" - echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" echo "::notice ::Dogfooding Nix installer from master (https://github.com/$DOGFOOD_REPO/actions/runs/$RUN_ID)" env: GH_TOKEN: ${{ inputs.github_token }} DOGFOOD_REPO: "NixOS/nix" - - name: "Gather system info for experimental installer" - shell: bash - if: ${{ inputs.experimental-installer == 'true' }} - run: | - echo "::notice Using experimental installer from $EXPERIMENTAL_INSTALLER_REPO (https://github.com/$EXPERIMENTAL_INSTALLER_REPO)" - - if [ "$RUNNER_OS" == "Linux" ]; then - EXPERIMENTAL_INSTALLER_SYSTEM="linux" - echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - elif [ "$RUNNER_OS" == "macOS" ]; then - EXPERIMENTAL_INSTALLER_SYSTEM="darwin" - echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - else - echo "::error ::Unsupported RUNNER_OS: $RUNNER_OS" - exit 1 - fi - - if [ "$RUNNER_ARCH" == "X64" ]; then - EXPERIMENTAL_INSTALLER_ARCH=x86_64 - echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV" - elif [ "$RUNNER_ARCH" == "ARM64" ]; then - EXPERIMENTAL_INSTALLER_ARCH=aarch64 - echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV" - else - echo "::error ::Unsupported RUNNER_ARCH: $RUNNER_ARCH" - exit 1 - fi - - echo "EXPERIMENTAL_INSTALLER_ARTIFACT=nix-installer-$EXPERIMENTAL_INSTALLER_ARCH-$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV" - env: - EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer" - - name: "Download latest experimental installer" - shell: bash - id: download-latest-experimental-installer - if: ${{ inputs.experimental-installer == 'true' && inputs.experimental-installer-version == 'latest' }} - run: | - RUN_ID=$(gh run list --repo "$EXPERIMENTAL_INSTALLER_REPO" --workflow ci.yml --branch main --status success --json databaseId --jq ".[0].databaseId") - - EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR="$GITHUB_WORKSPACE/$EXPERIMENTAL_INSTALLER_ARTIFACT" - mkdir -p "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" - - gh run download "$RUN_ID" --repo "$EXPERIMENTAL_INSTALLER_REPO" -n "$EXPERIMENTAL_INSTALLER_ARTIFACT" -D "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" - # Executable permissions are lost in artifacts - find $EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR -type f -exec chmod +x {} + - echo "installer-path=$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT" - env: - GH_TOKEN: ${{ inputs.github_token }} - EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer" - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1 - if: ${{ inputs.experimental-installer != 'true' }} with: # Ternary operator in GHA: https://www.github.com/actions/runner/issues/409#issuecomment-752775072 install_url: ${{ inputs.dogfood == 'true' && format('{0}/install', steps.download-nix-installer.outputs.installer-path) || inputs.install_url }} install_options: ${{ inputs.dogfood == 'true' && format('--tarball-url-prefix {0}', steps.download-nix-installer.outputs.installer-path) || '' }} extra_nix_config: ${{ inputs.extra_nix_config }} - - uses: DeterminateSystems/nix-installer-action@786fff0690178f1234e4e1fe9b536e94f5433196 # v20 - if: ${{ inputs.experimental-installer == 'true' }} - with: - diagnostic-endpoint: "" - # TODO: It'd be nice to use `artifacts.nixos.org` for both of these, maybe through an `/experimental-installer/latest` endpoint? or `/commit/`? - local-root: ${{ inputs.experimental-installer-version == 'latest' && steps.download-latest-experimental-installer.outputs.installer-path || '' }} - source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }} - nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }} - extra-conf: ${{ inputs.extra_nix_config }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be8cb0f0c4..07329d16a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,19 +164,19 @@ jobs: - scenario: on ubuntu runs-on: ubuntu-24.04 os: linux - experimental-installer: false + rust-installer: false - scenario: on macos runs-on: macos-14 os: darwin - experimental-installer: false - - scenario: on ubuntu (experimental) + rust-installer: false + - scenario: on ubuntu (rust) runs-on: ubuntu-24.04 os: linux - experimental-installer: true - - scenario: on macos (experimental) + rust-installer: true + - scenario: on macos (rust) runs-on: macos-14 os: darwin - experimental-installer: true + rust-installer: true name: installer test ${{ matrix.scenario }} runs-on: ${{ matrix.runs-on }} steps: @@ -188,22 +188,19 @@ jobs: path: out - name: Looking up the installer tarball URL id: installer-tarball-url - run: | - echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" - echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" + run: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 - if: ${{ !matrix.experimental-installer }} + if: ${{ !matrix.rust-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} - - uses: ./.github/actions/install-nix-action - if: ${{ matrix.experimental-installer }} - with: - dogfood: false - experimental-installer: true - tarball_url: ${{ steps.installer-tarball-url.outputs.tarball-path }} - github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Run rust installer + if: ${{ matrix.rust-installer }} + run: | + chmod +x out/nix-installer + ./out/nix-installer install --no-confirm + env: + RUST_BACKTRACE: full - run: sudo apt install fish zsh if: matrix.os == 'linux' - run: brew install fish diff --git a/ci/gha/tests/prepare-installer-for-github-actions b/ci/gha/tests/prepare-installer-for-github-actions index 0fbecf25c2..e240e56fca 100755 --- a/ci/gha/tests/prepare-installer-for-github-actions +++ b/ci/gha/tests/prepare-installer-for-github-actions @@ -2,10 +2,14 @@ set -euo pipefail -nix build -L ".#installerScriptForGHA" ".#binaryTarball" +nix build -L \ + ".#installerScriptForGHA" \ + ".#binaryTarball" \ + ".#rustInstaller" mkdir -p out cp ./result/install "out/install" name="$(basename "$(realpath ./result-1)")" # everything before the first dash cp -r ./result-1 "out/${name%%-*}" +cp ./result-2/bin/nix-installer "out/nix-installer" From 23d53de80a631bc2f5fefe2ed1d2f5b865755bc6 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Apr 2026 02:09:52 +0300 Subject: [PATCH 305/927] PosixDirectorySourceAccessor: Correct the SymlinkNotAllowed error message with cached parent dirfds This wouldn't display the prefix that's already cached. --- src/libutil/posix-source-accessor.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index da18d398e8..1fe3bf1346 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -333,10 +333,14 @@ std::pair> PosixDirectorySourceAccessor } } - AutoCloseFD parentFdOwning = - openFileEnsureBeneathNoSymlinks(startFd, relPath, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, std::move(cb)); - - return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; + try { + AutoCloseFD parentFdOwning = + openFileEnsureBeneathNoSymlinks(startFd, relPath, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, std::move(cb)); + return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; + } catch (SymlinkNotAllowed & e) { + /* Need to fixup the error message to include the actual path relative to the (possibly) cached fd. */ + throw SymlinkNotAllowed(anchor / e.path, "path '%s' is a symlink", showPath(anchor / e.path)); + } } std::optional PosixDirectorySourceAccessor::maybeLstat(const CanonPath & path) From 3e458a7af4544925624d7b00513571e178d78c5a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Apr 2026 02:35:06 +0300 Subject: [PATCH 306/927] PosixDirectorySourceAccessor: Improve dirFd caching for readFile/openSubdirectory Those didn't do caching without openat2 and did a bit more syscalls that ideal. With openat2 this is a slight regression, but overall not too meaningful. --- src/libutil/posix-source-accessor.cc | 93 +++++++++++++++++----------- 1 file changed, 57 insertions(+), 36 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 1fe3bf1346..661922cd39 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -207,6 +207,13 @@ class PosixDirectorySourceAccessor : public detail::PosixSourceAccessorBase globalDirFdCount.fetch_add(cache->size() - before, std::memory_order_relaxed); } + /** + * Helper for opening the parent directory used for other FD-relative operations down the line. + * Also caches the resulting dirFd. Throws FileNotFound if some intermediate directory or the parent + * doesn't exist. + */ + std::pair> openParentAndUpsert(const CanonPath & path, bool ignoreMissing); + /** * Get the parent directory of path. The second pair element might be an owning file descriptor * if path.parent().isRoot() is false. @@ -343,25 +350,39 @@ std::pair> PosixDirectorySourceAccessor } } +std::pair> +PosixDirectorySourceAccessor::openParentAndUpsert(const CanonPath & path, bool ignoreMissing) +{ + auto [parentFd, parentFdOwning] = openParent(path); + if (parentFd == INVALID_DESCRIPTOR) { + if (errno == ENOENT || errno == ENOTDIR) /* Intermediate component might not exist. */ { + if (ignoreMissing) + return {INVALID_DESCRIPTOR, {}}; + else + throw FileNotFound("path '%s' does not exist", showPath(path)); + } + throw SysError("opening directory '%1%'", showPath(path.parent().value())); + } + + if (dirFdCache && parentFdOwning) { + assert(*parentFdOwning); + insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); + } + + return {parentFd, parentFdOwning}; +} + std::optional PosixDirectorySourceAccessor::maybeLstat(const CanonPath & path) -try { +{ PosixStat st; if (path.isRoot()) { /* Must never fail - we already have the file descriptor for the directory. */ st = nix::fstat(dirFd.get()); } else { - auto [parentFd, parentFdOwning] = openParent(path); - if (parentFd == INVALID_DESCRIPTOR) { - if (errno == ENOENT || errno == ENOTDIR) - return std::nullopt; - throw SysError("opening directory '%1%'", showPath(path.parent().value())); - } - - if (dirFdCache && parentFdOwning) { - assert(*parentFdOwning); - insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); - } + auto [parentFd, parentFdOwning] = openParentAndUpsert(path, /*ignoreMissing=*/true); + if (parentFd == INVALID_DESCRIPTOR) + return std::nullopt; /* We know that CanonPath returns a NUL-terminated string_view, so the use of ->data() here is safe. */ if (::fstatat(parentFd, path.baseName()->data(), &st, AT_SYMLINK_NOFOLLOW) == -1) { @@ -373,20 +394,26 @@ try { maybeUpdateMtime(st.st_mtime); return sourceAccessorStatFromPosixStat(st); -} catch (SymlinkNotAllowed & e) { - throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } void PosixDirectorySourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) -try { +{ if (path.isRoot()) throw NotARegularFile("'%s' is not a regular file", showPath(path)); - AutoCloseFD fileFd = - openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_CLOEXEC, /*mode=*/0, makeDirFdCallback()); + /* TODO: We can do better when we have openat2. */ + auto [parentFd, parentFdOwning] = openParentAndUpsert(path, /*ignoreMissing=*/false); + AutoCloseFD fileFd; + + try { + fileFd = openFileEnsureBeneathNoSymlinks(parentFd, path.baseName().value(), O_RDONLY | O_CLOEXEC); + } catch (SymlinkNotAllowed & e) { + auto parent = path.parent().value(); + throw SymlinkNotAllowed(parent / e.path, "path '%s' is a symlink", showPath(parent / e.path)); + } if (!fileFd) { - if (errno == ENOENT || errno == ENOTDIR) /* Intermediate component might not exist. */ + if (errno == ENOENT) throw FileNotFound("file '%s' does not exist", showPath(path)); throw SysError("opening '%s'", showPath(path)); } @@ -397,12 +424,10 @@ try { PosixFileSourceAccessor fileAccessor(std::move(fileFd), fsPath / path.rel(), trackLastModified, st); maybeUpdateMtime(st.st_mtime); fileAccessor.readFile(CanonPath::root, sink, sizeCallback); -} catch (SymlinkNotAllowed & e) { - throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } AutoCloseFD PosixDirectorySourceAccessor::openSubdirectory(const CanonPath & path) -try { +{ AutoCloseFD dirFdOwning; if (path.isRoot()) { @@ -411,8 +436,16 @@ try { if (!dirFdOwning) throw SysError("opening directory '%s'", showPath(path)); } else { - dirFdOwning = openFileEnsureBeneathNoSymlinks( - dirFd.get(), path, O_DIRECTORY | O_RDONLY | O_CLOEXEC, /*mode=*/0, makeDirFdCallback()); + /* TODO: We can do better when we have openat2. */ + auto [parentFd, parentFdOwning] = openParentAndUpsert(path, /*ignoreMissing=*/false); + + try { + dirFdOwning = + openFileEnsureBeneathNoSymlinks(parentFd, path.baseName().value(), O_DIRECTORY | O_RDONLY | O_CLOEXEC); + } catch (SymlinkNotAllowed & e) { + auto parent = path.parent().value(); + throw SymlinkNotAllowed(parent / e.path, "path '%s' is a symlink", showPath(parent / e.path)); + } if (!dirFdOwning) { if (errno == ENOTDIR) @@ -422,8 +455,6 @@ try { } return dirFdOwning; -} catch (SymlinkNotAllowed & e) { - throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path)); } SourceAccessor::DirEntries PosixDirectorySourceAccessor::readDirectory(const CanonPath & path) @@ -497,17 +528,7 @@ try { if (path.isRoot()) throw NotASymlink("file '%s' is not a symlink", showPath(path)); - auto [parentFd, parentFdOwning] = openParent(path); - if (parentFd == INVALID_DESCRIPTOR) { - if (errno == ENOENT || errno == ENOTDIR) - throw FileNotFound("path '%s' does not exist", showPath(path)); - throw SysError("opening directory '%1%'", showPath(path.parent().value())); - } - - if (dirFdCache && parentFdOwning) { - assert(*parentFdOwning); - insertIntoDirFdCache(path.parent().value(), ref(parentFdOwning)); - } + auto [parentFd, parentFdOwning] = openParentAndUpsert(path, /*ignoreMissing=*/false); try { return readLinkAt(parentFd, CanonPath(path.baseName().value())); From 794f00a5776c03160147a3a58df5abffd9d9f2ee Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Apr 2026 03:01:40 +0300 Subject: [PATCH 307/927] Add CachingSourceAccessor for the sake of the evaluator, cache positive lstat/readlinks This avoids the footgun and issues around cache invalidation, while also keeping lookups more efficient. The evaluator already assumes a mostly immutable view of the filesystem (and we do nuke the caches during :r repl command). We used to cache lstat results because we do symlink resolution manually. Also adds the readDirectory override to the LocalStoreAccessor. --- src/libexpr/eval.cc | 2 + src/libstore/local-fs-store.cc | 8 ++ src/libutil/caching-source-accessor.cc | 102 ++++++++++++++++++ .../include/nix/util/source-accessor.hh | 6 ++ src/libutil/meson.build | 1 + 5 files changed, 119 insertions(+) create mode 100644 src/libutil/caching-source-accessor.cc diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index fb5d8d7a5e..7eb27f45b8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -277,6 +277,8 @@ EvalState::EvalState( mounted fetchTree. */ auto accessor = settings.pureEval ? storeFS.cast() : makeUnionSourceAccessor({getFSSourceAccessor(), storeFS}); + /* Cache positive lstat/readlink results to speed up resolveSymlinks. */ + accessor = makeCachingSourceAccessor(accessor); /* Apply access control if needed. */ if (settings.restrictEval || settings.pureEval) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 12818f32a4..3fc724f5fe 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -74,6 +74,14 @@ struct LocalStoreAccessor : SourceAccessor return accessor->readDirectory(path); } + void readDirectory( + const CanonPath & dirPath, + std::function callback) override + { + requireStoreObject(dirPath); + return accessor->readDirectory(dirPath, std::move(callback)); + } + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override { requireStoreObject(path); diff --git a/src/libutil/caching-source-accessor.cc b/src/libutil/caching-source-accessor.cc new file mode 100644 index 0000000000..041fab0d62 --- /dev/null +++ b/src/libutil/caching-source-accessor.cc @@ -0,0 +1,102 @@ +#include "nix/util/source-accessor.hh" + +#include + +namespace nix { + +class CachingSourceAccessor : public SourceAccessor +{ + ref next; + + boost::concurrent_flat_map lstatCache; + boost::concurrent_flat_map readLinkCache; + +public: + CachingSourceAccessor(ref next_) + : next(std::move(next_)) + { + displayPrefix.clear(); + } + + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override + { + next->readFile(path, sink, sizeCallback); + } + + std::optional maybeLstat(const CanonPath & path) override + { + if (auto res = getConcurrent(lstatCache, path)) + return *res; + + auto st = next->maybeLstat(path); + if (!st) + return std::nullopt; + + /* Never evict, the evaluator better keep positive lookups cached. */ + lstatCache.emplace(path, *st); + return st; + } + + Stat lstat(const CanonPath & path) override + { + if (auto res = getConcurrent(lstatCache, path)) + return *res; + + auto st = next->lstat(path); + /* Never evict, the evaluator better keep positive lookups cached. */ + lstatCache.emplace(path, st); + return st; + } + + DirEntries readDirectory(const CanonPath & path) override + { + return next->readDirectory(path); + } + + void readDirectory( + const CanonPath & dirPath, + std::function callback) override + { + return next->readDirectory(dirPath, std::move(callback)); + } + + std::string readLink(const CanonPath & path) override + { + if (auto res = getConcurrent(readLinkCache, path)) + return *res; + + auto target = next->readLink(path); + /* Never evict, the evaluator better keep positive lookups cached. */ + readLinkCache.emplace(path, target); + return target; + } + + std::string showPath(const CanonPath & path) override + { + return next->showPath(path); + } + + void invalidateCache() override + { + lstatCache.clear(); + readLinkCache.clear(); + next->invalidateCache(); + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return next->getPhysicalPath(path); + } + + std::pair> getFingerprint(const CanonPath & path) override + { + return next->getFingerprint(path); + } +}; + +ref makeCachingSourceAccessor(ref next) +{ + return make_ref(std::move(next)); +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index a2fbaac349..cc076054e7 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -288,4 +288,10 @@ ref makeFSSourceAccessor( */ ref makeUnionSourceAccessor(std::vector> && accessors); +/** + * Make a wrapper source accessor that caches positive lookup results. + * Useful for the evaluator which already assumes a mostly immutable view of the filesystem. + */ +ref makeCachingSourceAccessor(ref next); + } // namespace nix diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 50ade66887..d132ce67c7 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -136,6 +136,7 @@ sources = [ config_priv_h ] + files( 'base-n.cc', 'base-nix-32.cc', 'bump-memory-resource.cc', + 'caching-source-accessor.cc', 'canon-path.cc', 'compression-algo.cc', 'compression-settings.cc', From 6c2f1b446c9f6d0acf09703d28ba10b895de0c6c Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Apr 2026 22:48:01 +0300 Subject: [PATCH 308/927] libexpr: Cache findFile lookups better We were doing a bunch of pointless I/O (even before dirfd accessor) for non-existent paths in nixPath for stuff like and repeated lookups. This caches positive and negative lookups and nukes the cache in resetFileCache for the purposes of repl. --- src/libexpr/eval.cc | 34 ++++++++++++++++++++++------ src/libexpr/include/nix/expr/eval.hh | 15 +++++++++--- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7eb27f45b8..700d9a1ebe 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1173,6 +1173,7 @@ void EvalState::resetFileCache() importResolutionCache->clear(); fileEvalCache->clear(); inputCache->clear(); + lookupPathResolved->clear(); positions.clear(); rootFS->invalidateCache(); } @@ -3269,14 +3270,28 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ continue; auto r = *rOpt; - auto res = (r / CanonPath(suffix)).resolveSymlinks(); - if (res.pathExists()) + auto suffixPath = CanonPath(suffix); + if (auto cachedRes = getConcurrent(*rOpt->resolvedPaths, suffixPath)) { + if (*cachedRes) + return **cachedRes; + else + // Cached negative lookup. + continue; + } + + auto res = (r.path / suffixPath).resolveSymlinks(); + if (res.pathExists()) { + r.resolvedPaths->emplace(suffixPath, res); return res; + } // Backward compatibility hack: throw an exception if access // to this path is not allowed. if (auto accessor = res.accessor.dynamic_pointer_cast()) accessor->checkAccess(res.path); + + // Cache negative lookups too. + r.resolvedPaths->emplace(suffixPath, std::nullopt); } if (hasPrefix(path, "nix/")) @@ -3290,17 +3305,22 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ .debugThrow(); } -std::optional EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) +std::shared_ptr +EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) { auto & value = value0.s; if (auto cached = getConcurrent(*lookupPathResolved, value)) return *cached; - auto finish = [&](std::optional res) { - if (res) - debug("resolved search path element '%s' to '%s'", value, *res); - else + auto finish = [&](std::optional maybePath) { + std::shared_ptr res; + if (maybePath) { + debug("resolved search path element '%s' to '%s'", value, *maybePath); + res = std::make_shared( + *maybePath, make_ref()); + } else { debug("failed to resolve search path element '%s'", value); + } lookupPathResolved->emplace(std::string(value), res); return res; }; diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 89e2d5099d..20765f8554 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -489,7 +489,15 @@ private: LookupPath lookupPath; - const ref, StringViewHash, std::equal_to<>>> + struct LookupPathResolvedState + { + SourcePath path; + const ref>> resolvedPaths; + }; + + const ref< + boost:: + concurrent_flat_map, StringViewHash, std::equal_to<>>> lookupPathResolved; /** @@ -626,9 +634,10 @@ public: * * If the specified search path element is a URI, download it. * - * If it is not found, return `std::nullopt`. + * If it is not found, return `nullptr`. */ - std::optional resolveLookupPathPath(const LookupPath::Path & elem, bool initAccessControl = false); + std::shared_ptr + resolveLookupPathPath(const LookupPath::Path & elem, bool initAccessControl = false); /** * Evaluate an expression to normal form From bae48a5ba2ac21d4936f2a8d68aba9c603746a00 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Apr 2026 15:47:56 +0200 Subject: [PATCH 309/927] Input::getAccessorUnchecked(): Wrap fetches in a path lock This prevents multiple processes (like nix-eval-jobs instances) from fetching the same input at the same time. That doesn't matter for correctness, but it can cause a lot of redundant downloads. --- src/libfetchers/fetchers.cc | 16 ++++++++++++++++ tests/functional/tarball.sh | 14 ++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index fb87f9b945..c47b07c43b 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -7,6 +7,9 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/url.hh" +#include "nix/util/users.hh" +#include "nix/store/pathlocks.hh" +#include "nix/util/environment-variables.hh" #include @@ -301,6 +304,19 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); + /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up + * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. */ + auto lockFilePath = + getCacheDir() / "fetcher-locks" + / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false); + std::filesystem::create_directories(lockFilePath.parent_path()); + PathLocks lock( + {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); + + static auto inTest = getEnv("_NIX_TEST_CONCURRENT_FETCHES") == "1"; + if (inTest) + std::this_thread::sleep_for(std::chrono::seconds(1)); + /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. We only do this for final diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index e7d9b96fda..6deb7b96b7 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -116,3 +116,17 @@ path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" # Test that unpacking an empty file does not segfault (see https://github.com/NixOS/nix/issues/15116). touch "$TEST_ROOT/empty" expectStderr 1 nix store prefetch-file --unpack "file://$TEST_ROOT/empty" | grepQuiet "archive.*is empty" + +# Test that concurrent invocations of Nix will fetch the tarball only once. +rm -rf "$TEST_HOME/.cache" +store="$TEST_ROOT/prefetch-store" +nix-store --store "$store" --init # needed because concurrent creation of the store can give SQLite errors +_NIX_TEST_CONCURRENT_FETCHES=1 _NIX_FORCE_HTTP=1 nix flake prefetch --store "$store" -v "tarball+file://$TEST_ROOT/tar.tar" 2> "$TEST_ROOT/log1" & +pid1="$!" +_NIX_TEST_CONCURRENT_FETCHES=1 _NIX_FORCE_HTTP=1 nix flake prefetch --store "$store" -v "tarball+file://$TEST_ROOT/tar.tar" 2> "$TEST_ROOT/log2" & +pid2="$!" +wait "$pid1" +wait "$pid2" +[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "Download.*to") -eq 2 ]] +[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "downloading.*tar.tar") -eq 1 ]] +[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "waiting for another Nix process to finish fetching input") -eq 1 ]] From eb5c6775512df79d557687fe259f9223fbc5e348 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 23 Apr 2026 23:35:26 +0300 Subject: [PATCH 310/927] libfetchers: Acquire fetcher-locks only when input is not substituted When substituting an input, the store in question already acquires internal PathLocks on the to-be-substituted store path. Acquiring a lock eagerly is pessimising the case where multiple stores are substituted to concurrenty (but with the same cache dir). Also changes std::filesystem::create_directories to createDirs which wraps exceptions nicely and adds a setDeletion() call to clean up lock files. --- src/libfetchers/fetchers.cc | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c47b07c43b..4eedafa22a 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -304,19 +304,6 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); - /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up - * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. */ - auto lockFilePath = - getCacheDir() / "fetcher-locks" - / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false); - std::filesystem::create_directories(lockFilePath.parent_path()); - PathLocks lock( - {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); - - static auto inTest = getEnv("_NIX_TEST_CONCURRENT_FETCHES") == "1"; - if (inTest) - std::this_thread::sleep_for(std::chrono::seconds(1)); - /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. We only do this for final @@ -358,6 +345,21 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings } } + /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up + * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. Note that we don't need + * this when substituting the input. */ + auto lockFilePath = + getCacheDir() / "fetcher-locks" + / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false); + createDirs(lockFilePath.parent_path()); + PathLocks lock( + {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); + lock.setDeletion(true); + + static auto inTest = getEnv("_NIX_TEST_CONCURRENT_FETCHES") == "1"; + if (inTest) + std::this_thread::sleep_for(std::chrono::seconds(1)); + auto [accessor, result] = scheme->getAccessor(settings, store, *this); if (!accessor->fingerprint) From 5f5fb7a3a538525200505d65762d44bc6df7ffad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Apr 2026 18:14:10 +0200 Subject: [PATCH 311/927] RemoteStore::addToStore(): Fix version comparison --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 2def1bf87a..88c3847bb3 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -460,7 +460,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, Repair void RemoteStore::addMultipleToStore( PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) { - if (getConnection()->protoVersion < WorkerProto::Version{.number = {1, 32}}) { + if (getConnection()->protoVersion.number < WorkerProto::Version::Number{1, 32}) { Store::addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs); return; } From f63e1383143ac15e4e47b975c629f4298a8d9e0a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 24 Apr 2026 22:51:07 +0300 Subject: [PATCH 312/927] Tune error messages --- src/libutil/posix-source-accessor.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 661922cd39..b5921381d3 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -346,7 +346,7 @@ std::pair> PosixDirectorySourceAccessor return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; } catch (SymlinkNotAllowed & e) { /* Need to fixup the error message to include the actual path relative to the (possibly) cached fd. */ - throw SymlinkNotAllowed(anchor / e.path, "path '%s' is a symlink", showPath(anchor / e.path)); + throw SymlinkNotAllowed(anchor / e.path, "path '%s' (or its ancestor) is a symlink", showPath(anchor / e.path)); } } @@ -420,7 +420,7 @@ void PosixDirectorySourceAccessor::readFile(const CanonPath & path, Sink & sink, auto st = nix::fstat(fileFd.get()); if (!S_ISREG(st.st_mode)) - throw Error("file '%s' has an unsupported type", showPath(path)); + throw NotARegularFile("file '%s' is not a regular file", showPath(path)); PosixFileSourceAccessor fileAccessor(std::move(fileFd), fsPath / path.rel(), trackLastModified, st); maybeUpdateMtime(st.st_mtime); fileAccessor.readFile(CanonPath::root, sink, sizeCallback); From fb4d488dd55119703411aced25385a7346791ff2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 14:26:17 +0300 Subject: [PATCH 313/927] libflake: Drop unused NixStringContext in getFlake This was some leftovers from detnix cherry-pick, we ban string contexts in getFlake and have always. --- src/libflake/flake-primops.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 3ad6661172..1fc4f7ccbd 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -42,7 +42,6 @@ PrimOp getFlake(const Settings & settings) auto path = state.realisePath(pos, *args[0]); callFlake(state, lockFlake(settings, state, path, lockFlags), v); } else { - NixStringContext context; std::string flakeRefS( state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); From 569ee752c341db5de5fa63962d01f6a5d4cb995e Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 03:20:34 +0300 Subject: [PATCH 314/927] libexpr: Add a way to collect string context from ValuePrinter --- src/libexpr/include/nix/expr/print.hh | 13 +++++++++++-- src/libexpr/print.cc | 15 ++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/libexpr/include/nix/expr/print.hh b/src/libexpr/include/nix/expr/print.hh index 229f7159d1..8e6d0f9bf0 100644 --- a/src/libexpr/include/nix/expr/print.hh +++ b/src/libexpr/include/nix/expr/print.hh @@ -10,6 +10,7 @@ #include #include "nix/util/fmt.hh" +#include "nix/expr/value/context.hh" #include "nix/expr/print-options.hh" namespace nix { @@ -64,7 +65,12 @@ bool isReservedKeyword(const std::string_view str); */ std::ostream & printIdentifier(std::ostream & o, std::string_view s); -void printValue(EvalState & state, std::ostream & str, Value & v, PrintOptions options = PrintOptions{}); +void printValue( + EvalState & state, + std::ostream & str, + Value & v, + PrintOptions options = PrintOptions{}, + NixStringContext * context = nullptr); /** * A partially-applied form of `printValue` which can be formatted using `<<` @@ -77,12 +83,15 @@ private: EvalState & state; Value & value; PrintOptions options; + NixStringContext * context; public: - ValuePrinter(EvalState & state, Value & value, PrintOptions options = PrintOptions{}) + ValuePrinter( + EvalState & state, Value & value, PrintOptions options = PrintOptions{}, NixStringContext * context = nullptr) : state(state) , value(value) , options(options) + , context(context) { } }; diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index f2f62a6369..0c95ae60c5 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -162,6 +162,7 @@ class Printer std::ostream & output; EvalState & state; PrintOptions options; + NixStringContext * context; std::optional seen; size_t totalAttrsPrinted = 0; size_t totalListItemsPrinted = 0; @@ -577,9 +578,12 @@ class Printer printBool(v); break; - case nString: + case nString: { printString(v); + if (context) + copyContext(v, *context); break; + } case nPath: printPath(v); @@ -632,10 +636,11 @@ class Printer } public: - Printer(std::ostream & output, EvalState & state, PrintOptions options) + Printer(std::ostream & output, EvalState & state, PrintOptions options, NixStringContext * context) : output(output) , state(state) , options(options) + , context(context) { } @@ -656,14 +661,14 @@ class Printer } }; -void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options) +void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options, NixStringContext * context) { - Printer(output, state, options).print(v); + Printer(output, state, options, context).print(v); } std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer) { - printValue(printer.state, output, printer.value, printer.options); + printValue(printer.state, output, printer.value, printer.options, printer.context); return output; } From a8e2e0022732edc2602ad6849dd333213def2621 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Sat, 25 Apr 2026 13:26:37 -0700 Subject: [PATCH 315/927] linux-derivation-builder: Also block *listxattr Not blocking *listxattr not only leaks information from the host filesystem (e.g. leaks the fact that SELinux is enabled through the presence of security.selinux in the xattrs) but can confuse applications that assume that if *listxattr succeeds, xattrs are supported. For example, mkfs.ubifs will call llistxattr to get a list of attributes with a check for errno == EOPNOTSUPP to detect xattrs being unsupported. If that succeeds and it finds xattrs (which it will on an SELinux system), it calls lgetxattr on the individual attributes, which fails, causing mkfs.ubifs to fail. --- src/libstore/unix/build/linux-derivation-builder.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index 0bc32a91e6..1f636ec56f 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -116,7 +116,10 @@ static void setupSeccomp(const LocalSettings & localSettings) /* Prevent builders from using EAs or ACLs. Not all filesystems support these, and they're not allowed in the Nix store because they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(listxattr), 0) != 0 + || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(llistxattr), 0) != 0 + || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(flistxattr), 0) != 0 + || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 From 891ef140b8564a7848a3d75976e172c3e15ec14b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 19 Apr 2026 03:46:58 +0300 Subject: [PATCH 316/927] Don't copy flakes to the store unnecessarily This builds on top of https://github.com/NixOS/nix/pull/14050 to actually make flakes get lazily copied to the store. This repurposes a slightly less lazy (but also more deterministic) approach than determinate nix has taken. We do still pay to cost of hashing an input once to compute the store path and narHash daemon-client-side. This could be improved in follow-ups in case we don't actually need to check the narHash (like during local development). We need certain backwards compatibility hacks for getFlake with a discarded string context, those are similar to what detnix does. See: https://github.com/DeterminateSystems/nix-src/pull/422 See: https://github.com/DeterminateSystems/nix-src/pull/402 Co-authored-by: Eelco Dolstra --- src/libcmd/installable-value.cc | 5 +- src/libexpr/include/nix/expr/eval.hh | 27 ++++++++++- .../include/nix/expr/print-ambiguous.hh | 8 +++- src/libexpr/paths.cc | 47 ++++++++++++++++++- src/libexpr/primops.cc | 36 ++++++++++---- src/libexpr/print-ambiguous.cc | 14 ++++-- src/libflake/flake-primops.cc | 17 +++++++ src/nix/app.cc | 2 + src/nix/eval.cc | 16 ++++--- src/nix/flake.cc | 11 +++-- src/nix/nix-instantiate/nix-instantiate.cc | 4 +- tests/functional/flakes/flakes.sh | 1 - 12 files changed, 159 insertions(+), 29 deletions(-) diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 3a167af3db..92811c1d01 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -54,8 +54,11 @@ InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std:: } else if (v.type() == nString) { + auto path = state->coerceToSingleDerivedPath(pos, v, errorCtx); + if (auto o = std::get_if(&path.raw())) + state->ensureLazyPathCopied(o->path); return {{ - .path = DerivedPath::fromSingle(state->coerceToSingleDerivedPath(pos, v, errorCtx)), + .path = DerivedPath::fromSingle(path), .info = make_ref(), }}; } diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 20765f8554..1283b27cf8 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -738,6 +738,26 @@ public: std::optional tryAttrsToString( const PosIdx pos, Value & v, NixStringContext & context, bool coerceMore = false, bool copyToStore = true); + enum class CopyLazyPaths : bool { + PreserveLazy = false, + Copy = true, + }; + + /** + * For efficiency reasons, some store paths (as seen by the evaluator) in + * the storeFS at their content-addressed locations don't get copied to the + * store eagerly. This saves on needless I/O and possibly IPC if all the + * evaluator does is just evaluate nix expressions from those locations. + * This function copies such store objects to the store if they aren't already valid. + */ + void ensureLazyPathCopied(const StorePath & path); + + /** + * Ensure that all NixStringContextElem::Opaque context elements get fetched + * to the store. + */ + void ensureLazyPathsCopied(const NixStringContext & context); + /** * String coercion. * @@ -1044,9 +1064,14 @@ public: /** * Coerce `v` to a path and realise it, i.e. build anything in the value's string context using `realiseContext()`. + * @param copyLazyPaths When encountering a lazy path (i.e. a string with Opaque context that's also "mounted" on + * the storeFS), fetch the store path to the store. */ SourcePath realisePath( - const PosIdx pos, Value & v, std::optional resolveSymlinks = SymlinkResolution::Full); + const PosIdx pos, + Value & v, + std::optional resolveSymlinks = SymlinkResolution::Full, + CopyLazyPaths copyLazyPaths = CopyLazyPaths::PreserveLazy); /** * Realise the given string with context, and return the string with outputs instead of downstream output diff --git a/src/libexpr/include/nix/expr/print-ambiguous.hh b/src/libexpr/include/nix/expr/print-ambiguous.hh index 7e44a6b66e..07fcc337e3 100644 --- a/src/libexpr/include/nix/expr/print-ambiguous.hh +++ b/src/libexpr/include/nix/expr/print-ambiguous.hh @@ -17,6 +17,12 @@ class EvalState; * * See: https://github.com/NixOS/nix/issues/9730 */ -void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, size_t depth = 0); +void printAmbiguous( + EvalState & state, + Value & v, + std::ostream & str, + std::set * seen, + NixStringContext * context = nullptr, + size_t depth = 0); } // namespace nix diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index ca30320817..ff09b7aaa5 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -22,10 +22,55 @@ SourcePath EvalState::storePath(const StorePath & path) return {rootFS, CanonPath{store->printStorePath(path)}}; } +void EvalState::ensureLazyPathCopied(const StorePath & path) +{ + if (settings.isReadOnly()) + return; + + auto mount = storeFS->getMount(CanonPath(store->printStorePath(path))); + if (!mount) + return; + + /* TODO: We could memoise this in-memory if necessary. */ + auto storePath = fetchToStore( + fetchSettings, + *store, + SourcePath{ref(mount)}, + /* Force a copy. mountInput does a dryRun to just calculate the storePath and narHash. */ + FetchMode::Copy, + path.name()); + + /* Catch hash mismatches more loudly. This is more likely caused by unsound + caching of different accessor types that fetch the same repo with + the same git revision, but with different kinds of accessors (think + tarball-based fetchers vs local/remote git accessors). */ + if (storePath != path) { + panic(fmt( + "hashed store path computed by the evaluator ('%1%') does not match what was computed when copying to the store ('%2%'), this is a bug", + store->printStorePath(path), + store->printStorePath(storePath))); + } +} + +void EvalState::ensureLazyPathsCopied(const NixStringContext & context) +{ + for (const auto & c : context) + if (auto * o = std::get_if(&c.raw)) + /* TODO: This could be done in parallel. */ + ensureLazyPathCopied(o->path); +} + StorePath EvalState::mountInput(fetchers::Input & input, const fetchers::Input & originalInput, ref accessor) { - auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); + /* To mount the input, dryRun is sufficient. We still compute the narHash (to check for mismatches) and the store + path to figure out where to mount it. TODO: This could be relaxed in the future by making outPath and narHash + lazier. Good code that doesn't do `toString ./.` or otherwise inspects the outPath string and only uses it for + doing relative imports does not even require computing the store path. That is a big invasive change though and + would require having a special "LazyStorePathString" thunk. narHash also doesn't need to be computed eagerly in + case it's not actually specified (like during local development with a dirty tree) - in that case narHash could + also become a lazy app/thunk that shares the state with the storePath delayed computation. */ + auto [storePath, narHash] = fetchToStore2(fetchSettings, *store, accessor, FetchMode::DryRun, input.getName()); allowPath(storePath); // FIXME: should just whitelist the entire virtual store diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4b7680be1f..0c472fb102 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,6 +10,7 @@ #include "nix/store/names.hh" #include "nix/store/path-references.hh" #include "nix/store/store-api.hh" +#include "nix/util/mounted-source-accessor.hh" #include "nix/util/util.hh" #include "nix/util/os-string.hh" #include "nix/util/processes.hh" @@ -63,7 +64,7 @@ std::string EvalState::realiseString(Value & s, StorePathSet * storePathsOutMayb nix::NixStringContext stringContext; auto rawStr = coerceToString(pos, s, stringContext, "while realising a string").toOwned(); auto rewrites = realiseContext(stringContext, storePathsOutMaybe, isIFD); - + ensureLazyPathsCopied(stringContext); return nix::rewriteStrings(rawStr, rewrites); } @@ -88,7 +89,11 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { - ensureValid(o.path); + /* If the path happens to be mounted on the storeFS, that means it's lazy path string and would get + copied to the store on-demand (when referenced in a derivation). The string is equal to final + store path where the store object would end up (the path is hashed before mounting). */ + if (!storeFS->getMount(CanonPath(store->printStorePath(o.path)))) + ensureValid(o.path); if (maybePathsOut) maybePathsOut->emplace(o.path); }, @@ -158,7 +163,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS return res; } -SourcePath EvalState::realisePath(const PosIdx pos, Value & v, std::optional resolveSymlinks) +SourcePath EvalState::realisePath( + const PosIdx pos, Value & v, std::optional resolveSymlinks, CopyLazyPaths copyLazyPaths) { NixStringContext context; @@ -167,6 +173,8 @@ SourcePath EvalState::realisePath(const PosIdx pos, Value & v, std::optional(&c.raw)) + if (auto p = std::get_if(&c.raw)) { + state.ensureLazyPathCopied(p->path); refs.insert(p->path); - else + } else state .error( "files created by %1% may not reference derivations, but %2% references %3%", diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index ed91cad85a..b0a5224f26 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -7,7 +7,13 @@ namespace nix { // See: https://github.com/NixOS/nix/issues/9730 -void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, size_t depth) +void printAmbiguous( + EvalState & state, + Value & v, + std::ostream & str, + std::set * seen, + NixStringContext * context, + size_t depth) { checkInterrupt(); @@ -22,6 +28,8 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::setlexicographicOrder(state.symbols)) { str << state.symbols[i->name] << " = "; - printAmbiguous(state, *i->value, str, seen, depth + 1); + printAmbiguous(state, *i->value, str, seen, context, depth + 1); str << "; "; } str << "}"; @@ -52,7 +60,7 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::setisInStore(sourcePath->string())) { + auto [storePath, subPath] = state.store->toStorePath(sourcePath->string()); + if (auto mount = state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath)))) { + auto path = state.storePath(storePath) / CanonPath(subPath); + if (!flakeRef.subdir.empty()) + path = path / flakeRef.subdir; + return callFlake(state, lockFlake(settings, state, path, lockFlags), v); + } + } + callFlake(state, lockFlake(settings, state, flakeRef, lockFlags), v); } }; diff --git a/src/nix/app.cc b/src/nix/app.cc index 634db04f3f..167ffd8d86 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -95,6 +95,8 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) c.raw)); } + state.ensureLazyPathsCopied(context); + return UnresolvedApp{App{ .context = std::move(context2), .program = program, diff --git a/src/nix/eval.cc b/src/nix/eval.cc index ed6fe41be6..6b8c0ba122 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -83,10 +83,10 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption [&](this const auto & recurse, Value & v, const PosIdx pos, const std::filesystem::path & path) -> void { state->forceValue(v, pos); - if (v.type() == nString) - // FIXME: disallow strings with contexts? + if (v.type() == nString) { + copyContext(v, context); writeFile(path, v.string_view()); - else if (v.type() == nAttrs) { + } else if (v.type() == nAttrs) { [[maybe_unused]] bool directoryCreated = std::filesystem::create_directory(path); // Directory should not already exist assert(directoryCreated); @@ -110,9 +110,8 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption else if (raw) { logger->stop(); - writeFull( - getStandardOutput(), - *state->coerceToString(noPos, *v, context, "while generating the eval command output")); + auto string = state->coerceToString(noPos, *v, context, "while generating the eval command output"); + writeFull(getStandardOutput(), *string); } else if (json) { @@ -120,8 +119,11 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption } else { - logger->cout("%s", ValuePrinter(*state, *v, PrintOptions{.force = true, .derivationPaths = true})); + ValuePrinter printer(*state, *v, PrintOptions{.force = true, .derivationPaths = true}, &context); + logger->cout("%s", printer); } + + state->ensureLazyPathsCopied(context); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2a5239549e..53719f5ec7 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,6 +7,7 @@ #include "nix/expr/get-drvs.hh" #include "nix/util/os-string.hh" #include "nix/util/signals.hh" +#include "nix/util/mounted-source-accessor.hh" #include "nix/store/store-open.hh" #include "nix/store/derivations.hh" #include "nix/store/outputs-spec.hh" @@ -215,8 +216,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON auto lockedFlake = lockFlake(); auto & flake = lockedFlake.flake; - // Currently, all flakes are in the Nix store via the rootFS accessor. - auto storePath = store->printStorePath(store->toStorePath(flake.path.path.abs()).first); + /* Flakes do not get copied to the store, but are instead mounted at + their expected store paths in storeFS. Querying metadata does not + force copying to the store, as one would expect. */ + auto storePath = store->toStorePath(flake.path.path.abs()).first; if (json) { nlohmann::json j; @@ -238,7 +241,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = storePath; + j["path"] = store->printStorePath(storePath); j["locks"] = lockedFlake.lockFile.toJSON().first; if (auto fingerprint = lockedFlake.getFingerprint(*store, fetchSettings)) j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); @@ -249,7 +252,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout(ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", flake.lockedRef.to_string()); if (flake.description) logger->cout(ANSI_BOLD "Description:" ANSI_NORMAL " %s", *flake.description); - logger->cout(ANSI_BOLD "Path:" ANSI_NORMAL " %s", storePath); + logger->cout(ANSI_BOLD "Path:" ANSI_NORMAL " %s", store->printStorePath(storePath)); if (auto rev = flake.lockedRef.input.getRev()) logger->cout(ANSI_BOLD "Revision:" ANSI_NORMAL " %s", rev->to_string(HashFormat::Base16, false)); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) diff --git a/src/nix/nix-instantiate/nix-instantiate.cc b/src/nix/nix-instantiate/nix-instantiate.cc index 27a11767b0..dee79bcbfc 100644 --- a/src/nix/nix-instantiate/nix-instantiate.cc +++ b/src/nix/nix-instantiate/nix-instantiate.cc @@ -66,7 +66,7 @@ void processExpr( if (strict) state.forceValueDeep(vRes); std::set seen; - printAmbiguous(state, vRes, std::cout, &seen); + printAmbiguous(state, vRes, std::cout, &seen, &context); std::cout << std::endl; } } else { @@ -94,6 +94,8 @@ void processExpr( std::cout << fmt("%s%s\n", drvPathS, (outputName != "out" ? "!" + outputName : "")); } } + + state.ensureLazyPathsCopied(context); } } diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 8d95f61240..fec80cc895 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -69,7 +69,6 @@ nix flake metadata "$flake1Dir" | grepQuiet 'URL:.*flake1.*' # Test 'nix flake metadata --json'. json=$(nix flake metadata flake1 --json | jq .) [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] -[[ -d $(echo "$json" | jq -r .path) ]] [[ $(echo "$json" | jq -r .lastModified) = $(git -C "$flake1Dir" log -n1 --format=%ct) ]] hash1=$(echo "$json" | jq -r .revision) [[ -n $(echo "$json" | jq -r .fingerprint) ]] From d4ea1a07d2c14042e9e2cb359fa4efb39a244127 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Thu, 23 Apr 2026 22:21:48 -0400 Subject: [PATCH 317/927] Enable more clang-tidy checks, related fixes Signed-off-by: Lisanna Dettwyler --- .../common/clang-tidy/.clang-tidy | 17 +++++++++++------ src/libexpr-tests/nix_api_expr.cc | 4 ++-- src/libexpr/eval-cache.cc | 2 +- src/libexpr/eval-error.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/get-drvs.cc | 6 +++--- src/libexpr/include/nix/expr/diagnose.hh | 4 ++-- src/libexpr/include/nix/expr/nixexpr.hh | 2 +- src/libexpr/include/nix/expr/parser-state.hh | 2 +- src/libexpr/primops.cc | 2 +- src/libstore/build/derivation-goal.cc | 4 ++++ src/libstore/build/goal.cc | 4 ++-- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/build/worker.cc | 2 +- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/include/nix/store/build/goal.hh | 4 ++-- src/libstore/include/nix/store/sqlite.hh | 2 +- src/libstore/misc.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/sqlite.cc | 2 +- src/libstore/store-api.cc | 18 ++++++++---------- src/libstore/unix/build/child.cc | 2 +- src/libstore/unix/build/derivation-builder.cc | 4 ++-- src/libutil-tests/file-descriptor.cc | 4 ++-- src/libutil/file-descriptor.cc | 1 + src/libutil/include/nix/util/async.hh | 4 ++-- src/libutil/include/nix/util/closure.hh | 2 +- src/libutil/include/nix/util/configuration.hh | 2 +- src/libutil/include/nix/util/error.hh | 2 +- .../include/nix/util/file-descriptor.hh | 1 + src/libutil/include/nix/util/serialise.hh | 1 + src/libutil/include/nix/util/sort.hh | 12 ++++++------ src/libutil/include/nix/util/topo-sort.hh | 2 +- src/libutil/include/nix/util/util.hh | 2 +- src/libutil/linux/linux-namespaces.cc | 4 ++-- src/libutil/logging.cc | 12 +++++++++--- src/libutil/unix/file-system.cc | 2 +- 37 files changed, 80 insertions(+), 64 deletions(-) diff --git a/nix-meson-build-support/common/clang-tidy/.clang-tidy b/nix-meson-build-support/common/clang-tidy/.clang-tidy index daf4c7f7c4..b0cc92429c 100644 --- a/nix-meson-build-support/common/clang-tidy/.clang-tidy +++ b/nix-meson-build-support/common/clang-tidy/.clang-tidy @@ -51,11 +51,6 @@ Checks: - -bugprone-macro-parentheses # 1 warning - increment/decrement in conditions - -bugprone-inc-dec-in-conditions - # 2 warnings - std::move on forwarding reference (auto&&) in ranges lambdas - - -bugprone-move-forwarding-reference - # 2 warnings - coroutine pattern: co_await await(std::move(waitees)) then reuse. - # Relies on moved-from containers being empty (holds for libstdc++/libc++). - - -bugprone-use-after-move # 2 warnings - sorts Value* by ->string_view(), not by pointer value (false positive) - -bugprone-nondeterministic-pointer-iteration-order # 9 warnings - intentional std::bit_cast/memcpy on Value* arrays (evaluator hot path) @@ -68,7 +63,7 @@ Checks: # template; fires when T=unsigned char but that instantiation is correct. - -bugprone-unintended-char-ostream-output # - # Non-bugprone checks (also disabled to pass on current codebase): + # Non-bugprone checks (some disabled to pass on current codebase): # # 4 warnings - exceptions not derived from std::exception # All thrown exceptions must derive from std::exception @@ -77,6 +72,14 @@ Checks: # - cppcoreguidelines-pro-type-cstyle-cast # 11 warnings - coroutine lambdas with captures (intentional pattern in async goal/store code) # - cppcoreguidelines-avoid-capturing-lambda-coroutines + - performance-noexcept-swap + - performance-noexcept-move-constructor + - performance-noexcept-destructor + - performance-use-std-move + - misc-throw-by-value-catch-by-reference + - cppcoreguidelines-missing-std-forward + - android-cloexec-open + - android-cloexec-pipe2 # Custom nix checks (when added) - nix-* @@ -93,3 +96,5 @@ CheckOptions: bugprone-unsafe-functions.CustomFunctions: > ::std::filesystem::create_directories, nix::createDirs, "Use nix::createDirs (it wraps exceptions)"; ::std::filesystem::remove_all, nix::deletePath, "Use nix::deletePath (remove_all is not TOCTOU safe)"; + +ExtraArgs: ["-Werror=unnecessary-virtual-specifier"] diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index c3a3f2dd53..8362e68508 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -20,8 +20,8 @@ TEST_F(nix_api_expr_test, nix_eval_state_lookup_path) auto delTmpDir = std::make_unique(tmpDir, true); auto nixpkgs = tmpDir / "pkgs"; auto nixos = tmpDir / "cfg"; - std::filesystem::create_directories(nixpkgs); - std::filesystem::create_directories(nixos); + nix::createDirs(nixpkgs); + nix::createDirs(nixos); std::string nixpkgsEntry = "nixpkgs=" + nixpkgs.string(); std::string nixosEntry = "nixos-config=" + nixos.string(); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index a419530c6d..63f0c14887 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -106,7 +106,7 @@ struct AttrDb } template - AttrId doSQLite(F && fun) + AttrId doSQLite(const F & fun) { if (failed) return 0; diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 67e8d5ea1d..22c622526c 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -97,7 +97,7 @@ void EvalErrorBuilder::debugThrow() auto error = std::move(this->error); delete this; - throw error; + throw std::move(error); } template diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 700d9a1ebe..4cd90a2a5f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3465,7 +3465,7 @@ void forceNoNullByte(std::string_view s, std::function pos) if (pos) { error.atPos(pos()); } - throw error; + throw std::move(error); } } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 693b1946ee..1acc591b05 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -162,14 +162,14 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) - throw errMsg; + throw std::move(errMsg); Outputs result; for (auto elem : outTI->listView()) { if (elem->type() != nString) - throw errMsg; + throw std::move(errMsg); auto out = outputs.find(elem->string_view()); if (out == outputs.end()) - throw errMsg; + throw std::move(errMsg); result.insert(*out); } return result; diff --git a/src/libexpr/include/nix/expr/diagnose.hh b/src/libexpr/include/nix/expr/diagnose.hh index 68c8f6543f..4a360970ba 100644 --- a/src/libexpr/include/nix/expr/diagnose.hh +++ b/src/libexpr/include/nix/expr/diagnose.hh @@ -44,7 +44,7 @@ NIX_DECLARE_CONFIG_SERIALISER(Diagnose) * @throws The error returned by mkError if level is `Fatal` and mkError returns a value */ template -void diagnose(const Setting & setting, F && mkError) +void diagnose(const Setting & setting, const F & mkError) { auto withError = [&](bool fatal, auto && handler) { auto maybeError = mkError(fatal); @@ -64,7 +64,7 @@ void diagnose(const Setting & setting, F && mkError) withError(false, [](auto && error) { logWarning(error.info()); }); return; case Diagnose::Fatal: - withError(true, [](auto && error) { throw std::move(error); }); + withError(true, [](auto && error) { throw std::forward(error); }); return; } } diff --git a/src/libexpr/include/nix/expr/nixexpr.hh b/src/libexpr/include/nix/expr/nixexpr.hh index 07fbed403c..b13c00ad54 100644 --- a/src/libexpr/include/nix/expr/nixexpr.hh +++ b/src/libexpr/include/nix/expr/nixexpr.hh @@ -557,7 +557,7 @@ public: std::numeric_limits::max()); if (pos) err.atPos(positions[pos]); - throw err; + throw std::move(err); } std::uninitialized_copy_n(formals.formals.begin(), nFormals, formalsStart); }; diff --git a/src/libexpr/include/nix/expr/parser-state.hh b/src/libexpr/include/nix/expr/parser-state.hh index f9bd06589e..2482d53ea0 100644 --- a/src/libexpr/include/nix/expr/parser-state.hh +++ b/src/libexpr/include/nix/expr/parser-state.hh @@ -89,7 +89,7 @@ public: * @see https://github.com/NixOS/nix/issues/14642 */ template - void visit(F && f) + void visit(const F & f) { std::visit( overloaded{ diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4b7680be1f..5683d13d5c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -694,7 +694,7 @@ static RegisterPrimOp primop_isPath({ }); template -static inline void withExceptionContext(Trace trace, Callable && func) +static inline void withExceptionContext(Trace trace, const Callable & func) { try { func(); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6d76cd9d27..fc4c8a5bb5 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -92,6 +92,8 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) co_await await(std::move(waitees)); if (nrFailed == 0) { + // optimization depending on moved containers being empty afterwards + // NOLINTNEXTLINE(bugprone-use-after-move) waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath))); co_await await(std::move(waitees)); @@ -111,6 +113,8 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) } } + // optimization depending on moved containers being empty afterwards + // NOLINTNEXTLINE(bugprone-use-after-move) co_await await(std::move(waitees)); trace("all outputs substituted (maybe)"); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 7651282e3e..af245d9018 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -58,13 +58,13 @@ Goal::ChildEvent Goal::ChildEvents::popChildEvent() using handle_type = nix::Goal::handle_type; using Suspend = nix::Goal::Suspend; -Co::Co(Co && rhs) +Co::Co(Co && rhs) noexcept { this->handle = rhs.handle; rhs.handle = nullptr; } -Co & Co::operator=(Co && rhs) +Co & Co::operator=(Co && rhs) noexcept { if (handle) { handle.promise().alive = false; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 4cb42975fe..90273493e2 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -146,7 +146,7 @@ Goal::Co PathSubstitutionGoal::init() } if (lastStoresException.has_value()) { if (!worker.settings.tryFallback) { - throw *lastStoresException; + throw std::move(*lastStoresException); } else logError(lastStoresException->info()); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 002bb1a30d..3fe196d080 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -61,7 +61,7 @@ std::shared_ptr Worker::initGoalIfNeeded(std::weak_ptr & goal_weak, Args & if (auto goal = goal_weak.lock()) return goal; - auto goal = std::make_shared(args...); + auto goal = std::make_shared(std::forward(args)...); goal_weak = goal; wakeUp(goal); return goal; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index ff5a89f135..713de4969a 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -214,7 +214,7 @@ void HttpBinaryCacheStore::upsertFile( } catch (FileTransferError & e) { UploadToHTTP err(e.message()); err.addTrace({}, "while uploading to HTTP binary cache at '%s'", config->cacheUri.to_string()); - throw err; + throw std::move(err); } } diff --git a/src/libstore/include/nix/store/build/goal.hh b/src/libstore/include/nix/store/build/goal.hh index 6ddc73250d..3e5a3139d5 100644 --- a/src/libstore/include/nix/store/build/goal.hh +++ b/src/libstore/include/nix/store/build/goal.hh @@ -255,8 +255,8 @@ public: explicit Co(handle_type handle) : handle(handle) {}; - Co & operator=(Co &&); - Co(Co && rhs); + Co & operator=(Co &&) noexcept; + Co(Co && rhs) noexcept; Co & operator=(const Co &) = delete; Co(const Co & rhs) = delete; ~Co(); diff --git a/src/libstore/include/nix/store/sqlite.hh b/src/libstore/include/nix/store/sqlite.hh index 789e821746..0a2d21b128 100644 --- a/src/libstore/include/nix/store/sqlite.hh +++ b/src/libstore/include/nix/store/sqlite.hh @@ -207,7 +207,7 @@ void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning); * database is busy. */ template -T retrySQLite(F && fun) +T retrySQLite(const F & fun) { time_t nextWarning = time(nullptr) + 1; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 51708a2cbc..8db977b6fc 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -149,7 +149,7 @@ querySubstitutablePathInfosAsync(Store & store, const StorePathCAMap & paths, Su } if (lastStoresException.has_value()) { if (!settings.getWorkerSettings().tryFallback) { - throw *lastStoresException; + throw std::move(*lastStoresException); } else logError(lastStoresException->info()); } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 7331575241..f6d300c093 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -173,7 +173,7 @@ void S3BinaryCacheStore::upsertFile( } catch (FileTransferError & e) { UploadToS3 err(e.message()); err.addTrace({}, "while uploading to S3 binary cache at '%s'", config->cacheUri.to_string()); - throw err; + throw std::move(err); } } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 3fd1f798ea..c65fc240c6 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -47,7 +47,7 @@ SQLiteError::SQLiteError( exp.err.msg = HintFmt( err == SQLITE_PROTOCOL ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" : "SQLite database '%s' is busy", path ? path : "(in-memory)"); - throw exp; + throw std::move(exp); } else throw SQLiteError(path, errMsg, err, exterr, offset, std::move(hf)); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3f367b19a1..3c23156769 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -55,21 +55,19 @@ StoreConfigBase::StoreDirSetting::StoreDirSetting(Config * options, FilePathType switch (pathType) { case FilePathType::Unix: - return canonStoreDir( - envOverrides.transform([](auto && s) { return os_string_to_string(std::move(s)); }) - .value_or(NIX_STORE_DIR)); + return canonStoreDir(envOverrides.transform([](const auto & s) { return os_string_to_string(s); }) + .value_or(NIX_STORE_DIR)); case FilePathType::Native: - return canonStoreDir( - envOverrides.transform([](auto && s) { return std::filesystem::path(std::move(s)); }) - .or_else([]() -> std::optional { + return canonStoreDir(envOverrides.transform([](const auto & s) { return std::filesystem::path(s); }) + .or_else([]() -> std::optional { #ifdef _WIN32 - return windows::known_folders::getProgramData() / "nix" / "store"; + return windows::known_folders::getProgramData() / "nix" / "store"; #else - return std::filesystem::path{NIX_STORE_DIR}; + return std::filesystem::path{NIX_STORE_DIR}; #endif - }) - .value()); + }) + .value()); } assert(false); }(), diff --git a/src/libstore/unix/build/child.cc b/src/libstore/unix/build/child.cc index 3a704e6edf..f603b671e4 100644 --- a/src/libstore/unix/build/child.cc +++ b/src/libstore/unix/build/child.cc @@ -26,7 +26,7 @@ void commonChildInit() throw SysError("cannot dup stderr into stdout"); /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR | O_CLOEXEC); if (fdDevNull == -1) throw SysError("cannot open '%1%'", pathNullDevice); if (dup2(fdDevNull, STDIN_FILENO) == -1) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 7d348af319..3e9ad54347 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -974,7 +974,7 @@ void DerivationBuilderImpl::openSlave() { std::string slaveName = getPtsName(builderOut.get()); - AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY | O_CLOEXEC); if (!builderOut) throw SysError("opening pseudoterminal slave"); @@ -1056,7 +1056,7 @@ void DerivationBuilderImpl::processSandboxSetupMessages() FdSource source(builderOut.get()); auto ex = readError(source); ex.addTrace({}, "while setting up the build environment"); - throw ex; + throw std::move(ex); } debug("sandbox setup: " + msg); msgs.push_back(std::move(msg)); diff --git a/src/libutil-tests/file-descriptor.cc b/src/libutil-tests/file-descriptor.cc index 808bb31d8e..c385d2960f 100644 --- a/src/libutil-tests/file-descriptor.cc +++ b/src/libutil-tests/file-descriptor.cc @@ -130,7 +130,7 @@ TEST(ReadLine, TreatsEioAsEof) ASSERT_EQ(unlockpt(master), 0); // Open and immediately close the slave to trigger EIO on the master. - int slave = open(ptsname(master), O_RDWR | O_NOCTTY); + int slave = open(ptsname(master), O_RDWR | O_NOCTTY | O_CLOEXEC); ASSERT_NE(slave, -1); close(slave); @@ -153,7 +153,7 @@ TEST(ReadLine, PartialLineBeforeEio) ASSERT_EQ(grantpt(master), 0); ASSERT_EQ(unlockpt(master), 0); - int slave = open(ptsname(master), O_RDWR | O_NOCTTY); + int slave = open(ptsname(master), O_RDWR | O_NOCTTY | O_CLOEXEC); ASSERT_NE(slave, -1); // Write a partial line (no terminator) from the slave, then close it. diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index acd273ca79..eff35721b2 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -237,6 +237,7 @@ AutoCloseFD::AutoCloseFD(AutoCloseFD && that) noexcept that.fd = INVALID_DESCRIPTOR; } +// NOLINTNEXTLINE(performance-noexcept-move-constructor) - technically can throw AutoCloseFD & AutoCloseFD::operator=(AutoCloseFD && that) { close(); diff --git a/src/libutil/include/nix/util/async.hh b/src/libutil/include/nix/util/async.hh index 7abd446f73..3f6be9b48c 100644 --- a/src/libutil/include/nix/util/async.hh +++ b/src/libutil/include/nix/util/async.hh @@ -88,7 +88,7 @@ asio::awaitable callbackToAwaitable(F && initiate) } template -asio::awaitable forEachAsync(Range && range, F && f) +asio::awaitable forEachAsync(Range && range, const F & f) { /* This code only runs on a strand - we don't do multithreaded executors, so no need for synchronisation. */ @@ -102,7 +102,7 @@ asio::awaitable forEachAsync(Range && range, F && f) co_await asio::async_initiate( [&](auto handler) { auto h = std::make_shared(std::move(handler)); - for (auto && elt : range) { + for (auto && elt : std::forward(range)) { asio::co_spawn(executor, f(elt), [executor, h, &err, &pending](std::exception_ptr ex) { if (ex && !err) err = ex; diff --git a/src/libutil/include/nix/util/closure.hh b/src/libutil/include/nix/util/closure.hh index 586acfff7e..5078a02d07 100644 --- a/src/libutil/include/nix/util/closure.hh +++ b/src/libutil/include/nix/util/closure.hh @@ -26,7 +26,7 @@ template using GetEdgesAsync = fun>(const T & elt)>; template -auto computeClosure(std::set startElts, std::set & res, GetEdgesAsync getEdges, CompletionToken && token) +auto computeClosure(std::set startElts, std::set & res, GetEdgesAsync getEdges, CompletionToken token) { auto initiator = [&res, startElts = std::move(startElts), getEdges = std::move(getEdges)](auto handler) { auto executor = asio::make_strand(asio::get_associated_executor(handler)); diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 19d678601b..5dc98904a4 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -578,7 +578,7 @@ struct ExperimentalFeatureSettings : Config */ template requires std::invocable && std::convertible_to, std::string> - void require(const ExperimentalFeature & feature, GetReason && getReason) const + void require(const ExperimentalFeature & feature, const GetReason & getReason) const { if (isEnabled(feature)) return; diff --git a/src/libutil/include/nix/util/error.hh b/src/libutil/include/nix/util/error.hh index b01c18e58d..2a846fed17 100644 --- a/src/libutil/include/nix/util/error.hh +++ b/src/libutil/include/nix/util/error.hh @@ -126,7 +126,7 @@ protected: public: BaseError(const BaseError &) = default; BaseError & operator=(const BaseError &) = default; - BaseError & operator=(BaseError &&) = default; + BaseError & operator=(BaseError &&) noexcept = default; template BaseError(unsigned int status, Args &&... args) diff --git a/src/libutil/include/nix/util/file-descriptor.hh b/src/libutil/include/nix/util/file-descriptor.hh index f92e737fcd..a6796f2269 100644 --- a/src/libutil/include/nix/util/file-descriptor.hh +++ b/src/libutil/include/nix/util/file-descriptor.hh @@ -262,6 +262,7 @@ public: AutoCloseFD(AutoCloseFD && fd) noexcept; ~AutoCloseFD(); AutoCloseFD & operator=(const AutoCloseFD & fd) = delete; + // NOLINTNEXTLINE(performance-noexcept-move-constructor) - technically can throw because of close() AutoCloseFD & operator=(AutoCloseFD && fd); Descriptor get() const; explicit operator bool() const; diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index 761a0fe6ed..ed84c767e8 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -177,6 +177,7 @@ struct FdSink : BufferedSink FdSink(const FdSink &) = delete; FdSink & operator=(const FdSink &) = delete; + // NOLINTNEXTLINE(performance-noexcept-move-constructor) - can throw FdSink & operator=(FdSink && s) { flush(); diff --git a/src/libutil/include/nix/util/sort.hh b/src/libutil/include/nix/util/sort.hh index 2a4eb6e7c9..77a3ea59a5 100644 --- a/src/libutil/include/nix/util/sort.hh +++ b/src/libutil/include/nix/util/sort.hh @@ -123,7 +123,7 @@ void insertionsort(Iter begin, Iter end, Comparator comp = {}) * to the specified comparator. */ template>> -Iter strictlyDecreasingPrefix(Iter begin, Iter end, Comparator && comp = {}) +Iter strictlyDecreasingPrefix(Iter begin, Iter end, const Comparator & comp = {}) { if (begin == end) return begin; @@ -138,7 +138,7 @@ Iter strictlyDecreasingPrefix(Iter begin, Iter end, Comparator && comp = {}) * to the specified comparator. */ template>> -Iter strictlyDecreasingSuffix(Iter begin, Iter end, Comparator && comp = {}) +Iter strictlyDecreasingSuffix(Iter begin, Iter end, const Comparator & comp = {}) { if (begin == end) return end; @@ -153,9 +153,9 @@ Iter strictlyDecreasingSuffix(Iter begin, Iter end, Comparator && comp = {}) * to the specified comparator. */ template>> -Iter weaklyIncreasingPrefix(Iter begin, Iter end, Comparator && comp = {}) +Iter weaklyIncreasingPrefix(Iter begin, Iter end, const Comparator & comp = {}) { - return strictlyDecreasingPrefix(begin, end, std::not_fn(std::forward(comp))); + return strictlyDecreasingPrefix(begin, end, std::not_fn(comp)); } /** @@ -163,9 +163,9 @@ Iter weaklyIncreasingPrefix(Iter begin, Iter end, Comparator && comp = {}) * to the specified comparator. */ template>> -Iter weaklyIncreasingSuffix(Iter begin, Iter end, Comparator && comp = {}) +Iter weaklyIncreasingSuffix(Iter begin, Iter end, const Comparator & comp = {}) { - return strictlyDecreasingSuffix(begin, end, std::not_fn(std::forward(comp))); + return strictlyDecreasingSuffix(begin, end, std::not_fn(comp)); } /** diff --git a/src/libutil/include/nix/util/topo-sort.hh b/src/libutil/include/nix/util/topo-sort.hh index 6218b66a50..12031e0715 100644 --- a/src/libutil/include/nix/util/topo-sort.hh +++ b/src/libutil/include/nix/util/topo-sort.hh @@ -20,7 +20,7 @@ using TopoSortResult = std::variant, Cycle>; template F> requires std::same_as>, std::set> -TopoSortResult topoSort(std::set items, F && getChildren) +TopoSortResult topoSort(std::set items, const F & getChildren) { std::vector sorted; decltype(items) visited, parents; diff --git a/src/libutil/include/nix/util/util.hh b/src/libutil/include/nix/util/util.hh index 144c83cc0c..8d26ab1a1e 100644 --- a/src/libutil/include/nix/util/util.hh +++ b/src/libutil/include/nix/util/util.hh @@ -31,7 +31,7 @@ template auto concatStrings(Parts &&... parts) -> std::enable_if_t<(... && std::is_convertible_v), std::string> { - std::string_view views[sizeof...(parts)] = {parts...}; + std::string_view views[sizeof...(parts)] = {std::forward(parts)...}; return concatStringsSep({}, views); } diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index 9c96c5bcb7..26a7479050 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -95,11 +95,11 @@ void saveMountNamespace() { static std::once_flag done; std::call_once(done, []() { - fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY | O_CLOEXEC); if (!fdSavedMountNamespace) throw SysError("saving parent mount namespace"); - fdSavedRoot = open("/proc/self/root", O_RDONLY); + fdSavedRoot = open("/proc/self/root", O_RDONLY | O_CLOEXEC); }); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 58c78df349..66038e25e2 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -359,9 +359,15 @@ std::unique_ptr makeJSONLogger(const std::filesystem::path & path, bool } }; - AutoCloseFD fd = std::filesystem::is_socket(path) - ? connect(path) - : toDescriptor(open(path.string().c_str(), O_CREAT | O_APPEND | O_WRONLY, 0644)); + AutoCloseFD fd = std::filesystem::is_socket(path) ? connect(path) + : toDescriptor(open( + path.string().c_str(), + O_CREAT | O_APPEND | O_WRONLY +#ifndef _WIN32 + | O_CLOEXEC +#endif + , + 0644)); if (!fd) throw SysError("opening log file %1%", PathFmt(path)); diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc index 9181f4466e..f9417bb7e9 100644 --- a/src/libutil/unix/file-system.cc +++ b/src/libutil/unix/file-system.cc @@ -211,7 +211,7 @@ static void _deletePath( throw; } - int fd = openat(parentfd, name.rel_c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW); + int fd = openat(parentfd, name.rel_c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); if (fd == -1) throw SysError("opening directory %1%", PathFmt(path)); AutoCloseDir dir(fdopendir(fd)); From 8b974a35bddd047b8893d432ed951790765c4377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 26 Apr 2026 20:03:10 +0200 Subject: [PATCH 318/927] tests/functional/json: fix script(1) invocation for util-linux 2.42 util-linux 2.42 (commit 7268e79b) added "+" to the getopt string of script(1), so option parsing stops at the first non-option argument. The previous `script -e -q /dev/null -c CMD` ordering therefore treats `-c CMD` as extra positional arguments and fails with "unexpected number of arguments". Place all options before the argument, which works on both old and new util-linux. --- tests/functional/json.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/json.sh b/tests/functional/json.sh index 49992e0d93..157395e113 100644 --- a/tests/functional/json.sh +++ b/tests/functional/json.sh @@ -51,7 +51,7 @@ if type script &>/dev/null; then if [[ $acceptsCommandFlag -eq 0 ]]; then script -e -q /dev/null "$@" else - script -e -q /dev/null -c "$(shellEscapeArray "$@")" + script -e -q -c "$(shellEscapeArray "$@")" /dev/null fi } runScript nix eval --json --expr "{ a.b.c = true; }" > "$TEST_HOME/actual.json" From 456764609d0c696ce78758514cbeabab9917459d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 26 Apr 2026 20:53:13 +0200 Subject: [PATCH 319/927] doc/rl-next: add release note for blocking *listxattr in sandbox The seccomp filter change is user-visible: builds on SELinux hosts may now behave differently (correctly) and tools probing xattr support via listxattr will see ENOTSUP. Document this so users can trace behavior changes back to this PR. --- doc/manual/rl-next/seccomp-block-listxattr.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/manual/rl-next/seccomp-block-listxattr.md diff --git a/doc/manual/rl-next/seccomp-block-listxattr.md b/doc/manual/rl-next/seccomp-block-listxattr.md new file mode 100644 index 0000000000..2455ed4f82 --- /dev/null +++ b/doc/manual/rl-next/seccomp-block-listxattr.md @@ -0,0 +1,10 @@ +--- +synopsis: "Linux sandbox: also block `listxattr` syscalls" +prs: [15743] +--- + +The Linux sandbox now also returns `ENOTSUP` for `listxattr`, +`llistxattr` and `flistxattr`, matching the existing treatment of +`getxattr`/`setxattr`/`removexattr`. This prevents host xattrs (e.g. +`security.selinux`) from leaking into builds and fixes tools such as +`mkfs.ubifs` that probe xattr support via `listxattr`. From 02df27e1cff20c140ea216d0f7ff50325c4a9c40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 27 Apr 2026 18:54:01 +0200 Subject: [PATCH 320/927] rust-installer: 2.34.5 -> 2.34.6 2.34.6 dropped build.rs/env!() embedding in favour of appending the closure via scripts/pack post-build, so the Rust compile is independent of the embedded Nix and stays cacheable across revisions. Adapt the packaging accordingly. --- packaging/rust-installer/default.nix | 98 +++++++++++++++++----------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/packaging/rust-installer/default.nix b/packaging/rust-installer/default.nix index dcfbf4badc..98aec45e9e 100644 --- a/packaging/rust-installer/default.nix +++ b/packaging/rust-installer/default.nix @@ -4,61 +4,85 @@ { lib, stdenv, + buildPackages, + runCommand, rustPlatform, fetchFromGitHub, tarball, }: let - installerVersion = "2.34.5"; + installerVersion = "2.34.6"; src = fetchFromGitHub { owner = "NixOS"; repo = "nix-installer"; tag = installerVersion; - hash = "sha256-+gM241qQOzQlOnP0a7d47z3iRf9+yNjbBJCLIWWNX+c="; + hash = "sha256-aTaz8EtHexvke7tGr5MfeKy9g7AraIAFN+dPApm+fds="; }; -in -rustPlatform.buildRustPackage { - pname = "nix-installer"; - version = tarball.passthru.nixVersion; + # Bare binary: no Nix closure yet. Appended below via `pack`, so the + # (expensive) Rust compile is independent of the embedded Nix and + # stays cacheable across Nix revisions. + bare = rustPlatform.buildRustPackage { + pname = "nix-installer-bare"; + version = installerVersion; - inherit src; + inherit src; - cargoHash = "sha256-6pt2f7wznH672L5+SkbA5GA6Sxvk1KamAf3erGqZlLU="; + cargoHash = "sha256-/mNXkeZVuYsqd0TiUa7bzSP4xpKh0Fqga9EpasPbrzU="; - doCheck = false; + doCheck = false; - env = { - NIX_TARBALL_PATH = "${tarball}/nix.tar.zst"; - NIX_STORE_PATH = tarball.passthru.nixStorePath; - NSS_CACERT_STORE_PATH = tarball.passthru.cacertStorePath; - NIX_VERSION = tarball.passthru.nixVersion; - } - // lib.optionalAttrs stdenv.hostPlatform.isDarwin { - # Drop the unused libiconv dylib the darwin stdenv injects; the - # binary must run before `/nix/store` exists. - NIX_LDFLAGS = "-dead_strip_dylibs"; + env = lib.optionalAttrs stdenv.hostPlatform.isDarwin { + # Drop the unused libiconv dylib the darwin stdenv injects; the + # binary must run before `/nix/store` exists. + NIX_LDFLAGS = "-dead_strip_dylibs"; + }; + + postInstall = '' + install -m755 nix-installer.sh $out/bin/nix-installer.sh + ''; }; +in - postInstall = '' - install -m755 nix-installer.sh $out/bin/nix-installer.sh +runCommand "nix-installer-${tarball.passthru.nixVersion}" + { + nativeBuildInputs = [ + buildPackages.python3 + ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + buildPackages.darwin.sigtool + buildPackages.darwin.cctools + ]; - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix-installer" >> $out/nix-support/hydra-build-products - echo "file binary-dist $out/bin/nix-installer.sh" >> $out/nix-support/hydra-build-products - ''; + # The appended payload contains store-path strings on purpose; don't + # let the reference scanner pull the whole Nix closure into this + # derivation's runtime closure. + __structuredAttrs = true; + unsafeDiscardReferences.out = true; - # The binary embeds store-path strings (`NIX_STORE_PATH`, …) on - # purpose; don't let the reference scanner pull the whole Nix - # closure into this derivation's runtime closure. - __structuredAttrs = true; - unsafeDiscardReferences.out = true; + passthru = { inherit bare; }; - meta = { - description = "Rust-based Nix installer with an embedded Nix ${tarball.passthru.nixVersion}"; - homepage = "https://github.com/NixOS/nix-installer"; - license = lib.licenses.lgpl21Only; - mainProgram = "nix-installer"; - }; -} + meta = { + description = "Rust-based Nix installer with an embedded Nix ${tarball.passthru.nixVersion}"; + homepage = "https://github.com/NixOS/nix-installer"; + license = lib.licenses.lgpl21Only; + mainProgram = "nix-installer"; + }; + } + '' + mkdir -p $out/bin $out/nix-support + + python3 ${src}/scripts/pack \ + --input ${bare}/bin/nix-installer \ + --tarball ${tarball}/nix.tar.zst \ + --nix-store-path ${tarball.passthru.nixStorePath} \ + --cacert-store-path ${tarball.passthru.cacertStorePath} \ + --nix-version ${tarball.passthru.nixVersion} \ + --output $out/bin/nix-installer + + install -m755 ${bare}/bin/nix-installer.sh $out/bin/nix-installer.sh + + echo "file binary-dist $out/bin/nix-installer" >> $out/nix-support/hydra-build-products + echo "file binary-dist $out/bin/nix-installer.sh" >> $out/nix-support/hydra-build-products + '' From 498f96d1457de6d96b7e2b6aeeaee64c34c3a1f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 27 Apr 2026 21:35:04 +0200 Subject: [PATCH 321/927] libutil: Use poll() in FdSource::hasData() to avoid fd_set overflow FD_SET writes past the stack fd_set when fd >= FD_SETSIZE. hasData() runs before every frame in withFramedSink(), so clients with many open fds would corrupt the stack (or abort under glibc _FORTIFY_SOURCE) during addToStore(). poll() has no such limit; Windows keeps select() since its fd_set is a bounded handle array. --- src/libutil/serialise.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 6c77c15fe5..4ff2d63f2f 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -224,6 +224,9 @@ bool FdSource::hasData() return true; while (true) { +#ifdef _WIN32 + /* Windows' fd_set is a bounded handle array, so FD_SET can't + overflow; on Unix use poll() since fd may exceed FD_SETSIZE. */ fd_set fds; FD_ZERO(&fds); Socket sock = toSocket(fd); @@ -240,6 +243,20 @@ bool FdSource::hasData() throw SysError("polling file descriptor"); } return FD_ISSET(sock, &fds); +#else + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + pfd.revents = 0; + + auto n = poll(&pfd, 1, 0); + if (n < 0) { + if (errno == EINTR) + continue; + throw SysError("polling file descriptor"); + } + return n > 0 && (pfd.revents & (POLLIN | POLLHUP | POLLERR)) != 0; +#endif } } From 934a7af2877ac60207453bdf353092cf267a6ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 27 Apr 2026 23:27:08 +0200 Subject: [PATCH 322/927] dependabot: enable Nix ecosystem for flake.lock updates Dependabot recently gained support for updating Nix flake inputs. Enable it so we get automated weekly bumps of flake.lock instead of having to update inputs manually. Group all flake inputs into a single pull request to avoid a flood of one-PR-per-input updates and to keep CI load and review effort low. --- .github/dependabot.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5ace4600a1..1880a89da5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,11 @@ updates: directory: "/" schedule: interval: "weekly" + - package-ecosystem: "nix" + directory: "/" + schedule: + interval: "weekly" + groups: + flake-inputs: + patterns: + - "*" From f2720c2037a2fdb241699c855d8d1cd387032bf3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 22:54:30 +0000 Subject: [PATCH 323/927] build(deps): bump cachix/install-nix-action from 31.10.4 to 31.10.5 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.10.4 to 31.10.5. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/616559265b40713947b9c190a8ff4b507b5df49b...ab739621df7a23f52766f9ccc97f38da6b7af14f) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.10.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07329d16a2..49fff49b97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,7 +189,7 @@ jobs: - name: Looking up the installer tarball URL id: installer-tarball-url run: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 + - uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5 if: ${{ !matrix.rust-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} @@ -255,7 +255,7 @@ jobs: id: installer-tarball-url run: | echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@616559265b40713947b9c190a8ff4b507b5df49b # v31.10.4 + - uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5 with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} From 741de5d7481207eb2d761c17c3642711235d589e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 22:55:24 +0000 Subject: [PATCH 324/927] build(deps): bump the flake-inputs group with 2 updates Bumps the flake-inputs group with 2 updates: [flake-parts](https://github.com/hercules-ci/flake-parts) and [git-hooks-nix](https://github.com/cachix/git-hooks.nix). Updates `flake-parts` from `205b12d` to `3107b77` - [Commits](https://github.com/hercules-ci/flake-parts/compare/205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9...3107b77cd68437b9a76194f0f7f9c55f2329ca5b) Updates `git-hooks-nix` from `aa9f40c` to `3cfd774` - [Commits](https://github.com/cachix/git-hooks.nix/compare/aa9f40c906904ebd83da78e7f328cd8aeaeae785...3cfd774b0a530725a077e17354fbdb87ea1c4aad) --- updated-dependencies: - dependency-name: flake-parts dependency-version: 3107b77cd68437b9a76194f0f7f9c55f2329ca5b dependency-type: direct:production dependency-group: flake-inputs - dependency-name: git-hooks-nix dependency-version: 3cfd774b0a530725a077e17354fbdb87ea1c4aad dependency-type: direct:production dependency-group: flake-inputs ... Signed-off-by: dependabot[bot] --- flake.lock | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 4c0bf91927..1212049c3b 100644 --- a/flake.lock +++ b/flake.lock @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1733312601, - "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", + "lastModified": 1775087534, + "narHash": "sha256-91qqW8lhL7TLwgQWijoGBbiD4t7/q75KTi8NxjVmSmA=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", + "rev": "3107b77cd68437b9a76194f0f7f9c55f2329ca5b", "type": "github" }, "original": { @@ -42,17 +42,14 @@ "gitignore": [], "nixpkgs": [ "nixpkgs" - ], - "nixpkgs-stable": [ - "nixpkgs" ] }, "locked": { - "lastModified": 1734279981, - "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", + "lastModified": 1776796298, + "narHash": "sha256-PcRvlWayisPSjd0UcRQbhG8Oqw78AcPE6x872cPRHN8=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", + "rev": "3cfd774b0a530725a077e17354fbdb87ea1c4aad", "type": "github" }, "original": { From f63f6037c921af7da9cf4eaa1bb12f9a30981251 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 22 Apr 2026 15:45:23 -0400 Subject: [PATCH 325/927] Misc gc/deletion improvements / fixes - Improve deletion logging - Fix informational messages from being duplicated - Add some debugging messages useful for diagnosing why a path wasn't deleted - Use `.contains` instead of `.count` - Add --skip-live alias to match Lix (be forgiving to people used to Lix flags and support --skip-live as an alias for --skip-alive. - Exclude outputs and drvs from deletion if passed a StorePathSet. - gcDeleteSpecific is no longer an indicator of operating on specific paths, rather the presence of a StorePathSet in pathsToDelete is. - Clarify gc comments around keep-derivations and keep-outputs Signed-off-by: Lisanna Dettwyler --- src/libstore/gc.cc | 56 ++++++++++++++++++++++++----------------- src/nix/store-delete.cc | 1 + 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 835d068646..701fc66e69 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -385,8 +385,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Using `--ignore-liveness' with `--delete' can have unintended consequences if `keep-outputs' or `keep-derivations' are true (the garbage collector will recurse into deleting the outputs - or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + or derivers, respectively, even if they aren't in the + pathsToDelete). So disable them. */ + if (std::holds_alternative(options.pathsToDelete) && options.ignoreLiveness) { keepOutputs = false; keepDerivations = false; } @@ -608,14 +609,15 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Bail out if we've previously discovered that this path is alive. */ - if (alive.count(*path)) { + if (alive.contains(*path)) { + debug("cannot delete '%s' because '%s' is alive", printStorePath(start), printStorePath(*path)); alive.insert(start); return; } /* If we've previously deleted this path, we don't have to handle it again. */ - if (dead.count(*path)) + if (dead.contains(*path)) continue; auto markAlive = [&]() { @@ -636,19 +638,24 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) }; /* If this is a root, bail out. */ - if (roots.count(*path)) { + if (roots.contains(*path)) { debug("cannot delete '%s' because it's a root", printStorePath(*path)); return markAlive(); } if (std::holds_alternative(options.pathsToDelete) - && !std::get(options.pathsToDelete).contains(*path)) + && !std::get(options.pathsToDelete).contains(*path)) { + debug( + "cannot delete '%s' because '%s' is not in the specified paths to delete", + printStorePath(start), + printStorePath(*path)); return; + } { auto hashPart = path->hashPart(); auto shared(_shared.lock()); - if (shared->tempRoots.count(hashPart)) { + if (shared->tempRoots.contains(hashPart)) { debug("cannot delete '%s' because it's a temporary root", printStorePath(*path)); return markAlive(); } @@ -668,8 +675,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) for (auto & p : i->second) enqueue(p); - /* If keep-derivations is set and this is a - derivation, then visit the derivation outputs. */ + /* If keep-derivations is set and this is a derivation, then we only want to delete this derivation if + * we can also delete all its outputs, so visit the derivation outputs. */ if (keepDerivations && path->isDerivation()) { for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) if (maybeOutPath && isValidPath(*maybeOutPath) @@ -677,7 +684,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) enqueue(*maybeOutPath); } - /* If keep-outputs is set, then visit the derivers. */ + /* If keep-outputs is set, we only want to delete this path if we + * can also delete its derivers, so visit the derivers. */ if (keepOutputs) { auto derivers = queryValidDerivers(*path); for (auto & i : derivers) @@ -707,27 +715,29 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) std::visit( overloaded{ [&](const StorePathSet & paths) { - for (auto & i : paths) { - switch (options.action) { - case GCOptions::gcDeleteDead: - printInfo("deleting garbage within specified paths..."); - break; - case GCOptions::gcDeleteSpecific: - printInfo("deleting specified paths..."); - break; - case GCOptions::gcReturnDead: - case GCOptions::gcReturnLive: - printInfo("determining live/dead paths..."); - } + switch (options.action) { + case GCOptions::gcDeleteDead: + printInfo("deleting garbage within specified paths..."); + break; + case GCOptions::gcDeleteSpecific: + printInfo("deleting specified paths..."); + break; + case GCOptions::gcReturnDead: + case GCOptions::gcReturnLive: + printInfo("determining live/dead paths..."); + } + for (auto & i : paths) { maybeDeleteReferrersClosure(i); - if (options.action == GCOptions::gcDeleteSpecific && !dead.count(i)) + if (options.action == GCOptions::gcDeleteSpecific && !dead.contains(i)) throw Error( "Cannot delete path '%1%' since it is still alive. " "To find out why, use: " "nix-store --query --roots and nix-store --query --referrers", printStorePath(i)); + else if (!dead.contains(i)) + debug("cannot delete '%s' because it's still alive", printStorePath(i)); } }, [&](const GCOptions::WholeStore & _) { diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 5eef2d1ad2..a1a3878984 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -20,6 +20,7 @@ struct CmdStoreDelete : StorePathsCommand addFlag({ .longName = "skip-alive", + .aliases = {"skip-live"}, .description = "Do not emit errors when attempting to delete something that is still alive, useful with --recursive.", .handler = {&options.action, GCOptions::gcDeleteDead}, From 87032c6dc4b539a0fc9bb50964965bc6c3554532 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 28 Apr 2026 23:52:19 +0300 Subject: [PATCH 326/927] libstore: Drop queryMissing from Worker::run This is no longer needed, since PathSubstitutionGoal now queries ValidPathInfo asynchronously and we can start the builds right away without querying the whole closure at once. This can significantly speed up the build startup. This was added in back in bbdf08bc0facb5157a10c794712dae7e5902be03 when pathinfo queries were done sequentially and in a blocking manner - this is not the case anymore. The asynchronous queries don't wait for a build slot to start, so the concurrency there is not limited and all path queries can complete as the worker loop naturally. This also acts as natural rate limiting to avoid allocating too many coroutines up-front and blowing up the memory usage (which also slows down the whole build to a crawl because fork() starts taking increasingly longer to complete due large anonymous memory mappings). --- src/libstore/build/worker.cc | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 3fe196d080..208bf5ca3e 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -295,28 +295,11 @@ void Worker::waitForCompletion(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - std::vector topPaths; - - for (auto & i : _topGoals) { - topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back( - DerivedPath::Built{ - .drvPath = goal->drvReq, - .outputs = goal->wantedOutputs, - }); - } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Opaque{goal->storePath}); - } - } - - /* Call queryMissing() to efficiently query substitutes. */ - store.queryMissing(topPaths); - debug("entered goal loop"); + for (std::shared_ptr goal : _topGoals) + topGoals.insert(std::move(goal)); while (1) { - checkInterrupt(); // TODO GC interface? From 8c03190cea1b479246514006a55eae4bdfd844c3 Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Sun, 26 Apr 2026 18:47:53 +0200 Subject: [PATCH 327/927] include in src/libfetchers/fetchers.cc `sleep_for` is defined in --- src/libfetchers/fetchers.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 4eedafa22a..310373a19a 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -11,6 +11,7 @@ #include "nix/store/pathlocks.hh" #include "nix/util/environment-variables.hh" +#include #include namespace nix::fetchers { From 53a90079c1293fc07293b7f2e5df2c69c5df79a3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 29 Apr 2026 20:52:06 +0300 Subject: [PATCH 328/927] libstore: Anchor all Store and StoreConfig vtables Classes with virtual functions defined in headers must have at least one out-of-line definition of a virtual function (also called "key function" [3]) to prevent having the vtable from having weak (vague [2]) linkage and causing a bunch of headaches on Darwin. For now this just limits to fixes for what would be required for nix-heuristic-gc, which compiles with pybind11 which uses hidden visibility for the dylib. In the future we'd certainly want to enable -Werror=weak-vtables to prevent such footguns. The pattern used here is basically the same as what LLVM does [1]. [1]: https://github.com/llvm/llvm-project/blob/710c297289b751857bdc0a8677437d575b403922/clang/lib/ExtractAPI/API.cpp#L177-L200 [2]: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#vague [3]: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#vague-vtable --- src/libstore/binary-cache-store.cc | 4 ++++ src/libstore/common-ssh-store-config.cc | 2 ++ src/libstore/dummy-store.cc | 10 ++++++++++ src/libstore/http-binary-cache-store.cc | 4 ++++ .../include/nix/store/binary-cache-store.hh | 6 ++++++ .../include/nix/store/common-ssh-store-config.hh | 4 ++++ .../include/nix/store/dummy-store-impl.hh | 4 ++++ src/libstore/include/nix/store/dummy-store.hh | 4 ++++ src/libstore/include/nix/store/gc-store.hh | 4 ++++ .../include/nix/store/http-binary-cache-store.hh | 6 ++++++ .../include/nix/store/indirect-root-store.hh | 4 ++++ .../include/nix/store/legacy-ssh-store.hh | 8 ++++++++ .../nix/store/local-binary-cache-store.hh | 4 ++++ src/libstore/include/nix/store/local-fs-store.hh | 6 ++++++ .../include/nix/store/local-overlay-store.hh | 6 ++++++ src/libstore/include/nix/store/local-store.hh | 6 +++++- src/libstore/include/nix/store/log-store.hh | 4 ++++ src/libstore/include/nix/store/remote-store.hh | 8 ++++++++ src/libstore/include/nix/store/ssh-store.hh | 8 ++++++++ src/libstore/include/nix/store/store-api.hh | 10 ++++++++++ .../include/nix/store/uds-remote-store.hh | 8 ++++++++ src/libstore/indirect-root-store.cc | 2 ++ src/libstore/legacy-ssh-store.cc | 4 ++++ src/libstore/local-binary-cache-store.cc | 8 ++++++++ src/libstore/local-fs-store.cc | 4 ++++ src/libstore/local-overlay-store.cc | 4 ++++ src/libstore/local-store.cc | 8 ++++++++ src/libstore/log-store.cc | 2 ++ src/libstore/remote-store.cc | 4 ++++ src/libstore/restricted-store.cc | 6 ++++++ src/libstore/ssh-store.cc | 16 ++++++++++++++++ src/libstore/uds-remote-store.cc | 4 ++++ 32 files changed, 181 insertions(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 64fe33536b..8ceb8f2151 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -22,6 +22,10 @@ namespace nix { +void BinaryCacheStoreConfig::anchor() {} + +void BinaryCacheStore::anchor() {} + BinaryCacheStore::BinaryCacheStore(Config & config) : config{config} { diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index ee1d3bf8ac..db36771514 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -9,6 +9,8 @@ CommonSSHStoreConfig::CommonSSHStoreConfig(const ParsedURL::Authority & authorit { } +void CommonSSHStoreConfig::anchor() {} + SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const { return { diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 052ec9b128..2ee093d85b 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -10,6 +10,10 @@ namespace nix { +void DummyStoreConfig::anchor() {} + +void DummyStore::anchor() {} + std::string DummyStoreConfig::doc() { return @@ -126,6 +130,10 @@ bool DummyStoreConfig::getReadOnly() const struct DummyStoreImpl : DummyStore { +private: + void anchor() override; + +public: using Config = DummyStoreConfig; /** @@ -378,6 +386,8 @@ struct DummyStoreImpl : DummyStore } }; +void DummyStoreImpl::anchor() {} + ref DummyStore::Config::openDummyStore() const { return make_ref(ref{shared_from_this()}); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 713de4969a..8f0c22f998 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -23,6 +23,10 @@ StringSet HttpBinaryCacheStoreConfig::uriSchemes() return ret; } +void HttpBinaryCacheStoreConfig::anchor() {} + +void HttpBinaryCacheStore::anchor() {} + HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig(ParsedURL _cacheUri, const Params & params) : StoreConfig(params, FilePathType::Unix) , BinaryCacheStoreConfig(params) diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 7871ad03c8..3e8cf886c8 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -16,6 +16,10 @@ class RemoteFSAccessor; struct BinaryCacheStoreConfig : virtual StoreConfig { +private: + void anchor() override; + +public: BinaryCacheStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) { @@ -92,6 +96,8 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ Config & config; private: + void anchor() override; + std::vector> signers; protected: diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index 1e90c94afc..b622e82e26 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -10,6 +10,10 @@ class SSHMaster; struct CommonSSHStoreConfig : virtual StoreConfig { +private: + void anchor() override; + +public: CommonSSHStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) { diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh index 8fdeeb3625..bec77a6bee 100644 --- a/src/libstore/include/nix/store/dummy-store-impl.hh +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -15,6 +15,10 @@ struct MemorySourceAccessor; */ struct DummyStore : virtual Store { +private: + void anchor() override; + +public: using Config = DummyStoreConfig; ref config; diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index f76fb3d5c2..c8a212c756 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -12,6 +12,10 @@ struct DummyStore; struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig { +private: + void anchor() override; + +public: DummyStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) { diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 5e23f20524..015478e79f 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -106,6 +106,10 @@ struct GCResults */ struct GcStore : public virtual Store { +private: + void anchor() override; + +public: inline static std::string operationName = "Garbage collection"; /** diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index 748daec646..12465261ca 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -14,6 +14,10 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, virtual CommonSSHStoreConfig { +private: + void anchor() override; + +public: LegacySSHStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) , CommonSSHStoreConfig(params) @@ -63,6 +67,10 @@ struct LegacySSHStoreConfig : std::enable_shared_from_this struct LegacySSHStore : public virtual Store { +private: + void anchor() override; + +public: using Config = LegacySSHStoreConfig; ref config; diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index 69a4bac1c8..181b33e4bd 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -9,6 +9,10 @@ struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this> makeRootDirSetting(LocalFSStoreConfig & self, std::optional defaultValue) { @@ -87,6 +89,10 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ virtual GcStore, virtual LogStore { +private: + void anchor() override; + +public: using Config = LocalFSStoreConfig; const Config & config; diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index dfb1fb184a..10b04937c3 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -7,6 +7,10 @@ namespace nix { */ struct LocalOverlayStoreConfig : virtual LocalStoreConfig { +private: + void anchor() override; + +public: LocalOverlayStoreConfig(const StringMap & params) : LocalOverlayStoreConfig("", params) { @@ -119,6 +123,8 @@ struct LocalOverlayStore : virtual LocalStore LocalOverlayStore(ref); private: + void anchor() override; + /** * The store beneath us. * diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index bf3437e957..e5ecaf8a59 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -35,8 +35,9 @@ struct LocalSettings; struct LocalBuildStoreConfig : virtual LocalFSStoreConfig { - private: + void anchor() override; + /** Input for computing the build directory. See `getBuildDir()`. */ @@ -89,6 +90,7 @@ struct LocalStoreConfig : std::enable_shared_from_this, LocalStoreConfig(const std::filesystem::path & path, const Params & params); private: + void anchor() override; /** * An indirection so that we don't need to refer to global settings @@ -177,6 +179,8 @@ public: class LocalStore : public virtual IndirectRootStore, public virtual GcStore { + void anchor() override; + public: using Config = LocalStoreConfig; diff --git a/src/libstore/include/nix/store/log-store.hh b/src/libstore/include/nix/store/log-store.hh index 2d81d02b10..e0acd9a04f 100644 --- a/src/libstore/include/nix/store/log-store.hh +++ b/src/libstore/include/nix/store/log-store.hh @@ -7,6 +7,10 @@ namespace nix { struct LogStore : public virtual Store { +private: + void anchor() override; + +public: inline static std::string operationName = "Build log storage and retrieval"; /** diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index 57beb9135f..144b4b8e43 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -23,6 +23,10 @@ class RemoteFSAccessor; struct RemoteStoreConfig : virtual StoreConfig { +private: + void anchor() override; + +public: RemoteStoreConfig(const Params & params, FilePathType pathType) : StoreConfig(params, pathType) { @@ -44,6 +48,10 @@ struct RemoteStoreConfig : virtual StoreConfig */ struct RemoteStore : public virtual Store, public virtual GcStore, public virtual LogStore { +private: + void anchor() override; + +public: using Config = RemoteStoreConfig; const Config & config; diff --git a/src/libstore/include/nix/store/ssh-store.hh b/src/libstore/include/nix/store/ssh-store.hh index 4ab88ca74c..324e85eb60 100644 --- a/src/libstore/include/nix/store/ssh-store.hh +++ b/src/libstore/include/nix/store/ssh-store.hh @@ -12,6 +12,10 @@ struct SSHStoreConfig : std::enable_shared_from_this, virtual RemoteStoreConfig, virtual CommonSSHStoreConfig { +private: + void anchor() override; + +public: SSHStoreConfig(const Params & params) : StoreConfig(params, FilePathType::Unix) , RemoteStoreConfig(params, FilePathType::Unix) @@ -43,6 +47,10 @@ struct SSHStoreConfig : std::enable_shared_from_this, struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig { +private: + void anchor() override; + +public: MountedSSHStoreConfig(StringMap params); MountedSSHStoreConfig(const ParsedURL::Authority & authority, StringMap params); diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 97251eaee9..bfd4ffce21 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -229,6 +229,12 @@ public: */ struct StoreConfig : public StoreConfigBase, public StoreDirConfig { +private: + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor() = 0; + +public: using Params = StoreReference::Params; StoreConfig(const Params & params, FilePathType pathType); @@ -380,6 +386,10 @@ struct StoreConfig : public StoreConfigBase, public StoreDirConfig */ class Store : public std::enable_shared_from_this, public StoreDirConfig { + /* VTable anchor to avoid weak linkage of the vtable - it breaks + dynamic_cast across shared libraries on Darwin. */ + virtual void anchor() = 0; + public: using Config = StoreConfig; diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index 43931c628a..8359f8fdf4 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -27,6 +27,10 @@ struct UDSRemoteStoreConfig : std::enable_shared_from_this virtual LocalFSStoreConfig, virtual RemoteStoreConfig { +private: + void anchor() override; + +public: UDSRemoteStoreConfig(const std::filesystem::path & path, const Params & params); UDSRemoteStoreConfig(const Params & params); @@ -57,6 +61,10 @@ struct UDSRemoteStoreConfig : std::enable_shared_from_this struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore { +private: + void anchor() override; + +public: using Config = UDSRemoteStoreConfig; ref config; diff --git a/src/libstore/indirect-root-store.cc b/src/libstore/indirect-root-store.cc index 7384456e28..b203afb63d 100644 --- a/src/libstore/indirect-root-store.cc +++ b/src/libstore/indirect-root-store.cc @@ -2,6 +2,8 @@ namespace nix { +void IndirectRootStore::anchor() {} + void IndirectRootStore::makeSymlink(const std::filesystem::path & link, const std::filesystem::path & target) { /* Create directories up to `gcRoot'. */ diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a00914c449..cbcc42dbed 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -24,6 +24,8 @@ LegacySSHStoreConfig::LegacySSHStoreConfig(const ParsedURL::Authority & authorit { } +void LegacySSHStoreConfig::anchor() {} + std::string LegacySSHStoreConfig::doc() { return @@ -37,6 +39,8 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; +void LegacySSHStore::anchor() {} + LegacySSHStore::LegacySSHStore(ref config) : Store{*config} , config{config} diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 60298efdbb..4d79f613e4 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -52,6 +52,10 @@ StoreReference LocalBinaryCacheStoreConfig::getReference() const struct LocalBinaryCacheStore : virtual BinaryCacheStore { +private: + void anchor() override; + +public: using Config = LocalBinaryCacheStoreConfig; ref config; @@ -139,6 +143,10 @@ StringSet LocalBinaryCacheStoreConfig::uriSchemes() return {"file"}; } +void LocalBinaryCacheStoreConfig::anchor() {} + +void LocalBinaryCacheStore::anchor() {} + ref LocalBinaryCacheStoreConfig::openStore() const { auto store = make_ref( diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 3fc724f5fe..77525d5416 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -5,6 +5,10 @@ namespace nix { +void LocalFSStoreConfig::anchor() {} + +void LocalFSStore::anchor() {} + LocalFSStoreConfig::LocalFSStoreConfig(const std::filesystem::path & rootDir, const Params & params) : StoreConfig(params, FilePathType::Native) /* Default `?root` from `rootDir` if non set diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 8d1a16f912..c0fee83d7d 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -10,6 +10,10 @@ namespace nix { +void LocalOverlayStoreConfig::anchor() {} + +void LocalOverlayStore::anchor() {} + std::string LocalOverlayStoreConfig::doc() { return diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 75f485c31f..53d7456f94 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -56,6 +56,14 @@ namespace nix { +void LocalStoreConfig::anchor() {} + +void LocalBuildStoreConfig::anchor() {} + +void LocalStore::anchor() {} + +void GcStore::anchor() {} + LocalStoreConfig::LocalStoreConfig(const std::filesystem::path & path, const Params & params) : StoreConfig(params, FilePathType::Native) , LocalFSStoreConfig(path, params) diff --git a/src/libstore/log-store.cc b/src/libstore/log-store.cc index fd03bb30ea..23e6563991 100644 --- a/src/libstore/log-store.cc +++ b/src/libstore/log-store.cc @@ -2,6 +2,8 @@ namespace nix { +void LogStore::anchor() {} + std::optional LogStore::getBuildLog(const StorePath & path) { auto maybePath = getBuildDerivationPath(path); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 88c3847bb3..a2fb3a10d2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -30,6 +30,8 @@ namespace nix { +void RemoteStoreConfig::anchor() {} + /* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Config & config) : Store{config} @@ -59,6 +61,8 @@ RemoteStore::RemoteStore(const Config & config) { } +void RemoteStore::anchor() {} + ref RemoteStore::openConnectionWrapper() { if (failed) { diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 03c130da96..eaeca59a2b 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -38,6 +38,10 @@ bool RestrictionContext::isAllowed(const DerivedPath & req) */ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStore { +private: + void anchor() override; + +public: ref config; ref next; @@ -157,6 +161,8 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor } }; +void RestrictedStore::anchor() {} + ref makeRestrictedStore(ref config, ref next, RestrictionContext & context) { return make_ref(config, next, context); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 945271b1a9..4138be6e38 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -17,6 +17,10 @@ SSHStoreConfig::SSHStoreConfig(const ParsedURL::Authority & authority, const Par { } +void SSHStoreConfig::anchor() {} + +void MountedSSHStoreConfig::anchor() {} + std::string SSHStoreConfig::doc() { return @@ -39,6 +43,10 @@ StoreReference SSHStoreConfig::getReference() const struct alignas(8) /* Work around ASAN failures on i686-linux. */ SSHStore : virtual RemoteStore { +private: + void anchor() override; + +public: using Config = SSHStoreConfig; ref config; @@ -87,6 +95,8 @@ struct alignas(8) /* Work around ASAN failures on i686-linux. */ }; }; +void SSHStore::anchor() {} + MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) : StoreConfig(params, FilePathType::Native) , RemoteStoreConfig(params, FilePathType::Native) @@ -128,6 +138,10 @@ std::string MountedSSHStoreConfig::doc() */ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore { +private: + void anchor() override; + +public: using Config = MountedSSHStoreConfig; MountedSSHStore(ref config) @@ -187,6 +201,8 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore } }; +void MountedSSHStore::anchor() {} + ref SSHStore::Config::openStore() const { return make_ref(ref{shared_from_this()}); diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index e0b2d68ec3..c4137df7c6 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -26,6 +26,10 @@ std::filesystem::path getDaemonSocketPath(const Store::Config & config) .value_or(config.getStateDir() / "daemon-socket" / "socket"); } +void UDSRemoteStoreConfig::anchor() {} + +void UDSRemoteStore::anchor() {} + UDSRemoteStoreConfig::UDSRemoteStoreConfig(const std::filesystem::path & path, const StoreReference::Params & params) : Store::Config{params, FilePathType::Native} , LocalFSStore::Config{params} From 80416579477c30db201b6f9f0e0d19428eab839b Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Tue, 28 Apr 2026 19:36:45 +0200 Subject: [PATCH 329/927] Migrate C++ error trace tests into functional/lang tests. Migrated 138 language-level error trace tests from C++ unit tests to functional characterization tests in tests/functional/lang/. Some tests were commented out due to insufficiant MACROs, but they work well as functional lang tests. Some tests had duplicates, only one copy is kept. The remaning C++ tests (TraceBuilder and NestedThrows) were kept as is, while ~1200 lines were removed from src/libexpr-tests/error_traces.cc. --- src/libexpr-tests/error_traces.cc | 1266 ----------------- tests/functional/lang/eval-fail-add-1.err.exp | 10 + tests/functional/lang/eval-fail-add-1.nix | 1 + tests/functional/lang/eval-fail-add-2.err.exp | 10 + tests/functional/lang/eval-fail-add-2.nix | 1 + tests/functional/lang/eval-fail-all-1.err.exp | 10 + tests/functional/lang/eval-fail-all-1.nix | 1 + tests/functional/lang/eval-fail-all-2.err.exp | 10 + tests/functional/lang/eval-fail-all-2.nix | 1 + tests/functional/lang/eval-fail-all-3.err.exp | 10 + tests/functional/lang/eval-fail-all-3.nix | 1 + tests/functional/lang/eval-fail-any-1.err.exp | 10 + tests/functional/lang/eval-fail-any-1.nix | 1 + tests/functional/lang/eval-fail-any-2.err.exp | 10 + tests/functional/lang/eval-fail-any-2.nix | 1 + tests/functional/lang/eval-fail-any-3.err.exp | 10 + tests/functional/lang/eval-fail-any-3.nix | 1 + .../lang/eval-fail-attrNames-1.err.exp | 10 + .../functional/lang/eval-fail-attrNames-1.nix | 1 + .../lang/eval-fail-attrValues-1.err.exp | 10 + .../lang/eval-fail-attrValues-1.nix | 1 + .../lang/eval-fail-baseNameOf-1.err.exp | 10 + .../lang/eval-fail-baseNameOf-1.nix | 1 + .../lang/eval-fail-bitAnd-1.err.exp | 10 + tests/functional/lang/eval-fail-bitAnd-1.nix | 1 + .../lang/eval-fail-bitAnd-2.err.exp | 10 + tests/functional/lang/eval-fail-bitAnd-2.nix | 1 + .../functional/lang/eval-fail-bitOr-1.err.exp | 10 + tests/functional/lang/eval-fail-bitOr-1.nix | 1 + .../functional/lang/eval-fail-bitOr-2.err.exp | 10 + tests/functional/lang/eval-fail-bitOr-2.nix | 1 + .../lang/eval-fail-bitXor-1.err.exp | 10 + tests/functional/lang/eval-fail-bitXor-1.nix | 1 + .../lang/eval-fail-bitXor-2.err.exp | 10 + tests/functional/lang/eval-fail-bitXor-2.nix | 1 + .../lang/eval-fail-catAttrs-1.err.exp | 10 + .../functional/lang/eval-fail-catAttrs-1.nix | 1 + .../lang/eval-fail-catAttrs-2.err.exp | 10 + .../functional/lang/eval-fail-catAttrs-2.nix | 1 + .../lang/eval-fail-catAttrs-3.err.exp | 10 + .../functional/lang/eval-fail-catAttrs-3.nix | 1 + .../lang/eval-fail-catAttrs-4.err.exp | 10 + .../functional/lang/eval-fail-catAttrs-4.nix | 5 + .../functional/lang/eval-fail-ceil-1.err.exp | 10 + tests/functional/lang/eval-fail-ceil-1.nix | 1 + .../lang/eval-fail-compareVersions-1.err.exp | 10 + .../lang/eval-fail-compareVersions-1.nix | 1 + .../lang/eval-fail-compareVersions-2.err.exp | 10 + .../lang/eval-fail-compareVersions-2.nix | 1 + .../lang/eval-fail-concatLists-1.err.exp | 10 + .../lang/eval-fail-concatLists-1.nix | 1 + .../lang/eval-fail-concatLists-2.err.exp | 10 + .../lang/eval-fail-concatLists-2.nix | 1 + .../lang/eval-fail-concatLists-3.err.exp | 10 + .../lang/eval-fail-concatLists-3.nix | 4 + .../lang/eval-fail-concatMap-1.err.exp | 10 + .../functional/lang/eval-fail-concatMap-1.nix | 1 + .../lang/eval-fail-concatMap-2.err.exp | 10 + .../functional/lang/eval-fail-concatMap-2.nix | 1 + .../lang/eval-fail-concatMap-3.err.exp | 14 + .../functional/lang/eval-fail-concatMap-3.nix | 1 + .../lang/eval-fail-concatMap-4.err.exp | 14 + .../functional/lang/eval-fail-concatMap-4.nix | 4 + .../lang/eval-fail-concatStringsSep-1.err.exp | 10 + .../lang/eval-fail-concatStringsSep-1.nix | 1 + .../lang/eval-fail-concatStringsSep-2.err.exp | 10 + .../lang/eval-fail-concatStringsSep-2.nix | 1 + .../lang/eval-fail-concatStringsSep-3.err.exp | 10 + .../lang/eval-fail-concatStringsSep-3.nix | 5 + .../lang/eval-fail-derivationStrict-1.err.exp | 10 + .../lang/eval-fail-derivationStrict-1.nix | 1 + .../eval-fail-derivationStrict-10.err.exp | 18 + .../lang/eval-fail-derivationStrict-10.nix | 6 + .../eval-fail-derivationStrict-11.err.exp | 18 + .../lang/eval-fail-derivationStrict-11.nix | 6 + .../eval-fail-derivationStrict-12.err.exp | 18 + .../lang/eval-fail-derivationStrict-12.nix | 5 + .../eval-fail-derivationStrict-13.err.exp | 18 + .../lang/eval-fail-derivationStrict-13.nix | 6 + .../eval-fail-derivationStrict-14.err.exp | 18 + .../lang/eval-fail-derivationStrict-14.nix | 6 + .../eval-fail-derivationStrict-15.err.exp | 18 + .../lang/eval-fail-derivationStrict-15.nix | 9 + .../eval-fail-derivationStrict-16.err.exp | 18 + .../lang/eval-fail-derivationStrict-16.nix | 7 + .../eval-fail-derivationStrict-17.err.exp | 18 + .../lang/eval-fail-derivationStrict-17.nix | 7 + .../eval-fail-derivationStrict-19.err.exp | 18 + .../lang/eval-fail-derivationStrict-19.nix | 7 + .../lang/eval-fail-derivationStrict-2.err.exp | 10 + .../lang/eval-fail-derivationStrict-2.nix | 1 + .../eval-fail-derivationStrict-20.err.exp | 20 + .../lang/eval-fail-derivationStrict-20.nix | 7 + .../eval-fail-derivationStrict-21.err.exp | 20 + .../lang/eval-fail-derivationStrict-21.nix | 10 + .../eval-fail-derivationStrict-22.err.exp | 18 + .../lang/eval-fail-derivationStrict-22.nix | 7 + .../lang/eval-fail-derivationStrict-3.err.exp | 16 + .../lang/eval-fail-derivationStrict-3.nix | 1 + .../lang/eval-fail-derivationStrict-4.err.exp | 11 + .../lang/eval-fail-derivationStrict-4.nix | 1 + .../lang/eval-fail-derivationStrict-5.err.exp | 13 + .../lang/eval-fail-derivationStrict-5.nix | 5 + .../lang/eval-fail-derivationStrict-6.err.exp | 13 + .../lang/eval-fail-derivationStrict-6.nix | 5 + .../lang/eval-fail-derivationStrict-7.err.exp | 18 + .../lang/eval-fail-derivationStrict-7.nix | 5 + .../lang/eval-fail-derivationStrict-8.err.exp | 18 + .../lang/eval-fail-derivationStrict-8.nix | 5 + .../lang/eval-fail-derivationStrict-9.err.exp | 18 + .../lang/eval-fail-derivationStrict-9.nix | 5 + tests/functional/lang/eval-fail-div-1.err.exp | 10 + tests/functional/lang/eval-fail-div-1.nix | 1 + tests/functional/lang/eval-fail-div-2.err.exp | 10 + tests/functional/lang/eval-fail-div-2.nix | 1 + tests/functional/lang/eval-fail-div-3.err.exp | 8 + tests/functional/lang/eval-fail-div-3.nix | 1 + .../functional/lang/eval-fail-elem-1.err.exp | 10 + tests/functional/lang/eval-fail-elem-1.nix | 1 + .../lang/eval-fail-elemAt-1.err.exp | 10 + tests/functional/lang/eval-fail-elemAt-1.nix | 1 + .../lang/eval-fail-elemAt-2.err.exp | 8 + tests/functional/lang/eval-fail-elemAt-2.nix | 1 + .../lang/eval-fail-elemAt-3.err.exp | 8 + tests/functional/lang/eval-fail-elemAt-3.nix | 1 + .../lang/eval-fail-filter-1.err.exp | 10 + tests/functional/lang/eval-fail-filter-1.nix | 1 + .../lang/eval-fail-filter-2.err.exp | 10 + tests/functional/lang/eval-fail-filter-2.nix | 1 + .../lang/eval-fail-filter-3.err.exp | 10 + tests/functional/lang/eval-fail-filter-3.nix | 1 + .../lang/eval-fail-filterSource-1.err.exp | 10 + .../lang/eval-fail-filterSource-1.nix | 1 + .../lang/eval-fail-filterSource-2.err.exp | 10 + .../lang/eval-fail-filterSource-2.nix | 1 + .../lang/eval-fail-filterSource-3.err.exp | 10 + .../lang/eval-fail-filterSource-3.nix | 1 + .../lang/eval-fail-filterSource-4.err.exp | 10 + .../lang/eval-fail-filterSource-4.nix | 1 + .../lang/eval-fail-filterSource-5.err.exp | 12 + .../lang/eval-fail-filterSource-5.nix | 1 + .../functional/lang/eval-fail-floor-1.err.exp | 10 + tests/functional/lang/eval-fail-floor-1.nix | 1 + .../lang/eval-fail-foldlPrime-1.err.exp | 10 + .../lang/eval-fail-foldlPrime-1.nix | 1 + .../lang/eval-fail-foldlPrime-2.err.exp | 10 + .../lang/eval-fail-foldlPrime-2.nix | 1 + .../lang/eval-fail-foldlPrime-3.err.exp | 8 + .../lang/eval-fail-foldlPrime-3.nix | 1 + .../lang/eval-fail-foldlPrime-4.err.exp | 24 + .../lang/eval-fail-foldlPrime-4.nix | 1 + .../lang/eval-fail-functionArgs-1.err.exp | 8 + .../lang/eval-fail-functionArgs-1.nix | 1 + .../lang/eval-fail-genList-1.err.exp | 10 + tests/functional/lang/eval-fail-genList-1.nix | 1 + .../lang/eval-fail-genList-2.err.exp | 10 + tests/functional/lang/eval-fail-genList-2.nix | 1 + .../lang/eval-fail-genList-3.err.exp | 20 + tests/functional/lang/eval-fail-genList-3.nix | 1 + .../lang/eval-fail-genList-4.err.exp | 8 + tests/functional/lang/eval-fail-genList-4.nix | 1 + .../lang/eval-fail-getAttr-1.err.exp | 10 + tests/functional/lang/eval-fail-getAttr-1.nix | 1 + .../lang/eval-fail-getAttr-2.err.exp | 10 + tests/functional/lang/eval-fail-getAttr-2.nix | 1 + .../lang/eval-fail-getAttr-3.err.exp | 10 + tests/functional/lang/eval-fail-getAttr-3.nix | 1 + .../lang/eval-fail-getEnv-1.err.exp | 10 + tests/functional/lang/eval-fail-getEnv-1.nix | 1 + .../lang/eval-fail-groupBy-1.err.exp | 10 + tests/functional/lang/eval-fail-groupBy-1.nix | 1 + .../lang/eval-fail-groupBy-2.err.exp | 10 + tests/functional/lang/eval-fail-groupBy-2.nix | 1 + .../lang/eval-fail-groupBy-3.err.exp | 10 + tests/functional/lang/eval-fail-groupBy-3.nix | 5 + .../lang/eval-fail-hasAttr-1.err.exp | 10 + tests/functional/lang/eval-fail-hasAttr-1.nix | 1 + .../lang/eval-fail-hasAttr-2.err.exp | 10 + tests/functional/lang/eval-fail-hasAttr-2.nix | 1 + .../lang/eval-fail-hashString-1.err.exp | 10 + .../lang/eval-fail-hashString-1.nix | 1 + .../lang/eval-fail-hashString-2.err.exp | 9 + .../lang/eval-fail-hashString-2.nix | 1 + .../lang/eval-fail-hashString-3.err.exp | 10 + .../lang/eval-fail-hashString-3.nix | 1 + .../functional/lang/eval-fail-head-1.err.exp | 10 + tests/functional/lang/eval-fail-head-1.nix | 1 + .../functional/lang/eval-fail-head-2.err.exp | 8 + tests/functional/lang/eval-fail-head-2.nix | 1 + .../lang/eval-fail-intersectAttrs-1.err.exp | 10 + .../lang/eval-fail-intersectAttrs-1.nix | 1 + .../lang/eval-fail-intersectAttrs-2.err.exp | 10 + .../lang/eval-fail-intersectAttrs-2.nix | 1 + .../lang/eval-fail-length-1.err.exp | 10 + tests/functional/lang/eval-fail-length-1.nix | 1 + .../lang/eval-fail-length-2.err.exp | 10 + tests/functional/lang/eval-fail-length-2.nix | 1 + .../lang/eval-fail-lessThan-1.err.exp | 8 + .../functional/lang/eval-fail-lessThan-1.nix | 1 + .../lang/eval-fail-lessThan-2.err.exp | 8 + .../functional/lang/eval-fail-lessThan-2.nix | 1 + .../lang/eval-fail-lessThan-3.err.exp | 10 + .../functional/lang/eval-fail-lessThan-3.nix | 1 + .../lang/eval-fail-listToAttrs-1.err.exp | 10 + .../lang/eval-fail-listToAttrs-1.nix | 1 + .../lang/eval-fail-listToAttrs-2.err.exp | 10 + .../lang/eval-fail-listToAttrs-2.nix | 1 + .../lang/eval-fail-listToAttrs-3.err.exp | 10 + .../lang/eval-fail-listToAttrs-3.nix | 1 + .../lang/eval-fail-listToAttrs-4.err.exp | 18 + .../lang/eval-fail-listToAttrs-4.nix | 1 + .../lang/eval-fail-listToAttrs-5.err.exp | 10 + .../lang/eval-fail-listToAttrs-5.nix | 1 + tests/functional/lang/eval-fail-map-1.err.exp | 10 + tests/functional/lang/eval-fail-map-1.nix | 1 + tests/functional/lang/eval-fail-map-2.err.exp | 10 + tests/functional/lang/eval-fail-map-2.nix | 1 + .../lang/eval-fail-mapAttrs-1.err.exp | 10 + .../functional/lang/eval-fail-mapAttrs-1.nix | 1 + .../lang/eval-fail-mapAttrs-2.err.exp | 4 + .../functional/lang/eval-fail-mapAttrs-2.nix | 1 + .../lang/eval-fail-mapAttrs-3.err.exp | 4 + .../functional/lang/eval-fail-mapAttrs-3.nix | 1 + .../lang/eval-fail-mapAttrs-4.err.exp | 16 + .../functional/lang/eval-fail-mapAttrs-4.nix | 1 + .../functional/lang/eval-fail-match-1.err.exp | 10 + tests/functional/lang/eval-fail-match-1.nix | 1 + .../functional/lang/eval-fail-match-2.err.exp | 10 + tests/functional/lang/eval-fail-match-2.nix | 1 + .../functional/lang/eval-fail-match-3.err.exp | 8 + tests/functional/lang/eval-fail-match-3.nix | 1 + tests/functional/lang/eval-fail-mul-1.err.exp | 10 + tests/functional/lang/eval-fail-mul-1.nix | 1 + tests/functional/lang/eval-fail-mul-2.err.exp | 10 + tests/functional/lang/eval-fail-mul-2.nix | 1 + .../lang/eval-fail-parseDrvName-1.err.exp | 10 + .../lang/eval-fail-parseDrvName-1.nix | 1 + .../lang/eval-fail-partition-1.err.exp | 10 + .../functional/lang/eval-fail-partition-1.nix | 1 + .../lang/eval-fail-partition-2.err.exp | 10 + .../functional/lang/eval-fail-partition-2.nix | 1 + .../lang/eval-fail-partition-3.err.exp | 10 + .../functional/lang/eval-fail-partition-3.nix | 1 + .../lang/eval-fail-pathExists-1.err.exp | 10 + .../lang/eval-fail-pathExists-1.nix | 1 + .../lang/eval-fail-pathExists-2.err.exp | 10 + .../lang/eval-fail-pathExists-2.nix | 1 + .../lang/eval-fail-placeholder-1.err.exp | 10 + .../lang/eval-fail-placeholder-1.nix | 1 + .../lang/eval-fail-removeAttrs-1.err.exp | 10 + .../lang/eval-fail-removeAttrs-1.nix | 1 + .../lang/eval-fail-removeAttrs-2.err.exp | 10 + .../lang/eval-fail-removeAttrs-2.nix | 1 + .../lang/eval-fail-removeAttrs-3.err.exp | 10 + .../lang/eval-fail-removeAttrs-3.nix | 1 + .../lang/eval-fail-replaceStrings-1.err.exp | 10 + .../lang/eval-fail-replaceStrings-1.nix | 1 + .../lang/eval-fail-replaceStrings-2.err.exp | 10 + .../lang/eval-fail-replaceStrings-2.nix | 1 + .../lang/eval-fail-replaceStrings-3.err.exp | 8 + .../lang/eval-fail-replaceStrings-3.nix | 1 + .../lang/eval-fail-replaceStrings-4.err.exp | 10 + .../lang/eval-fail-replaceStrings-4.nix | 1 + .../lang/eval-fail-replaceStrings-5.err.exp | 10 + .../lang/eval-fail-replaceStrings-5.nix | 1 + .../lang/eval-fail-replaceStrings-6.err.exp | 10 + .../lang/eval-fail-replaceStrings-6.nix | 1 + .../functional/lang/eval-fail-sort-1.err.exp | 10 + tests/functional/lang/eval-fail-sort-1.nix | 1 + .../functional/lang/eval-fail-sort-2.err.exp | 10 + tests/functional/lang/eval-fail-sort-2.nix | 1 + .../functional/lang/eval-fail-sort-3.err.exp | 8 + tests/functional/lang/eval-fail-sort-3.nix | 4 + .../functional/lang/eval-fail-sort-4.err.exp | 10 + tests/functional/lang/eval-fail-sort-4.nix | 4 + .../functional/lang/eval-fail-sort-5.err.exp | 26 + tests/functional/lang/eval-fail-sort-5.nix | 4 + .../functional/lang/eval-fail-sort-6.err.exp | 26 + tests/functional/lang/eval-fail-sort-6.nix | 4 + .../functional/lang/eval-fail-split-1.err.exp | 10 + tests/functional/lang/eval-fail-split-1.nix | 1 + .../functional/lang/eval-fail-split-2.err.exp | 10 + tests/functional/lang/eval-fail-split-2.nix | 1 + .../functional/lang/eval-fail-split-3.err.exp | 8 + tests/functional/lang/eval-fail-split-3.nix | 1 + .../lang/eval-fail-splitVersion-1.err.exp | 10 + .../lang/eval-fail-splitVersion-1.nix | 1 + .../lang/eval-fail-storePath-1.err.exp | 10 + .../functional/lang/eval-fail-storePath-1.nix | 1 + .../lang/eval-fail-stringLength-1.err.exp | 10 + .../lang/eval-fail-stringLength-1.nix | 1 + tests/functional/lang/eval-fail-sub-1.err.exp | 10 + tests/functional/lang/eval-fail-sub-1.nix | 1 + tests/functional/lang/eval-fail-sub-2.err.exp | 10 + tests/functional/lang/eval-fail-sub-2.nix | 1 + .../lang/eval-fail-substring-1.err.exp | 10 + .../functional/lang/eval-fail-substring-1.nix | 1 + .../lang/eval-fail-substring-2.err.exp | 10 + .../functional/lang/eval-fail-substring-2.nix | 1 + .../lang/eval-fail-substring-3.err.exp | 10 + .../functional/lang/eval-fail-substring-3.nix | 1 + .../lang/eval-fail-substring-4.err.exp | 8 + .../functional/lang/eval-fail-substring-4.nix | 1 + .../functional/lang/eval-fail-tail-1.err.exp | 10 + tests/functional/lang/eval-fail-tail-1.nix | 1 + .../functional/lang/eval-fail-tail-2.err.exp | 8 + tests/functional/lang/eval-fail-tail-2.nix | 1 + .../lang/eval-fail-toPath-1.err.exp | 10 + tests/functional/lang/eval-fail-toPath-1.nix | 1 + .../lang/eval-fail-toPath-2.err.exp | 10 + tests/functional/lang/eval-fail-toPath-2.nix | 1 + .../lang/eval-fail-toString-1.err.exp | 10 + .../functional/lang/eval-fail-toString-1.nix | 1 + .../lang/eval-fail-zipAttrsWith-1.err.exp | 10 + .../lang/eval-fail-zipAttrsWith-1.nix | 1 + .../lang/eval-fail-zipAttrsWith-2.err.exp | 10 + .../lang/eval-fail-zipAttrsWith-2.nix | 1 + .../lang/eval-fail-zipAttrsWith-3.err.exp | 8 + .../lang/eval-fail-zipAttrsWith-3.nix | 1 + .../lang/eval-fail-zipAttrsWith-4.err.exp | 22 + .../lang/eval-fail-zipAttrsWith-4.nix | 4 + 321 files changed, 2068 insertions(+), 1266 deletions(-) create mode 100644 tests/functional/lang/eval-fail-add-1.err.exp create mode 100644 tests/functional/lang/eval-fail-add-1.nix create mode 100644 tests/functional/lang/eval-fail-add-2.err.exp create mode 100644 tests/functional/lang/eval-fail-add-2.nix create mode 100644 tests/functional/lang/eval-fail-all-1.err.exp create mode 100644 tests/functional/lang/eval-fail-all-1.nix create mode 100644 tests/functional/lang/eval-fail-all-2.err.exp create mode 100644 tests/functional/lang/eval-fail-all-2.nix create mode 100644 tests/functional/lang/eval-fail-all-3.err.exp create mode 100644 tests/functional/lang/eval-fail-all-3.nix create mode 100644 tests/functional/lang/eval-fail-any-1.err.exp create mode 100644 tests/functional/lang/eval-fail-any-1.nix create mode 100644 tests/functional/lang/eval-fail-any-2.err.exp create mode 100644 tests/functional/lang/eval-fail-any-2.nix create mode 100644 tests/functional/lang/eval-fail-any-3.err.exp create mode 100644 tests/functional/lang/eval-fail-any-3.nix create mode 100644 tests/functional/lang/eval-fail-attrNames-1.err.exp create mode 100644 tests/functional/lang/eval-fail-attrNames-1.nix create mode 100644 tests/functional/lang/eval-fail-attrValues-1.err.exp create mode 100644 tests/functional/lang/eval-fail-attrValues-1.nix create mode 100644 tests/functional/lang/eval-fail-baseNameOf-1.err.exp create mode 100644 tests/functional/lang/eval-fail-baseNameOf-1.nix create mode 100644 tests/functional/lang/eval-fail-bitAnd-1.err.exp create mode 100644 tests/functional/lang/eval-fail-bitAnd-1.nix create mode 100644 tests/functional/lang/eval-fail-bitAnd-2.err.exp create mode 100644 tests/functional/lang/eval-fail-bitAnd-2.nix create mode 100644 tests/functional/lang/eval-fail-bitOr-1.err.exp create mode 100644 tests/functional/lang/eval-fail-bitOr-1.nix create mode 100644 tests/functional/lang/eval-fail-bitOr-2.err.exp create mode 100644 tests/functional/lang/eval-fail-bitOr-2.nix create mode 100644 tests/functional/lang/eval-fail-bitXor-1.err.exp create mode 100644 tests/functional/lang/eval-fail-bitXor-1.nix create mode 100644 tests/functional/lang/eval-fail-bitXor-2.err.exp create mode 100644 tests/functional/lang/eval-fail-bitXor-2.nix create mode 100644 tests/functional/lang/eval-fail-catAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-catAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-catAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-catAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-catAttrs-3.err.exp create mode 100644 tests/functional/lang/eval-fail-catAttrs-3.nix create mode 100644 tests/functional/lang/eval-fail-catAttrs-4.err.exp create mode 100644 tests/functional/lang/eval-fail-catAttrs-4.nix create mode 100644 tests/functional/lang/eval-fail-ceil-1.err.exp create mode 100644 tests/functional/lang/eval-fail-ceil-1.nix create mode 100644 tests/functional/lang/eval-fail-compareVersions-1.err.exp create mode 100644 tests/functional/lang/eval-fail-compareVersions-1.nix create mode 100644 tests/functional/lang/eval-fail-compareVersions-2.err.exp create mode 100644 tests/functional/lang/eval-fail-compareVersions-2.nix create mode 100644 tests/functional/lang/eval-fail-concatLists-1.err.exp create mode 100644 tests/functional/lang/eval-fail-concatLists-1.nix create mode 100644 tests/functional/lang/eval-fail-concatLists-2.err.exp create mode 100644 tests/functional/lang/eval-fail-concatLists-2.nix create mode 100644 tests/functional/lang/eval-fail-concatLists-3.err.exp create mode 100644 tests/functional/lang/eval-fail-concatLists-3.nix create mode 100644 tests/functional/lang/eval-fail-concatMap-1.err.exp create mode 100644 tests/functional/lang/eval-fail-concatMap-1.nix create mode 100644 tests/functional/lang/eval-fail-concatMap-2.err.exp create mode 100644 tests/functional/lang/eval-fail-concatMap-2.nix create mode 100644 tests/functional/lang/eval-fail-concatMap-3.err.exp create mode 100644 tests/functional/lang/eval-fail-concatMap-3.nix create mode 100644 tests/functional/lang/eval-fail-concatMap-4.err.exp create mode 100644 tests/functional/lang/eval-fail-concatMap-4.nix create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-1.err.exp create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-1.nix create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-2.err.exp create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-2.nix create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-3.err.exp create mode 100644 tests/functional/lang/eval-fail-concatStringsSep-3.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-1.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-1.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-10.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-10.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-11.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-11.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-12.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-12.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-13.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-13.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-14.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-14.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-15.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-15.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-16.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-16.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-17.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-17.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-19.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-19.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-2.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-2.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-20.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-20.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-21.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-21.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-22.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-22.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-3.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-3.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-4.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-4.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-5.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-5.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-6.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-6.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-7.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-7.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-8.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-8.nix create mode 100644 tests/functional/lang/eval-fail-derivationStrict-9.err.exp create mode 100644 tests/functional/lang/eval-fail-derivationStrict-9.nix create mode 100644 tests/functional/lang/eval-fail-div-1.err.exp create mode 100644 tests/functional/lang/eval-fail-div-1.nix create mode 100644 tests/functional/lang/eval-fail-div-2.err.exp create mode 100644 tests/functional/lang/eval-fail-div-2.nix create mode 100644 tests/functional/lang/eval-fail-div-3.err.exp create mode 100644 tests/functional/lang/eval-fail-div-3.nix create mode 100644 tests/functional/lang/eval-fail-elem-1.err.exp create mode 100644 tests/functional/lang/eval-fail-elem-1.nix create mode 100644 tests/functional/lang/eval-fail-elemAt-1.err.exp create mode 100644 tests/functional/lang/eval-fail-elemAt-1.nix create mode 100644 tests/functional/lang/eval-fail-elemAt-2.err.exp create mode 100644 tests/functional/lang/eval-fail-elemAt-2.nix create mode 100644 tests/functional/lang/eval-fail-elemAt-3.err.exp create mode 100644 tests/functional/lang/eval-fail-elemAt-3.nix create mode 100644 tests/functional/lang/eval-fail-filter-1.err.exp create mode 100644 tests/functional/lang/eval-fail-filter-1.nix create mode 100644 tests/functional/lang/eval-fail-filter-2.err.exp create mode 100644 tests/functional/lang/eval-fail-filter-2.nix create mode 100644 tests/functional/lang/eval-fail-filter-3.err.exp create mode 100644 tests/functional/lang/eval-fail-filter-3.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-1.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-1.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-2.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-2.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-3.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-3.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-4.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-4.nix create mode 100644 tests/functional/lang/eval-fail-filterSource-5.err.exp create mode 100644 tests/functional/lang/eval-fail-filterSource-5.nix create mode 100644 tests/functional/lang/eval-fail-floor-1.err.exp create mode 100644 tests/functional/lang/eval-fail-floor-1.nix create mode 100644 tests/functional/lang/eval-fail-foldlPrime-1.err.exp create mode 100644 tests/functional/lang/eval-fail-foldlPrime-1.nix create mode 100644 tests/functional/lang/eval-fail-foldlPrime-2.err.exp create mode 100644 tests/functional/lang/eval-fail-foldlPrime-2.nix create mode 100644 tests/functional/lang/eval-fail-foldlPrime-3.err.exp create mode 100644 tests/functional/lang/eval-fail-foldlPrime-3.nix create mode 100644 tests/functional/lang/eval-fail-foldlPrime-4.err.exp create mode 100644 tests/functional/lang/eval-fail-foldlPrime-4.nix create mode 100644 tests/functional/lang/eval-fail-functionArgs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-functionArgs-1.nix create mode 100644 tests/functional/lang/eval-fail-genList-1.err.exp create mode 100644 tests/functional/lang/eval-fail-genList-1.nix create mode 100644 tests/functional/lang/eval-fail-genList-2.err.exp create mode 100644 tests/functional/lang/eval-fail-genList-2.nix create mode 100644 tests/functional/lang/eval-fail-genList-3.err.exp create mode 100644 tests/functional/lang/eval-fail-genList-3.nix create mode 100644 tests/functional/lang/eval-fail-genList-4.err.exp create mode 100644 tests/functional/lang/eval-fail-genList-4.nix create mode 100644 tests/functional/lang/eval-fail-getAttr-1.err.exp create mode 100644 tests/functional/lang/eval-fail-getAttr-1.nix create mode 100644 tests/functional/lang/eval-fail-getAttr-2.err.exp create mode 100644 tests/functional/lang/eval-fail-getAttr-2.nix create mode 100644 tests/functional/lang/eval-fail-getAttr-3.err.exp create mode 100644 tests/functional/lang/eval-fail-getAttr-3.nix create mode 100644 tests/functional/lang/eval-fail-getEnv-1.err.exp create mode 100644 tests/functional/lang/eval-fail-getEnv-1.nix create mode 100644 tests/functional/lang/eval-fail-groupBy-1.err.exp create mode 100644 tests/functional/lang/eval-fail-groupBy-1.nix create mode 100644 tests/functional/lang/eval-fail-groupBy-2.err.exp create mode 100644 tests/functional/lang/eval-fail-groupBy-2.nix create mode 100644 tests/functional/lang/eval-fail-groupBy-3.err.exp create mode 100644 tests/functional/lang/eval-fail-groupBy-3.nix create mode 100644 tests/functional/lang/eval-fail-hasAttr-1.err.exp create mode 100644 tests/functional/lang/eval-fail-hasAttr-1.nix create mode 100644 tests/functional/lang/eval-fail-hasAttr-2.err.exp create mode 100644 tests/functional/lang/eval-fail-hasAttr-2.nix create mode 100644 tests/functional/lang/eval-fail-hashString-1.err.exp create mode 100644 tests/functional/lang/eval-fail-hashString-1.nix create mode 100644 tests/functional/lang/eval-fail-hashString-2.err.exp create mode 100644 tests/functional/lang/eval-fail-hashString-2.nix create mode 100644 tests/functional/lang/eval-fail-hashString-3.err.exp create mode 100644 tests/functional/lang/eval-fail-hashString-3.nix create mode 100644 tests/functional/lang/eval-fail-head-1.err.exp create mode 100644 tests/functional/lang/eval-fail-head-1.nix create mode 100644 tests/functional/lang/eval-fail-head-2.err.exp create mode 100644 tests/functional/lang/eval-fail-head-2.nix create mode 100644 tests/functional/lang/eval-fail-intersectAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-intersectAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-intersectAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-intersectAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-length-1.err.exp create mode 100644 tests/functional/lang/eval-fail-length-1.nix create mode 100644 tests/functional/lang/eval-fail-length-2.err.exp create mode 100644 tests/functional/lang/eval-fail-length-2.nix create mode 100644 tests/functional/lang/eval-fail-lessThan-1.err.exp create mode 100644 tests/functional/lang/eval-fail-lessThan-1.nix create mode 100644 tests/functional/lang/eval-fail-lessThan-2.err.exp create mode 100644 tests/functional/lang/eval-fail-lessThan-2.nix create mode 100644 tests/functional/lang/eval-fail-lessThan-3.err.exp create mode 100644 tests/functional/lang/eval-fail-lessThan-3.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-3.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-3.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-4.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-4.nix create mode 100644 tests/functional/lang/eval-fail-listToAttrs-5.err.exp create mode 100644 tests/functional/lang/eval-fail-listToAttrs-5.nix create mode 100644 tests/functional/lang/eval-fail-map-1.err.exp create mode 100644 tests/functional/lang/eval-fail-map-1.nix create mode 100644 tests/functional/lang/eval-fail-map-2.err.exp create mode 100644 tests/functional/lang/eval-fail-map-2.nix create mode 100644 tests/functional/lang/eval-fail-mapAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-mapAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-mapAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-mapAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-mapAttrs-3.err.exp create mode 100644 tests/functional/lang/eval-fail-mapAttrs-3.nix create mode 100644 tests/functional/lang/eval-fail-mapAttrs-4.err.exp create mode 100644 tests/functional/lang/eval-fail-mapAttrs-4.nix create mode 100644 tests/functional/lang/eval-fail-match-1.err.exp create mode 100644 tests/functional/lang/eval-fail-match-1.nix create mode 100644 tests/functional/lang/eval-fail-match-2.err.exp create mode 100644 tests/functional/lang/eval-fail-match-2.nix create mode 100644 tests/functional/lang/eval-fail-match-3.err.exp create mode 100644 tests/functional/lang/eval-fail-match-3.nix create mode 100644 tests/functional/lang/eval-fail-mul-1.err.exp create mode 100644 tests/functional/lang/eval-fail-mul-1.nix create mode 100644 tests/functional/lang/eval-fail-mul-2.err.exp create mode 100644 tests/functional/lang/eval-fail-mul-2.nix create mode 100644 tests/functional/lang/eval-fail-parseDrvName-1.err.exp create mode 100644 tests/functional/lang/eval-fail-parseDrvName-1.nix create mode 100644 tests/functional/lang/eval-fail-partition-1.err.exp create mode 100644 tests/functional/lang/eval-fail-partition-1.nix create mode 100644 tests/functional/lang/eval-fail-partition-2.err.exp create mode 100644 tests/functional/lang/eval-fail-partition-2.nix create mode 100644 tests/functional/lang/eval-fail-partition-3.err.exp create mode 100644 tests/functional/lang/eval-fail-partition-3.nix create mode 100644 tests/functional/lang/eval-fail-pathExists-1.err.exp create mode 100644 tests/functional/lang/eval-fail-pathExists-1.nix create mode 100644 tests/functional/lang/eval-fail-pathExists-2.err.exp create mode 100644 tests/functional/lang/eval-fail-pathExists-2.nix create mode 100644 tests/functional/lang/eval-fail-placeholder-1.err.exp create mode 100644 tests/functional/lang/eval-fail-placeholder-1.nix create mode 100644 tests/functional/lang/eval-fail-removeAttrs-1.err.exp create mode 100644 tests/functional/lang/eval-fail-removeAttrs-1.nix create mode 100644 tests/functional/lang/eval-fail-removeAttrs-2.err.exp create mode 100644 tests/functional/lang/eval-fail-removeAttrs-2.nix create mode 100644 tests/functional/lang/eval-fail-removeAttrs-3.err.exp create mode 100644 tests/functional/lang/eval-fail-removeAttrs-3.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-1.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-1.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-2.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-2.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-3.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-3.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-4.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-4.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-5.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-5.nix create mode 100644 tests/functional/lang/eval-fail-replaceStrings-6.err.exp create mode 100644 tests/functional/lang/eval-fail-replaceStrings-6.nix create mode 100644 tests/functional/lang/eval-fail-sort-1.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-1.nix create mode 100644 tests/functional/lang/eval-fail-sort-2.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-2.nix create mode 100644 tests/functional/lang/eval-fail-sort-3.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-3.nix create mode 100644 tests/functional/lang/eval-fail-sort-4.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-4.nix create mode 100644 tests/functional/lang/eval-fail-sort-5.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-5.nix create mode 100644 tests/functional/lang/eval-fail-sort-6.err.exp create mode 100644 tests/functional/lang/eval-fail-sort-6.nix create mode 100644 tests/functional/lang/eval-fail-split-1.err.exp create mode 100644 tests/functional/lang/eval-fail-split-1.nix create mode 100644 tests/functional/lang/eval-fail-split-2.err.exp create mode 100644 tests/functional/lang/eval-fail-split-2.nix create mode 100644 tests/functional/lang/eval-fail-split-3.err.exp create mode 100644 tests/functional/lang/eval-fail-split-3.nix create mode 100644 tests/functional/lang/eval-fail-splitVersion-1.err.exp create mode 100644 tests/functional/lang/eval-fail-splitVersion-1.nix create mode 100644 tests/functional/lang/eval-fail-storePath-1.err.exp create mode 100644 tests/functional/lang/eval-fail-storePath-1.nix create mode 100644 tests/functional/lang/eval-fail-stringLength-1.err.exp create mode 100644 tests/functional/lang/eval-fail-stringLength-1.nix create mode 100644 tests/functional/lang/eval-fail-sub-1.err.exp create mode 100644 tests/functional/lang/eval-fail-sub-1.nix create mode 100644 tests/functional/lang/eval-fail-sub-2.err.exp create mode 100644 tests/functional/lang/eval-fail-sub-2.nix create mode 100644 tests/functional/lang/eval-fail-substring-1.err.exp create mode 100644 tests/functional/lang/eval-fail-substring-1.nix create mode 100644 tests/functional/lang/eval-fail-substring-2.err.exp create mode 100644 tests/functional/lang/eval-fail-substring-2.nix create mode 100644 tests/functional/lang/eval-fail-substring-3.err.exp create mode 100644 tests/functional/lang/eval-fail-substring-3.nix create mode 100644 tests/functional/lang/eval-fail-substring-4.err.exp create mode 100644 tests/functional/lang/eval-fail-substring-4.nix create mode 100644 tests/functional/lang/eval-fail-tail-1.err.exp create mode 100644 tests/functional/lang/eval-fail-tail-1.nix create mode 100644 tests/functional/lang/eval-fail-tail-2.err.exp create mode 100644 tests/functional/lang/eval-fail-tail-2.nix create mode 100644 tests/functional/lang/eval-fail-toPath-1.err.exp create mode 100644 tests/functional/lang/eval-fail-toPath-1.nix create mode 100644 tests/functional/lang/eval-fail-toPath-2.err.exp create mode 100644 tests/functional/lang/eval-fail-toPath-2.nix create mode 100644 tests/functional/lang/eval-fail-toString-1.err.exp create mode 100644 tests/functional/lang/eval-fail-toString-1.nix create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-1.err.exp create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-1.nix create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-2.err.exp create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-2.nix create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-3.err.exp create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-3.nix create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-4.err.exp create mode 100644 tests/functional/lang/eval-fail-zipAttrsWith-4.nix diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index e722cc4849..9f2d1f92fa 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -54,1270 +54,4 @@ TEST_F(ErrorTraceTest, NestedThrows) } } -#define ASSERT_TRACE1(args, type, message) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 1u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE2(args, type, message, context) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 2u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE3(args, type, message, context1, context2) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 3u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context1)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context2)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -#define ASSERT_TRACE4(args, type, message, context1, context2, context3) \ - ASSERT_THROW( \ - std::string expr(args); std::string name = expr.substr(0, expr.find(" ")); try { \ - Value v = eval("builtins." args); \ - state.forceValueDeep(v); \ - } catch (BaseError & e) { \ - ASSERT_EQ(PrintToString(e.info().msg), PrintToString(message)); \ - ASSERT_EQ(e.info().traces.size(), 4u) << "while testing " args << std::endl << e.what(); \ - auto trace = e.info().traces.rbegin(); \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context1)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context2)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(context3)); \ - ++trace; \ - ASSERT_EQ(PrintToString(trace->hint), PrintToString(HintFmt("while calling the '%s' builtin", name))); \ - throw; \ - }, \ - type) - -// We assume that expr starts with "builtins.derivationStrict { name =", -// otherwise the name attribute position (1, 29) would be invalid. -#define DERIVATION_TRACE_HINTFMT(name) \ - HintFmt( \ - "while evaluating derivation '%s'\n" \ - " whose name attribute is located at %s", \ - name, \ - Pos(1, 29, Pos::String{.source = make_ref(expr)})) - -// To keep things simple, we also assume that derivation name is "foo". -#define ASSERT_DERIVATION_TRACE1(args, type, message) \ - ASSERT_TRACE2(args, type, message, DERIVATION_TRACE_HINTFMT("foo")) -#define ASSERT_DERIVATION_TRACE2(args, type, message, context) \ - ASSERT_TRACE3(args, type, message, context, DERIVATION_TRACE_HINTFMT("foo")) -#define ASSERT_DERIVATION_TRACE3(args, type, message, context1, context2) \ - ASSERT_TRACE4(args, type, message, context1, context2, DERIVATION_TRACE_HINTFMT("foo")) - -TEST_F(ErrorTraceTest, replaceStrings) -{ - ASSERT_TRACE2( - "replaceStrings 0 0 {}", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [] 0 {}", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.replaceStrings")); - - ASSERT_TRACE1( - "replaceStrings [ 0 ] [] {}", - EvalError, - HintFmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); - - ASSERT_TRACE2( - "replaceStrings [ 1 ] [ \"new\" ] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [ \"oo\" ] [ true ] \"foo\"", - TypeError, - HintFmt("expected a string but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); - - ASSERT_TRACE2( - "replaceStrings [ \"old\" ] [ \"new\" ] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the third argument passed to builtins.replaceStrings")); -} - -TEST_F(ErrorTraceTest, scopedImport) {} - -TEST_F(ErrorTraceTest, import) {} - -TEST_F(ErrorTraceTest, typeOf) {} - -TEST_F(ErrorTraceTest, isNull) {} - -TEST_F(ErrorTraceTest, isFunction) {} - -TEST_F(ErrorTraceTest, isInt) {} - -TEST_F(ErrorTraceTest, isFloat) {} - -TEST_F(ErrorTraceTest, isString) {} - -TEST_F(ErrorTraceTest, isBool) {} - -TEST_F(ErrorTraceTest, isPath) {} - -TEST_F(ErrorTraceTest, break) {} - -TEST_F(ErrorTraceTest, abort) {} - -TEST_F(ErrorTraceTest, throw) {} - -TEST_F(ErrorTraceTest, addErrorContext) {} - -TEST_F(ErrorTraceTest, ceil) -{ - ASSERT_TRACE2( - "ceil \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.ceil")); -} - -TEST_F(ErrorTraceTest, floor) -{ - ASSERT_TRACE2( - "floor \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.floor")); -} - -TEST_F(ErrorTraceTest, tryEval) {} - -TEST_F(ErrorTraceTest, getEnv) -{ - ASSERT_TRACE2( - "getEnv [ ]", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.getEnv")); -} - -TEST_F(ErrorTraceTest, seq) {} - -TEST_F(ErrorTraceTest, deepSeq) {} - -TEST_F(ErrorTraceTest, trace) {} - -TEST_F(ErrorTraceTest, placeholder) -{ - ASSERT_TRACE2( - "placeholder []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.placeholder")); -} - -TEST_F(ErrorTraceTest, toPath) -{ - ASSERT_TRACE2( - "toPath []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.toPath")); - - ASSERT_TRACE2( - "toPath \"foo\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "foo"), - HintFmt("while evaluating the first argument passed to builtins.toPath")); -} - -TEST_F(ErrorTraceTest, storePath) -{ - ASSERT_TRACE2( - "storePath true", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.storePath'")); -} - -TEST_F(ErrorTraceTest, pathExists) -{ - ASSERT_TRACE2( - "pathExists []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while realising the context of a path")); - - ASSERT_TRACE2( - "pathExists \"zorglub\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "zorglub"), - HintFmt("while realising the context of a path")); -} - -TEST_F(ErrorTraceTest, baseNameOf) -{ - ASSERT_TRACE2( - "baseNameOf []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.baseNameOf")); -} - -TEST_F(ErrorTraceTest, dirOf) {} - -TEST_F(ErrorTraceTest, readFile) {} - -TEST_F(ErrorTraceTest, findFile) {} - -TEST_F(ErrorTraceTest, hashFile) {} - -TEST_F(ErrorTraceTest, readDir) {} - -TEST_F(ErrorTraceTest, toXML) {} - -TEST_F(ErrorTraceTest, toJSON) {} - -TEST_F(ErrorTraceTest, fromJSON) {} - -TEST_F(ErrorTraceTest, toFile) {} - -TEST_F(ErrorTraceTest, filterSource) -{ - ASSERT_TRACE2( - "filterSource [] []", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); - - ASSERT_TRACE2( - "filterSource [] \"foo\"", - EvalError, - HintFmt("string '%s' doesn't represent an absolute path", "foo"), - HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); - - ASSERT_TRACE2( - "filterSource [] ./.", - TypeError, - HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.filterSource")); - - // Unsupported by store "dummy" - - // ASSERT_TRACE2("filterSource (_: 1) ./.", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "an integer"), - // HintFmt("while adding path '/home/layus/projects/nix'")); - - // ASSERT_TRACE2("filterSource (_: _: 1) ./.", - // TypeError, - // HintFmt("expected a Boolean but found %s: %s", "an integer", "1"), - // HintFmt("while evaluating the return value of the path filter function")); -} - -TEST_F(ErrorTraceTest, path) {} - -TEST_F(ErrorTraceTest, attrNames) -{ - ASSERT_TRACE2( - "attrNames []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the argument passed to builtins.attrNames")); -} - -TEST_F(ErrorTraceTest, attrValues) -{ - ASSERT_TRACE2( - "attrValues []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the argument passed to builtins.attrValues")); -} - -TEST_F(ErrorTraceTest, getAttr) -{ - ASSERT_TRACE2( - "getAttr [] []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.getAttr")); - - ASSERT_TRACE2( - "getAttr \"foo\" []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.getAttr")); - - ASSERT_TRACE2( - "getAttr \"foo\" {}", - TypeError, - HintFmt("attribute '%s' missing", "foo"), - HintFmt("in the attribute set under consideration")); -} - -TEST_F(ErrorTraceTest, unsafeGetAttrPos) {} - -TEST_F(ErrorTraceTest, hasAttr) -{ - ASSERT_TRACE2( - "hasAttr [] []", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.hasAttr")); - - ASSERT_TRACE2( - "hasAttr \"foo\" []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.hasAttr")); -} - -TEST_F(ErrorTraceTest, isAttrs) {} - -TEST_F(ErrorTraceTest, removeAttrs) -{ - ASSERT_TRACE2( - "removeAttrs \"\" \"\"", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); - - ASSERT_TRACE2( - "removeAttrs \"\" [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); - - ASSERT_TRACE2( - "removeAttrs \"\" [ \"1\" ]", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); -} - -TEST_F(ErrorTraceTest, listToAttrs) -{ - ASSERT_TRACE2( - "listToAttrs 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the argument passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element of the list passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ {} ]", - TypeError, - HintFmt("attribute '%s' missing", "name"), - HintFmt("in a {name=...; value=...;} pair")); - - ASSERT_TRACE2( - "listToAttrs [ { name = 1; } ]", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); - - ASSERT_TRACE2( - "listToAttrs [ { name = \"foo\"; } ]", - TypeError, - HintFmt("attribute '%s' missing", "value"), - HintFmt("in a {name=...; value=...;} pair")); -} - -TEST_F(ErrorTraceTest, intersectAttrs) -{ - ASSERT_TRACE2( - "intersectAttrs [] []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.intersectAttrs")); - - ASSERT_TRACE2( - "intersectAttrs {} []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.intersectAttrs")); -} - -TEST_F(ErrorTraceTest, catAttrs) -{ - ASSERT_TRACE2( - "catAttrs [] {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" {}", - TypeError, - HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); - - ASSERT_TRACE2( - "catAttrs \"foo\" [ { foo = 1; } 1 { bar = 5;} ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); -} - -TEST_F(ErrorTraceTest, functionArgs) -{ - ASSERT_TRACE1("functionArgs {}", TypeError, HintFmt("'functionArgs' requires a function")); -} - -TEST_F(ErrorTraceTest, mapAttrs) -{ - ASSERT_TRACE2( - "mapAttrs [] []", - TypeError, - HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the second argument passed to builtins.mapAttrs")); - - // XXX: deferred - // ASSERT_TRACE2("mapAttrs \"\" { foo.bar = 1; }", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "a string"), - // HintFmt("while evaluating the attribute 'foo'")); - - // ASSERT_TRACE2("mapAttrs (x: x + \"1\") { foo.bar = 1; }", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "a string"), - // HintFmt("while evaluating the attribute 'foo'")); - - // ASSERT_TRACE2("mapAttrs (x: y: x + 1) { foo.bar = 1; }", - // TypeError, - // HintFmt("cannot coerce %s to a string", "an integer"), - // HintFmt("while evaluating a path segment")); -} - -TEST_F(ErrorTraceTest, zipAttrsWith) -{ - ASSERT_TRACE2( - "zipAttrsWith [] [ 1 ]", - TypeError, - HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), - HintFmt("while evaluating the first argument passed to builtins.zipAttrsWith")); - - ASSERT_TRACE2( - "zipAttrsWith (_: 1) [ 1 ]", - TypeError, - HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); - - // XXX: How to properly tell that the function takes two arguments ? - // The same question also applies to sort, and maybe others. - // Due to laziness, we only create a thunk, and it fails later on. - // ASSERT_TRACE2("zipAttrsWith (_: 1) [ { foo = 1; } ]", - // TypeError, - // HintFmt("attempt to call something which is not a function but %s", "an integer"), - // HintFmt("while evaluating the attribute 'foo'")); - - // XXX: Also deferred deeply - // ASSERT_TRACE2("zipAttrsWith (a: b: a + b) [ { foo = 1; } { foo = 2; } ]", - // TypeError, - // HintFmt("cannot coerce %s to a string", "a list"), - // HintFmt("while evaluating a path segment")); -} - -TEST_F(ErrorTraceTest, isList) {} - -TEST_F(ErrorTraceTest, elemAt) -{ - ASSERT_TRACE2( - "elemAt \"foo\" (-1)", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.elemAt'")); - - ASSERT_TRACE1( - "elemAt [] (-1)", Error, HintFmt("'builtins.elemAt' called with index %d on a list of size %d", -1, 0)); - - ASSERT_TRACE1( - "elemAt [\"foo\"] 3", Error, HintFmt("'builtins.elemAt' called with index %d on a list of size %d", 3, 1)); -} - -TEST_F(ErrorTraceTest, head) -{ - ASSERT_TRACE2( - "head 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.head'")); - - ASSERT_TRACE1("head []", Error, HintFmt("'builtins.head' called on an empty list")); -} - -TEST_F(ErrorTraceTest, tail) -{ - ASSERT_TRACE2( - "tail 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to 'builtins.tail'")); - - ASSERT_TRACE1("tail []", Error, HintFmt("'builtins.tail' called on an empty list")); -} - -TEST_F(ErrorTraceTest, map) -{ - ASSERT_TRACE2( - "map 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.map")); - - ASSERT_TRACE2( - "map 1 [ 1 ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.map")); -} - -TEST_F(ErrorTraceTest, filter) -{ - ASSERT_TRACE2( - "filter 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.filter")); - - ASSERT_TRACE2( - "filter 1 [ \"foo\" ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.filter")); - - ASSERT_TRACE2( - "filter (_: 5) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "5" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the filtering function passed to builtins.filter")); -} - -TEST_F(ErrorTraceTest, elem) -{ - ASSERT_TRACE2( - "elem 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.elem")); -} - -TEST_F(ErrorTraceTest, concatLists) -{ - ASSERT_TRACE2( - "concatLists 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.concatLists")); - - ASSERT_TRACE2( - "concatLists [ 1 ]", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed to builtins.concatLists")); - - ASSERT_TRACE2( - "concatLists [ [1] \"foo\" ]", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating a value of the list passed to builtins.concatLists")); -} - -TEST_F(ErrorTraceTest, length) -{ - ASSERT_TRACE2( - "length 1", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.length")); - - ASSERT_TRACE2( - "length \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.length")); -} - -TEST_F(ErrorTraceTest, foldlPrime) -{ - ASSERT_TRACE2( - "foldl' 1 \"foo\" true", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.foldlStrict")); - - ASSERT_TRACE2( - "foldl' (_: 1) \"foo\" true", - TypeError, - HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), - HintFmt("while evaluating the third argument passed to builtins.foldlStrict")); - - ASSERT_TRACE1( - "foldl' (_: 1) \"foo\" [ true ]", - TypeError, - HintFmt( - "attempt to call something which is not a function but %s: %s", - "an integer", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); - - ASSERT_TRACE2( - "foldl' (a: b: a && b) \"foo\" [ true ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("in the left operand of the AND (&&) operator")); -} - -TEST_F(ErrorTraceTest, any) -{ - ASSERT_TRACE2( - "any 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.any")); - - ASSERT_TRACE2( - "any (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.any")); - - ASSERT_TRACE2( - "any (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.any")); -} - -TEST_F(ErrorTraceTest, all) -{ - ASSERT_TRACE2( - "all 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.all")); - - ASSERT_TRACE2( - "all (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.all")); - - ASSERT_TRACE2( - "all (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.all")); -} - -TEST_F(ErrorTraceTest, genList) -{ - ASSERT_TRACE2( - "genList 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.genList")); - - ASSERT_TRACE2( - "genList 1 2", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.genList")); - - // XXX: deferred - // ASSERT_TRACE2("genList (x: x + \"foo\") 2 #TODO", - // TypeError, - // HintFmt("cannot add %s to an integer", "a string"), - // HintFmt("while evaluating anonymous lambda")); - - ASSERT_TRACE1("genList false (-3)", EvalError, HintFmt("cannot create list of size %d", -3)); -} - -TEST_F(ErrorTraceTest, sort) -{ - ASSERT_TRACE2( - "sort 1 \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.sort")); - - ASSERT_TRACE2( - "sort 1 [ \"foo\" ]", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.sort")); - - ASSERT_TRACE1( - "sort (_: 1) [ \"foo\" \"bar\" ]", - TypeError, - HintFmt( - "attempt to call something which is not a function but %s: %s", - "an integer", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); - - ASSERT_TRACE2( - "sort (_: _: 1) [ \"foo\" \"bar\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the sorting function passed to builtins.sort")); - - // XXX: Trace too deep, need better asserts - // ASSERT_TRACE1("sort (a: b: a <= b) [ \"foo\" {} ] # TODO", - // TypeError, - // HintFmt("cannot compare %s with %s", "a string", "a set")); - - // ASSERT_TRACE1("sort (a: b: a <= b) [ {} {} ] # TODO", - // TypeError, - // HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); -} - -TEST_F(ErrorTraceTest, partition) -{ - ASSERT_TRACE2( - "partition 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.partition")); - - ASSERT_TRACE2( - "partition (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.partition")); - - ASSERT_TRACE2( - "partition (_: 1) [ \"foo\" ]", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the partition function passed to builtins.partition")); -} - -TEST_F(ErrorTraceTest, groupBy) -{ - ASSERT_TRACE2( - "groupBy 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.groupBy")); - - ASSERT_TRACE2( - "groupBy (_: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.groupBy")); - - ASSERT_TRACE2( - "groupBy (x: x) [ \"foo\" \"bar\" 1 ]", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); -} - -TEST_F(ErrorTraceTest, concatMap) -{ - ASSERT_TRACE2( - "concatMap 1 \"foo\"", - TypeError, - HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: 1) \"foo\"", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: 1) [ \"foo\" ] # TODO", - TypeError, - HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); - - ASSERT_TRACE2( - "concatMap (x: \"foo\") [ 1 2 ] # TODO", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); -} - -TEST_F(ErrorTraceTest, add) -{ - ASSERT_TRACE2( - "add \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the addition")); - - ASSERT_TRACE2( - "add 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the addition")); -} - -TEST_F(ErrorTraceTest, sub) -{ - ASSERT_TRACE2( - "sub \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the subtraction")); - - ASSERT_TRACE2( - "sub 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the subtraction")); -} - -TEST_F(ErrorTraceTest, mul) -{ - ASSERT_TRACE2( - "mul \"foo\" 1", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument of the multiplication")); - - ASSERT_TRACE2( - "mul 1 \"foo\"", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument of the multiplication")); -} - -TEST_F(ErrorTraceTest, div) -{ - ASSERT_TRACE2( - "div \"foo\" 1 # TODO: an integer was expected -> a number", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first operand of the division")); - - ASSERT_TRACE2( - "div 1 \"foo\"", - TypeError, - HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second operand of the division")); - - ASSERT_TRACE1("div \"foo\" 0", EvalError, HintFmt("division by zero")); -} - -TEST_F(ErrorTraceTest, bitAnd) -{ - ASSERT_TRACE2( - "bitAnd 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitAnd")); - - ASSERT_TRACE2( - "bitAnd 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitAnd")); -} - -TEST_F(ErrorTraceTest, bitOr) -{ - ASSERT_TRACE2( - "bitOr 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitOr")); - - ASSERT_TRACE2( - "bitOr 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitOr")); -} - -TEST_F(ErrorTraceTest, bitXor) -{ - ASSERT_TRACE2( - "bitXor 1.1 2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.bitXor")); - - ASSERT_TRACE2( - "bitXor 1 2.2", - TypeError, - HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), - HintFmt("while evaluating the second argument passed to builtins.bitXor")); -} - -TEST_F(ErrorTraceTest, lessThan) -{ - ASSERT_TRACE1( - "lessThan 1 \"foo\"", - EvalError, - HintFmt( - "cannot compare %s with %s; values are %s and %s", - "an integer", - "a string", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL), - Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL))); - - ASSERT_TRACE1( - "lessThan {} {}", - EvalError, - HintFmt( - "cannot compare %s with %s; values of that type are incomparable (values are %s and %s)", - "a set", - "a set", - Uncolored("{ }"), - Uncolored("{ }"))); - - ASSERT_TRACE2( - "lessThan [ 1 2 ] [ \"foo\" ]", - EvalError, - HintFmt( - "cannot compare %s with %s; values are %s and %s", - "an integer", - "a string", - Uncolored(ANSI_CYAN "1" ANSI_NORMAL), - Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while comparing two list elements")); -} - -TEST_F(ErrorTraceTest, toString) -{ - ASSERT_TRACE2( - "toString { a = 1; }", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ a = " ANSI_CYAN "1" ANSI_NORMAL "; }")), - HintFmt("while evaluating the first argument passed to builtins.toString")); -} - -TEST_F(ErrorTraceTest, substring) -{ - ASSERT_TRACE2( - "substring {} \"foo\" true", - TypeError, - HintFmt("expected an integer but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the first argument (the start offset) passed to builtins.substring")); - - ASSERT_TRACE2( - "substring 3 \"foo\" true", - TypeError, - HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the second argument (the substring length) passed to builtins.substring")); - - ASSERT_TRACE2( - "substring 0 3 {}", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the third argument (the string) passed to builtins.substring")); - - ASSERT_TRACE1("substring (-3) 3 \"sometext\"", EvalError, HintFmt("negative start position in 'substring'")); -} - -TEST_F(ErrorTraceTest, stringLength) -{ - ASSERT_TRACE2( - "stringLength {} # TODO: context is missing ???", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the argument passed to builtins.stringLength")); -} - -TEST_F(ErrorTraceTest, hashString) -{ - ASSERT_TRACE2( - "hashString 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.hashString")); - - ASSERT_TRACE1( - "hashString \"foo\" \"content\"", - UsageError, - HintFmt("unknown hash algorithm '%s', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); - - ASSERT_TRACE2( - "hashString \"sha256\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.hashString")); -} - -TEST_F(ErrorTraceTest, match) -{ - ASSERT_TRACE2( - "match 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.match")); - - ASSERT_TRACE2( - "match \"foo\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.match")); - - ASSERT_TRACE1("match \"(.*\" \"\"", EvalError, HintFmt("invalid regular expression '%s'", "(.*")); -} - -TEST_F(ErrorTraceTest, split) -{ - ASSERT_TRACE2( - "split 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.split")); - - ASSERT_TRACE2( - "split \"foo\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.split")); - - ASSERT_TRACE1("split \"f(o*o\" \"1foo2\"", EvalError, HintFmt("invalid regular expression '%s'", "f(o*o")); -} - -TEST_F(ErrorTraceTest, concatStringsSep) -{ - ASSERT_TRACE2( - "concatStringsSep 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); - - ASSERT_TRACE2( - "concatStringsSep \"foo\" {}", - TypeError, - HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), - HintFmt( - "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); - - ASSERT_TRACE2( - "concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", - TypeError, - HintFmt("cannot coerce %s to a string: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); -} - -TEST_F(ErrorTraceTest, parseDrvName) -{ - ASSERT_TRACE2( - "parseDrvName 1", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.parseDrvName")); -} - -TEST_F(ErrorTraceTest, compareVersions) -{ - ASSERT_TRACE2( - "compareVersions 1 {}", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.compareVersions")); - - ASSERT_TRACE2( - "compareVersions \"abd\" {}", - TypeError, - HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), - HintFmt("while evaluating the second argument passed to builtins.compareVersions")); -} - -TEST_F(ErrorTraceTest, splitVersion) -{ - ASSERT_TRACE2( - "splitVersion 1", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.splitVersion")); -} - -TEST_F(ErrorTraceTest, traceVerbose) {} - -TEST_F(ErrorTraceTest, derivationStrict) -{ - ASSERT_TRACE2( - "derivationStrict \"\"", - TypeError, - HintFmt("expected a set but found %s: %s", "a string", "\"\""), - HintFmt("while evaluating the argument passed to builtins.derivationStrict")); - - ASSERT_TRACE2( - "derivationStrict {}", - TypeError, - HintFmt("attribute '%s' missing", "name"), - HintFmt("in the attrset passed as argument to builtins.derivationStrict")); - - ASSERT_TRACE3( - "derivationStrict { name = 1; }", - TypeError, - HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the `name` attribute passed to builtins.derivationStrict"), - HintFmt("while evaluating the derivation attribute 'name'")); - - ASSERT_DERIVATION_TRACE1( - "derivationStrict { name = \"foo\"; }", EvalError, HintFmt("required attribute 'builder' missing")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; __structuredAttrs = 15; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), - HintFmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; __ignoreNulls = 15; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), - HintFmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; outputHashMode = 15; }", - EvalError, - HintFmt("invalid value '%s' for 'outputHashMode' attribute", "15"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; outputHashMode = \"custom\"; }", - EvalError, - HintFmt("invalid value '%s' for 'outputHashMode' attribute", "custom"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "system", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drvPath\"; }", - EvalError, - HintFmt("invalid derivation output name 'drvPath'"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; outputs = \"out\"; __structuredAttrs = true; }", - EvalError, - HintFmt("expected a list but found %s: %s", "a string", "\"out\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = []; }", - EvalError, - HintFmt("derivation cannot have an empty set of outputs"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"drvPath\" ]; }", - EvalError, - HintFmt("invalid derivation output name 'drvPath'"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE2( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"out\" \"out\" ]; }", - EvalError, - HintFmt("duplicate derivation output '%s'", "out"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __contentAddressed = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__contentAddressed", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", - TypeError, - HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = \"foo\"; }", - TypeError, - HintFmt("expected a list but found %s: %s", "a string", "\"foo\""), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt("while evaluating an element of the argument list"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt("while evaluating an element of the argument list"), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); - - ASSERT_DERIVATION_TRACE3( - "derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", - TypeError, - HintFmt("cannot coerce %s to a string: { }", "a set"), - HintFmt(""), - HintFmt("while evaluating attribute '%s' of derivation '%s'", "FOO", "foo")); -} - } /* namespace nix */ diff --git a/tests/functional/lang/eval-fail-add-1.err.exp b/tests/functional/lang/eval-fail-add-1.err.exp new file mode 100644 index 0000000000..4a72eba125 --- /dev/null +++ b/tests/functional/lang/eval-fail-add-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'add' builtin + at /pwd/lang/eval-fail-add-1.nix:1:1: + 1| builtins.add "foo" 1 + | ^ + 2| + + … while evaluating the first argument of the addition + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-add-1.nix b/tests/functional/lang/eval-fail-add-1.nix new file mode 100644 index 0000000000..4a59d94e68 --- /dev/null +++ b/tests/functional/lang/eval-fail-add-1.nix @@ -0,0 +1 @@ +builtins.add "foo" 1 diff --git a/tests/functional/lang/eval-fail-add-2.err.exp b/tests/functional/lang/eval-fail-add-2.err.exp new file mode 100644 index 0000000000..429636348d --- /dev/null +++ b/tests/functional/lang/eval-fail-add-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'add' builtin + at /pwd/lang/eval-fail-add-2.nix:1:1: + 1| builtins.add 1 "foo" + | ^ + 2| + + … while evaluating the second argument of the addition + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-add-2.nix b/tests/functional/lang/eval-fail-add-2.nix new file mode 100644 index 0000000000..1b785aba27 --- /dev/null +++ b/tests/functional/lang/eval-fail-add-2.nix @@ -0,0 +1 @@ +builtins.add 1 "foo" diff --git a/tests/functional/lang/eval-fail-all-1.err.exp b/tests/functional/lang/eval-fail-all-1.err.exp new file mode 100644 index 0000000000..95e8735f8c --- /dev/null +++ b/tests/functional/lang/eval-fail-all-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'all' builtin + at /pwd/lang/eval-fail-all-1.nix:1:1: + 1| builtins.all 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.all + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-all-1.nix b/tests/functional/lang/eval-fail-all-1.nix new file mode 100644 index 0000000000..c1ae6ff5e1 --- /dev/null +++ b/tests/functional/lang/eval-fail-all-1.nix @@ -0,0 +1 @@ +builtins.all 1 "foo" diff --git a/tests/functional/lang/eval-fail-all-2.err.exp b/tests/functional/lang/eval-fail-all-2.err.exp new file mode 100644 index 0000000000..584543f5e8 --- /dev/null +++ b/tests/functional/lang/eval-fail-all-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'all' builtin + at /pwd/lang/eval-fail-all-2.nix:1:1: + 1| builtins.all (_: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.all + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-all-2.nix b/tests/functional/lang/eval-fail-all-2.nix new file mode 100644 index 0000000000..b8ec8c8712 --- /dev/null +++ b/tests/functional/lang/eval-fail-all-2.nix @@ -0,0 +1 @@ +builtins.all (_: 1) "foo" diff --git a/tests/functional/lang/eval-fail-all-3.err.exp b/tests/functional/lang/eval-fail-all-3.err.exp new file mode 100644 index 0000000000..692cebe3c2 --- /dev/null +++ b/tests/functional/lang/eval-fail-all-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'all' builtin + at /pwd/lang/eval-fail-all-3.nix:1:1: + 1| builtins.all (_: 1) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the function passed to builtins.all + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-all-3.nix b/tests/functional/lang/eval-fail-all-3.nix new file mode 100644 index 0000000000..68f6ce6567 --- /dev/null +++ b/tests/functional/lang/eval-fail-all-3.nix @@ -0,0 +1 @@ +builtins.all (_: 1) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-any-1.err.exp b/tests/functional/lang/eval-fail-any-1.err.exp new file mode 100644 index 0000000000..2d81dae425 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'any' builtin + at /pwd/lang/eval-fail-any-1.nix:1:1: + 1| builtins.any 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.any + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-any-1.nix b/tests/functional/lang/eval-fail-any-1.nix new file mode 100644 index 0000000000..aa59791b73 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-1.nix @@ -0,0 +1 @@ +builtins.any 1 "foo" diff --git a/tests/functional/lang/eval-fail-any-2.err.exp b/tests/functional/lang/eval-fail-any-2.err.exp new file mode 100644 index 0000000000..a03a9e7c34 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'any' builtin + at /pwd/lang/eval-fail-any-2.nix:1:1: + 1| builtins.any (_: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.any + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-any-2.nix b/tests/functional/lang/eval-fail-any-2.nix new file mode 100644 index 0000000000..64a5d3acc8 --- /dev/null +++ b/tests/functional/lang/eval-fail-any-2.nix @@ -0,0 +1 @@ +builtins.any (_: 1) "foo" diff --git a/tests/functional/lang/eval-fail-any-3.err.exp b/tests/functional/lang/eval-fail-any-3.err.exp new file mode 100644 index 0000000000..b06d27abdf --- /dev/null +++ b/tests/functional/lang/eval-fail-any-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'any' builtin + at /pwd/lang/eval-fail-any-3.nix:1:1: + 1| builtins.any (_: 1) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the function passed to builtins.any + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-any-3.nix b/tests/functional/lang/eval-fail-any-3.nix new file mode 100644 index 0000000000..906d08252a --- /dev/null +++ b/tests/functional/lang/eval-fail-any-3.nix @@ -0,0 +1 @@ +builtins.any (_: 1) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-attrNames-1.err.exp b/tests/functional/lang/eval-fail-attrNames-1.err.exp new file mode 100644 index 0000000000..46f71a4910 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrNames-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'attrNames' builtin + at /pwd/lang/eval-fail-attrNames-1.nix:1:1: + 1| builtins.attrNames [ ] + | ^ + 2| + + … while evaluating the argument passed to builtins.attrNames + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-attrNames-1.nix b/tests/functional/lang/eval-fail-attrNames-1.nix new file mode 100644 index 0000000000..38f68e4d67 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrNames-1.nix @@ -0,0 +1 @@ +builtins.attrNames [ ] diff --git a/tests/functional/lang/eval-fail-attrValues-1.err.exp b/tests/functional/lang/eval-fail-attrValues-1.err.exp new file mode 100644 index 0000000000..c7018784f1 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrValues-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'attrValues' builtin + at /pwd/lang/eval-fail-attrValues-1.nix:1:1: + 1| builtins.attrValues [ ] + | ^ + 2| + + … while evaluating the argument passed to builtins.attrValues + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-attrValues-1.nix b/tests/functional/lang/eval-fail-attrValues-1.nix new file mode 100644 index 0000000000..c3154854c7 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrValues-1.nix @@ -0,0 +1 @@ +builtins.attrValues [ ] diff --git a/tests/functional/lang/eval-fail-baseNameOf-1.err.exp b/tests/functional/lang/eval-fail-baseNameOf-1.err.exp new file mode 100644 index 0000000000..4116a1a063 --- /dev/null +++ b/tests/functional/lang/eval-fail-baseNameOf-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'baseNameOf' builtin + at /pwd/lang/eval-fail-baseNameOf-1.nix:1:1: + 1| builtins.baseNameOf [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.baseNameOf + + error: cannot coerce a list to a string: [ ] diff --git a/tests/functional/lang/eval-fail-baseNameOf-1.nix b/tests/functional/lang/eval-fail-baseNameOf-1.nix new file mode 100644 index 0000000000..3099c87f57 --- /dev/null +++ b/tests/functional/lang/eval-fail-baseNameOf-1.nix @@ -0,0 +1 @@ +builtins.baseNameOf [ ] diff --git a/tests/functional/lang/eval-fail-bitAnd-1.err.exp b/tests/functional/lang/eval-fail-bitAnd-1.err.exp new file mode 100644 index 0000000000..04ae5c8540 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitAnd-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitAnd' builtin + at /pwd/lang/eval-fail-bitAnd-1.nix:1:1: + 1| builtins.bitAnd 1.1 2 + | ^ + 2| + + … while evaluating the first argument passed to builtins.bitAnd + + error: expected an integer but found a float: 1.1 diff --git a/tests/functional/lang/eval-fail-bitAnd-1.nix b/tests/functional/lang/eval-fail-bitAnd-1.nix new file mode 100644 index 0000000000..24c5d60bdd --- /dev/null +++ b/tests/functional/lang/eval-fail-bitAnd-1.nix @@ -0,0 +1 @@ +builtins.bitAnd 1.1 2 diff --git a/tests/functional/lang/eval-fail-bitAnd-2.err.exp b/tests/functional/lang/eval-fail-bitAnd-2.err.exp new file mode 100644 index 0000000000..f845462f12 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitAnd-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitAnd' builtin + at /pwd/lang/eval-fail-bitAnd-2.nix:1:1: + 1| builtins.bitAnd 1 2.2 + | ^ + 2| + + … while evaluating the second argument passed to builtins.bitAnd + + error: expected an integer but found a float: 2.2 diff --git a/tests/functional/lang/eval-fail-bitAnd-2.nix b/tests/functional/lang/eval-fail-bitAnd-2.nix new file mode 100644 index 0000000000..12ea1451f5 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitAnd-2.nix @@ -0,0 +1 @@ +builtins.bitAnd 1 2.2 diff --git a/tests/functional/lang/eval-fail-bitOr-1.err.exp b/tests/functional/lang/eval-fail-bitOr-1.err.exp new file mode 100644 index 0000000000..38b498da46 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitOr-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitOr' builtin + at /pwd/lang/eval-fail-bitOr-1.nix:1:1: + 1| builtins.bitOr 1.1 2 + | ^ + 2| + + … while evaluating the first argument passed to builtins.bitOr + + error: expected an integer but found a float: 1.1 diff --git a/tests/functional/lang/eval-fail-bitOr-1.nix b/tests/functional/lang/eval-fail-bitOr-1.nix new file mode 100644 index 0000000000..2eab4871ca --- /dev/null +++ b/tests/functional/lang/eval-fail-bitOr-1.nix @@ -0,0 +1 @@ +builtins.bitOr 1.1 2 diff --git a/tests/functional/lang/eval-fail-bitOr-2.err.exp b/tests/functional/lang/eval-fail-bitOr-2.err.exp new file mode 100644 index 0000000000..a1ca655add --- /dev/null +++ b/tests/functional/lang/eval-fail-bitOr-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitOr' builtin + at /pwd/lang/eval-fail-bitOr-2.nix:1:1: + 1| builtins.bitOr 1 2.2 + | ^ + 2| + + … while evaluating the second argument passed to builtins.bitOr + + error: expected an integer but found a float: 2.2 diff --git a/tests/functional/lang/eval-fail-bitOr-2.nix b/tests/functional/lang/eval-fail-bitOr-2.nix new file mode 100644 index 0000000000..f66e3a3635 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitOr-2.nix @@ -0,0 +1 @@ +builtins.bitOr 1 2.2 diff --git a/tests/functional/lang/eval-fail-bitXor-1.err.exp b/tests/functional/lang/eval-fail-bitXor-1.err.exp new file mode 100644 index 0000000000..f051ef6220 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitXor-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitXor' builtin + at /pwd/lang/eval-fail-bitXor-1.nix:1:1: + 1| builtins.bitXor 1.1 2 + | ^ + 2| + + … while evaluating the first argument passed to builtins.bitXor + + error: expected an integer but found a float: 1.1 diff --git a/tests/functional/lang/eval-fail-bitXor-1.nix b/tests/functional/lang/eval-fail-bitXor-1.nix new file mode 100644 index 0000000000..c1b0f1b4e8 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitXor-1.nix @@ -0,0 +1 @@ +builtins.bitXor 1.1 2 diff --git a/tests/functional/lang/eval-fail-bitXor-2.err.exp b/tests/functional/lang/eval-fail-bitXor-2.err.exp new file mode 100644 index 0000000000..3b5ab1317a --- /dev/null +++ b/tests/functional/lang/eval-fail-bitXor-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'bitXor' builtin + at /pwd/lang/eval-fail-bitXor-2.nix:1:1: + 1| builtins.bitXor 1 2.2 + | ^ + 2| + + … while evaluating the second argument passed to builtins.bitXor + + error: expected an integer but found a float: 2.2 diff --git a/tests/functional/lang/eval-fail-bitXor-2.nix b/tests/functional/lang/eval-fail-bitXor-2.nix new file mode 100644 index 0000000000..b9f7ad9c28 --- /dev/null +++ b/tests/functional/lang/eval-fail-bitXor-2.nix @@ -0,0 +1 @@ +builtins.bitXor 1 2.2 diff --git a/tests/functional/lang/eval-fail-catAttrs-1.err.exp b/tests/functional/lang/eval-fail-catAttrs-1.err.exp new file mode 100644 index 0000000000..e669dc39a6 --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'catAttrs' builtin + at /pwd/lang/eval-fail-catAttrs-1.nix:1:1: + 1| builtins.catAttrs [ ] { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.catAttrs + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-catAttrs-1.nix b/tests/functional/lang/eval-fail-catAttrs-1.nix new file mode 100644 index 0000000000..97d06f5b63 --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-1.nix @@ -0,0 +1 @@ +builtins.catAttrs [ ] { } diff --git a/tests/functional/lang/eval-fail-catAttrs-2.err.exp b/tests/functional/lang/eval-fail-catAttrs-2.err.exp new file mode 100644 index 0000000000..2f6c1425d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'catAttrs' builtin + at /pwd/lang/eval-fail-catAttrs-2.nix:1:1: + 1| builtins.catAttrs "foo" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.catAttrs + + error: expected a list but found a set: { } diff --git a/tests/functional/lang/eval-fail-catAttrs-2.nix b/tests/functional/lang/eval-fail-catAttrs-2.nix new file mode 100644 index 0000000000..caf616b20b --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-2.nix @@ -0,0 +1 @@ +builtins.catAttrs "foo" { } diff --git a/tests/functional/lang/eval-fail-catAttrs-3.err.exp b/tests/functional/lang/eval-fail-catAttrs-3.err.exp new file mode 100644 index 0000000000..2643d37f9e --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'catAttrs' builtin + at /pwd/lang/eval-fail-catAttrs-3.nix:1:1: + 1| builtins.catAttrs "foo" [ 1 ] + | ^ + 2| + + … while evaluating an element in the list passed as second argument to builtins.catAttrs + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-catAttrs-3.nix b/tests/functional/lang/eval-fail-catAttrs-3.nix new file mode 100644 index 0000000000..82595b94ed --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-3.nix @@ -0,0 +1 @@ +builtins.catAttrs "foo" [ 1 ] diff --git a/tests/functional/lang/eval-fail-catAttrs-4.err.exp b/tests/functional/lang/eval-fail-catAttrs-4.err.exp new file mode 100644 index 0000000000..ffd5fcec7c --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-4.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'catAttrs' builtin + at /pwd/lang/eval-fail-catAttrs-4.nix:1:1: + 1| builtins.catAttrs "foo" [ + | ^ + 2| { foo = 1; } + + … while evaluating an element in the list passed as second argument to builtins.catAttrs + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-catAttrs-4.nix b/tests/functional/lang/eval-fail-catAttrs-4.nix new file mode 100644 index 0000000000..1f294a693c --- /dev/null +++ b/tests/functional/lang/eval-fail-catAttrs-4.nix @@ -0,0 +1,5 @@ +builtins.catAttrs "foo" [ + { foo = 1; } + 1 + { bar = 5; } +] diff --git a/tests/functional/lang/eval-fail-ceil-1.err.exp b/tests/functional/lang/eval-fail-ceil-1.err.exp new file mode 100644 index 0000000000..96d8b1df7a --- /dev/null +++ b/tests/functional/lang/eval-fail-ceil-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'ceil' builtin + at /pwd/lang/eval-fail-ceil-1.nix:1:1: + 1| builtins.ceil "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.ceil + + error: expected a float but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-ceil-1.nix b/tests/functional/lang/eval-fail-ceil-1.nix new file mode 100644 index 0000000000..0ea67754c3 --- /dev/null +++ b/tests/functional/lang/eval-fail-ceil-1.nix @@ -0,0 +1 @@ +builtins.ceil "foo" diff --git a/tests/functional/lang/eval-fail-compareVersions-1.err.exp b/tests/functional/lang/eval-fail-compareVersions-1.err.exp new file mode 100644 index 0000000000..8632cf237e --- /dev/null +++ b/tests/functional/lang/eval-fail-compareVersions-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'compareVersions' builtin + at /pwd/lang/eval-fail-compareVersions-1.nix:1:1: + 1| builtins.compareVersions 1 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.compareVersions + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-compareVersions-1.nix b/tests/functional/lang/eval-fail-compareVersions-1.nix new file mode 100644 index 0000000000..f408381cf4 --- /dev/null +++ b/tests/functional/lang/eval-fail-compareVersions-1.nix @@ -0,0 +1 @@ +builtins.compareVersions 1 { } diff --git a/tests/functional/lang/eval-fail-compareVersions-2.err.exp b/tests/functional/lang/eval-fail-compareVersions-2.err.exp new file mode 100644 index 0000000000..bcd246460f --- /dev/null +++ b/tests/functional/lang/eval-fail-compareVersions-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'compareVersions' builtin + at /pwd/lang/eval-fail-compareVersions-2.nix:1:1: + 1| builtins.compareVersions "abd" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.compareVersions + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-compareVersions-2.nix b/tests/functional/lang/eval-fail-compareVersions-2.nix new file mode 100644 index 0000000000..49c9f0a887 --- /dev/null +++ b/tests/functional/lang/eval-fail-compareVersions-2.nix @@ -0,0 +1 @@ +builtins.compareVersions "abd" { } diff --git a/tests/functional/lang/eval-fail-concatLists-1.err.exp b/tests/functional/lang/eval-fail-concatLists-1.err.exp new file mode 100644 index 0000000000..15193daae1 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatLists' builtin + at /pwd/lang/eval-fail-concatLists-1.nix:1:1: + 1| builtins.concatLists 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.concatLists + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatLists-1.nix b/tests/functional/lang/eval-fail-concatLists-1.nix new file mode 100644 index 0000000000..97e171cb6f --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-1.nix @@ -0,0 +1 @@ +builtins.concatLists 1 diff --git a/tests/functional/lang/eval-fail-concatLists-2.err.exp b/tests/functional/lang/eval-fail-concatLists-2.err.exp new file mode 100644 index 0000000000..35fbc09bfe --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatLists' builtin + at /pwd/lang/eval-fail-concatLists-2.nix:1:1: + 1| builtins.concatLists [ 1 ] + | ^ + 2| + + … while evaluating a value of the list passed to builtins.concatLists + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatLists-2.nix b/tests/functional/lang/eval-fail-concatLists-2.nix new file mode 100644 index 0000000000..0ad98a8eb4 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-2.nix @@ -0,0 +1 @@ +builtins.concatLists [ 1 ] diff --git a/tests/functional/lang/eval-fail-concatLists-3.err.exp b/tests/functional/lang/eval-fail-concatLists-3.err.exp new file mode 100644 index 0000000000..d762eb55bd --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatLists' builtin + at /pwd/lang/eval-fail-concatLists-3.nix:1:1: + 1| builtins.concatLists [ + | ^ + 2| [ 1 ] + + … while evaluating a value of the list passed to builtins.concatLists + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-concatLists-3.nix b/tests/functional/lang/eval-fail-concatLists-3.nix new file mode 100644 index 0000000000..2366f314e1 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatLists-3.nix @@ -0,0 +1,4 @@ +builtins.concatLists [ + [ 1 ] + "foo" +] diff --git a/tests/functional/lang/eval-fail-concatMap-1.err.exp b/tests/functional/lang/eval-fail-concatMap-1.err.exp new file mode 100644 index 0000000000..e43c160a38 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatMap' builtin + at /pwd/lang/eval-fail-concatMap-1.nix:1:1: + 1| builtins.concatMap 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.concatMap + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatMap-1.nix b/tests/functional/lang/eval-fail-concatMap-1.nix new file mode 100644 index 0000000000..68c9c3560f --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-1.nix @@ -0,0 +1 @@ +builtins.concatMap 1 "foo" diff --git a/tests/functional/lang/eval-fail-concatMap-2.err.exp b/tests/functional/lang/eval-fail-concatMap-2.err.exp new file mode 100644 index 0000000000..52c51fd015 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatMap' builtin + at /pwd/lang/eval-fail-concatMap-2.nix:1:1: + 1| builtins.concatMap (x: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.concatMap + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-concatMap-2.nix b/tests/functional/lang/eval-fail-concatMap-2.nix new file mode 100644 index 0000000000..b98860470c --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-2.nix @@ -0,0 +1 @@ +builtins.concatMap (x: 1) "foo" diff --git a/tests/functional/lang/eval-fail-concatMap-3.err.exp b/tests/functional/lang/eval-fail-concatMap-3.err.exp new file mode 100644 index 0000000000..21d13118f0 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-3.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'concatMap' builtin + at /pwd/lang/eval-fail-concatMap-3.nix:1:1: + 1| builtins.concatMap (x: 1) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the function passed to builtins.concatMap + at /pwd/lang/eval-fail-concatMap-3.nix:1:21: + 1| builtins.concatMap (x: 1) [ "foo" ] + | ^ + 2| + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatMap-3.nix b/tests/functional/lang/eval-fail-concatMap-3.nix new file mode 100644 index 0000000000..f4a5190603 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-3.nix @@ -0,0 +1 @@ +builtins.concatMap (x: 1) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-concatMap-4.err.exp b/tests/functional/lang/eval-fail-concatMap-4.err.exp new file mode 100644 index 0000000000..1b8e865fd3 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-4.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'concatMap' builtin + at /pwd/lang/eval-fail-concatMap-4.nix:1:1: + 1| builtins.concatMap (x: "foo") [ + | ^ + 2| 1 + + … while evaluating the return value of the function passed to builtins.concatMap + at /pwd/lang/eval-fail-concatMap-4.nix:1:21: + 1| builtins.concatMap (x: "foo") [ + | ^ + 2| 1 + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-concatMap-4.nix b/tests/functional/lang/eval-fail-concatMap-4.nix new file mode 100644 index 0000000000..0f678005bc --- /dev/null +++ b/tests/functional/lang/eval-fail-concatMap-4.nix @@ -0,0 +1,4 @@ +builtins.concatMap (x: "foo") [ + 1 + 2 +] diff --git a/tests/functional/lang/eval-fail-concatStringsSep-1.err.exp b/tests/functional/lang/eval-fail-concatStringsSep-1.err.exp new file mode 100644 index 0000000000..4b40db46d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatStringsSep' builtin + at /pwd/lang/eval-fail-concatStringsSep-1.nix:1:1: + 1| builtins.concatStringsSep 1 { } + | ^ + 2| + + … while evaluating the first argument (the separator string) passed to builtins.concatStringsSep + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-concatStringsSep-1.nix b/tests/functional/lang/eval-fail-concatStringsSep-1.nix new file mode 100644 index 0000000000..0c10cea346 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-1.nix @@ -0,0 +1 @@ +builtins.concatStringsSep 1 { } diff --git a/tests/functional/lang/eval-fail-concatStringsSep-2.err.exp b/tests/functional/lang/eval-fail-concatStringsSep-2.err.exp new file mode 100644 index 0000000000..9cc806f495 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatStringsSep' builtin + at /pwd/lang/eval-fail-concatStringsSep-2.nix:1:1: + 1| builtins.concatStringsSep "foo" { } + | ^ + 2| + + … while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep + + error: expected a list but found a set: { } diff --git a/tests/functional/lang/eval-fail-concatStringsSep-2.nix b/tests/functional/lang/eval-fail-concatStringsSep-2.nix new file mode 100644 index 0000000000..ae6a5f1984 --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-2.nix @@ -0,0 +1 @@ +builtins.concatStringsSep "foo" { } diff --git a/tests/functional/lang/eval-fail-concatStringsSep-3.err.exp b/tests/functional/lang/eval-fail-concatStringsSep-3.err.exp new file mode 100644 index 0000000000..5c098d471e --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'concatStringsSep' builtin + at /pwd/lang/eval-fail-concatStringsSep-3.nix:1:1: + 1| builtins.concatStringsSep "foo" [ + | ^ + 2| 1 + + … while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep + + error: cannot coerce an integer to a string: 1 diff --git a/tests/functional/lang/eval-fail-concatStringsSep-3.nix b/tests/functional/lang/eval-fail-concatStringsSep-3.nix new file mode 100644 index 0000000000..f51ae1c7dd --- /dev/null +++ b/tests/functional/lang/eval-fail-concatStringsSep-3.nix @@ -0,0 +1,5 @@ +builtins.concatStringsSep "foo" [ + 1 + 2 + { } +] diff --git a/tests/functional/lang/eval-fail-derivationStrict-1.err.exp b/tests/functional/lang/eval-fail-derivationStrict-1.err.exp new file mode 100644 index 0000000000..468c789e8d --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-1.nix:1:1: + 1| builtins.derivationStrict "" + | ^ + 2| + + … while evaluating the argument passed to builtins.derivationStrict + + error: expected a set but found a string: "" diff --git a/tests/functional/lang/eval-fail-derivationStrict-1.nix b/tests/functional/lang/eval-fail-derivationStrict-1.nix new file mode 100644 index 0000000000..948230469d --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-1.nix @@ -0,0 +1 @@ +builtins.derivationStrict "" diff --git a/tests/functional/lang/eval-fail-derivationStrict-10.err.exp b/tests/functional/lang/eval-fail-derivationStrict-10.err.exp new file mode 100644 index 0000000000..21eca42fad --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-10.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-10.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-10.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-10.nix:5:3: + 4| system = 1; + 5| outputs = { }; + | ^ + 6| } + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-10.nix b/tests/functional/lang/eval-fail-derivationStrict-10.nix new file mode 100644 index 0000000000..410f48a465 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-10.nix @@ -0,0 +1,6 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = { }; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-11.err.exp b/tests/functional/lang/eval-fail-derivationStrict-11.err.exp new file mode 100644 index 0000000000..b0fe664e52 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-11.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-11.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-11.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-11.nix:5:3: + 4| system = 1; + 5| outputs = "drvPath"; + | ^ + 6| } + + error: invalid derivation output name 'drvPath' diff --git a/tests/functional/lang/eval-fail-derivationStrict-11.nix b/tests/functional/lang/eval-fail-derivationStrict-11.nix new file mode 100644 index 0000000000..60947b6e2e --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-11.nix @@ -0,0 +1,6 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "drvPath"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-12.err.exp b/tests/functional/lang/eval-fail-derivationStrict-12.err.exp new file mode 100644 index 0000000000..adfc0064bf --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-12.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-12.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-12.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-12.nix:3:3: + 2| name = "foo"; + 3| outputs = "out"; + | ^ + 4| __structuredAttrs = true; + + error: expected a list but found a string: "out" diff --git a/tests/functional/lang/eval-fail-derivationStrict-12.nix b/tests/functional/lang/eval-fail-derivationStrict-12.nix new file mode 100644 index 0000000000..516b8ba39a --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-12.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + outputs = "out"; + __structuredAttrs = true; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-13.err.exp b/tests/functional/lang/eval-fail-derivationStrict-13.err.exp new file mode 100644 index 0000000000..8a313333d5 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-13.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-13.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-13.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-13.nix:5:3: + 4| system = 1; + 5| outputs = [ ]; + | ^ + 6| } + + error: derivation cannot have an empty set of outputs diff --git a/tests/functional/lang/eval-fail-derivationStrict-13.nix b/tests/functional/lang/eval-fail-derivationStrict-13.nix new file mode 100644 index 0000000000..9a104d2253 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-13.nix @@ -0,0 +1,6 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = [ ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-14.err.exp b/tests/functional/lang/eval-fail-derivationStrict-14.err.exp new file mode 100644 index 0000000000..fb0c5423c4 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-14.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-14.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-14.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-14.nix:5:3: + 4| system = 1; + 5| outputs = [ "drvPath" ]; + | ^ + 6| } + + error: invalid derivation output name 'drvPath' diff --git a/tests/functional/lang/eval-fail-derivationStrict-14.nix b/tests/functional/lang/eval-fail-derivationStrict-14.nix new file mode 100644 index 0000000000..4d286d34c7 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-14.nix @@ -0,0 +1,6 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = [ "drvPath" ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-15.err.exp b/tests/functional/lang/eval-fail-derivationStrict-15.err.exp new file mode 100644 index 0000000000..8df0470691 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-15.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-15.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-15.nix:2:3 + + … while evaluating attribute 'outputs' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-15.nix:5:3: + 4| system = 1; + 5| outputs = [ + | ^ + 6| "out" + + error: duplicate derivation output 'out' diff --git a/tests/functional/lang/eval-fail-derivationStrict-15.nix b/tests/functional/lang/eval-fail-derivationStrict-15.nix new file mode 100644 index 0000000000..96e5a9de40 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-15.nix @@ -0,0 +1,9 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = [ + "out" + "out" + ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-16.err.exp b/tests/functional/lang/eval-fail-derivationStrict-16.err.exp new file mode 100644 index 0000000000..c969aa30d0 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-16.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-16.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-16.nix:2:3 + + … while evaluating attribute '__contentAddressed' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-16.nix:6:3: + 5| outputs = "out"; + 6| __contentAddressed = "true"; + | ^ + 7| } + + error: expected a Boolean but found a string: "true" diff --git a/tests/functional/lang/eval-fail-derivationStrict-16.nix b/tests/functional/lang/eval-fail-derivationStrict-16.nix new file mode 100644 index 0000000000..554ad09df1 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-16.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + __contentAddressed = "true"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-17.err.exp b/tests/functional/lang/eval-fail-derivationStrict-17.err.exp new file mode 100644 index 0000000000..ed7558c4cb --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-17.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-17.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-17.nix:2:3 + + … while evaluating attribute '__impure' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-17.nix:6:3: + 5| outputs = "out"; + 6| __impure = "true"; + | ^ + 7| } + + error: expected a Boolean but found a string: "true" diff --git a/tests/functional/lang/eval-fail-derivationStrict-17.nix b/tests/functional/lang/eval-fail-derivationStrict-17.nix new file mode 100644 index 0000000000..4a357ba12d --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-17.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + __impure = "true"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-19.err.exp b/tests/functional/lang/eval-fail-derivationStrict-19.err.exp new file mode 100644 index 0000000000..4861b1cd3a --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-19.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-19.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-19.nix:2:3 + + … while evaluating attribute 'args' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-19.nix:6:3: + 5| outputs = "out"; + 6| args = "foo"; + | ^ + 7| } + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-derivationStrict-19.nix b/tests/functional/lang/eval-fail-derivationStrict-19.nix new file mode 100644 index 0000000000..912be12c57 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-19.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + args = "foo"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-2.err.exp b/tests/functional/lang/eval-fail-derivationStrict-2.err.exp new file mode 100644 index 0000000000..7ba9fc8255 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-2.nix:1:1: + 1| builtins.derivationStrict { } + | ^ + 2| + + … in the attrset passed as argument to builtins.derivationStrict + + error: attribute 'name' missing diff --git a/tests/functional/lang/eval-fail-derivationStrict-2.nix b/tests/functional/lang/eval-fail-derivationStrict-2.nix new file mode 100644 index 0000000000..70a07b36d2 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-2.nix @@ -0,0 +1 @@ +builtins.derivationStrict { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-20.err.exp b/tests/functional/lang/eval-fail-derivationStrict-20.err.exp new file mode 100644 index 0000000000..df1805bef6 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-20.err.exp @@ -0,0 +1,20 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-20.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-20.nix:2:3 + + … while evaluating attribute 'args' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-20.nix:6:3: + 5| outputs = "out"; + 6| args = [ { } ]; + | ^ + 7| } + + … while evaluating an element of the argument list + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-20.nix b/tests/functional/lang/eval-fail-derivationStrict-20.nix new file mode 100644 index 0000000000..17bc3a26c9 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-20.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + args = [ { } ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-21.err.exp b/tests/functional/lang/eval-fail-derivationStrict-21.err.exp new file mode 100644 index 0000000000..0a6384a779 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-21.err.exp @@ -0,0 +1,20 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-21.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-21.nix:2:3 + + … while evaluating attribute 'args' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-21.nix:6:3: + 5| outputs = "out"; + 6| args = [ + | ^ + 7| "a" + + … while evaluating an element of the argument list + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-21.nix b/tests/functional/lang/eval-fail-derivationStrict-21.nix new file mode 100644 index 0000000000..7846874194 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-21.nix @@ -0,0 +1,10 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + args = [ + "a" + { } + ]; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-22.err.exp b/tests/functional/lang/eval-fail-derivationStrict-22.err.exp new file mode 100644 index 0000000000..91a44ce1e4 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-22.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-22.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-22.nix:2:3 + + … while evaluating attribute 'FOO' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-22.nix:6:3: + 5| outputs = "out"; + 6| FOO = { }; + | ^ + 7| } + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-22.nix b/tests/functional/lang/eval-fail-derivationStrict-22.nix new file mode 100644 index 0000000000..7b3b7c8248 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-22.nix @@ -0,0 +1,7 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = 1; + outputs = "out"; + FOO = { }; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-3.err.exp b/tests/functional/lang/eval-fail-derivationStrict-3.err.exp new file mode 100644 index 0000000000..f8c301a977 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-3.err.exp @@ -0,0 +1,16 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-3.nix:1:1: + 1| builtins.derivationStrict { name = 1; } + | ^ + 2| + + … while evaluating the derivation attribute 'name' + at /pwd/lang/eval-fail-derivationStrict-3.nix:1:29: + 1| builtins.derivationStrict { name = 1; } + | ^ + 2| + + … while evaluating the `name` attribute passed to builtins.derivationStrict + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-derivationStrict-3.nix b/tests/functional/lang/eval-fail-derivationStrict-3.nix new file mode 100644 index 0000000000..a18edebd87 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-3.nix @@ -0,0 +1 @@ +builtins.derivationStrict { name = 1; } diff --git a/tests/functional/lang/eval-fail-derivationStrict-4.err.exp b/tests/functional/lang/eval-fail-derivationStrict-4.err.exp new file mode 100644 index 0000000000..b598f0f11c --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-4.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-4.nix:1:1: + 1| builtins.derivationStrict { name = "foo"; } + | ^ + 2| + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-4.nix:1:29 + + error: required attribute 'builder' missing diff --git a/tests/functional/lang/eval-fail-derivationStrict-4.nix b/tests/functional/lang/eval-fail-derivationStrict-4.nix new file mode 100644 index 0000000000..f13692bc5e --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-4.nix @@ -0,0 +1 @@ +builtins.derivationStrict { name = "foo"; } diff --git a/tests/functional/lang/eval-fail-derivationStrict-5.err.exp b/tests/functional/lang/eval-fail-derivationStrict-5.err.exp new file mode 100644 index 0000000000..930fda3a95 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-5.err.exp @@ -0,0 +1,13 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-5.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-5.nix:2:3 + + … while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict + + error: expected a Boolean but found an integer: 15 diff --git a/tests/functional/lang/eval-fail-derivationStrict-5.nix b/tests/functional/lang/eval-fail-derivationStrict-5.nix new file mode 100644 index 0000000000..e74a8caf45 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-5.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + __structuredAttrs = 15; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-6.err.exp b/tests/functional/lang/eval-fail-derivationStrict-6.err.exp new file mode 100644 index 0000000000..323db3b379 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-6.err.exp @@ -0,0 +1,13 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-6.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-6.nix:2:3 + + … while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict + + error: expected a Boolean but found an integer: 15 diff --git a/tests/functional/lang/eval-fail-derivationStrict-6.nix b/tests/functional/lang/eval-fail-derivationStrict-6.nix new file mode 100644 index 0000000000..feda92b00d --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-6.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + __ignoreNulls = 15; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-7.err.exp b/tests/functional/lang/eval-fail-derivationStrict-7.err.exp new file mode 100644 index 0000000000..78ddd15739 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-7.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-7.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-7.nix:2:3 + + … while evaluating attribute 'outputHashMode' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-7.nix:4:3: + 3| builder = 1; + 4| outputHashMode = 15; + | ^ + 5| } + + error: invalid value '15' for 'outputHashMode' attribute diff --git a/tests/functional/lang/eval-fail-derivationStrict-7.nix b/tests/functional/lang/eval-fail-derivationStrict-7.nix new file mode 100644 index 0000000000..1ffdb6fdbe --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-7.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + outputHashMode = 15; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-8.err.exp b/tests/functional/lang/eval-fail-derivationStrict-8.err.exp new file mode 100644 index 0000000000..ba4e0109c6 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-8.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-8.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-8.nix:2:3 + + … while evaluating attribute 'outputHashMode' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-8.nix:4:3: + 3| builder = 1; + 4| outputHashMode = "custom"; + | ^ + 5| } + + error: invalid value 'custom' for 'outputHashMode' attribute diff --git a/tests/functional/lang/eval-fail-derivationStrict-8.nix b/tests/functional/lang/eval-fail-derivationStrict-8.nix new file mode 100644 index 0000000000..38243416fd --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-8.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + outputHashMode = "custom"; +} diff --git a/tests/functional/lang/eval-fail-derivationStrict-9.err.exp b/tests/functional/lang/eval-fail-derivationStrict-9.err.exp new file mode 100644 index 0000000000..6bfbbc65b8 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-9.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'derivationStrict' builtin + at /pwd/lang/eval-fail-derivationStrict-9.nix:1:1: + 1| builtins.derivationStrict { + | ^ + 2| name = "foo"; + + … while evaluating derivation 'foo' + whose name attribute is located at /pwd/lang/eval-fail-derivationStrict-9.nix:2:3 + + … while evaluating attribute 'system' of derivation 'foo' + at /pwd/lang/eval-fail-derivationStrict-9.nix:4:3: + 3| builder = 1; + 4| system = { }; + | ^ + 5| } + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-derivationStrict-9.nix b/tests/functional/lang/eval-fail-derivationStrict-9.nix new file mode 100644 index 0000000000..a3a567a14c --- /dev/null +++ b/tests/functional/lang/eval-fail-derivationStrict-9.nix @@ -0,0 +1,5 @@ +builtins.derivationStrict { + name = "foo"; + builder = 1; + system = { }; +} diff --git a/tests/functional/lang/eval-fail-div-1.err.exp b/tests/functional/lang/eval-fail-div-1.err.exp new file mode 100644 index 0000000000..286e1f79c3 --- /dev/null +++ b/tests/functional/lang/eval-fail-div-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'div' builtin + at /pwd/lang/eval-fail-div-1.nix:1:1: + 1| builtins.div "foo" 1 + | ^ + 2| + + … while evaluating the first operand of the division + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-div-1.nix b/tests/functional/lang/eval-fail-div-1.nix new file mode 100644 index 0000000000..c6edce3697 --- /dev/null +++ b/tests/functional/lang/eval-fail-div-1.nix @@ -0,0 +1 @@ +builtins.div "foo" 1 diff --git a/tests/functional/lang/eval-fail-div-2.err.exp b/tests/functional/lang/eval-fail-div-2.err.exp new file mode 100644 index 0000000000..20e6e710c0 --- /dev/null +++ b/tests/functional/lang/eval-fail-div-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'div' builtin + at /pwd/lang/eval-fail-div-2.nix:1:1: + 1| builtins.div 1 "foo" + | ^ + 2| + + … while evaluating the second operand of the division + + error: expected a float but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-div-2.nix b/tests/functional/lang/eval-fail-div-2.nix new file mode 100644 index 0000000000..71d9947719 --- /dev/null +++ b/tests/functional/lang/eval-fail-div-2.nix @@ -0,0 +1 @@ +builtins.div 1 "foo" diff --git a/tests/functional/lang/eval-fail-div-3.err.exp b/tests/functional/lang/eval-fail-div-3.err.exp new file mode 100644 index 0000000000..6d263c956a --- /dev/null +++ b/tests/functional/lang/eval-fail-div-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'div' builtin + at /pwd/lang/eval-fail-div-3.nix:1:1: + 1| builtins.div "foo" 0 + | ^ + 2| + + error: division by zero diff --git a/tests/functional/lang/eval-fail-div-3.nix b/tests/functional/lang/eval-fail-div-3.nix new file mode 100644 index 0000000000..11c606baac --- /dev/null +++ b/tests/functional/lang/eval-fail-div-3.nix @@ -0,0 +1 @@ +builtins.div "foo" 0 diff --git a/tests/functional/lang/eval-fail-elem-1.err.exp b/tests/functional/lang/eval-fail-elem-1.err.exp new file mode 100644 index 0000000000..cd0a228141 --- /dev/null +++ b/tests/functional/lang/eval-fail-elem-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'elem' builtin + at /pwd/lang/eval-fail-elem-1.nix:1:1: + 1| builtins.elem 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.elem + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-elem-1.nix b/tests/functional/lang/eval-fail-elem-1.nix new file mode 100644 index 0000000000..eecc965cb4 --- /dev/null +++ b/tests/functional/lang/eval-fail-elem-1.nix @@ -0,0 +1 @@ +builtins.elem 1 "foo" diff --git a/tests/functional/lang/eval-fail-elemAt-1.err.exp b/tests/functional/lang/eval-fail-elemAt-1.err.exp new file mode 100644 index 0000000000..3ea5b0aef0 --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'elemAt' builtin + at /pwd/lang/eval-fail-elemAt-1.nix:1:1: + 1| builtins.elemAt "foo" (-1) + | ^ + 2| + + … while evaluating the first argument passed to 'builtins.elemAt' + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-elemAt-1.nix b/tests/functional/lang/eval-fail-elemAt-1.nix new file mode 100644 index 0000000000..27de601eba --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-1.nix @@ -0,0 +1 @@ +builtins.elemAt "foo" (-1) diff --git a/tests/functional/lang/eval-fail-elemAt-2.err.exp b/tests/functional/lang/eval-fail-elemAt-2.err.exp new file mode 100644 index 0000000000..6d746c7739 --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-2.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'elemAt' builtin + at /pwd/lang/eval-fail-elemAt-2.nix:1:1: + 1| builtins.elemAt [ ] (-1) + | ^ + 2| + + error: 'builtins.elemAt' called with index -1 on a list of size 0 diff --git a/tests/functional/lang/eval-fail-elemAt-2.nix b/tests/functional/lang/eval-fail-elemAt-2.nix new file mode 100644 index 0000000000..cf48f089f2 --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-2.nix @@ -0,0 +1 @@ +builtins.elemAt [ ] (-1) diff --git a/tests/functional/lang/eval-fail-elemAt-3.err.exp b/tests/functional/lang/eval-fail-elemAt-3.err.exp new file mode 100644 index 0000000000..1f34afb458 --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'elemAt' builtin + at /pwd/lang/eval-fail-elemAt-3.nix:1:1: + 1| builtins.elemAt [ "foo" ] 3 + | ^ + 2| + + error: 'builtins.elemAt' called with index 3 on a list of size 1 diff --git a/tests/functional/lang/eval-fail-elemAt-3.nix b/tests/functional/lang/eval-fail-elemAt-3.nix new file mode 100644 index 0000000000..8493015def --- /dev/null +++ b/tests/functional/lang/eval-fail-elemAt-3.nix @@ -0,0 +1 @@ +builtins.elemAt [ "foo" ] 3 diff --git a/tests/functional/lang/eval-fail-filter-1.err.exp b/tests/functional/lang/eval-fail-filter-1.err.exp new file mode 100644 index 0000000000..d3c55d9f65 --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filter' builtin + at /pwd/lang/eval-fail-filter-1.nix:1:1: + 1| builtins.filter 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.filter + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-filter-1.nix b/tests/functional/lang/eval-fail-filter-1.nix new file mode 100644 index 0000000000..37b379f87d --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-1.nix @@ -0,0 +1 @@ +builtins.filter 1 "foo" diff --git a/tests/functional/lang/eval-fail-filter-2.err.exp b/tests/functional/lang/eval-fail-filter-2.err.exp new file mode 100644 index 0000000000..5e5d0ecc5a --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filter' builtin + at /pwd/lang/eval-fail-filter-2.nix:1:1: + 1| builtins.filter 1 [ "foo" ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.filter + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-filter-2.nix b/tests/functional/lang/eval-fail-filter-2.nix new file mode 100644 index 0000000000..177adfa82c --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-2.nix @@ -0,0 +1 @@ +builtins.filter 1 [ "foo" ] diff --git a/tests/functional/lang/eval-fail-filter-3.err.exp b/tests/functional/lang/eval-fail-filter-3.err.exp new file mode 100644 index 0000000000..ba5a87e275 --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filter' builtin + at /pwd/lang/eval-fail-filter-3.nix:1:1: + 1| builtins.filter (_: 5) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the filtering function passed to builtins.filter + + error: expected a Boolean but found an integer: 5 diff --git a/tests/functional/lang/eval-fail-filter-3.nix b/tests/functional/lang/eval-fail-filter-3.nix new file mode 100644 index 0000000000..c262f25bf6 --- /dev/null +++ b/tests/functional/lang/eval-fail-filter-3.nix @@ -0,0 +1 @@ +builtins.filter (_: 5) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-filterSource-1.err.exp b/tests/functional/lang/eval-fail-filterSource-1.err.exp new file mode 100644 index 0000000000..cf2d153f0d --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-1.nix:1:1: + 1| builtins.filterSource [ ] [ ] + | ^ + 2| + + … while evaluating the second argument (the path to filter) passed to 'builtins.filterSource' + + error: cannot coerce a list to a string: [ ] diff --git a/tests/functional/lang/eval-fail-filterSource-1.nix b/tests/functional/lang/eval-fail-filterSource-1.nix new file mode 100644 index 0000000000..5833e0de59 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-1.nix @@ -0,0 +1 @@ +builtins.filterSource [ ] [ ] diff --git a/tests/functional/lang/eval-fail-filterSource-2.err.exp b/tests/functional/lang/eval-fail-filterSource-2.err.exp new file mode 100644 index 0000000000..e3302f032f --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-2.nix:1:1: + 1| builtins.filterSource [ ] "foo" + | ^ + 2| + + … while evaluating the second argument (the path to filter) passed to 'builtins.filterSource' + + error: string 'foo' doesn't represent an absolute path diff --git a/tests/functional/lang/eval-fail-filterSource-2.nix b/tests/functional/lang/eval-fail-filterSource-2.nix new file mode 100644 index 0000000000..5fc0b68cb0 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-2.nix @@ -0,0 +1 @@ +builtins.filterSource [ ] "foo" diff --git a/tests/functional/lang/eval-fail-filterSource-3.err.exp b/tests/functional/lang/eval-fail-filterSource-3.err.exp new file mode 100644 index 0000000000..5750f8198c --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-3.nix:1:1: + 1| builtins.filterSource [ ] ./. + | ^ + 2| + + … while evaluating the first argument passed to builtins.filterSource + + error: expected a function but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-filterSource-3.nix b/tests/functional/lang/eval-fail-filterSource-3.nix new file mode 100644 index 0000000000..6c4124aadb --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-3.nix @@ -0,0 +1 @@ +builtins.filterSource [ ] ./. diff --git a/tests/functional/lang/eval-fail-filterSource-4.err.exp b/tests/functional/lang/eval-fail-filterSource-4.err.exp new file mode 100644 index 0000000000..4318616ee8 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-4.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-4.nix:1:1: + 1| builtins.filterSource (_: 1) ./. + | ^ + 2| + + … while adding path '/pwd/lang' + + error: attempt to call something which is not a function but an integer: 1 diff --git a/tests/functional/lang/eval-fail-filterSource-4.nix b/tests/functional/lang/eval-fail-filterSource-4.nix new file mode 100644 index 0000000000..28a729c614 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-4.nix @@ -0,0 +1 @@ +builtins.filterSource (_: 1) ./. diff --git a/tests/functional/lang/eval-fail-filterSource-5.err.exp b/tests/functional/lang/eval-fail-filterSource-5.err.exp new file mode 100644 index 0000000000..fe017b2efb --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-5.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'filterSource' builtin + at /pwd/lang/eval-fail-filterSource-5.nix:1:1: + 1| builtins.filterSource (_: _: 1) ./. + | ^ + 2| + + … while adding path '/pwd/lang' + + … while evaluating the return value of the path filter function + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-filterSource-5.nix b/tests/functional/lang/eval-fail-filterSource-5.nix new file mode 100644 index 0000000000..dbb0decee0 --- /dev/null +++ b/tests/functional/lang/eval-fail-filterSource-5.nix @@ -0,0 +1 @@ +builtins.filterSource (_: _: 1) ./. diff --git a/tests/functional/lang/eval-fail-floor-1.err.exp b/tests/functional/lang/eval-fail-floor-1.err.exp new file mode 100644 index 0000000000..2e5fb0aae3 --- /dev/null +++ b/tests/functional/lang/eval-fail-floor-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'floor' builtin + at /pwd/lang/eval-fail-floor-1.nix:1:1: + 1| builtins.floor "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.floor + + error: expected a float but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-floor-1.nix b/tests/functional/lang/eval-fail-floor-1.nix new file mode 100644 index 0000000000..1d4e1dcf5b --- /dev/null +++ b/tests/functional/lang/eval-fail-floor-1.nix @@ -0,0 +1 @@ +builtins.floor "foo" diff --git a/tests/functional/lang/eval-fail-foldlPrime-1.err.exp b/tests/functional/lang/eval-fail-foldlPrime-1.err.exp new file mode 100644 index 0000000000..45540098ea --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlPrime-1.nix:1:1: + 1| builtins.foldl' 1 "foo" true + | ^ + 2| + + … while evaluating the first argument passed to builtins.foldlStrict + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-foldlPrime-1.nix b/tests/functional/lang/eval-fail-foldlPrime-1.nix new file mode 100644 index 0000000000..e7790d4191 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-1.nix @@ -0,0 +1 @@ +builtins.foldl' 1 "foo" true diff --git a/tests/functional/lang/eval-fail-foldlPrime-2.err.exp b/tests/functional/lang/eval-fail-foldlPrime-2.err.exp new file mode 100644 index 0000000000..7597793eb2 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlPrime-2.nix:1:1: + 1| builtins.foldl' (_: 1) "foo" true + | ^ + 2| + + … while evaluating the third argument passed to builtins.foldlStrict + + error: expected a list but found a Boolean: true diff --git a/tests/functional/lang/eval-fail-foldlPrime-2.nix b/tests/functional/lang/eval-fail-foldlPrime-2.nix new file mode 100644 index 0000000000..9599f33cd2 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-2.nix @@ -0,0 +1 @@ +builtins.foldl' (_: 1) "foo" true diff --git a/tests/functional/lang/eval-fail-foldlPrime-3.err.exp b/tests/functional/lang/eval-fail-foldlPrime-3.err.exp new file mode 100644 index 0000000000..8ba0a3dfb8 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlPrime-3.nix:1:1: + 1| builtins.foldl' (_: 1) "foo" [ true ] + | ^ + 2| + + error: attempt to call something which is not a function but an integer: 1 diff --git a/tests/functional/lang/eval-fail-foldlPrime-3.nix b/tests/functional/lang/eval-fail-foldlPrime-3.nix new file mode 100644 index 0000000000..6e570d9c70 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-3.nix @@ -0,0 +1 @@ +builtins.foldl' (_: 1) "foo" [ true ] diff --git a/tests/functional/lang/eval-fail-foldlPrime-4.err.exp b/tests/functional/lang/eval-fail-foldlPrime-4.err.exp new file mode 100644 index 0000000000..9ead5b3929 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-4.err.exp @@ -0,0 +1,24 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlPrime-4.nix:1:1: + 1| builtins.foldl' (a: b: a && b) "foo" [ true ] + | ^ + 2| + + … while calling anonymous lambda + at /pwd/lang/eval-fail-foldlPrime-4.nix:1:21: + 1| builtins.foldl' (a: b: a && b) "foo" [ true ] + | ^ + 2| + + … in the left operand of the AND (&&) operator + at /pwd/lang/eval-fail-foldlPrime-4.nix:1:26: + 1| builtins.foldl' (a: b: a && b) "foo" [ true ] + | ^ + 2| + + error: expected a Boolean but found a string: "foo" + at /pwd/lang/eval-fail-foldlPrime-4.nix:1:26: + 1| builtins.foldl' (a: b: a && b) "foo" [ true ] + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-foldlPrime-4.nix b/tests/functional/lang/eval-fail-foldlPrime-4.nix new file mode 100644 index 0000000000..c34df35dd1 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlPrime-4.nix @@ -0,0 +1 @@ +builtins.foldl' (a: b: a && b) "foo" [ true ] diff --git a/tests/functional/lang/eval-fail-functionArgs-1.err.exp b/tests/functional/lang/eval-fail-functionArgs-1.err.exp new file mode 100644 index 0000000000..4f15c0655d --- /dev/null +++ b/tests/functional/lang/eval-fail-functionArgs-1.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'functionArgs' builtin + at /pwd/lang/eval-fail-functionArgs-1.nix:1:1: + 1| builtins.functionArgs { } + | ^ + 2| + + error: 'functionArgs' requires a function diff --git a/tests/functional/lang/eval-fail-functionArgs-1.nix b/tests/functional/lang/eval-fail-functionArgs-1.nix new file mode 100644 index 0000000000..151e9fa976 --- /dev/null +++ b/tests/functional/lang/eval-fail-functionArgs-1.nix @@ -0,0 +1 @@ +builtins.functionArgs { } diff --git a/tests/functional/lang/eval-fail-genList-1.err.exp b/tests/functional/lang/eval-fail-genList-1.err.exp new file mode 100644 index 0000000000..deea7e7ead --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'genList' builtin + at /pwd/lang/eval-fail-genList-1.nix:1:1: + 1| builtins.genList 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.genList + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-genList-1.nix b/tests/functional/lang/eval-fail-genList-1.nix new file mode 100644 index 0000000000..276147c1d3 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-1.nix @@ -0,0 +1 @@ +builtins.genList 1 "foo" diff --git a/tests/functional/lang/eval-fail-genList-2.err.exp b/tests/functional/lang/eval-fail-genList-2.err.exp new file mode 100644 index 0000000000..dbbab7d525 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'genList' builtin + at /pwd/lang/eval-fail-genList-2.nix:1:1: + 1| builtins.genList 1 2 + | ^ + 2| + + … while evaluating the first argument passed to builtins.genList + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-genList-2.nix b/tests/functional/lang/eval-fail-genList-2.nix new file mode 100644 index 0000000000..0f5bb6a3b6 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-2.nix @@ -0,0 +1 @@ +builtins.genList 1 2 diff --git a/tests/functional/lang/eval-fail-genList-3.err.exp b/tests/functional/lang/eval-fail-genList-3.err.exp new file mode 100644 index 0000000000..62878d88eb --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-3.err.exp @@ -0,0 +1,20 @@ +error: + … while evaluating list element at index 0 + + … from call site + at /pwd/lang/eval-fail-genList-3.nix:1:19: + 1| builtins.genList (x: x + "foo") 2 + | ^ + 2| + + … while calling anonymous lambda + at /pwd/lang/eval-fail-genList-3.nix:1:19: + 1| builtins.genList (x: x + "foo") 2 + | ^ + 2| + + error: cannot add a string to an integer + at /pwd/lang/eval-fail-genList-3.nix:1:26: + 1| builtins.genList (x: x + "foo") 2 + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-genList-3.nix b/tests/functional/lang/eval-fail-genList-3.nix new file mode 100644 index 0000000000..82b8b30bc1 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-3.nix @@ -0,0 +1 @@ +builtins.genList (x: x + "foo") 2 diff --git a/tests/functional/lang/eval-fail-genList-4.err.exp b/tests/functional/lang/eval-fail-genList-4.err.exp new file mode 100644 index 0000000000..a60ffe6d77 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-4.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'genList' builtin + at /pwd/lang/eval-fail-genList-4.nix:1:1: + 1| builtins.genList false (-3) + | ^ + 2| + + error: cannot create list of size -3 diff --git a/tests/functional/lang/eval-fail-genList-4.nix b/tests/functional/lang/eval-fail-genList-4.nix new file mode 100644 index 0000000000..b4f4ee6188 --- /dev/null +++ b/tests/functional/lang/eval-fail-genList-4.nix @@ -0,0 +1 @@ +builtins.genList false (-3) diff --git a/tests/functional/lang/eval-fail-getAttr-1.err.exp b/tests/functional/lang/eval-fail-getAttr-1.err.exp new file mode 100644 index 0000000000..d2ff63a710 --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'getAttr' builtin + at /pwd/lang/eval-fail-getAttr-1.nix:1:1: + 1| builtins.getAttr [ ] [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.getAttr + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-getAttr-1.nix b/tests/functional/lang/eval-fail-getAttr-1.nix new file mode 100644 index 0000000000..80b92f45de --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-1.nix @@ -0,0 +1 @@ +builtins.getAttr [ ] [ ] diff --git a/tests/functional/lang/eval-fail-getAttr-2.err.exp b/tests/functional/lang/eval-fail-getAttr-2.err.exp new file mode 100644 index 0000000000..e4a8afe799 --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'getAttr' builtin + at /pwd/lang/eval-fail-getAttr-2.nix:1:1: + 1| builtins.getAttr "foo" [ ] + | ^ + 2| + + … while evaluating the second argument passed to builtins.getAttr + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-getAttr-2.nix b/tests/functional/lang/eval-fail-getAttr-2.nix new file mode 100644 index 0000000000..6d836c0332 --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-2.nix @@ -0,0 +1 @@ +builtins.getAttr "foo" [ ] diff --git a/tests/functional/lang/eval-fail-getAttr-3.err.exp b/tests/functional/lang/eval-fail-getAttr-3.err.exp new file mode 100644 index 0000000000..e3a3dab058 --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'getAttr' builtin + at /pwd/lang/eval-fail-getAttr-3.nix:1:1: + 1| builtins.getAttr "foo" { } + | ^ + 2| + + … in the attribute set under consideration + + error: attribute 'foo' missing diff --git a/tests/functional/lang/eval-fail-getAttr-3.nix b/tests/functional/lang/eval-fail-getAttr-3.nix new file mode 100644 index 0000000000..248a616bf3 --- /dev/null +++ b/tests/functional/lang/eval-fail-getAttr-3.nix @@ -0,0 +1 @@ +builtins.getAttr "foo" { } diff --git a/tests/functional/lang/eval-fail-getEnv-1.err.exp b/tests/functional/lang/eval-fail-getEnv-1.err.exp new file mode 100644 index 0000000000..92db4e5e42 --- /dev/null +++ b/tests/functional/lang/eval-fail-getEnv-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'getEnv' builtin + at /pwd/lang/eval-fail-getEnv-1.nix:1:1: + 1| builtins.getEnv [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.getEnv + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-getEnv-1.nix b/tests/functional/lang/eval-fail-getEnv-1.nix new file mode 100644 index 0000000000..5e7cb3ea1b --- /dev/null +++ b/tests/functional/lang/eval-fail-getEnv-1.nix @@ -0,0 +1 @@ +builtins.getEnv [ ] diff --git a/tests/functional/lang/eval-fail-groupBy-1.err.exp b/tests/functional/lang/eval-fail-groupBy-1.err.exp new file mode 100644 index 0000000000..02d0415b2b --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'groupBy' builtin + at /pwd/lang/eval-fail-groupBy-1.nix:1:1: + 1| builtins.groupBy 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.groupBy + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-groupBy-1.nix b/tests/functional/lang/eval-fail-groupBy-1.nix new file mode 100644 index 0000000000..8fb41ada13 --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-1.nix @@ -0,0 +1 @@ +builtins.groupBy 1 "foo" diff --git a/tests/functional/lang/eval-fail-groupBy-2.err.exp b/tests/functional/lang/eval-fail-groupBy-2.err.exp new file mode 100644 index 0000000000..4a9b7b6daa --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'groupBy' builtin + at /pwd/lang/eval-fail-groupBy-2.nix:1:1: + 1| builtins.groupBy (_: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.groupBy + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-groupBy-2.nix b/tests/functional/lang/eval-fail-groupBy-2.nix new file mode 100644 index 0000000000..fe91a7bae8 --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-2.nix @@ -0,0 +1 @@ +builtins.groupBy (_: 1) "foo" diff --git a/tests/functional/lang/eval-fail-groupBy-3.err.exp b/tests/functional/lang/eval-fail-groupBy-3.err.exp new file mode 100644 index 0000000000..4adc418338 --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'groupBy' builtin + at /pwd/lang/eval-fail-groupBy-3.nix:1:1: + 1| builtins.groupBy (x: x) [ + | ^ + 2| "foo" + + … while evaluating the return value of the grouping function passed to builtins.groupBy + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-groupBy-3.nix b/tests/functional/lang/eval-fail-groupBy-3.nix new file mode 100644 index 0000000000..a4165783d9 --- /dev/null +++ b/tests/functional/lang/eval-fail-groupBy-3.nix @@ -0,0 +1,5 @@ +builtins.groupBy (x: x) [ + "foo" + "bar" + 1 +] diff --git a/tests/functional/lang/eval-fail-hasAttr-1.err.exp b/tests/functional/lang/eval-fail-hasAttr-1.err.exp new file mode 100644 index 0000000000..b44ae1c0c7 --- /dev/null +++ b/tests/functional/lang/eval-fail-hasAttr-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'hasAttr' builtin + at /pwd/lang/eval-fail-hasAttr-1.nix:1:1: + 1| builtins.hasAttr [ ] [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.hasAttr + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-hasAttr-1.nix b/tests/functional/lang/eval-fail-hasAttr-1.nix new file mode 100644 index 0000000000..8bdfbaaa71 --- /dev/null +++ b/tests/functional/lang/eval-fail-hasAttr-1.nix @@ -0,0 +1 @@ +builtins.hasAttr [ ] [ ] diff --git a/tests/functional/lang/eval-fail-hasAttr-2.err.exp b/tests/functional/lang/eval-fail-hasAttr-2.err.exp new file mode 100644 index 0000000000..cd52370acf --- /dev/null +++ b/tests/functional/lang/eval-fail-hasAttr-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'hasAttr' builtin + at /pwd/lang/eval-fail-hasAttr-2.nix:1:1: + 1| builtins.hasAttr "foo" [ ] + | ^ + 2| + + … while evaluating the second argument passed to builtins.hasAttr + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-hasAttr-2.nix b/tests/functional/lang/eval-fail-hasAttr-2.nix new file mode 100644 index 0000000000..d49909c763 --- /dev/null +++ b/tests/functional/lang/eval-fail-hasAttr-2.nix @@ -0,0 +1 @@ +builtins.hasAttr "foo" [ ] diff --git a/tests/functional/lang/eval-fail-hashString-1.err.exp b/tests/functional/lang/eval-fail-hashString-1.err.exp new file mode 100644 index 0000000000..ecce1a5d84 --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'hashString' builtin + at /pwd/lang/eval-fail-hashString-1.nix:1:1: + 1| builtins.hashString 1 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.hashString + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-hashString-1.nix b/tests/functional/lang/eval-fail-hashString-1.nix new file mode 100644 index 0000000000..bbca23d43f --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-1.nix @@ -0,0 +1 @@ +builtins.hashString 1 { } diff --git a/tests/functional/lang/eval-fail-hashString-2.err.exp b/tests/functional/lang/eval-fail-hashString-2.err.exp new file mode 100644 index 0000000000..edbf5e23ff --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-2.err.exp @@ -0,0 +1,9 @@ +error: + … while calling the 'hashString' builtin + at /pwd/lang/eval-fail-hashString-2.nix:1:1: + 1| builtins.hashString "foo" "content" + | ^ + 2| + + error: unknown hash algorithm 'foo', expect 'blake3', 'md5', 'sha1', 'sha256', or 'sha512' +Try 'nix-instantiate --help' for more information. diff --git a/tests/functional/lang/eval-fail-hashString-2.nix b/tests/functional/lang/eval-fail-hashString-2.nix new file mode 100644 index 0000000000..9439b8b41d --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-2.nix @@ -0,0 +1 @@ +builtins.hashString "foo" "content" diff --git a/tests/functional/lang/eval-fail-hashString-3.err.exp b/tests/functional/lang/eval-fail-hashString-3.err.exp new file mode 100644 index 0000000000..2126b00625 --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'hashString' builtin + at /pwd/lang/eval-fail-hashString-3.nix:1:1: + 1| builtins.hashString "sha256" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.hashString + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-hashString-3.nix b/tests/functional/lang/eval-fail-hashString-3.nix new file mode 100644 index 0000000000..50b960c8d3 --- /dev/null +++ b/tests/functional/lang/eval-fail-hashString-3.nix @@ -0,0 +1 @@ +builtins.hashString "sha256" { } diff --git a/tests/functional/lang/eval-fail-head-1.err.exp b/tests/functional/lang/eval-fail-head-1.err.exp new file mode 100644 index 0000000000..3fcbbe29b6 --- /dev/null +++ b/tests/functional/lang/eval-fail-head-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'head' builtin + at /pwd/lang/eval-fail-head-1.nix:1:1: + 1| builtins.head 1 + | ^ + 2| + + … while evaluating the first argument passed to 'builtins.head' + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-head-1.nix b/tests/functional/lang/eval-fail-head-1.nix new file mode 100644 index 0000000000..11aa724870 --- /dev/null +++ b/tests/functional/lang/eval-fail-head-1.nix @@ -0,0 +1 @@ +builtins.head 1 diff --git a/tests/functional/lang/eval-fail-head-2.err.exp b/tests/functional/lang/eval-fail-head-2.err.exp new file mode 100644 index 0000000000..68cecc8fa1 --- /dev/null +++ b/tests/functional/lang/eval-fail-head-2.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'head' builtin + at /pwd/lang/eval-fail-head-2.nix:1:1: + 1| builtins.head [ ] + | ^ + 2| + + error: 'builtins.head' called on an empty list diff --git a/tests/functional/lang/eval-fail-head-2.nix b/tests/functional/lang/eval-fail-head-2.nix new file mode 100644 index 0000000000..c7a77da4e2 --- /dev/null +++ b/tests/functional/lang/eval-fail-head-2.nix @@ -0,0 +1 @@ +builtins.head [ ] diff --git a/tests/functional/lang/eval-fail-intersectAttrs-1.err.exp b/tests/functional/lang/eval-fail-intersectAttrs-1.err.exp new file mode 100644 index 0000000000..098084a3f4 --- /dev/null +++ b/tests/functional/lang/eval-fail-intersectAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'intersectAttrs' builtin + at /pwd/lang/eval-fail-intersectAttrs-1.nix:1:1: + 1| builtins.intersectAttrs [ ] [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.intersectAttrs + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-intersectAttrs-1.nix b/tests/functional/lang/eval-fail-intersectAttrs-1.nix new file mode 100644 index 0000000000..5a8c133163 --- /dev/null +++ b/tests/functional/lang/eval-fail-intersectAttrs-1.nix @@ -0,0 +1 @@ +builtins.intersectAttrs [ ] [ ] diff --git a/tests/functional/lang/eval-fail-intersectAttrs-2.err.exp b/tests/functional/lang/eval-fail-intersectAttrs-2.err.exp new file mode 100644 index 0000000000..663df4f614 --- /dev/null +++ b/tests/functional/lang/eval-fail-intersectAttrs-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'intersectAttrs' builtin + at /pwd/lang/eval-fail-intersectAttrs-2.nix:1:1: + 1| builtins.intersectAttrs { } [ ] + | ^ + 2| + + … while evaluating the second argument passed to builtins.intersectAttrs + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-intersectAttrs-2.nix b/tests/functional/lang/eval-fail-intersectAttrs-2.nix new file mode 100644 index 0000000000..413ce5d58b --- /dev/null +++ b/tests/functional/lang/eval-fail-intersectAttrs-2.nix @@ -0,0 +1 @@ +builtins.intersectAttrs { } [ ] diff --git a/tests/functional/lang/eval-fail-length-1.err.exp b/tests/functional/lang/eval-fail-length-1.err.exp new file mode 100644 index 0000000000..7161a2fbf2 --- /dev/null +++ b/tests/functional/lang/eval-fail-length-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'length' builtin + at /pwd/lang/eval-fail-length-1.nix:1:1: + 1| builtins.length 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.length + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-length-1.nix b/tests/functional/lang/eval-fail-length-1.nix new file mode 100644 index 0000000000..972eb72c76 --- /dev/null +++ b/tests/functional/lang/eval-fail-length-1.nix @@ -0,0 +1 @@ +builtins.length 1 diff --git a/tests/functional/lang/eval-fail-length-2.err.exp b/tests/functional/lang/eval-fail-length-2.err.exp new file mode 100644 index 0000000000..8bd66a9a70 --- /dev/null +++ b/tests/functional/lang/eval-fail-length-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'length' builtin + at /pwd/lang/eval-fail-length-2.nix:1:1: + 1| builtins.length "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.length + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-length-2.nix b/tests/functional/lang/eval-fail-length-2.nix new file mode 100644 index 0000000000..a42e1034ab --- /dev/null +++ b/tests/functional/lang/eval-fail-length-2.nix @@ -0,0 +1 @@ +builtins.length "foo" diff --git a/tests/functional/lang/eval-fail-lessThan-1.err.exp b/tests/functional/lang/eval-fail-lessThan-1.err.exp new file mode 100644 index 0000000000..09ec5d8ffd --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-1.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-lessThan-1.nix:1:1: + 1| builtins.lessThan 1 "foo" + | ^ + 2| + + error: cannot compare an integer with a string; values are 1 and "foo" diff --git a/tests/functional/lang/eval-fail-lessThan-1.nix b/tests/functional/lang/eval-fail-lessThan-1.nix new file mode 100644 index 0000000000..10cd489674 --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-1.nix @@ -0,0 +1 @@ +builtins.lessThan 1 "foo" diff --git a/tests/functional/lang/eval-fail-lessThan-2.err.exp b/tests/functional/lang/eval-fail-lessThan-2.err.exp new file mode 100644 index 0000000000..6a8a183fa7 --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-2.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-lessThan-2.nix:1:1: + 1| builtins.lessThan { } { } + | ^ + 2| + + error: cannot compare a set with a set; values of that type are incomparable (values are { } and { }) diff --git a/tests/functional/lang/eval-fail-lessThan-2.nix b/tests/functional/lang/eval-fail-lessThan-2.nix new file mode 100644 index 0000000000..a20eca6bba --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-2.nix @@ -0,0 +1 @@ +builtins.lessThan { } { } diff --git a/tests/functional/lang/eval-fail-lessThan-3.err.exp b/tests/functional/lang/eval-fail-lessThan-3.err.exp new file mode 100644 index 0000000000..4a66625c00 --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-lessThan-3.nix:1:1: + 1| builtins.lessThan [ 1 2 ] [ "foo" ] + | ^ + 2| + + … while comparing two list elements + + error: cannot compare an integer with a string; values are 1 and "foo" diff --git a/tests/functional/lang/eval-fail-lessThan-3.nix b/tests/functional/lang/eval-fail-lessThan-3.nix new file mode 100644 index 0000000000..bdea0fef17 --- /dev/null +++ b/tests/functional/lang/eval-fail-lessThan-3.nix @@ -0,0 +1 @@ +builtins.lessThan [ 1 2 ] [ "foo" ] diff --git a/tests/functional/lang/eval-fail-listToAttrs-1.err.exp b/tests/functional/lang/eval-fail-listToAttrs-1.err.exp new file mode 100644 index 0000000000..ce2ef38c8b --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-1.nix:1:1: + 1| builtins.listToAttrs 1 + | ^ + 2| + + … while evaluating the argument passed to builtins.listToAttrs + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-listToAttrs-1.nix b/tests/functional/lang/eval-fail-listToAttrs-1.nix new file mode 100644 index 0000000000..260f136e51 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-1.nix @@ -0,0 +1 @@ +builtins.listToAttrs 1 diff --git a/tests/functional/lang/eval-fail-listToAttrs-2.err.exp b/tests/functional/lang/eval-fail-listToAttrs-2.err.exp new file mode 100644 index 0000000000..9809d216a2 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-2.nix:1:1: + 1| builtins.listToAttrs [ 1 ] + | ^ + 2| + + … while evaluating an element of the list passed to builtins.listToAttrs + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-listToAttrs-2.nix b/tests/functional/lang/eval-fail-listToAttrs-2.nix new file mode 100644 index 0000000000..2db54c3c7b --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-2.nix @@ -0,0 +1 @@ +builtins.listToAttrs [ 1 ] diff --git a/tests/functional/lang/eval-fail-listToAttrs-3.err.exp b/tests/functional/lang/eval-fail-listToAttrs-3.err.exp new file mode 100644 index 0000000000..5c4886c174 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-3.nix:1:1: + 1| builtins.listToAttrs [ { } ] + | ^ + 2| + + … in a {name=...; value=...;} pair + + error: attribute 'name' missing diff --git a/tests/functional/lang/eval-fail-listToAttrs-3.nix b/tests/functional/lang/eval-fail-listToAttrs-3.nix new file mode 100644 index 0000000000..ce5b117981 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-3.nix @@ -0,0 +1 @@ +builtins.listToAttrs [ { } ] diff --git a/tests/functional/lang/eval-fail-listToAttrs-4.err.exp b/tests/functional/lang/eval-fail-listToAttrs-4.err.exp new file mode 100644 index 0000000000..cb284ad2cd --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-4.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-4.nix:1:1: + 1| builtins.listToAttrs [ { name = 1; } ] + | ^ + 2| + + … while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs + at /pwd/lang/eval-fail-listToAttrs-4.nix:1:26: + 1| builtins.listToAttrs [ { name = 1; } ] + | ^ + 2| + + error: expected a string but found an integer: 1 + at /pwd/lang/eval-fail-listToAttrs-4.nix:1:26: + 1| builtins.listToAttrs [ { name = 1; } ] + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-listToAttrs-4.nix b/tests/functional/lang/eval-fail-listToAttrs-4.nix new file mode 100644 index 0000000000..a4bc7fd11c --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-4.nix @@ -0,0 +1 @@ +builtins.listToAttrs [ { name = 1; } ] diff --git a/tests/functional/lang/eval-fail-listToAttrs-5.err.exp b/tests/functional/lang/eval-fail-listToAttrs-5.err.exp new file mode 100644 index 0000000000..a03834b49a --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-5.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'listToAttrs' builtin + at /pwd/lang/eval-fail-listToAttrs-5.nix:1:1: + 1| builtins.listToAttrs [ { name = "foo"; } ] + | ^ + 2| + + … in a {name=...; value=...;} pair + + error: attribute 'value' missing diff --git a/tests/functional/lang/eval-fail-listToAttrs-5.nix b/tests/functional/lang/eval-fail-listToAttrs-5.nix new file mode 100644 index 0000000000..7cca5d0c72 --- /dev/null +++ b/tests/functional/lang/eval-fail-listToAttrs-5.nix @@ -0,0 +1 @@ +builtins.listToAttrs [ { name = "foo"; } ] diff --git a/tests/functional/lang/eval-fail-map-1.err.exp b/tests/functional/lang/eval-fail-map-1.err.exp new file mode 100644 index 0000000000..65c0d5d3ab --- /dev/null +++ b/tests/functional/lang/eval-fail-map-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'map' builtin + at /pwd/lang/eval-fail-map-1.nix:1:1: + 1| builtins.map 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.map + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-map-1.nix b/tests/functional/lang/eval-fail-map-1.nix new file mode 100644 index 0000000000..65f5cf8a67 --- /dev/null +++ b/tests/functional/lang/eval-fail-map-1.nix @@ -0,0 +1 @@ +builtins.map 1 "foo" diff --git a/tests/functional/lang/eval-fail-map-2.err.exp b/tests/functional/lang/eval-fail-map-2.err.exp new file mode 100644 index 0000000000..cf9a3f6367 --- /dev/null +++ b/tests/functional/lang/eval-fail-map-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'map' builtin + at /pwd/lang/eval-fail-map-2.nix:1:1: + 1| builtins.map 1 [ 1 ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.map + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-map-2.nix b/tests/functional/lang/eval-fail-map-2.nix new file mode 100644 index 0000000000..16100c0a79 --- /dev/null +++ b/tests/functional/lang/eval-fail-map-2.nix @@ -0,0 +1 @@ +builtins.map 1 [ 1 ] diff --git a/tests/functional/lang/eval-fail-mapAttrs-1.err.exp b/tests/functional/lang/eval-fail-mapAttrs-1.err.exp new file mode 100644 index 0000000000..9a10bb92f8 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'mapAttrs' builtin + at /pwd/lang/eval-fail-mapAttrs-1.nix:1:1: + 1| builtins.mapAttrs [ ] [ ] + | ^ + 2| + + … while evaluating the second argument passed to builtins.mapAttrs + + error: expected a set but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-mapAttrs-1.nix b/tests/functional/lang/eval-fail-mapAttrs-1.nix new file mode 100644 index 0000000000..c913e838d8 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-1.nix @@ -0,0 +1 @@ +builtins.mapAttrs [ ] [ ] diff --git a/tests/functional/lang/eval-fail-mapAttrs-2.err.exp b/tests/functional/lang/eval-fail-mapAttrs-2.err.exp new file mode 100644 index 0000000000..9cc2b5495a --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-2.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the attribute 'foo' + + error: attempt to call something which is not a function but a string: "" diff --git a/tests/functional/lang/eval-fail-mapAttrs-2.nix b/tests/functional/lang/eval-fail-mapAttrs-2.nix new file mode 100644 index 0000000000..b58ed8c3a6 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-2.nix @@ -0,0 +1 @@ +builtins.mapAttrs "" { foo.bar = 1; } diff --git a/tests/functional/lang/eval-fail-mapAttrs-3.err.exp b/tests/functional/lang/eval-fail-mapAttrs-3.err.exp new file mode 100644 index 0000000000..f4c48ee9a4 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-3.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the attribute 'foo' + + error: attempt to call something which is not a function but a string: "foo1" diff --git a/tests/functional/lang/eval-fail-mapAttrs-3.nix b/tests/functional/lang/eval-fail-mapAttrs-3.nix new file mode 100644 index 0000000000..6fc4de8251 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-3.nix @@ -0,0 +1 @@ +builtins.mapAttrs (x: x + "1") { foo.bar = 1; } diff --git a/tests/functional/lang/eval-fail-mapAttrs-4.err.exp b/tests/functional/lang/eval-fail-mapAttrs-4.err.exp new file mode 100644 index 0000000000..bd18ceeac9 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-4.err.exp @@ -0,0 +1,16 @@ +error: + … while evaluating the attribute 'foo' + + … while calling anonymous lambda + at /pwd/lang/eval-fail-mapAttrs-4.nix:1:23: + 1| builtins.mapAttrs (x: y: x + 1) { foo.bar = 1; } + | ^ + 2| + + … while evaluating a path segment + at /pwd/lang/eval-fail-mapAttrs-4.nix:1:30: + 1| builtins.mapAttrs (x: y: x + 1) { foo.bar = 1; } + | ^ + 2| + + error: cannot coerce an integer to a string: 1 diff --git a/tests/functional/lang/eval-fail-mapAttrs-4.nix b/tests/functional/lang/eval-fail-mapAttrs-4.nix new file mode 100644 index 0000000000..2ad09b6024 --- /dev/null +++ b/tests/functional/lang/eval-fail-mapAttrs-4.nix @@ -0,0 +1 @@ +builtins.mapAttrs (x: y: x + 1) { foo.bar = 1; } diff --git a/tests/functional/lang/eval-fail-match-1.err.exp b/tests/functional/lang/eval-fail-match-1.err.exp new file mode 100644 index 0000000000..2f757a6834 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'match' builtin + at /pwd/lang/eval-fail-match-1.nix:1:1: + 1| builtins.match 1 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.match + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-match-1.nix b/tests/functional/lang/eval-fail-match-1.nix new file mode 100644 index 0000000000..3eb26c017c --- /dev/null +++ b/tests/functional/lang/eval-fail-match-1.nix @@ -0,0 +1 @@ +builtins.match 1 { } diff --git a/tests/functional/lang/eval-fail-match-2.err.exp b/tests/functional/lang/eval-fail-match-2.err.exp new file mode 100644 index 0000000000..e415f5ccd3 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'match' builtin + at /pwd/lang/eval-fail-match-2.nix:1:1: + 1| builtins.match "foo" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.match + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-match-2.nix b/tests/functional/lang/eval-fail-match-2.nix new file mode 100644 index 0000000000..35eb16cfbf --- /dev/null +++ b/tests/functional/lang/eval-fail-match-2.nix @@ -0,0 +1 @@ +builtins.match "foo" { } diff --git a/tests/functional/lang/eval-fail-match-3.err.exp b/tests/functional/lang/eval-fail-match-3.err.exp new file mode 100644 index 0000000000..d69577bd71 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'match' builtin + at /pwd/lang/eval-fail-match-3.nix:1:1: + 1| builtins.match "(.*" "" + | ^ + 2| + + error: invalid regular expression '(.*' diff --git a/tests/functional/lang/eval-fail-match-3.nix b/tests/functional/lang/eval-fail-match-3.nix new file mode 100644 index 0000000000..8115807f92 --- /dev/null +++ b/tests/functional/lang/eval-fail-match-3.nix @@ -0,0 +1 @@ +builtins.match "(.*" "" diff --git a/tests/functional/lang/eval-fail-mul-1.err.exp b/tests/functional/lang/eval-fail-mul-1.err.exp new file mode 100644 index 0000000000..c03284987c --- /dev/null +++ b/tests/functional/lang/eval-fail-mul-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-mul-1.nix:1:1: + 1| builtins.mul "foo" 1 + | ^ + 2| + + … while evaluating the first argument of the multiplication + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-mul-1.nix b/tests/functional/lang/eval-fail-mul-1.nix new file mode 100644 index 0000000000..a163338246 --- /dev/null +++ b/tests/functional/lang/eval-fail-mul-1.nix @@ -0,0 +1 @@ +builtins.mul "foo" 1 diff --git a/tests/functional/lang/eval-fail-mul-2.err.exp b/tests/functional/lang/eval-fail-mul-2.err.exp new file mode 100644 index 0000000000..16744abe73 --- /dev/null +++ b/tests/functional/lang/eval-fail-mul-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-mul-2.nix:1:1: + 1| builtins.mul 1 "foo" + | ^ + 2| + + … while evaluating the second argument of the multiplication + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-mul-2.nix b/tests/functional/lang/eval-fail-mul-2.nix new file mode 100644 index 0000000000..57e2ef6925 --- /dev/null +++ b/tests/functional/lang/eval-fail-mul-2.nix @@ -0,0 +1 @@ +builtins.mul 1 "foo" diff --git a/tests/functional/lang/eval-fail-parseDrvName-1.err.exp b/tests/functional/lang/eval-fail-parseDrvName-1.err.exp new file mode 100644 index 0000000000..e5fbc2b6ec --- /dev/null +++ b/tests/functional/lang/eval-fail-parseDrvName-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'parseDrvName' builtin + at /pwd/lang/eval-fail-parseDrvName-1.nix:1:1: + 1| builtins.parseDrvName 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.parseDrvName + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-parseDrvName-1.nix b/tests/functional/lang/eval-fail-parseDrvName-1.nix new file mode 100644 index 0000000000..f524d500f5 --- /dev/null +++ b/tests/functional/lang/eval-fail-parseDrvName-1.nix @@ -0,0 +1 @@ +builtins.parseDrvName 1 diff --git a/tests/functional/lang/eval-fail-partition-1.err.exp b/tests/functional/lang/eval-fail-partition-1.err.exp new file mode 100644 index 0000000000..d989969bd5 --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'partition' builtin + at /pwd/lang/eval-fail-partition-1.nix:1:1: + 1| builtins.partition 1 "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.partition + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-partition-1.nix b/tests/functional/lang/eval-fail-partition-1.nix new file mode 100644 index 0000000000..0452d11101 --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-1.nix @@ -0,0 +1 @@ +builtins.partition 1 "foo" diff --git a/tests/functional/lang/eval-fail-partition-2.err.exp b/tests/functional/lang/eval-fail-partition-2.err.exp new file mode 100644 index 0000000000..e3fef6848f --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'partition' builtin + at /pwd/lang/eval-fail-partition-2.nix:1:1: + 1| builtins.partition (_: 1) "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.partition + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-partition-2.nix b/tests/functional/lang/eval-fail-partition-2.nix new file mode 100644 index 0000000000..1a7219c5f3 --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-2.nix @@ -0,0 +1 @@ +builtins.partition (_: 1) "foo" diff --git a/tests/functional/lang/eval-fail-partition-3.err.exp b/tests/functional/lang/eval-fail-partition-3.err.exp new file mode 100644 index 0000000000..a1257c7a6b --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'partition' builtin + at /pwd/lang/eval-fail-partition-3.nix:1:1: + 1| builtins.partition (_: 1) [ "foo" ] + | ^ + 2| + + … while evaluating the return value of the partition function passed to builtins.partition + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-partition-3.nix b/tests/functional/lang/eval-fail-partition-3.nix new file mode 100644 index 0000000000..07aa480fc1 --- /dev/null +++ b/tests/functional/lang/eval-fail-partition-3.nix @@ -0,0 +1 @@ +builtins.partition (_: 1) [ "foo" ] diff --git a/tests/functional/lang/eval-fail-pathExists-1.err.exp b/tests/functional/lang/eval-fail-pathExists-1.err.exp new file mode 100644 index 0000000000..3349247966 --- /dev/null +++ b/tests/functional/lang/eval-fail-pathExists-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'pathExists' builtin + at /pwd/lang/eval-fail-pathExists-1.nix:1:1: + 1| builtins.pathExists [ ] + | ^ + 2| + + … while realising the context of a path + + error: cannot coerce a list to a string: [ ] diff --git a/tests/functional/lang/eval-fail-pathExists-1.nix b/tests/functional/lang/eval-fail-pathExists-1.nix new file mode 100644 index 0000000000..83bda9afca --- /dev/null +++ b/tests/functional/lang/eval-fail-pathExists-1.nix @@ -0,0 +1 @@ +builtins.pathExists [ ] diff --git a/tests/functional/lang/eval-fail-pathExists-2.err.exp b/tests/functional/lang/eval-fail-pathExists-2.err.exp new file mode 100644 index 0000000000..184e47238d --- /dev/null +++ b/tests/functional/lang/eval-fail-pathExists-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'pathExists' builtin + at /pwd/lang/eval-fail-pathExists-2.nix:1:1: + 1| builtins.pathExists "zorglub" + | ^ + 2| + + … while realising the context of a path + + error: string 'zorglub' doesn't represent an absolute path diff --git a/tests/functional/lang/eval-fail-pathExists-2.nix b/tests/functional/lang/eval-fail-pathExists-2.nix new file mode 100644 index 0000000000..da594ca602 --- /dev/null +++ b/tests/functional/lang/eval-fail-pathExists-2.nix @@ -0,0 +1 @@ +builtins.pathExists "zorglub" diff --git a/tests/functional/lang/eval-fail-placeholder-1.err.exp b/tests/functional/lang/eval-fail-placeholder-1.err.exp new file mode 100644 index 0000000000..4360954480 --- /dev/null +++ b/tests/functional/lang/eval-fail-placeholder-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'placeholder' builtin + at /pwd/lang/eval-fail-placeholder-1.nix:1:1: + 1| builtins.placeholder [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.placeholder + + error: expected a string but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-placeholder-1.nix b/tests/functional/lang/eval-fail-placeholder-1.nix new file mode 100644 index 0000000000..ecc934a5e3 --- /dev/null +++ b/tests/functional/lang/eval-fail-placeholder-1.nix @@ -0,0 +1 @@ +builtins.placeholder [ ] diff --git a/tests/functional/lang/eval-fail-removeAttrs-1.err.exp b/tests/functional/lang/eval-fail-removeAttrs-1.err.exp new file mode 100644 index 0000000000..ddfde967f1 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'removeAttrs' builtin + at /pwd/lang/eval-fail-removeAttrs-1.nix:1:1: + 1| builtins.removeAttrs "" "" + | ^ + 2| + + … while evaluating the first argument passed to builtins.removeAttrs + + error: expected a set but found a string: "" diff --git a/tests/functional/lang/eval-fail-removeAttrs-1.nix b/tests/functional/lang/eval-fail-removeAttrs-1.nix new file mode 100644 index 0000000000..83fe63003c --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-1.nix @@ -0,0 +1 @@ +builtins.removeAttrs "" "" diff --git a/tests/functional/lang/eval-fail-removeAttrs-2.err.exp b/tests/functional/lang/eval-fail-removeAttrs-2.err.exp new file mode 100644 index 0000000000..ab668aaea0 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'removeAttrs' builtin + at /pwd/lang/eval-fail-removeAttrs-2.nix:1:1: + 1| builtins.removeAttrs "" [ 1 ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.removeAttrs + + error: expected a set but found a string: "" diff --git a/tests/functional/lang/eval-fail-removeAttrs-2.nix b/tests/functional/lang/eval-fail-removeAttrs-2.nix new file mode 100644 index 0000000000..ecedf26cf9 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-2.nix @@ -0,0 +1 @@ +builtins.removeAttrs "" [ 1 ] diff --git a/tests/functional/lang/eval-fail-removeAttrs-3.err.exp b/tests/functional/lang/eval-fail-removeAttrs-3.err.exp new file mode 100644 index 0000000000..1937cd1be8 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'removeAttrs' builtin + at /pwd/lang/eval-fail-removeAttrs-3.nix:1:1: + 1| builtins.removeAttrs "" [ "1" ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.removeAttrs + + error: expected a set but found a string: "" diff --git a/tests/functional/lang/eval-fail-removeAttrs-3.nix b/tests/functional/lang/eval-fail-removeAttrs-3.nix new file mode 100644 index 0000000000..4bc9e02394 --- /dev/null +++ b/tests/functional/lang/eval-fail-removeAttrs-3.nix @@ -0,0 +1 @@ +builtins.removeAttrs "" [ "1" ] diff --git a/tests/functional/lang/eval-fail-replaceStrings-1.err.exp b/tests/functional/lang/eval-fail-replaceStrings-1.err.exp new file mode 100644 index 0000000000..b0bc7c758c --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-1.nix:1:1: + 1| builtins.replaceStrings 0 0 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.replaceStrings + + error: expected a list but found an integer: 0 diff --git a/tests/functional/lang/eval-fail-replaceStrings-1.nix b/tests/functional/lang/eval-fail-replaceStrings-1.nix new file mode 100644 index 0000000000..7e9edc28d2 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-1.nix @@ -0,0 +1 @@ +builtins.replaceStrings 0 0 { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-2.err.exp b/tests/functional/lang/eval-fail-replaceStrings-2.err.exp new file mode 100644 index 0000000000..ee289eded6 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-2.nix:1:1: + 1| builtins.replaceStrings [ ] 0 { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.replaceStrings + + error: expected a list but found an integer: 0 diff --git a/tests/functional/lang/eval-fail-replaceStrings-2.nix b/tests/functional/lang/eval-fail-replaceStrings-2.nix new file mode 100644 index 0000000000..5731172020 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-2.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ ] 0 { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-3.err.exp b/tests/functional/lang/eval-fail-replaceStrings-3.err.exp new file mode 100644 index 0000000000..3e71cfa75c --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-3.nix:1:1: + 1| builtins.replaceStrings [ 0 ] [ ] { } + | ^ + 2| + + error: 'from' and 'to' arguments passed to builtins.replaceStrings have different lengths diff --git a/tests/functional/lang/eval-fail-replaceStrings-3.nix b/tests/functional/lang/eval-fail-replaceStrings-3.nix new file mode 100644 index 0000000000..c7cdac4a96 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-3.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ 0 ] [ ] { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-4.err.exp b/tests/functional/lang/eval-fail-replaceStrings-4.err.exp new file mode 100644 index 0000000000..f4e8340905 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-4.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-4.nix:1:1: + 1| builtins.replaceStrings [ 1 ] [ "new" ] { } + | ^ + 2| + + … while evaluating one of the strings to replace passed to builtins.replaceStrings + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-replaceStrings-4.nix b/tests/functional/lang/eval-fail-replaceStrings-4.nix new file mode 100644 index 0000000000..cc700ea54a --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-4.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ 1 ] [ "new" ] { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-5.err.exp b/tests/functional/lang/eval-fail-replaceStrings-5.err.exp new file mode 100644 index 0000000000..75e2b5e8b3 --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-5.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-5.nix:1:1: + 1| builtins.replaceStrings [ "oo" ] [ true ] "foo" + | ^ + 2| + + … while evaluating one of the replacement strings passed to builtins.replaceStrings + + error: expected a string but found a Boolean: true diff --git a/tests/functional/lang/eval-fail-replaceStrings-5.nix b/tests/functional/lang/eval-fail-replaceStrings-5.nix new file mode 100644 index 0000000000..40010d845e --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-5.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ "oo" ] [ true ] "foo" diff --git a/tests/functional/lang/eval-fail-replaceStrings-6.err.exp b/tests/functional/lang/eval-fail-replaceStrings-6.err.exp new file mode 100644 index 0000000000..5ced1affaf --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-6.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'replaceStrings' builtin + at /pwd/lang/eval-fail-replaceStrings-6.nix:1:1: + 1| builtins.replaceStrings [ "old" ] [ "new" ] { } + | ^ + 2| + + … while evaluating the third argument passed to builtins.replaceStrings + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-replaceStrings-6.nix b/tests/functional/lang/eval-fail-replaceStrings-6.nix new file mode 100644 index 0000000000..2953b4888b --- /dev/null +++ b/tests/functional/lang/eval-fail-replaceStrings-6.nix @@ -0,0 +1 @@ +builtins.replaceStrings [ "old" ] [ "new" ] { } diff --git a/tests/functional/lang/eval-fail-sort-1.err.exp b/tests/functional/lang/eval-fail-sort-1.err.exp new file mode 100644 index 0000000000..2797d344d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-1.nix:1:1: + 1| builtins.sort 1 "foo" + | ^ + 2| + + … while evaluating the second argument passed to builtins.sort + + error: expected a list but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-sort-1.nix b/tests/functional/lang/eval-fail-sort-1.nix new file mode 100644 index 0000000000..12d3a10287 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-1.nix @@ -0,0 +1 @@ +builtins.sort 1 "foo" diff --git a/tests/functional/lang/eval-fail-sort-2.err.exp b/tests/functional/lang/eval-fail-sort-2.err.exp new file mode 100644 index 0000000000..462ade0e44 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-2.nix:1:1: + 1| builtins.sort 1 [ "foo" ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.sort + + error: expected a function but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-sort-2.nix b/tests/functional/lang/eval-fail-sort-2.nix new file mode 100644 index 0000000000..27ffb1d363 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-2.nix @@ -0,0 +1 @@ +builtins.sort 1 [ "foo" ] diff --git a/tests/functional/lang/eval-fail-sort-3.err.exp b/tests/functional/lang/eval-fail-sort-3.err.exp new file mode 100644 index 0000000000..0d86a7f4d1 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-3.nix:1:1: + 1| builtins.sort (_: 1) [ + | ^ + 2| "foo" + + error: attempt to call something which is not a function but an integer: 1 diff --git a/tests/functional/lang/eval-fail-sort-3.nix b/tests/functional/lang/eval-fail-sort-3.nix new file mode 100644 index 0000000000..1fcd434d07 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-3.nix @@ -0,0 +1,4 @@ +builtins.sort (_: 1) [ + "foo" + "bar" +] diff --git a/tests/functional/lang/eval-fail-sort-4.err.exp b/tests/functional/lang/eval-fail-sort-4.err.exp new file mode 100644 index 0000000000..517eefeb5c --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-4.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-4.nix:1:1: + 1| builtins.sort (_: _: 1) [ + | ^ + 2| "foo" + + … while evaluating the return value of the sorting function passed to builtins.sort + + error: expected a Boolean but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-sort-4.nix b/tests/functional/lang/eval-fail-sort-4.nix new file mode 100644 index 0000000000..89868de9ab --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-4.nix @@ -0,0 +1,4 @@ +builtins.sort (_: _: 1) [ + "foo" + "bar" +] diff --git a/tests/functional/lang/eval-fail-sort-5.err.exp b/tests/functional/lang/eval-fail-sort-5.err.exp new file mode 100644 index 0000000000..f4c66dbd46 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-5.err.exp @@ -0,0 +1,26 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-5.nix:1:1: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| "foo" + + … while calling anonymous lambda + at /pwd/lang/eval-fail-sort-5.nix:1:19: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| "foo" + + … in the argument of the not operator + at /pwd/lang/eval-fail-sort-5.nix:1:24: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| "foo" + + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-sort-5.nix:1:24: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| "foo" + + error: cannot compare a string with a set; values are "foo" and { } diff --git a/tests/functional/lang/eval-fail-sort-5.nix b/tests/functional/lang/eval-fail-sort-5.nix new file mode 100644 index 0000000000..bd50e0b325 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-5.nix @@ -0,0 +1,4 @@ +builtins.sort (a: b: a <= b) [ + "foo" + { } +] diff --git a/tests/functional/lang/eval-fail-sort-6.err.exp b/tests/functional/lang/eval-fail-sort-6.err.exp new file mode 100644 index 0000000000..302a27f236 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-6.err.exp @@ -0,0 +1,26 @@ +error: + … while calling the 'sort' builtin + at /pwd/lang/eval-fail-sort-6.nix:1:1: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| { } + + … while calling anonymous lambda + at /pwd/lang/eval-fail-sort-6.nix:1:19: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| { } + + … in the argument of the not operator + at /pwd/lang/eval-fail-sort-6.nix:1:24: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| { } + + … while calling the 'lessThan' builtin + at /pwd/lang/eval-fail-sort-6.nix:1:24: + 1| builtins.sort (a: b: a <= b) [ + | ^ + 2| { } + + error: cannot compare a set with a set; values of that type are incomparable (values are { } and { }) diff --git a/tests/functional/lang/eval-fail-sort-6.nix b/tests/functional/lang/eval-fail-sort-6.nix new file mode 100644 index 0000000000..3c527596d8 --- /dev/null +++ b/tests/functional/lang/eval-fail-sort-6.nix @@ -0,0 +1,4 @@ +builtins.sort (a: b: a <= b) [ + { } + { } +] diff --git a/tests/functional/lang/eval-fail-split-1.err.exp b/tests/functional/lang/eval-fail-split-1.err.exp new file mode 100644 index 0000000000..c8ac708574 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'split' builtin + at /pwd/lang/eval-fail-split-1.nix:1:1: + 1| builtins.split 1 { } + | ^ + 2| + + … while evaluating the first argument passed to builtins.split + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-split-1.nix b/tests/functional/lang/eval-fail-split-1.nix new file mode 100644 index 0000000000..ad702a1a3a --- /dev/null +++ b/tests/functional/lang/eval-fail-split-1.nix @@ -0,0 +1 @@ +builtins.split 1 { } diff --git a/tests/functional/lang/eval-fail-split-2.err.exp b/tests/functional/lang/eval-fail-split-2.err.exp new file mode 100644 index 0000000000..a912771d06 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'split' builtin + at /pwd/lang/eval-fail-split-2.nix:1:1: + 1| builtins.split "foo" { } + | ^ + 2| + + … while evaluating the second argument passed to builtins.split + + error: expected a string but found a set: { } diff --git a/tests/functional/lang/eval-fail-split-2.nix b/tests/functional/lang/eval-fail-split-2.nix new file mode 100644 index 0000000000..43489c5374 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-2.nix @@ -0,0 +1 @@ +builtins.split "foo" { } diff --git a/tests/functional/lang/eval-fail-split-3.err.exp b/tests/functional/lang/eval-fail-split-3.err.exp new file mode 100644 index 0000000000..a7cfd39418 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-3.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'split' builtin + at /pwd/lang/eval-fail-split-3.nix:1:1: + 1| builtins.split "f(o*o" "1foo2" + | ^ + 2| + + error: invalid regular expression 'f(o*o' diff --git a/tests/functional/lang/eval-fail-split-3.nix b/tests/functional/lang/eval-fail-split-3.nix new file mode 100644 index 0000000000..f45294e930 --- /dev/null +++ b/tests/functional/lang/eval-fail-split-3.nix @@ -0,0 +1 @@ +builtins.split "f(o*o" "1foo2" diff --git a/tests/functional/lang/eval-fail-splitVersion-1.err.exp b/tests/functional/lang/eval-fail-splitVersion-1.err.exp new file mode 100644 index 0000000000..f7eb7614d8 --- /dev/null +++ b/tests/functional/lang/eval-fail-splitVersion-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'splitVersion' builtin + at /pwd/lang/eval-fail-splitVersion-1.nix:1:1: + 1| builtins.splitVersion 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.splitVersion + + error: expected a string but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-splitVersion-1.nix b/tests/functional/lang/eval-fail-splitVersion-1.nix new file mode 100644 index 0000000000..43941d1b66 --- /dev/null +++ b/tests/functional/lang/eval-fail-splitVersion-1.nix @@ -0,0 +1 @@ +builtins.splitVersion 1 diff --git a/tests/functional/lang/eval-fail-storePath-1.err.exp b/tests/functional/lang/eval-fail-storePath-1.err.exp new file mode 100644 index 0000000000..711ed13ed8 --- /dev/null +++ b/tests/functional/lang/eval-fail-storePath-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'storePath' builtin + at /pwd/lang/eval-fail-storePath-1.nix:1:1: + 1| builtins.storePath true + | ^ + 2| + + … while evaluating the first argument passed to 'builtins.storePath' + + error: cannot coerce a Boolean to a string: true diff --git a/tests/functional/lang/eval-fail-storePath-1.nix b/tests/functional/lang/eval-fail-storePath-1.nix new file mode 100644 index 0000000000..d4ce341022 --- /dev/null +++ b/tests/functional/lang/eval-fail-storePath-1.nix @@ -0,0 +1 @@ +builtins.storePath true diff --git a/tests/functional/lang/eval-fail-stringLength-1.err.exp b/tests/functional/lang/eval-fail-stringLength-1.err.exp new file mode 100644 index 0000000000..37d63fd493 --- /dev/null +++ b/tests/functional/lang/eval-fail-stringLength-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'stringLength' builtin + at /pwd/lang/eval-fail-stringLength-1.nix:1:1: + 1| builtins.stringLength { } + | ^ + 2| + + … while evaluating the argument passed to builtins.stringLength + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-stringLength-1.nix b/tests/functional/lang/eval-fail-stringLength-1.nix new file mode 100644 index 0000000000..0a5ba6d8a8 --- /dev/null +++ b/tests/functional/lang/eval-fail-stringLength-1.nix @@ -0,0 +1 @@ +builtins.stringLength { } diff --git a/tests/functional/lang/eval-fail-sub-1.err.exp b/tests/functional/lang/eval-fail-sub-1.err.exp new file mode 100644 index 0000000000..769bf252bb --- /dev/null +++ b/tests/functional/lang/eval-fail-sub-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sub' builtin + at /pwd/lang/eval-fail-sub-1.nix:1:1: + 1| builtins.sub "foo" 1 + | ^ + 2| + + … while evaluating the first argument of the subtraction + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-sub-1.nix b/tests/functional/lang/eval-fail-sub-1.nix new file mode 100644 index 0000000000..cbff4ca1ed --- /dev/null +++ b/tests/functional/lang/eval-fail-sub-1.nix @@ -0,0 +1 @@ +builtins.sub "foo" 1 diff --git a/tests/functional/lang/eval-fail-sub-2.err.exp b/tests/functional/lang/eval-fail-sub-2.err.exp new file mode 100644 index 0000000000..ee5ed4618a --- /dev/null +++ b/tests/functional/lang/eval-fail-sub-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'sub' builtin + at /pwd/lang/eval-fail-sub-2.nix:1:1: + 1| builtins.sub 1 "foo" + | ^ + 2| + + … while evaluating the second argument of the subtraction + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-sub-2.nix b/tests/functional/lang/eval-fail-sub-2.nix new file mode 100644 index 0000000000..4a3cfc6e02 --- /dev/null +++ b/tests/functional/lang/eval-fail-sub-2.nix @@ -0,0 +1 @@ +builtins.sub 1 "foo" diff --git a/tests/functional/lang/eval-fail-substring-1.err.exp b/tests/functional/lang/eval-fail-substring-1.err.exp new file mode 100644 index 0000000000..84e61c17c0 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring-1.nix:1:1: + 1| builtins.substring { } "foo" true + | ^ + 2| + + … while evaluating the first argument (the start offset) passed to builtins.substring + + error: expected an integer but found a set: { } diff --git a/tests/functional/lang/eval-fail-substring-1.nix b/tests/functional/lang/eval-fail-substring-1.nix new file mode 100644 index 0000000000..c12228550b --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-1.nix @@ -0,0 +1 @@ +builtins.substring { } "foo" true diff --git a/tests/functional/lang/eval-fail-substring-2.err.exp b/tests/functional/lang/eval-fail-substring-2.err.exp new file mode 100644 index 0000000000..641fde5a83 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring-2.nix:1:1: + 1| builtins.substring 3 "foo" true + | ^ + 2| + + … while evaluating the second argument (the substring length) passed to builtins.substring + + error: expected an integer but found a string: "foo" diff --git a/tests/functional/lang/eval-fail-substring-2.nix b/tests/functional/lang/eval-fail-substring-2.nix new file mode 100644 index 0000000000..358b0d0835 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-2.nix @@ -0,0 +1 @@ +builtins.substring 3 "foo" true diff --git a/tests/functional/lang/eval-fail-substring-3.err.exp b/tests/functional/lang/eval-fail-substring-3.err.exp new file mode 100644 index 0000000000..67b12cab27 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-3.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring-3.nix:1:1: + 1| builtins.substring 0 3 { } + | ^ + 2| + + … while evaluating the third argument (the string) passed to builtins.substring + + error: cannot coerce a set to a string: { } diff --git a/tests/functional/lang/eval-fail-substring-3.nix b/tests/functional/lang/eval-fail-substring-3.nix new file mode 100644 index 0000000000..f52921e481 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-3.nix @@ -0,0 +1 @@ +builtins.substring 0 3 { } diff --git a/tests/functional/lang/eval-fail-substring-4.err.exp b/tests/functional/lang/eval-fail-substring-4.err.exp new file mode 100644 index 0000000000..cdaa11b202 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-4.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring-4.nix:1:1: + 1| builtins.substring (-3) 3 "sometext" + | ^ + 2| + + error: negative start position in 'substring' diff --git a/tests/functional/lang/eval-fail-substring-4.nix b/tests/functional/lang/eval-fail-substring-4.nix new file mode 100644 index 0000000000..82813b2c58 --- /dev/null +++ b/tests/functional/lang/eval-fail-substring-4.nix @@ -0,0 +1 @@ +builtins.substring (-3) 3 "sometext" diff --git a/tests/functional/lang/eval-fail-tail-1.err.exp b/tests/functional/lang/eval-fail-tail-1.err.exp new file mode 100644 index 0000000000..2473767896 --- /dev/null +++ b/tests/functional/lang/eval-fail-tail-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'tail' builtin + at /pwd/lang/eval-fail-tail-1.nix:1:1: + 1| builtins.tail 1 + | ^ + 2| + + … while evaluating the first argument passed to 'builtins.tail' + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-tail-1.nix b/tests/functional/lang/eval-fail-tail-1.nix new file mode 100644 index 0000000000..229ab7c8d7 --- /dev/null +++ b/tests/functional/lang/eval-fail-tail-1.nix @@ -0,0 +1 @@ +builtins.tail 1 diff --git a/tests/functional/lang/eval-fail-tail-2.err.exp b/tests/functional/lang/eval-fail-tail-2.err.exp new file mode 100644 index 0000000000..63abbf2cba --- /dev/null +++ b/tests/functional/lang/eval-fail-tail-2.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'tail' builtin + at /pwd/lang/eval-fail-tail-2.nix:1:1: + 1| builtins.tail [ ] + | ^ + 2| + + error: 'builtins.tail' called on an empty list diff --git a/tests/functional/lang/eval-fail-tail-2.nix b/tests/functional/lang/eval-fail-tail-2.nix new file mode 100644 index 0000000000..3bd318d1c3 --- /dev/null +++ b/tests/functional/lang/eval-fail-tail-2.nix @@ -0,0 +1 @@ +builtins.tail [ ] diff --git a/tests/functional/lang/eval-fail-toPath-1.err.exp b/tests/functional/lang/eval-fail-toPath-1.err.exp new file mode 100644 index 0000000000..6e977c6a2b --- /dev/null +++ b/tests/functional/lang/eval-fail-toPath-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'toPath' builtin + at /pwd/lang/eval-fail-toPath-1.nix:1:1: + 1| builtins.toPath [ ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.toPath + + error: cannot coerce a list to a string: [ ] diff --git a/tests/functional/lang/eval-fail-toPath-1.nix b/tests/functional/lang/eval-fail-toPath-1.nix new file mode 100644 index 0000000000..cd96c880b8 --- /dev/null +++ b/tests/functional/lang/eval-fail-toPath-1.nix @@ -0,0 +1 @@ +builtins.toPath [ ] diff --git a/tests/functional/lang/eval-fail-toPath-2.err.exp b/tests/functional/lang/eval-fail-toPath-2.err.exp new file mode 100644 index 0000000000..f7d421b76d --- /dev/null +++ b/tests/functional/lang/eval-fail-toPath-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'toPath' builtin + at /pwd/lang/eval-fail-toPath-2.nix:1:1: + 1| builtins.toPath "foo" + | ^ + 2| + + … while evaluating the first argument passed to builtins.toPath + + error: string 'foo' doesn't represent an absolute path diff --git a/tests/functional/lang/eval-fail-toPath-2.nix b/tests/functional/lang/eval-fail-toPath-2.nix new file mode 100644 index 0000000000..4bcb7fc147 --- /dev/null +++ b/tests/functional/lang/eval-fail-toPath-2.nix @@ -0,0 +1 @@ +builtins.toPath "foo" diff --git a/tests/functional/lang/eval-fail-toString-1.err.exp b/tests/functional/lang/eval-fail-toString-1.err.exp new file mode 100644 index 0000000000..d73b9ad84c --- /dev/null +++ b/tests/functional/lang/eval-fail-toString-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'toString' builtin + at /pwd/lang/eval-fail-toString-1.nix:1:1: + 1| builtins.toString { a = 1; } + | ^ + 2| + + … while evaluating the first argument passed to builtins.toString + + error: cannot coerce a set to a string: { a = 1; } diff --git a/tests/functional/lang/eval-fail-toString-1.nix b/tests/functional/lang/eval-fail-toString-1.nix new file mode 100644 index 0000000000..e4b2d8d795 --- /dev/null +++ b/tests/functional/lang/eval-fail-toString-1.nix @@ -0,0 +1 @@ +builtins.toString { a = 1; } diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-1.err.exp b/tests/functional/lang/eval-fail-zipAttrsWith-1.err.exp new file mode 100644 index 0000000000..7aced51b97 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-1.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'zipAttrsWith' builtin + at /pwd/lang/eval-fail-zipAttrsWith-1.nix:1:1: + 1| builtins.zipAttrsWith [ ] [ 1 ] + | ^ + 2| + + … while evaluating the first argument passed to builtins.zipAttrsWith + + error: expected a function but found a list: [ ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-1.nix b/tests/functional/lang/eval-fail-zipAttrsWith-1.nix new file mode 100644 index 0000000000..1f44609cd2 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-1.nix @@ -0,0 +1 @@ +builtins.zipAttrsWith [ ] [ 1 ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-2.err.exp b/tests/functional/lang/eval-fail-zipAttrsWith-2.err.exp new file mode 100644 index 0000000000..0a27e2d019 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-2.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'zipAttrsWith' builtin + at /pwd/lang/eval-fail-zipAttrsWith-2.nix:1:1: + 1| builtins.zipAttrsWith (_: 1) [ 1 ] + | ^ + 2| + + … while evaluating a value of the list passed as second argument to builtins.zipAttrsWith + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-2.nix b/tests/functional/lang/eval-fail-zipAttrsWith-2.nix new file mode 100644 index 0000000000..331da91a53 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-2.nix @@ -0,0 +1 @@ +builtins.zipAttrsWith (_: 1) [ 1 ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-3.err.exp b/tests/functional/lang/eval-fail-zipAttrsWith-3.err.exp new file mode 100644 index 0000000000..780643b9df --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-3.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the attribute 'foo' + + error: attempt to call something which is not a function but an integer: 1 + at /pwd/lang/eval-fail-zipAttrsWith-3.nix:1:24: + 1| builtins.zipAttrsWith (_: 1) [ { foo = 1; } ] + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-3.nix b/tests/functional/lang/eval-fail-zipAttrsWith-3.nix new file mode 100644 index 0000000000..6ac25c0110 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-3.nix @@ -0,0 +1 @@ +builtins.zipAttrsWith (_: 1) [ { foo = 1; } ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-4.err.exp b/tests/functional/lang/eval-fail-zipAttrsWith-4.err.exp new file mode 100644 index 0000000000..034d842541 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-4.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the attribute 'foo' + + … from call site + at /pwd/lang/eval-fail-zipAttrsWith-4.nix:1:24: + 1| builtins.zipAttrsWith (a: b: a + b) [ + | ^ + 2| { foo = 1; } + + … while calling anonymous lambda + at /pwd/lang/eval-fail-zipAttrsWith-4.nix:1:27: + 1| builtins.zipAttrsWith (a: b: a + b) [ + | ^ + 2| { foo = 1; } + + … while evaluating a path segment + at /pwd/lang/eval-fail-zipAttrsWith-4.nix:1:34: + 1| builtins.zipAttrsWith (a: b: a + b) [ + | ^ + 2| { foo = 1; } + + error: cannot coerce a list to a string: [ 1 2 ] diff --git a/tests/functional/lang/eval-fail-zipAttrsWith-4.nix b/tests/functional/lang/eval-fail-zipAttrsWith-4.nix new file mode 100644 index 0000000000..dbe63d0b56 --- /dev/null +++ b/tests/functional/lang/eval-fail-zipAttrsWith-4.nix @@ -0,0 +1,4 @@ +builtins.zipAttrsWith (a: b: a + b) [ + { foo = 1; } + { foo = 2; } +] From 940825bce6d2496e58e6f892125ad02038267ccd Mon Sep 17 00:00:00 2001 From: Artemis Tosini Date: Thu, 30 Apr 2026 12:34:46 -0400 Subject: [PATCH 330/927] Filter systemd socket activation sockets by name When available, use the `LISTEN_FDNAMES` environment variable provided by systemd to listen only to the correct sockets in `serveUnixSocket`. Filtering makes it possible to differentiate different activation sockets connecting to the same daemon process. --- src/libcmd/include/nix/cmd/unix-socket-server.hh | 7 +++++++ src/libcmd/unix/unix-socket-server.cc | 8 ++++++++ src/nix/unix/daemon.cc | 1 + src/nix/unix/store-roots-daemon.cc | 1 + 4 files changed, 17 insertions(+) diff --git a/src/libcmd/include/nix/cmd/unix-socket-server.hh b/src/libcmd/include/nix/cmd/unix-socket-server.hh index 7a0d9fa793..544b988e2d 100644 --- a/src/libcmd/include/nix/cmd/unix-socket-server.hh +++ b/src/libcmd/include/nix/cmd/unix-socket-server.hh @@ -55,6 +55,13 @@ struct ServeUnixSocketOptions mode_t socketMode = 0666; #ifndef _WIN32 + /** + * Name of the socket for socket activation, as included in `LISTEN_FDNAMES` + * Ordinarily the name of the socket unit, e.g. `nix-daemon.socket` + * If this field is empty, no name filtering will be performed. + */ + std::string activationName = ""; + /** * Additional file descriptor to poll. Useful for doing a self-pipe trick * https://cr.yp.to/docs/selfpipe.html. diff --git a/src/libcmd/unix/unix-socket-server.cc b/src/libcmd/unix/unix-socket-server.cc index 5d1fba4622..1794148b6f 100644 --- a/src/libcmd/unix/unix-socket-server.cc +++ b/src/libcmd/unix/unix-socket-server.cc @@ -5,6 +5,7 @@ #include "nix/util/file-system.hh" #include "nix/util/logging.hh" #include "nix/util/signals.hh" +#include "nix/util/strings.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/util.hh" @@ -65,9 +66,16 @@ PeerInfo getPeerInfo(Descriptor remote) if (listenFds) { if (getEnv("LISTEN_PID") != std::to_string(getpid())) throw Error("unexpected systemd environment variables"); + + auto fdNames = tokenizeString>(getEnv("LISTEN_FDNAMES").value_or(""), ":"); auto count = string2Int(*listenFds); assert(count); for (unsigned int i = 0; i < count; ++i) { + // Not all implementations of LISTEN_FDS will implement names, + // listen anyway if we do not have enough names + if (i < fdNames.size() && options.activationName != "" && fdNames[i] != options.activationName) + continue; + AutoCloseFD fdSocket(SD_LISTEN_FDS_START + i); closeOnExec(fdSocket.get()); listeningSockets.push_back(std::move(fdSocket)); diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 05e47f79c3..c50c9f5942 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -285,6 +285,7 @@ static void daemonLoop( { .socketPath = std::move(socketPath), .socketMode = 0666, + .activationName = "nix-daemon.socket", .auxiliaryFd = sigChldPipe.pipe.readSide.get(), .onAuxiliaryFdPollin = []() { diff --git a/src/nix/unix/store-roots-daemon.cc b/src/nix/unix/store-roots-daemon.cc index 85b0a67a91..b803c8cea6 100644 --- a/src/nix/unix/store-roots-daemon.cc +++ b/src/nix/unix/store-roots-daemon.cc @@ -44,6 +44,7 @@ struct CmdRootsDaemon : StoreConfigCommand { .socketPath = gcSocketPath, .socketMode = 0666, + .activationName = "nix-roots-daemon.socket", }, [&](AutoCloseFD remote, std::function closeListeners) { std::thread([&, remote = std::move(remote)]() mutable { From a79b0f4dc1521d27ea5c9829ff422914843775ee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Apr 2026 20:06:48 +0200 Subject: [PATCH 331/927] LocalStore::addToStore(): Handle negative path info cache entry If we have a negative path info cache entry for the path being added (e.g. due to a prior call to maybeQueryPathInfo()), then we need to use isValidPathUncached(), otherwise addToStore() will fail. --- src/libstore/local-store.cc | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 53d7456f94..0bc7b6e1b6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1013,7 +1013,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ - pathInfoCache->lock()->erase(path); + invalidatePathInfoCacheFor(path); } const PublicKeys & LocalStore::getPublicKeys() @@ -1049,17 +1049,18 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF auto realPath = toRealPath(info.path); /* Lock the output path. But don't lock if we're being called - from a build hook (whose parent process already acquired a - lock on this path). */ + from a build hook (whose parent process already acquired a + lock on this path). */ if (!locksHeld.count(printStorePath(info.path))) outputLock.lockPaths({realPath}); - if (repair || !isValidPath(info.path)) { + /* The path may have been created by another process in the meantime, so check again. */ + if (repair || !isValidPathUncached(info.path)) { deletePath(realPath); /* While restoring the path from the NAR, compute the hash - of the NAR. */ + of the NAR. */ HashSink hashSink(HashAlgorithm::SHA256); TeeSource wrapperSource{source, hashSink}; @@ -1129,7 +1130,9 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF } registerValidPath(info); - } + } else + // We may have a negative cache entry for this path, so get rid of it. + invalidatePathInfoCacheFor(info.path); outputLock.setDeletion(true); } @@ -1246,7 +1249,8 @@ StorePath LocalStore::addToStoreFromDump( PathLocks outputLock({realPath}); - if (repair || !isValidPath(dstPath)) { + /* The path may have been created by another process in the meantime, so check again. */ + if (repair || !isValidPathUncached(dstPath)) { deletePath(realPath); @@ -1293,7 +1297,9 @@ StorePath LocalStore::addToStoreFromDump( auto info = ValidPathInfo::makeFromCA(*this, name, std::move(desc), narHash.hash); info.narSize = narHash.numBytesDigested; registerValidPath(info); - } + } else + // We may have a negative cache entry for this path, so get rid of it. + invalidatePathInfoCacheFor(dstPath); outputLock.setDeletion(true); } From 026e930912d459290fac3eabbf11b6eb31ce681d Mon Sep 17 00:00:00 2001 From: ryota2357 Date: Sat, 18 Oct 2025 11:55:40 +0900 Subject: [PATCH 332/927] nix-profile{,-daemon}.fish: set NIX_PROFILES to use $NIX_LINK --- scripts/nix-profile-daemon.fish.in | 2 +- scripts/nix-profile.fish.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 1a20dffd24..93cb3c45a5 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -53,7 +53,7 @@ end # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix -set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc if test -z "$XDG_DATA_DIRS" diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index abf716cec6..201a564389 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -58,7 +58,7 @@ end # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix -set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc if test -z "$XDG_DATA_DIRS" From 0072b3271f6f23429805a6f7ea878a9f1b682cb6 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Fri, 1 May 2026 14:16:00 -0400 Subject: [PATCH 333/927] Ignore keep-{outputs,derivations} for delete `keep-outputs` and `keep-derivations` were not meant to affect delete operations. This change makes them only read when using the `WholeStore` variant type. Signed-off-by: Lisanna Dettwyler --- doc/manual/rl-next/delete-keep.md | 8 +++ src/libstore/gc.cc | 65 ++++++++++--------- .../include/nix/store/local-settings.hh | 6 ++ tests/functional/delete-no-keep.sh | 32 +++++++++ tests/functional/meson.build | 1 + 5 files changed, 82 insertions(+), 30 deletions(-) create mode 100644 doc/manual/rl-next/delete-keep.md create mode 100755 tests/functional/delete-no-keep.sh diff --git a/doc/manual/rl-next/delete-keep.md b/doc/manual/rl-next/delete-keep.md new file mode 100644 index 0000000000..0332e0e393 --- /dev/null +++ b/doc/manual/rl-next/delete-keep.md @@ -0,0 +1,8 @@ +--- +synopsis: "Fixed a bug where keep-outputs and keep-derivations can interfere with delete commands" +prs: [15776] +--- + +Setting `keep-derivations = true` and trying to delete a derivation with realised outputs would previously fail. +Same with `keep-outputs = true` and trying to delete an output that still has derivers. +These options no longer affect the deletion commands, and are now documented as such. diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 701fc66e69..3b9e8dbf5c 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -357,8 +357,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) const auto & gcSettings = config->getLocalSettings().getGCSettings(); bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; - bool keepOutputs = gcSettings.keepOutputs; - bool keepDerivations = gcSettings.keepDerivations; boost::unordered_flat_set> roots, dead, alive; @@ -382,16 +380,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) std::condition_variable wakeup; - /* Using `--ignore-liveness' with `--delete' can have unintended - consequences if `keep-outputs' or `keep-derivations' are true - (the garbage collector will recurse into deleting the outputs - or derivers, respectively, even if they aren't in the - pathsToDelete). So disable them. */ - if (std::holds_alternative(options.pathsToDelete) && options.ignoreLiveness) { - keepOutputs = false; - keepDerivations = false; - } - if (shouldDelete) deletePath(reservedPath); @@ -625,12 +613,23 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) alive.insert(start); try { StorePathSet closure; + bool includeOutputs = false; + bool includeDerivers = false; + std::visit( + overloaded{ + [&](const GCOptions::WholeStore &) { + includeOutputs = gcSettings.keepOutputs; + includeDerivers = gcSettings.keepDerivations; + }, + [](const StorePathSet &) {}, + }, + options.pathsToDelete); computeFSClosure( *path, closure, /* flipDirection */ false, - keepOutputs, - keepDerivations); + includeOutputs, + includeDerivers); for (auto & p : closure) alive.insert(p); } catch (InvalidPath &) { @@ -675,22 +674,28 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) for (auto & p : i->second) enqueue(p); - /* If keep-derivations is set and this is a derivation, then we only want to delete this derivation if - * we can also delete all its outputs, so visit the derivation outputs. */ - if (keepDerivations && path->isDerivation()) { - for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) - if (maybeOutPath && isValidPath(*maybeOutPath) - && queryPathInfo(*maybeOutPath)->deriver == *path) - enqueue(*maybeOutPath); - } - - /* If keep-outputs is set, we only want to delete this path if we - * can also delete its derivers, so visit the derivers. */ - if (keepOutputs) { - auto derivers = queryValidDerivers(*path); - for (auto & i : derivers) - enqueue(i); - } + std::visit( + overloaded{ + [&](const GCOptions::WholeStore &) { + /* If keep-derivations is set and this is a derivation, then we only want to delete this + * derivation if we can also delete all its outputs, so visit the derivation outputs. */ + if (gcSettings.keepDerivations && path->isDerivation()) + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) + if (maybeOutPath && isValidPath(*maybeOutPath) + && queryPathInfo(*maybeOutPath)->deriver == path) + enqueue(*maybeOutPath); + + /* If keep-outputs is set, we only want to delete this path if we + * can also delete its derivers, so visit the derivers. */ + if (gcSettings.keepOutputs) { + auto derivers = queryValidDerivers(*path); + for (auto & i : derivers) + enqueue(i); + } + }, + [](const StorePathSet &) {}, + }, + options.pathsToDelete); } } for (auto & path : topoSortPaths(visited)) { diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index 4fe28818d2..7381b5b8e7 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -61,6 +61,9 @@ struct GCSettings : public virtual Config collector still deletes store paths that are used only at build time (e.g., the C compiler, or source tarballs downloaded from the network). To prevent it from doing so, set this option to `true`. + + This option only applies to garbage collection of the whole store + and does not affect deleting explicit paths. )", {"gc-keep-outputs"}, }; @@ -80,6 +83,9 @@ struct GCSettings : public virtual Config store path was built), so by default this option is on. Turn it off to save a bit of disk space (or a lot if `keep-outputs` is also turned on). + + This option only applies to garbage collection of the whole store + and does not affect deleting explicit paths. )", {"gc-keep-derivations"}, }; diff --git a/tests/functional/delete-no-keep.sh b/tests/functional/delete-no-keep.sh new file mode 100755 index 0000000000..8ac64fd444 --- /dev/null +++ b/tests/functional/delete-no-keep.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +deleteNoKeep() { + keep="$1" + + clearStore + drvPath=$(nix-instantiate simple.nix) + outPath=$(nix build -f simple.nix --no-link --print-out-paths) + + { + echo "keep-outputs = false" + echo "keep-derivations = false" + echo "keep-$keep = true" + } >> "$test_nix_conf" + + if [[ "$keep" = "outputs" ]]; then + nix store delete "$outPath" + [[ ! -e "$outPath" ]] || fail "$outPath should have been deleted" + else + nix store delete "$drvPath" + [[ ! -e "$drvPath" ]] || fail "$drvPath should have been deleted" + fi +} + +if isDaemonNewer "2.35pre"; then + deleteNoKeep outputs + deleteNoKeep derivations +fi diff --git a/tests/functional/meson.build b/tests/functional/meson.build index e2668c716e..ecb9ef87f6 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -179,6 +179,7 @@ suites = [ 'help.sh', 'symlinks.sh', 'external-builders.sh', + 'delete-no-keep.sh', ], 'workdir' : meson.current_source_dir(), }, From 6bf83e259a83d0324584977db81d0b66d9e649af Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 22 Apr 2026 15:38:15 -0400 Subject: [PATCH 334/927] Add `--also-referrers` to `nix store delete` This makes collecting garbage within a closure more ergonomic. If you build `hello` from scratch, it'll also build `glibc` and `glibc-static`. `glibc-static` depends on `glibc`, but is not part of `hello`'s closure. This normally prevents deletion of `glibc`, but using this flag allows `glibc-static` to also be deleted as long as it is dead. Signed-off-by: Lisanna Dettwyler --- doc/manual/rl-next/closure-gc.md | 7 ++-- src/libstore/daemon.cc | 12 ++++-- src/libstore/gc.cc | 37 +++++++++++------- src/libstore/include/nix/store/gc-store.hh | 12 +++++- .../include/nix/store/worker-protocol.hh | 8 +++- src/libstore/remote-store.cc | 12 ++++-- src/libstore/worker-protocol.cc | 22 +++++++++-- src/nix/nix-store/nix-store.cc | 4 +- src/nix/store-delete.cc | 12 +++++- tests/functional/dependencies2.nix | 39 +++++++++++++++++++ tests/functional/gc-closure.sh | 34 ++++++++++++---- 11 files changed, 159 insertions(+), 40 deletions(-) create mode 100644 tests/functional/dependencies2.nix diff --git a/doc/manual/rl-next/closure-gc.md b/doc/manual/rl-next/closure-gc.md index fc24d5e08b..945a78dc9f 100644 --- a/doc/manual/rl-next/closure-gc.md +++ b/doc/manual/rl-next/closure-gc.md @@ -1,9 +1,8 @@ --- synopsis: "Added `--skip-alive` option to `nix store delete` for collecting garbage within a closure" issues: 7239 -prs: 15236 +prs: [15236, 15727] --- -`nix store delete --recursive --skip-alive` can be used to collect garbage -within a closure, in which case it will only collect the dead paths that are -part of the closure of its arguments. +`nix store delete --recursive --skip-alive` can be used to collect garbage within a closure, in which case it will only collect the dead paths that are part of the closure of its arguments. +The additional option `--also-referrers` is added to support this mode, which allows referrers of paths in the closure to also be deleted. diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 52e1c121c3..8a02ea6ae1 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -746,14 +746,17 @@ static void performOp( case WorkerProto::Op::CollectGarbage: { GCOptions options; options.action = WorkerProto::Serialise::read(*store, rconn); - if (rconn.version.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + if (rconn.version.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers)) { options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); } else { auto paths = WorkerProto::Serialise::read(*store, rconn); if (options.action != GCAction::gcDeleteSpecific && paths.empty()) options.pathsToDelete = GCOptions::WholeStore{}; else - options.pathsToDelete = paths; + options.pathsToDelete = GCOptions::SpecificPaths{ + .paths = paths, + .deleteReferrers = false, + }; } conn.from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields @@ -761,8 +764,9 @@ static void performOp( readInt(conn.from); readInt(conn.from); - if (options.action == GCAction::gcDeleteDead && std::holds_alternative(options.pathsToDelete) - && !conn.protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + if (options.action == GCAction::gcDeleteDead + && std::holds_alternative(options.pathsToDelete) + && !conn.protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers)) { throw Error( "Garbage collecting specific paths requested but it is not supported by the negotiated protocol"); } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 3b9e8dbf5c..645c4c6f97 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -361,8 +361,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) boost::unordered_flat_set> roots, dead, alive; /* Return early if nothing to delete */ - if (std::holds_alternative(options.pathsToDelete) - && std::get(options.pathsToDelete).empty()) + if (std::visit( + overloaded{ + [](const GCOptions::SpecificPaths & pathsToDelete) { return pathsToDelete.paths.empty(); }, + [](const GCOptions::WholeStore & _) { return false; }}, + options.pathsToDelete)) return; struct Shared @@ -621,7 +624,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) includeOutputs = gcSettings.keepOutputs; includeDerivers = gcSettings.keepDerivations; }, - [](const StorePathSet &) {}, + [](const GCOptions::SpecificPaths &) {}, }, options.pathsToDelete); computeFSClosure( @@ -642,14 +645,22 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) return markAlive(); } - if (std::holds_alternative(options.pathsToDelete) - && !std::get(options.pathsToDelete).contains(*path)) { - debug( - "cannot delete '%s' because '%s' is not in the specified paths to delete", - printStorePath(start), - printStorePath(*path)); + if (std::visit( + overloaded{ + [&](const GCOptions::SpecificPaths & pathsToDelete) { + if (!pathsToDelete.deleteReferrers && !pathsToDelete.paths.contains(*path)) { + debug( + "cannot delete '%s' because '%s' is not in the specified paths to delete", + printStorePath(start), + printStorePath(*path)); + return true; + } + return false; + }, + [](const GCOptions::WholeStore & _) { return false; }, + }, + options.pathsToDelete)) return; - } { auto hashPart = path->hashPart(); @@ -693,7 +704,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) enqueue(i); } }, - [](const StorePathSet &) {}, + [](const GCOptions::SpecificPaths &) {}, }, options.pathsToDelete); } @@ -719,7 +730,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Either delete all garbage paths, or just the specified paths. */ std::visit( overloaded{ - [&](const StorePathSet & paths) { + [&](const GCOptions::SpecificPaths & pathsToDelete) { switch (options.action) { case GCOptions::gcDeleteDead: printInfo("deleting garbage within specified paths..."); @@ -732,7 +743,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printInfo("determining live/dead paths..."); } - for (auto & i : paths) { + for (auto & i : pathsToDelete.paths) { maybeDeleteReferrersClosure(i); if (options.action == GCOptions::gcDeleteSpecific && !dead.contains(i)) diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 015478e79f..0a0335016b 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -42,6 +42,16 @@ struct GCOptions struct WholeStore {}; + struct SpecificPaths + { + StorePathSet paths; + + /** + * Allow dead referrers of candidate paths to also be deleted. + */ + bool deleteReferrers = false; + }; + GCAction action{gcDeleteDead}; /** @@ -55,7 +65,7 @@ struct GCOptions /** * The paths from which to delete. */ - using GCPaths = std::variant; + using GCPaths = std::variant; GCPaths pathsToDelete; /** diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index ea1cbb502d..64cab1494a 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -126,9 +126,9 @@ struct WorkerProto static constexpr std::string_view featureRealisationWithPath = "realisation-with-path-not-hash"; /** - * Feature for garbage collecting a specific set of paths. + * Feature for garbage collecting a specific set of paths and deleting referrers. */ - static constexpr std::string_view featureDeleteDeadSpecific = "delete-dead-specific"; + static constexpr std::string_view featureDeleteDeadSpecificReferrers = "delete-dead-specific-referrers"; /** * A unidirectional read connection, to be used by the read half of the @@ -345,6 +345,10 @@ template<> DECLARE_WORKER_SERIALISER(std::optional); template<> DECLARE_WORKER_SERIALISER(WorkerProto::ClientHandshakeInfo); + +template<> +DECLARE_WORKER_SERIALISER(GCOptions::SpecificPaths); + template<> DECLARE_WORKER_SERIALISER(GCOptions::GCPaths); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a2fb3a10d2..2c1ea0c638 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -686,18 +686,23 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { auto conn(getConnection()); - if (conn->protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecific)) { + bool supportsDeleteSpecificReferrers = + conn->protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers); + + if (supportsDeleteSpecificReferrers) { conn->to << WorkerProto::Op::CollectGarbage; WorkerProto::write(*this, *conn, options.action); WorkerProto::write(*this, *conn, options.pathsToDelete); } else { auto paths = std::visit( overloaded{ - [&](const StorePathSet & paths) { + [&](const GCOptions::SpecificPaths & paths) { if (options.action != GCOptions::gcDeleteSpecific) throw Error( "Your daemon version is too old to support garbage collecting a specific set of paths"); - return paths; + if (paths.deleteReferrers) + throw Error("Your daemon version is too old to support deleting referrers."); + return paths.paths; }, [](const GCOptions::WholeStore & _) { return StorePathSet{}; }, }, @@ -706,6 +711,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) WorkerProto::write(*this, *conn, options.action); WorkerProto::write(*this, *conn, paths); } + conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 31373a128f..37aa4e31be 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -28,7 +28,7 @@ const WorkerProto::Version WorkerProto::latest = { std::string{ WorkerProto::featureRealisationWithPath, }, - std::string{WorkerProto::featureDeleteDeadSpecific}, + std::string{WorkerProto::featureDeleteDeadSpecificReferrers}, }, }; @@ -533,13 +533,29 @@ void WorkerProto::Serialise::write(const StoreDirConfig & store, Wr WorkerProto::write(store, conn, static_cast(info)); } +GCOptions::SpecificPaths +WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + GCOptions::SpecificPaths paths; + paths.paths = WorkerProto::Serialise::read(store, conn); + conn.from >> paths.deleteReferrers; + return paths; +} + +void WorkerProto::Serialise::write( + const StoreDirConfig & store, WriteConn conn, const GCOptions::SpecificPaths & paths) +{ + WorkerProto::write(store, conn, paths.paths); + conn.to << paths.deleteReferrers; +} + GCOptions::GCPaths WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { uint8_t wholeStore; conn.from >> wholeStore; switch (wholeStore) { case 0: - return WorkerProto::Serialise::read(store, conn); + return WorkerProto::Serialise::read(store, conn); case 1: return GCOptions::WholeStore{}; default: @@ -552,7 +568,7 @@ void WorkerProto::Serialise::write( { std::visit( overloaded{ - [&](const StorePathSet paths) { + [&](const GCOptions::SpecificPaths paths) { conn.to << uint8_t{0}; WorkerProto::write(store, conn, paths); }, diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index 15a4e878f5..63952ba63a 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -727,7 +727,9 @@ static void opDelete(Strings opFlags, Strings opArgs) StorePathSet paths; for (auto & i : opArgs) paths.insert(store->followLinksToStorePath(i)); - options.pathsToDelete = std::move(paths); + options.pathsToDelete = GCOptions::SpecificPaths{ + .paths = std::move(paths), + }; auto & gcStore = require(*store); diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index a1a3878984..c2f649ff56 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -9,6 +9,7 @@ namespace nix { struct CmdStoreDelete : StorePathsCommand { GCOptions options{.action = GCOptions::gcDeleteSpecific}; + bool deleteReferrers = false; CmdStoreDelete() { @@ -25,6 +26,12 @@ struct CmdStoreDelete : StorePathsCommand "Do not emit errors when attempting to delete something that is still alive, useful with --recursive.", .handler = {&options.action, GCOptions::gcDeleteDead}, }); + + addFlag({ + .longName = "also-referrers", + .description = "Also allow deletion of any referrers of the specified paths.", + .handler = {&deleteReferrers, true}, + }); } std::string description() override @@ -46,7 +53,10 @@ struct CmdStoreDelete : StorePathsCommand StorePathSet paths; for (auto & path : storePaths) paths.insert(path); - options.pathsToDelete = std::move(paths); + options.pathsToDelete = GCOptions::SpecificPaths{ + .paths = std::move(paths), + .deleteReferrers = deleteReferrers, + }; GCResults results; Finally printer([&] { printFreed(false, results); }); diff --git a/tests/functional/dependencies2.nix b/tests/functional/dependencies2.nix new file mode 100644 index 0000000000..6300f0ac41 --- /dev/null +++ b/tests/functional/dependencies2.nix @@ -0,0 +1,39 @@ +with import ./config.nix; + +let + + input0 = mkDerivation { + name = "dependencies-input-0"; + buildCommand = "mkdir $out; echo foo > $out/bar"; + }; + + input1 = mkDerivation { + name = "dependencies-input-1"; + buildCommand = "mkdir $out; echo FOO > $out/foo"; + }; + + input2 = mkDerivation { + name = "dependencies-input-2"; + buildCommand = '' + mkdir $out + echo BAR > $out/bar + echo ${input0} > $out/input0 + echo "$out" > $out2 + ''; + outputs = [ + "out" + "out2" + ]; + }; + +in +mkDerivation { + name = "dependencies-top"; + builder = ./dependencies.builder0.sh + "/FOOBAR/../."; + input1 = input1 + "/."; + input2 = "${input2}/."; + input1_drv = input1; + input2_drv = input2; + input0_drv = input0; + meta.description = "Random test package"; +} diff --git a/tests/functional/gc-closure.sh b/tests/functional/gc-closure.sh index 03e2a0fc25..4b811e7797 100755 --- a/tests/functional/gc-closure.sh +++ b/tests/functional/gc-closure.sh @@ -5,25 +5,43 @@ source common.sh TODO_NixOS nix_gc_closure() { + ensureNoDeleteReferrer="${1}" + extraArg="${2:-""}" clearStore - nix build -f dependencies.nix input0_drv --out-link "$TEST_ROOT/gc-root" + nix build -f dependencies2.nix input0_drv --out-link "$TEST_ROOT/gc-root" input0=$(realpath "$TEST_ROOT/gc-root") - input1=$(nix build -f dependencies.nix input1_drv --no-link --print-out-paths) - input2=$(nix build -f dependencies.nix input2_drv --no-link --print-out-paths) - top=$(nix build -f dependencies.nix --no-link --print-out-paths) - somthing_else=$(nix store add-path ./dependencies.nix) + input1=$(nix build -f dependencies2.nix input1_drv --no-link --print-out-paths) + input2=$(nix build -f dependencies2.nix input2_drv --no-link --print-out-paths) + input2_out=$(printf "%s" "$input2" | head -n1) + input2_out2=$(printf "%s" "$input2" | tail -n1) + top=$(nix build -f dependencies2.nix --no-link --print-out-paths) + somthing_else=$(nix store add-path ./dependencies2.nix) if isDaemonNewer "2.35pre"; then + if [[ "$extraArg" != "--also-referrers" ]] && ! "$ensureNoDeleteReferrer"; then + nix store delete "$input2_out2" + fi # Check that nix store delete --recursive --skip-alive is best-effort (doesn't fail when some paths in the closure are alive) - nix store delete --recursive --skip-alive "$top" + # shellcheck disable=SC2086 # we want $extraArg to expand to nothing if unset + nix store delete --recursive --skip-alive $extraArg "$top" [[ ! -e "$top" ]] || fail "top should have been deleted" [[ -e "$input0" ]] || fail "input0 is a gc root, shouldn't have been deleted" - [[ ! -e "$input2" ]] || fail "input2 is not a gc root and is part of top's closure, it should have been deleted" [[ -e "$input1" ]] || fail "input1 is not in the closure of top, it shouldn't have been deleted" [[ -e "$somthing_else" ]] || fail "somthing_else is not in the closure of top, it shouldn't have been deleted" + if [[ "$extraArg" = "--also-referrers" ]]; then + [[ ! -e "$input2_out" ]] || fail "input2_out is part of top's closure and we can delete dead referrers, it should have been deleted" + elif "$ensureNoDeleteReferrer"; then + [[ -e "$input2_out" ]] || fail "input2_out is part of top's closure but we can't delete dead referrers, it shouldn't have been deleted" + else + [[ ! -e "$input2_out" ]] || fail "input2_out is not a gc root, is part of top's closure, and has no referrers, it should have been deleted" + fi + elif [[ "$extraArg" = "--also-referrers" ]]; then + expectStderr 1 nix store delete --recursive --also-referrers "$top" | grepQuiet "Your daemon version is too old to support deleting referrers" else expectStderr 1 nix store delete --recursive --skip-alive "$top" | grepQuiet "Your daemon version is too old to support garbage collecting a specific set of paths" fi } -nix_gc_closure +nix_gc_closure false +nix_gc_closure true +nix_gc_closure false --also-referrers From e6d05b4bab021a695efc0f26b706cf6df5ce182d Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Wed, 22 Apr 2026 15:50:10 -0400 Subject: [PATCH 335/927] Add some examples for GCing a closure Signed-off-by: Lisanna Dettwyler --- src/nix/store-delete.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md index 026dccd0f1..9b6fd9701d 100644 --- a/src/nix/store-delete.md +++ b/src/nix/store-delete.md @@ -8,6 +8,18 @@ R""( # nix store delete /nix/store/fdhrijyv3670djsgprx596nn89iwlj2s-hello-2.10 ``` +* Garbage collect a closure: + + ```console + # nix store delete --recursive --skip-alive nixpkgs#hello + ``` + +* Garbage collect a closure including dead referrers of the closure: + + ```console + # nix store delete --recursive --skip-alive --also-referrers nixpkgs#hello + ``` + # Description This command deletes the store paths specified by [*installables*](./nix.md#installables), From 66d9c62cde51b9c2e75f64c57a81b880467ef0fb Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 2 May 2026 03:16:09 +0300 Subject: [PATCH 336/927] Fix observe_string_cb to not assume a NUL terminated string With std::string_view::data() we really can't assume that the string ends up being NUL terminated. We already pass the length when needed, but we just don't use it in the test harness. All existing API consumers really should not be relying on this being the case in all situations, but this wasn't properly documented anywhere (though it is rather self-evident from the size parameter passed to the callback). --- src/libstore-tests/nix_api_store.cc | 4 ++-- src/libutil-c/nix_api_util.h | 1 + src/libutil-test-support/string_callback.cc | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 60626869a9..0162684daf 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -77,8 +77,8 @@ TEST_F(nix_api_store_test, ReturnsValidStorePath) { StorePath * result = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); ASSERT_NE(result, nullptr); - ASSERT_STREQ("name", result->path.name().data()); - ASSERT_STREQ(PATH_SUFFIX.substr(1).c_str(), result->path.to_string().data()); + ASSERT_EQ("name", result->path.name()); + ASSERT_EQ(PATH_SUFFIX.substr(1), result->path.to_string()); nix_store_path_free(result); } diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index 66ea175222..b48d9166d4 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -163,6 +163,7 @@ typedef struct nix_c_context nix_c_context; * @brief Called to get the value of a string owned by Nix. * * The `start` data is borrowed and the function must not assume that the buffer persists after it returns. + * @warning Don't assume that the string is NUL-terminated. * * @param[in] start the string to copy. * @param[in] n the string length. diff --git a/src/libutil-test-support/string_callback.cc b/src/libutil-test-support/string_callback.cc index b64389e4ad..70ff7ca7f4 100644 --- a/src/libutil-test-support/string_callback.cc +++ b/src/libutil-test-support/string_callback.cc @@ -5,7 +5,7 @@ namespace nix::testing { void observe_string_cb(const char * start, unsigned int n, void * user_data) { auto user_data_casted = reinterpret_cast(user_data); - *user_data_casted = std::string(start); + *user_data_casted = std::string(start, n); } } // namespace nix::testing From 5f90b0c5d5944638ead691ccc9437614ecce51f1 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 2 May 2026 03:53:20 +0300 Subject: [PATCH 337/927] Don't assume NUL terminated std::string_view in SQLiteStmt::Use::operator() std::string_view doesn't have to be NUL terminated - we were just getting lucky because everything that we passed in happened to be NUL terminated. This bug has existed since 7a9687ba30d579bc51e0aaf3193e0ab8d86400d2. --- src/libstore/sqlite.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index c65fc240c6..4f39b61c5d 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -177,7 +177,7 @@ SQLiteStmt::Use::~Use() SQLiteStmt::Use & SQLiteStmt::Use::operator()(std::string_view value, bool notNull) { if (notNull) { - if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmt, curArg++, value.data(), value.size(), SQLITE_TRANSIENT) != SQLITE_OK) SQLiteError::throw_(stmt.db, "binding argument"); } else bind(); From 77ecdaf6d184e553492574d10c7648f3b9332779 Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Sun, 26 Apr 2026 18:47:53 +0200 Subject: [PATCH 338/927] Add unit test to reproduce #15713. --- src/libstore-tests/meson.build | 1 + src/libstore-tests/outputs-query.cc | 114 ++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 src/libstore-tests/outputs-query.cc diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 78a1cc4a12..ab03bb2d2a 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -77,6 +77,7 @@ sources = files( 'nar-info-disk-cache.cc', 'nar-info.cc', 'nix_api_store.cc', + 'outputs-query.cc', 'outputs-spec.cc', 'path-info.cc', 'path.cc', diff --git a/src/libstore-tests/outputs-query.cc b/src/libstore-tests/outputs-query.cc new file mode 100644 index 0000000000..4be25e2ad3 --- /dev/null +++ b/src/libstore-tests/outputs-query.cc @@ -0,0 +1,114 @@ +// Regression tests for the functions in outputs-query.cc +// +// See https://github.com/NixOS/nix/issues/15713 + +#include + +#include "nix/store/outputs-query.hh" +#include "nix/store/derivations.hh" +#include "nix/store/dummy-store-impl.hh" +#include "nix/store/realisation.hh" +#include "nix/store/tests/libstore.hh" + +namespace nix { + +class OutputsQueryTest : public ::testing::Test +{ +public: + static void SetUpTestSuite() + { + initLibStore(false); + } + +protected: + EnableExperimentalFeature caFeature{"ca-derivations"}; + + ref store = [] { + auto cfg = make_ref(StoreReference::Params{}); + cfg->readOnly = false; + return cfg->openDummyStore(); + }(); + + static DerivationOutput caFloatingOutput() + { + return DerivationOutput{DerivationOutput::CAFloating{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}; + } + + /** + * Build a simple floating CA derivation with a given name and no input + * derivations. + */ + Derivation makeLeafDrv(std::string name) + { + Derivation drv; + drv.name = std::move(name); + drv.platform = "x86_64-linux"; + drv.builder = "/bin/sh"; + drv.outputs = {{"out", caFloatingOutput()}}; + return drv; + } +}; + +/** + * Regression test for https://github.com/NixOS/nix/issues/15713 + * + * In a Fibonacci-style chain of floating CA derivations, the resolution + * algorithm used to call queryRealisation O(Fib(N)) times. + * This test verifies that memoization reduces this to O(N). + */ +TEST_F(OutputsQueryTest, fibonacciChainQueryCount) +{ + constexpr static size_t N = 10; + std::vector drvPaths; + + // d0, d1: leaf derivations + for (int i = 0; i < 2; ++i) { + drvPaths.push_back(store->writeDerivation(makeLeafDrv("d" + std::to_string(i)))); + } + + // d_i depends on d_{i-1} and d_{i-2} + for (size_t i = 2; i <= N; ++i) { + Derivation drv = makeLeafDrv("d" + std::to_string(i)); + drv.inputDrvs.map[drvPaths[i - 1]].value.insert("out"); + drv.inputDrvs.map[drvPaths[i - 2]].value.insert("out"); + drvPaths.push_back(store->writeDerivation(drv)); + } + + // Tracker for queryRealisation calls. + std::map callCounts; + std::map outPaths; + + QueryRealisationFun queryRealisation = [&](const DrvOutput & id) -> std::shared_ptr { + assert(id.outputName == "out"); + callCounts[id.drvPath]++; + + // Memoize mock output paths. + auto it = outPaths.find(id.drvPath); + if (it == outPaths.end()) { + auto hash = hashString(HashAlgorithm::SHA1, "mock-output-" + std::to_string(outPaths.size())); + it = outPaths.emplace(id.drvPath, StorePath(hash, "out")).first; + } + + return std::make_shared(UnkeyedRealisation{.outPath = it->second}); + }; + + auto result = deepQueryPartialDerivationOutput(*store, drvPaths[N], "out", nullptr, queryRealisation); + + ASSERT_TRUE(result); + + int totalCalls = 0; + for (auto & [path, count] : callCounts) { + totalCalls += count; + if (count > 1) + ADD_FAILURE() << "Derivation at " << store->printStorePath(path) << " was queried " << count + << " times (expected 1)"; + } + + // With full memoization (ResolveCache + RealisationCache), each derivation should be queried exactly once. + EXPECT_EQ(totalCalls, N + 1) << "queryRealisation called " << totalCalls << " times; expected exactly " << (N + 1); +} + +} // namespace nix From 6974f9e1257c9c2371ed3307fc4f5f748d2e5883 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Sat, 2 May 2026 13:42:19 -0400 Subject: [PATCH 339/927] Fix FreeBSD non-unity build Adds missing `` include. Signed-off-by: Lisanna Dettwyler --- src/libutil-tests/unix/unix-domain-socket.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil-tests/unix/unix-domain-socket.cc b/src/libutil-tests/unix/unix-domain-socket.cc index 439d3188ac..b5050e5645 100644 --- a/src/libutil-tests/unix/unix-domain-socket.cc +++ b/src/libutil-tests/unix/unix-domain-socket.cc @@ -6,6 +6,7 @@ #include #include +#include namespace nix { From 2c26a23a9a055df33850de3106488fa7e54d0c9d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 2 May 2026 13:58:44 -0400 Subject: [PATCH 340/927] Remove Perl bindings `nix-serve` is being deprecated, and so Hydra is the only active user of the perl bindings. As such, they have been moved to the hydra repo. Now that that is done, we can remove them from this repo. Delete `src/perl/` and all references to `nix-perl-bindings` across the build system, packaging, CI, and documentation. The `fetchers-substitute` NixOS test still uses `nix-serve`, which depends on Perl bindings, but now uses the older Nix from nixpkgs rather than this project's bindings. This is a stop-gap until we stop using `nix-serve` in that test (because it is deprecated) completely. --- ci/gha/tests/default.nix | 2 - doc/manual/source/development/debugging.md | 1 - flake.nix | 10 - meson.build | 5 - meson.options | 7 - .../clang-tidy/build_required_targets.py | 2 - .../common/clang-tidy/clean_compdb.py | 3 - packaging/components.nix | 2 - packaging/dev-shell.nix | 15 +- packaging/everything.nix | 20 +- packaging/hydra.nix | 12 - src/perl/.version | 1 - src/perl/.yath.rc.in | 2 - src/perl/MANIFEST | 7 - src/perl/lib/Nix/Config.pm.in | 24 - src/perl/lib/Nix/CopyClosure.pm | 61 --- src/perl/lib/Nix/Manifest.pm | 325 ------------- src/perl/lib/Nix/SSH.pm | 110 ----- src/perl/lib/Nix/Store.pm | 45 -- src/perl/lib/Nix/Store.xs | 430 ------------------ src/perl/lib/Nix/Utils.pm | 47 -- src/perl/lib/Nix/meson.build | 61 --- src/perl/meson.build | 196 -------- src/perl/meson.options | 30 -- src/perl/package.nix | 82 ---- src/perl/t/init.t | 13 - src/perl/t/meson.build | 15 - tests/nixos/fetchers-substitute.nix | 18 +- 28 files changed, 4 insertions(+), 1542 deletions(-) delete mode 120000 src/perl/.version delete mode 100644 src/perl/.yath.rc.in delete mode 100644 src/perl/MANIFEST delete mode 100644 src/perl/lib/Nix/Config.pm.in delete mode 100644 src/perl/lib/Nix/CopyClosure.pm delete mode 100644 src/perl/lib/Nix/Manifest.pm delete mode 100644 src/perl/lib/Nix/SSH.pm delete mode 100644 src/perl/lib/Nix/Store.pm delete mode 100644 src/perl/lib/Nix/Store.xs delete mode 100644 src/perl/lib/Nix/Utils.pm delete mode 100644 src/perl/lib/Nix/meson.build delete mode 100644 src/perl/meson.build delete mode 100644 src/perl/meson.options delete mode 100644 src/perl/package.nix delete mode 100644 src/perl/t/init.t delete mode 100644 src/perl/t/meson.build diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index 5e1b23c7a3..8d2383a92f 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -57,8 +57,6 @@ rec { nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; }; mesonComponentOverrides = lib.composeManyExtensions componentOverrides; - # Unclear how to make Perl bindings work with a dynamically linked ASAN. - nix-perl-bindings = if withSanitizers then null else prev.nix-perl-bindings; } ); diff --git a/doc/manual/source/development/debugging.md b/doc/manual/source/development/debugging.md index 6578632d99..35e4c71ec3 100644 --- a/doc/manual/source/development/debugging.md +++ b/doc/manual/source/development/debugging.md @@ -26,7 +26,6 @@ or GCC. This is useful when debugging memory corruption issues. ```console [nix-shell]$ export mesonBuildType=debugoptimized [nix-shell]$ appendToVar mesonFlags "-Dlibexpr:gc=disabled" # Disable Boehm -[nix-shell]$ appendToVar mesonFlags "-Dbindings=false" # Disable nix-perl [nix-shell]$ appendToVar mesonFlags "-Db_sanitize=address,undefined" ``` diff --git a/flake.nix b/flake.nix index c9be5e1dd5..8a2fe16c27 100644 --- a/flake.nix +++ b/flake.nix @@ -328,12 +328,6 @@ // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; } - // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { - # Some perl dependencies are broken on i686-linux. - # Since the support is only best-effort there, disable the perl - # bindings - perlBindings = self.hydraJobs.perlBindings.${system}; - } # Add "passthru" tests // flatMapAttrs @@ -422,10 +416,6 @@ supportsCross = false; }; - "nix-perl-bindings" = { - supportsCross = false; - }; - "nix-clang-tidy-plugin" = { supportsCross = false; }; diff --git a/meson.build b/meson.build index bef2e6221a..b5d9434a1a 100644 --- a/meson.build +++ b/meson.build @@ -47,11 +47,6 @@ subproject('libmain-c') asan_enabled = 'address' in get_option('b_sanitize') -# Language Bindings -if get_option('bindings') and not meson.is_cross_build() and not asan_enabled - subproject('perl') -endif - # Testing if get_option('unit-tests') subproject('libutil-test-support') diff --git a/meson.options b/meson.options index a306a84252..7b847beba8 100644 --- a/meson.options +++ b/meson.options @@ -14,13 +14,6 @@ option( description : 'Build unit tests', ) -option( - 'bindings', - type : 'boolean', - value : true, - description : 'Build language bindings (e.g. Perl)', -) - option( 'benchmarks', type : 'boolean', diff --git a/nix-meson-build-support/common/clang-tidy/build_required_targets.py b/nix-meson-build-support/common/clang-tidy/build_required_targets.py index d55acd74a2..24e4f290c6 100755 --- a/nix-meson-build-support/common/clang-tidy/build_required_targets.py +++ b/nix-meson-build-support/common/clang-tidy/build_required_targets.py @@ -53,8 +53,6 @@ def main(): + [t for t in custom_commands if t.endswith(".gen.inc")] # Flex/Bison generated parsers + [t for t in custom_commands if t.endswith("-tab.cc")] - # Perl XS generated bindings - + [t for t in custom_commands if t.endswith(".cc") and "perl" in t.lower()] ) ninja_build(args.build_root, targets) diff --git a/nix-meson-build-support/common/clang-tidy/clean_compdb.py b/nix-meson-build-support/common/clang-tidy/clean_compdb.py index 659667209a..8087b0bf9b 100755 --- a/nix-meson-build-support/common/clang-tidy/clean_compdb.py +++ b/nix-meson-build-support/common/clang-tidy/clean_compdb.py @@ -48,9 +48,6 @@ def cmdfilter(item: dict) -> bool: # Filter out Flex/Bison generated parsers (generated code) if file.endswith("-tab.cc"): return False - # Filter out Perl XS generated bindings (generated code) - if "/perl/" in file and file.endswith(".cc"): - return False return True return [chomp(x) for x in compdb if cmdfilter(x)] diff --git a/packaging/components.nix b/packaging/components.nix index 112791c27d..fae41caf50 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -512,8 +512,6 @@ in */ nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { }; - nix-perl-bindings = callPackage ../src/perl/package.nix { }; - # The clang-tidy plugin is a build-time tool loaded into clang-tidy itself, # so it must be built with a clang stdenv for ABI compatibility with the # clang-tidy binary from the same llvmPackages set, regardless of the diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index a29d65cb2a..783818e0eb 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -127,7 +127,6 @@ nixComponents.callPackage ( rest = builtins.substring 2 (builtins.stringLength flag) flag; in "-D${prefix}:${rest}"; - havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; availableComponents = lib.filterAttrs ( @@ -170,12 +169,7 @@ nixComponents.callPackage ( # perhaps other things that are primarily for overriding and not the shell. config = { # Default getComponents - getComponents = - c: - builtins.removeAttrs c ( - lib.optionals (!havePerl) [ "nix-perl-bindings" ] - ++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ] - ); + getComponents = c: builtins.removeAttrs c (lib.optionals (!buildCanExecuteHost) [ "nix-manual" ]); }; /** @@ -211,7 +205,6 @@ nixComponents.callPackage ( "nix-fetchers-tests" "nix-flake-tests" "nix-functional-tests" - "nix-perl-bindings" ] (_: null)) c ); }; @@ -287,9 +280,6 @@ nixComponents.callPackage ( ++ map (transformFlag "libutil") (ignoreCrossFile nixComponents.nix-util.mesonFlags) ++ map (transformFlag "libstore") (ignoreCrossFile nixComponents.nix-store.mesonFlags) ++ map (transformFlag "libfetchers") (ignoreCrossFile nixComponents.nix-fetchers.mesonFlags) - ++ lib.optionals havePerl ( - map (transformFlag "perl") (ignoreCrossFile nixComponents.nix-perl-bindings.mesonFlags) - ) ++ map (transformFlag "libexpr") (ignoreCrossFile nixComponents.nix-expr.mesonFlags) ++ map (transformFlag "libcmd") (ignoreCrossFile nixComponents.nix-cmd.mesonFlags) ++ map (transformFlag "nix") (ignoreCrossFile nixComponents.nix-cli.mesonFlags); @@ -345,8 +335,7 @@ nixComponents.callPackage ( lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark ++ dedupByString (v: "${v}") ( lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) - ) - ++ lib.optional havePerl pkgs.perl; + ); propagatedBuildInputs = dedupByString (v: "${v}") ( lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents) diff --git a/packaging/everything.nix b/packaging/everything.nix index 751d861c9e..df7d57a858 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -41,8 +41,6 @@ nix-internal-api-docs, nix-external-api-docs, - nix-perl-bindings, - testers, patchedSrc ? null, @@ -65,16 +63,7 @@ let nix-main-c nix-cmd ; - } - // - lib.optionalAttrs - (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) - { - # Currently fails in static build - inherit - nix-perl-bindings - ; - }; + }; devdoc = buildEnv { name = "nix-${nix-cli.version}-devdoc"; @@ -144,13 +133,6 @@ stdenv.mkDerivation (finalAttrs: { lib.optionals (stdenv.hostPlatform.isLinux && stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ nix-util-tests.tests.run-without-new-syscalls - ] - ++ - lib.optionals (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) - [ - # Perl currently fails in static build - # TODO: Split out tests into a separate derivation? - nix-perl-bindings ]; nativeBuildInputs = [ diff --git a/packaging/hydra.nix b/packaging/hydra.nix index e3f8f1c1f1..e30b62eb57 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -33,7 +33,6 @@ let forAllPackages = forAllPackages' { }; forAllPackages' = { - enableBindings ? false, enableDocs ? false, # already have separate attrs for these }: lib.genAttrs ( @@ -66,9 +65,6 @@ let "nix-json-schema-checks" "nix-clang-tidy-plugin" ] - ++ lib.optionals enableBindings [ - "nix-perl-bindings" - ] ++ lib.optionals enableDocs [ "nix-manual" "nix-manual-manpages-only" @@ -85,7 +81,6 @@ rec { let arbitrarySystem = "x86_64-linux"; listedPkgs = forAllPackages' { - enableBindings = true; enableDocs = true; } (_: null); actualPkgs = lib.concatMapAttrs ( @@ -176,8 +171,6 @@ rec { # Build without unity to catch include issues. withUnityBuild = false; nix-expr = super.nix-expr.override { enableGC = false; }; - # Unclear how to make Perl bindings work with a dynamically linked ASAN. - nix-perl-bindings = null; } ) ); @@ -201,8 +194,6 @@ rec { pkgs.nixComponents2.overrideScope ( self: super: { withTSan = true; - # Dies at startup. - nix-perl-bindings = null; # TSan has issues with fork and threads. nix-functional-tests = super.nix-functional-tests.overrideAttrs { doCheck = false; }; } @@ -256,9 +247,6 @@ rec { ) (forAllSystems (system: components.${system}.${pkgName})) ); - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-perl-bindings); - # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. diff --git a/src/perl/.version b/src/perl/.version deleted file mode 120000 index b7badcd0cc..0000000000 --- a/src/perl/.version +++ /dev/null @@ -1 +0,0 @@ -../../.version \ No newline at end of file diff --git a/src/perl/.yath.rc.in b/src/perl/.yath.rc.in deleted file mode 100644 index e6f5f93ecd..0000000000 --- a/src/perl/.yath.rc.in +++ /dev/null @@ -1,2 +0,0 @@ -[test] --I=rel(@lib_dir@) diff --git a/src/perl/MANIFEST b/src/perl/MANIFEST deleted file mode 100644 index 08897647c9..0000000000 --- a/src/perl/MANIFEST +++ /dev/null @@ -1,7 +0,0 @@ -Changes -Makefile.PL -MANIFEST -Nix.xs -README -t/Nix.t -lib/Nix.pm diff --git a/src/perl/lib/Nix/Config.pm.in b/src/perl/lib/Nix/Config.pm.in deleted file mode 100644 index ad51cff3b2..0000000000 --- a/src/perl/lib/Nix/Config.pm.in +++ /dev/null @@ -1,24 +0,0 @@ -package Nix::Config; - -use MIME::Base64; -use Nix::Store; - -$version = "@PACKAGE_VERSION@"; - -$storeDir = Nix::Store::getStoreDir; - -%config = (); - -sub readConfig { - my $config = "$confDir/nix.conf"; - return unless -f $config; - - open CONFIG, "<$config" or die "cannot open '$config'"; - while () { - /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next; - $config{$1} = $2; - } - close CONFIG; -} - -return 1; diff --git a/src/perl/lib/Nix/CopyClosure.pm b/src/perl/lib/Nix/CopyClosure.pm deleted file mode 100644 index 902ee1a1bc..0000000000 --- a/src/perl/lib/Nix/CopyClosure.pm +++ /dev/null @@ -1,61 +0,0 @@ -package Nix::CopyClosure; - -use utf8; -use strict; -use Nix::Config; -use Nix::Store; -use Nix::SSH; -use List::Util qw(sum); -use IPC::Open2; - - -sub copyToOpen { - my ($from, $to, $sshHost, $storePaths, $includeOutputs, $dryRun, $useSubstitutes) = @_; - - $useSubstitutes = 0 if $dryRun || !defined $useSubstitutes; - - # Get the closure of this path. - my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, - map { followLinksToStorePath $_ } @{$storePaths}))); - - # Send the "query valid paths" command with the "lock" option - # enabled. This prevents a race where the remote host - # garbage-collect paths that are already there. Optionally, ask - # the remote host to substitute missing paths. - syswrite($to, pack("L{url} eq $info->{url}; - } - - push @{$narFileList}, $info if !$found; -} - - -sub addPatch { - my ($patches, $storePath, $patch) = @_; - - $$patches{$storePath} = [] - unless defined $$patches{$storePath}; - - my $patchList = $$patches{$storePath}; - - my $found = 0; - foreach my $patch2 (@{$patchList}) { - $found = 1 if - $patch2->{url} eq $patch->{url} && - $patch2->{basePath} eq $patch->{basePath}; - } - - push @{$patchList}, $patch if !$found; - - return !$found; -} - - -sub readManifest_ { - my ($manifest, $addNAR, $addPatch) = @_; - - # Decompress the manifest if necessary. - if ($manifest =~ /\.bz2$/) { - open MANIFEST, "$Nix::Config::bzip2 -d < $manifest |" - or die "cannot decompress '$manifest': $!"; - } else { - open MANIFEST, "<$manifest" - or die "cannot open '$manifest': $!"; - } - - my $inside = 0; - my $type; - - my $manifestVersion = 2; - - my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType); - my ($narHash, $narSize, $references, $deriver, $copyFrom, $system, $compressionType); - - while () { - chomp; - s/\#.*$//g; - next if (/^$/); - - if (!$inside) { - - if (/^\s*(\w*)\s*\{$/) { - $type = $1; - $type = "narfile" if $type eq ""; - $inside = 1; - undef $storePath; - undef $url; - undef $hash; - undef $size; - undef $narHash; - undef $narSize; - undef $basePath; - undef $baseHash; - undef $patchType; - undef $system; - $references = ""; - $deriver = ""; - $compressionType = "bzip2"; - } - - } else { - - if (/^\}$/) { - $inside = 0; - - if ($type eq "narfile") { - &$addNAR($storePath, - { url => $url, hash => $hash, size => $size - , narHash => $narHash, narSize => $narSize - , references => $references - , deriver => $deriver - , system => $system - , compressionType => $compressionType - }); - } - - elsif ($type eq "patch") { - &$addPatch($storePath, - { url => $url, hash => $hash, size => $size - , basePath => $basePath, baseHash => $baseHash - , narHash => $narHash, narSize => $narSize - , patchType => $patchType - }); - } - - } - - elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; } - elsif (/^\s*CopyFrom:\s*(\/\S+)\s*$/) { $copyFrom = $1; } - elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; } - elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; } - elsif (/^\s*Compression:\s*(\S+)\s*$/) { $compressionType = $1; } - elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; } - elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; } - elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; } - elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; } - elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; } - elsif (/^\s*NarSize:\s*(\d+)\s*$/) { $narSize = $1; } - elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; } - elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; } - elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; } - elsif (/^\s*System:\s*(\S+)\s*$/) { $system = $1; } - - # Compatibility; - elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; } - elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = "md5:$1"; } - - } - } - - close MANIFEST; - - return $manifestVersion; -} - - -sub readManifest { - my ($manifest, $narFiles, $patches) = @_; - readManifest_($manifest, - sub { addNAR($narFiles, @_); }, - sub { addPatch($patches, @_); } ); -} - - -sub writeManifest { - my ($manifest, $narFiles, $patches, $noCompress) = @_; - - open MANIFEST, ">$manifest.tmp"; # !!! check exclusive - - print MANIFEST "version {\n"; - print MANIFEST " ManifestVersion: 3\n"; - print MANIFEST "}\n"; - - foreach my $storePath (sort (keys %{$narFiles})) { - my $narFileList = $$narFiles{$storePath}; - foreach my $narFile (@{$narFileList}) { - print MANIFEST "{\n"; - print MANIFEST " StorePath: $storePath\n"; - print MANIFEST " NarURL: $narFile->{url}\n"; - print MANIFEST " Compression: $narFile->{compressionType}\n"; - print MANIFEST " Hash: $narFile->{hash}\n" if defined $narFile->{hash}; - print MANIFEST " Size: $narFile->{size}\n" if defined $narFile->{size}; - print MANIFEST " NarHash: $narFile->{narHash}\n"; - print MANIFEST " NarSize: $narFile->{narSize}\n" if $narFile->{narSize}; - print MANIFEST " References: $narFile->{references}\n" - if defined $narFile->{references} && $narFile->{references} ne ""; - print MANIFEST " Deriver: $narFile->{deriver}\n" - if defined $narFile->{deriver} && $narFile->{deriver} ne ""; - print MANIFEST " System: $narFile->{system}\n" if defined $narFile->{system}; - print MANIFEST "}\n"; - } - } - - foreach my $storePath (sort (keys %{$patches})) { - my $patchList = $$patches{$storePath}; - foreach my $patch (@{$patchList}) { - print MANIFEST "patch {\n"; - print MANIFEST " StorePath: $storePath\n"; - print MANIFEST " NarURL: $patch->{url}\n"; - print MANIFEST " Hash: $patch->{hash}\n"; - print MANIFEST " Size: $patch->{size}\n"; - print MANIFEST " NarHash: $patch->{narHash}\n"; - print MANIFEST " NarSize: $patch->{narSize}\n" if $patch->{narSize}; - print MANIFEST " BasePath: $patch->{basePath}\n"; - print MANIFEST " BaseHash: $patch->{baseHash}\n"; - print MANIFEST " Type: $patch->{patchType}\n"; - print MANIFEST "}\n"; - } - } - - - close MANIFEST; - - rename("$manifest.tmp", $manifest) - or die "cannot rename $manifest.tmp: $!"; - - - # Create a bzipped manifest. - unless (defined $noCompress) { - system("$Nix::Config::bzip2 < $manifest > $manifest.bz2.tmp") == 0 - or die "cannot compress manifest"; - - rename("$manifest.bz2.tmp", "$manifest.bz2") - or die "cannot rename $manifest.bz2.tmp: $!"; - } -} - - -# Return a fingerprint of a store path to be used in binary cache -# signatures. It contains the store path, the base-32 SHA-256 hash of -# the contents of the path, and the references. -sub fingerprintPath { - my ($storePath, $narHash, $narSize, $references) = @_; - die if substr($storePath, 0, length($Nix::Config::storeDir)) ne $Nix::Config::storeDir; - die if substr($narHash, 0, 7) ne "sha256:"; - # Convert hash from base-16 to base-32, if necessary. - $narHash = "sha256:" . convertHash("sha256", substr($narHash, 7), 1) - if length($narHash) == 71; - die if length($narHash) != 59; - foreach my $ref (@{$references}) { - die if substr($ref, 0, length($Nix::Config::storeDir)) ne $Nix::Config::storeDir; - } - return "1;" . $storePath . ";" . $narHash . ";" . $narSize . ";" . join(",", @{$references}); -} - - -# Parse a NAR info file. -sub parseNARInfo { - my ($storePath, $content, $requireValidSig, $location) = @_; - - my ($storePath2, $url, $fileHash, $fileSize, $narHash, $narSize, $deriver, $system, $sig); - my $compression = "bzip2"; - my @refs; - - foreach my $line (split "\n", $content) { - return undef unless $line =~ /^(.*): (.*)$/; - if ($1 eq "StorePath") { $storePath2 = $2; } - elsif ($1 eq "URL") { $url = $2; } - elsif ($1 eq "Compression") { $compression = $2; } - elsif ($1 eq "FileHash") { $fileHash = $2; } - elsif ($1 eq "FileSize") { $fileSize = int($2); } - elsif ($1 eq "NarHash") { $narHash = $2; } - elsif ($1 eq "NarSize") { $narSize = int($2); } - elsif ($1 eq "References") { @refs = split / /, $2; } - elsif ($1 eq "Deriver") { $deriver = $2; } - elsif ($1 eq "System") { $system = $2; } - elsif ($1 eq "Sig") { $sig = $2; } - } - - return undef if $storePath ne $storePath2 || !defined $url || !defined $narHash; - - my $res = - { url => $url - , compression => $compression - , fileHash => $fileHash - , fileSize => $fileSize - , narHash => $narHash - , narSize => $narSize - , refs => [ @refs ] - , deriver => $deriver - , system => $system - }; - - if ($requireValidSig) { - # FIXME: might be useful to support multiple signatures per .narinfo. - - if (!defined $sig) { - warn "NAR info file '$location' lacks a signature; ignoring\n"; - return undef; - } - my ($keyName, $sig64) = split ":", $sig; - return undef unless defined $keyName && defined $sig64; - - my $publicKey = $Nix::Config::binaryCachePublicKeys{$keyName}; - if (!defined $publicKey) { - warn "NAR info file '$location' is signed by unknown key '$keyName'; ignoring\n"; - return undef; - } - - my $fingerprint; - eval { - $fingerprint = fingerprintPath( - $storePath, $narHash, $narSize, - [ map { "$Nix::Config::storeDir/$_" } @refs ]); - }; - if ($@) { - warn "cannot compute fingerprint of '$location'; ignoring\n"; - return undef; - } - - if (!checkSignature($publicKey, decode_base64($sig64), $fingerprint)) { - warn "NAR info file '$location' has an incorrect signature; ignoring\n"; - return undef; - } - - $res->{signedBy} = $keyName; - } - - return $res; -} - - -return 1; diff --git a/src/perl/lib/Nix/SSH.pm b/src/perl/lib/Nix/SSH.pm deleted file mode 100644 index 490ba0ea99..0000000000 --- a/src/perl/lib/Nix/SSH.pm +++ /dev/null @@ -1,110 +0,0 @@ -package Nix::SSH; - -use utf8; -use strict; -use File::Temp qw(tempdir); -use IPC::Open2; - -our @ISA = qw(Exporter); -our @EXPORT = qw( - @globalSshOpts - readN readInt readString readStrings - writeInt writeString writeStrings - connectToRemoteNix -); - - -our @globalSshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or ""); - - -sub readN { - my ($bytes, $from) = @_; - my $res = ""; - while ($bytes > 0) { - my $s; - my $n = sysread($from, $s, $bytes); - die "I/O error reading from remote side\n" if !defined $n; - die "got EOF while expecting $bytes bytes from remote side\n" if !$n; - $bytes -= $n; - $res .= $s; - } - return $res; -} - - -sub readInt { - my ($from) = @_; - return unpack("L= 0x300; - - return ($from, $to, $pid); -} - - -1; diff --git a/src/perl/lib/Nix/Store.pm b/src/perl/lib/Nix/Store.pm deleted file mode 100644 index f2ae7e88f8..0000000000 --- a/src/perl/lib/Nix/Store.pm +++ /dev/null @@ -1,45 +0,0 @@ -package Nix::Store; - -use strict; -use warnings; - -require Exporter; - -our @ISA = qw(Exporter); - -our %EXPORT_TAGS = ( 'all' => [ qw( ) ] ); - -our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); - -our @EXPORT = qw( - StoreWrapper - StoreWrapper::new - StoreWrapper::isValidPath StoreWrapper::queryReferences StoreWrapper::queryPathInfo StoreWrapper::queryDeriver StoreWrapper::queryPathHash - StoreWrapper::queryPathFromHashPart - StoreWrapper::topoSortPaths StoreWrapper::computeFSClosure followLinksToStorePath StoreWrapper::exportPaths StoreWrapper::importPaths - StoreWrapper::addToStore StoreWrapper::makeFixedOutputPath - StoreWrapper::derivationFromPath - StoreWrapper::addTempRoot - StoreWrapper::queryRawRealisation - - hashPath hashFile hashString convertHash - signString checkSignature - getStoreDir - setVerbosity -); - -our $VERSION = '0.15'; - -sub backtick { - open(RES, "-|", @_) or die; - local $/; - my $res = || ""; - close RES or die; - return $res; -} - -require XSLoader; -XSLoader::load('Nix::Store', $VERSION); - -1; -__END__ diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs deleted file mode 100644 index 8b28b0e539..0000000000 --- a/src/perl/lib/Nix/Store.xs +++ /dev/null @@ -1,430 +0,0 @@ -#include "EXTERN.h" -#include "perl.h" -#include "XSUB.h" - -/* Prevent a clash between some Perl and libstdc++ macros. */ -#undef do_open -#undef do_close - -#include "nix/store/derivations.hh" -#include "nix/store/realisation.hh" -#include "nix/store/globals.hh" -#include "nix/store/store-open.hh" -#include "nix/util/posix-source-accessor.hh" -#include "nix/store/export-import.hh" - -#include -#include - -using namespace nix; - -static bool libStoreInitialized = false; - -struct StoreWrapper { - ref store; -}; - -MODULE = Nix::Store PACKAGE = Nix::Store -PROTOTYPES: ENABLE - -TYPEMAP: < _store; - try { - if (!libStoreInitialized) { - initLibStore(); - libStoreInitialized = true; - } - if (items == 1) { - _store = openStore(); - RETVAL = new StoreWrapper { - .store = ref{_store} - }; - } else { - RETVAL = new StoreWrapper { - .store = openStore(s) - }; - } - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -void init() - CODE: - if (!libStoreInitialized) { - initLibStore(); - libStoreInitialized = true; - } - - -void setVerbosity(int level) - CODE: - verbosity = (Verbosity) level; - - -int -StoreWrapper::isValidPath(char * path) - CODE: - try { - RETVAL = THIS->store->isValidPath(THIS->store->parseStorePath(path)); - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -SV * -StoreWrapper::queryReferences(char * path) - PPCODE: - try { - for (auto & i : THIS->store->queryPathInfo(THIS->store->parseStorePath(path))->references) - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::queryPathHash(char * path) - PPCODE: - try { - auto s = THIS->store->queryPathInfo(THIS->store->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::queryDeriver(char * path) - PPCODE: - try { - auto info = THIS->store->queryPathInfo(THIS->store->parseStorePath(path)); - if (!info->deriver) XSRETURN_UNDEF; - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(*info->deriver).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::queryPathInfo(char * path, int base32) - PPCODE: - try { - auto info = THIS->store->queryPathInfo(THIS->store->parseStorePath(path)); - if (!info->deriver) - XPUSHs(&PL_sv_undef); - else - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, true); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - mXPUSHi(info->registrationTime); - mXPUSHi(info->narSize); - AV * refs = newAV(); - for (auto & i : info->references) - av_push(refs, newSVpv(THIS->store->printStorePath(i).c_str(), 0)); - XPUSHs(sv_2mortal(newRV((SV *) refs))); - AV * sigs = newAV(); - for (auto & i : info->sigs) - av_push(sigs, newSVpv(i.to_string().c_str(), 0)); - XPUSHs(sv_2mortal(newRV((SV *) sigs))); - } catch (Error & e) { - croak("%s", e.what()); - } - -SV * -StoreWrapper::queryRawRealisation(char * drvPath, char * outputName) - PPCODE: - try { - auto realisation = THIS->store->queryRealisation(DrvOutput{ - .drvPath = THIS->store->parseStorePath(drvPath), - .outputName = outputName, - }); - if (realisation) - XPUSHs(sv_2mortal(newSVpv(static_cast(*realisation).dump().c_str(), 0))); - else - XPUSHs(sv_2mortal(newSVpv("", 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::queryPathFromHashPart(char * hashPart) - PPCODE: - try { - auto path = THIS->store->queryPathFromHashPart(hashPart); - XPUSHs(sv_2mortal(newSVpv(path ? THIS->store->printStorePath(*path).c_str() : "", 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::computeFSClosure(int flipDirection, int includeOutputs, ...) - PPCODE: - try { - StorePathSet paths; - for (int n = 3; n < items; ++n) - THIS->store->computeFSClosure(THIS->store->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs); - for (auto & i : paths) - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::topoSortPaths(...) - PPCODE: - try { - StorePathSet paths; - for (int n = 1; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); - auto sorted = THIS->store->topoSortPaths(paths); - for (auto & i : sorted) - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::followLinksToStorePath(char * path) - CODE: - try { - RETVAL = newSVpv(THIS->store->printStorePath(THIS->store->followLinksToStorePath(path)).c_str(), 0); - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -void -StoreWrapper::exportPaths(int fd, ...) - PPCODE: - try { - StorePathSet paths; - for (int n = 2; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); - FdSink sink(fd); - exportPaths(*THIS->store, paths, sink); - } catch (Error & e) { - croak("%s", e.what()); - } - - -void -StoreWrapper::importPaths(int fd, int dontCheckSigs) - PPCODE: - try { - FdSource source(fd); - importPaths(*THIS->store, source, dontCheckSigs ? NoCheckSigs : CheckSigs); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -hashPath(char * algo, int base32, char * path) - PPCODE: - try { - Hash h = hashPath( - makeFSSourceAccessor(absPath(path)), - FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first; - auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * hashFile(char * algo, int base32, char * path) - PPCODE: - try { - Hash h = hashFile(parseHashAlgo(algo), path); - auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * hashString(char * algo, int base32, char * s) - PPCODE: - try { - Hash h = hashString(parseHashAlgo(algo), s); - auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * convertHash(char * algo, char * s, int toBase32) - PPCODE: - try { - auto h = Hash::parseAny(s, parseHashAlgo(algo)); - auto s = h.to_string(toBase32 ? HashFormat::Nix32 : HashFormat::Base16, false); - XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * signString(char * secretKey_, char * msg) - PPCODE: - try { - auto sig = SecretKey(secretKey_).signDetached(msg).to_string(); - XPUSHs(sv_2mortal(newSVpv(sig.c_str(), sig.size()))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -int checkSignature(SV * publicKey_, SV * sig_, char * msg) - CODE: - try { - STRLEN publicKeyLen; - unsigned char * publicKey = (unsigned char *) SvPV(publicKey_, publicKeyLen); - if (publicKeyLen != crypto_sign_PUBLICKEYBYTES) - throw Error("public key is not valid"); - - STRLEN sigLen; - unsigned char * sig = (unsigned char *) SvPV(sig_, sigLen); - if (sigLen != crypto_sign_BYTES) - throw Error("signature is not valid"); - - RETVAL = crypto_sign_verify_detached(sig, (unsigned char *) msg, strlen(msg), publicKey) == 0; - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -SV * -StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) - PPCODE: - try { - auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat; - auto path = THIS->store->addToStore( - std::string(baseNameOf(srcPath)), - makeFSSourceAccessor(absPath(srcPath)), - method, parseHashAlgo(algo)); - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) - PPCODE: - try { - auto h = Hash::parseAny(hash, parseHashAlgo(algo)); - auto method = recursive ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; - auto path = THIS->store->makeFixedOutputPath(name, FixedOutputInfo { - .method = method, - .hash = h, - .references = {}, - }); - XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * -StoreWrapper::derivationFromPath(char * drvPath) - PREINIT: - HV *hash; - CODE: - try { - Derivation drv = THIS->store->derivationFromPath(THIS->store->parseStorePath(drvPath)); - hash = newHV(); - - HV * outputs = newHV(); - for (auto & i : drv.outputsAndOptPaths(*THIS->store)) { - hv_store( - outputs, i.first.c_str(), i.first.size(), - !i.second.second - ? newSV(0) /* null value */ - : newSVpv(THIS->store->printStorePath(*i.second.second).c_str(), 0), - 0); - } - hv_stores(hash, "outputs", newRV((SV *) outputs)); - - AV * inputDrvs = newAV(); - for (auto & i : drv.inputDrvs.map) - av_push(inputDrvs, newSVpv(THIS->store->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second - hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs)); - - AV * inputSrcs = newAV(); - for (auto & i : drv.inputSrcs) - av_push(inputSrcs, newSVpv(THIS->store->printStorePath(i).c_str(), 0)); - hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs)); - - hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0)); - hv_stores(hash, "builder", newSVpv(drv.builder.c_str(), 0)); - - AV * args = newAV(); - for (auto & i : drv.args) - av_push(args, newSVpv(i.c_str(), 0)); - hv_stores(hash, "args", newRV((SV *) args)); - - HV * env = newHV(); - for (auto & i : drv.env) - hv_store(env, i.first.c_str(), i.first.size(), newSVpv(i.second.c_str(), 0), 0); - hv_stores(hash, "env", newRV((SV *) env)); - - RETVAL = newRV_noinc((SV *)hash); - } catch (Error & e) { - croak("%s", e.what()); - } - OUTPUT: - RETVAL - - -void -StoreWrapper::addTempRoot(char * storePath) - PPCODE: - try { - THIS->store->addTempRoot(THIS->store->parseStorePath(storePath)); - } catch (Error & e) { - croak("%s", e.what()); - } - - -SV * getStoreDir() - PPCODE: - XPUSHs(sv_2mortal(newSVpv(resolveStoreConfig(StoreReference{settings.storeUri.get()})->storeDir.c_str(), 0))); diff --git a/src/perl/lib/Nix/Utils.pm b/src/perl/lib/Nix/Utils.pm deleted file mode 100644 index 44955a7069..0000000000 --- a/src/perl/lib/Nix/Utils.pm +++ /dev/null @@ -1,47 +0,0 @@ -package Nix::Utils; - -use utf8; -use File::Temp qw(tempdir); - -our @ISA = qw(Exporter); -our @EXPORT = qw(checkURL uniq writeFile readFile mkTempDir); - -$urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*]+ )"; - -sub checkURL { - my ($url) = @_; - die "invalid URL '$url'\n" unless $url =~ /^ $urlRE $ /x; -} - -sub uniq { - my %seen; - my @res; - foreach my $name (@_) { - next if $seen{$name}; - $seen{$name} = 1; - push @res, $name; - } - return @res; -} - -sub writeFile { - my ($fn, $s) = @_; - open TMP, ">$fn" or die "cannot create file '$fn': $!"; - print TMP "$s" or die; - close TMP or die; -} - -sub readFile { - local $/ = undef; - my ($fn) = @_; - open TMP, "<$fn" or die "cannot open file '$fn': $!"; - my $s = ; - close TMP or die; - return $s; -} - -sub mkTempDir { - my ($name) = @_; - return tempdir("$name.XXXXXX", CLEANUP => 1, DIR => $ENV{"TMPDIR"} // $ENV{"XDG_RUNTIME_DIR"} // "/tmp") - || die "cannot create a temporary directory"; -} diff --git a/src/perl/lib/Nix/meson.build b/src/perl/lib/Nix/meson.build deleted file mode 100644 index dd5560e21c..0000000000 --- a/src/perl/lib/Nix/meson.build +++ /dev/null @@ -1,61 +0,0 @@ -# Nix-Perl Scripts -#============================================================================ - - - -# Sources -#------------------------------------------------- - -nix_perl_store_xs = files('Store.xs') - -nix_perl_scripts = files( - 'CopyClosure.pm', - 'Manifest.pm', - 'SSH.pm', - 'Store.pm', - 'Utils.pm', -) - -nix_perl_scripts_copy_tgts = [] -foreach f : nix_perl_scripts - nix_perl_scripts_copy_tgts += fs.copyfile(f) -endforeach - - -# Targets -#--------------------------------------------------- - -nix_perl_scripts += configure_file( - output : 'Config.pm', - input : 'Config.pm.in', - configuration : nix_perl_conf, -) - -nix_perl_store_cc = custom_target( - 'Store.cc', - output : 'Store.cc', - input : nix_perl_store_xs, - command : [ xsubpp, '@INPUT@', '-output', '@OUTPUT@' ], -) - -# Build Nix::Store Library -#------------------------------------------------- -nix_perl_store_lib = library( - 'Store', - sources : nix_perl_store_cc, - name_prefix : '', - prelink : true, # For C++ static initializers - install : true, - install_mode : 'rwxr-xr-x', - install_dir : join_paths(nix_perl_install_dir, 'auto', 'Nix', 'Store'), - dependencies : nix_perl_store_dep_list, -) - - -# Install Scripts -#--------------------------------------------------- -install_data( - nix_perl_scripts, - install_mode : 'rw-r--r--', - install_dir : join_paths(nix_perl_install_dir, 'Nix'), -) diff --git a/src/perl/meson.build b/src/perl/meson.build deleted file mode 100644 index 59f2a66b83..0000000000 --- a/src/perl/meson.build +++ /dev/null @@ -1,196 +0,0 @@ -# Nix-Perl Meson build -#============================================================================ - - -# init project -#============================================================================ -project( - 'nix-perl', - 'cpp', - version : files('.version'), - meson_version : '>= 1.1', - license : 'LGPL-2.1-or-later', -) - -# setup env -#------------------------------------------------- -fs = import('fs') -cpp = meson.get_compiler('cpp') -nix_perl_conf = configuration_data() -nix_perl_conf.set('PACKAGE_VERSION', meson.project_version()) - - -# set error arguments -#------------------------------------------------- -error_args = [ - '-Wdeprecated-copy', - '-Wdeprecated-declarations', - '-Werror=suggest-override', - '-Werror=unused-result', - '-Wignored-qualifiers', - '-Wno-duplicate-decl-specifier', - '-Wno-literal-suffix', - '-Wno-missing-field-initializers', - '-Wno-non-virtual-dtor', - '-Wno-pedantic', - '-Wno-pointer-bool-conversion', - '-Wno-reserved-user-defined-literal', - '-Wno-unknown-warning-option', - '-Wno-unused-parameter', - '-Wno-unused-variable', - '-Wno-variadic-macros', -] - -add_project_arguments( - cpp.get_supported_arguments(error_args), - language : 'cpp', -) - - -# set install directories -#------------------------------------------------- -prefix = get_option('prefix') -libdir = join_paths(prefix, get_option('libdir')) - -# Dependencies -#============================================================================ - -# Required Programs -#------------------------------------------------- -find_program('xz') -xsubpp = find_program('xsubpp') -perl = find_program('perl') -find_program('curl') -yath = find_program('yath', required : false) - -# Required Libraries -#------------------------------------------------- -bzip2_dep = dependency('bzip2', required : false) -if not bzip2_dep.found() - bzip2_dep = cpp.find_library('bz2') - if not bzip2_dep.found() - error('No "bzip2" pkg-config or "bz2" library found') - endif -endif -curl_dep = dependency('libcurl') -libsodium_dep = dependency('libsodium') - -nix_store_dep = dependency('nix-store') - - -# Finding Perl Headers is a pain. as they do not have -# pkgconfig available, are not in a standard location, -# and are installed into a version folder. Use the -# Perl binary to give hints about perl include dir. -# -# Note that until we have a better solution for this, cross-compiling -# the perl bindings does not appear to be possible. -#------------------------------------------------- -perl_archname = run_command( - perl, - '-e', - 'use Config; print $Config{archname};', - check : true, -).stdout() -perl_version = run_command( - perl, - '-e', - 'use Config; print $Config{version};', - check : true, -).stdout() -perl_archlibexp = run_command( - perl, - '-e', - 'use Config; print $Config{archlibexp};', - check : true, -).stdout() -perl_site_libdir = run_command( - perl, - '-e', - 'use Config; print $Config{installsitearch};', - check : true, -).stdout() -nix_perl_install_dir = join_paths( - libdir, - 'perl5', - 'site_perl', - perl_version, - perl_archname, -) - - -# print perl hints for logs -#------------------------------------------------- -message('Perl archname: @0@'.format(perl_archname)) -message('Perl version: @0@'.format(perl_version)) -message('Perl archlibexp: @0@'.format(perl_archlibexp)) -message('Perl install site: @0@'.format(perl_site_libdir)) -message('Assumed Nix-Perl install dir: @0@'.format(nix_perl_install_dir)) - -# Now find perl modules -#------------------------------------------------- -perl_check_dbi = run_command( - perl, - '-e', - 'use DBI; use DBD::SQLite;', - '-I@0@'.format(get_option('dbi_path')), - '-I@0@'.format(get_option('dbd_sqlite_path')), - check : true, -) - -if perl_check_dbi.returncode() == 2 - error('The Perl modules DBI and/or DBD::SQLite are missing.') -else - message('Found Perl Modules: DBI, DBD::SQLite.') -endif - - - -# declare perl dependency -#------------------------------------------------- -perl_dep = declare_dependency( - dependencies : cpp.find_library( - 'perl', - has_headers : [ - join_paths(perl_archlibexp, 'CORE', 'perl.h'), - join_paths(perl_archlibexp, 'CORE', 'EXTERN.h'), - ], - dirs : [ - join_paths(perl_archlibexp, 'CORE'), - ], - ), - include_directories : join_paths(perl_archlibexp, 'CORE'), -) - -# declare dependencies -#------------------------------------------------- -nix_perl_store_dep_list = [ - perl_dep, - bzip2_dep, - curl_dep, - libsodium_dep, - nix_store_dep, -] - -# # build -# #------------------------------------------------- -lib_dir = join_paths('lib', 'Nix') -subdir(lib_dir) - -if get_option('tests').enabled() - yath_rc_conf = configuration_data() - yath_rc_conf.set('lib_dir', lib_dir) - configure_file( - output : '.yath.rc', - input : '.yath.rc.in', - configuration : yath_rc_conf, - ) - subdir('t') - test( - 'nix-perl-test', - yath, - args : [ 'test' ], - workdir : meson.current_build_dir(), - depends : [ nix_perl_store_lib ] + nix_perl_tests_copy_tgts + nix_perl_scripts_copy_tgts, - ) -endif diff --git a/src/perl/meson.options b/src/perl/meson.options deleted file mode 100644 index 03ddf57f14..0000000000 --- a/src/perl/meson.options +++ /dev/null @@ -1,30 +0,0 @@ -# Nix-Perl build options -#============================================================================ - - -# compiler args -#============================================================================ - -option( - 'tests', - type : 'feature', - value : 'disabled', - description : 'run nix-perl tests', -) - - -# Location of Perl Modules -#============================================================================ -option( - 'dbi_path', - type : 'string', - value : '/usr', - description : 'path to perl::dbi', -) - -option( - 'dbd_sqlite_path', - type : 'string', - value : '/usr', - description : 'path to perl::dbd-SQLite', -) diff --git a/src/perl/package.nix b/src/perl/package.nix deleted file mode 100644 index e25b2996c8..0000000000 --- a/src/perl/package.nix +++ /dev/null @@ -1,82 +0,0 @@ -{ - lib, - stdenv, - mkMesonDerivation, - pkg-config, - perl, - perlPackages, - nix-store, - version, - curl, - bzip2, - libsodium, -}: - -let - inherit (lib) fileset; -in - -perl.pkgs.toPerlModule ( - mkMesonDerivation (finalAttrs: { - pname = "nix-perl"; - inherit version; - - workDir = ./.; - fileset = fileset.unions ( - [ - ./.version - ../../.version - ./MANIFEST - ./lib - ./meson.build - ./meson.options - ] - ++ lib.optionals finalAttrs.finalPackage.doCheck [ - ./.yath.rc.in - ./t - ] - ); - - nativeBuildInputs = [ - pkg-config - perl - curl - ]; - - buildInputs = [ - nix-store - bzip2 - libsodium - perlPackages.DBI - perlPackages.DBDSQLite - ]; - - # `perlPackages.Test2Harness` is marked broken for Darwin - doCheck = !stdenv.isDarwin; - - nativeCheckInputs = [ - perlPackages.Test2Harness - ]; - - preConfigure = - # "Inline" .version so its not a symlink, and includes the suffix - '' - chmod u+w .version - echo ${finalAttrs.version} > .version - ''; - - mesonFlags = [ - (lib.mesonEnable "tests" finalAttrs.finalPackage.doCheck) - ]; - - mesonCheckFlags = [ - "--print-errorlogs" - ]; - - strictDeps = false; - - meta = { - platforms = lib.platforms.unix; - }; - }) -) diff --git a/src/perl/t/init.t b/src/perl/t/init.t deleted file mode 100644 index 80197e0137..0000000000 --- a/src/perl/t/init.t +++ /dev/null @@ -1,13 +0,0 @@ -use strict; -use warnings; -use Test2::V0; - -use Nix::Store; - -my $s = new Nix::Store("dummy://"); - -my $res = $s->isValidPath("/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"); - -ok(!$res, "should not have path"); - -done_testing; diff --git a/src/perl/t/meson.build b/src/perl/t/meson.build deleted file mode 100644 index f95bee2ffc..0000000000 --- a/src/perl/t/meson.build +++ /dev/null @@ -1,15 +0,0 @@ -# Nix-Perl Tests -#============================================================================ - - -# src -#--------------------------------------------------- - -nix_perl_tests = files( - 'init.t', -) - -nix_perl_tests_copy_tgts = [] -foreach f : nix_perl_tests - nix_perl_tests_copy_tgts += fs.copyfile(f) -endforeach diff --git a/tests/nixos/fetchers-substitute.nix b/tests/nixos/fetchers-substitute.nix index 7abadd43af..b61a3929af 100644 --- a/tests/nixos/fetchers-substitute.nix +++ b/tests/nixos/fetchers-substitute.nix @@ -1,26 +1,9 @@ -{ nixComponents, ... }: { name = "fetchers-substitute"; nodes.substituter = { pkgs, ... }: { - # nix-serve is broken while cross-compiling in nixpkgs 25.11. It's been - # fixed since, but while we're pinning 25.11 we use this workaround. - nixpkgs.overlays = [ - (final: prev: { - nix-serve = - final.lib.warnIf (final.lib.versions.majorMinor final.lib.version != "25.11") - "remove the hack in fetchers-substitute.nix when updating nixpkgs from 25.11" - ( - prev.nix-serve.override { - nix = prev.nix // { - libs.nix-perl-bindings = nixComponents.nix-perl-bindings; - }; - } - ); - }) - ]; virtualisation.writableStore = true; nix.settings.extra-experimental-features = [ @@ -30,6 +13,7 @@ networking.firewall.allowedTCPPorts = [ 5000 ]; + # TODO stop using this, because it has to depend on an older version of Nix that still has the perl bindings. services.nix-serve = { enable = true; secretKeyFile = From 5172f041353f0b26454264fbc5e163518f76a956 Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Fri, 1 May 2026 20:35:35 +0200 Subject: [PATCH 341/927] Deduplicate terminal realisation queries in CA derivation resolution This change introduces RealisationCache to ensure each unique derivation output is only queried once from the store during graph traversal. Internal helper functions are moved into an anonymous namespace. --- src/libstore/include/nix/store/realisation.hh | 21 +++ src/libstore/outputs-query.cc | 131 ++++++++++++++---- 2 files changed, 125 insertions(+), 27 deletions(-) diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index 33159b6dc8..ef89d290aa 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -4,6 +4,7 @@ #include #include "nix/util/hash.hh" +#include "nix/util/std-hash.hh" #include "nix/store/path.hh" #include "nix/store/derived-path.hh" #include @@ -178,6 +179,26 @@ public: } // namespace nix +template<> +struct std::hash +{ + std::size_t operator()(const nix::DrvOutput & id) const noexcept + { + std::size_t h = 0; + nix::hash_combine(h, id.drvPath, id.outputName); + return h; + } +}; + +namespace nix { + +inline std::size_t hash_value(const DrvOutput & id) +{ + return std::hash{}(id); +} + +} // namespace nix + JSON_IMPL(nix::DrvOutput) JSON_IMPL(nix::UnkeyedRealisation) JSON_IMPL(nix::Realisation) diff --git a/src/libstore/outputs-query.cc b/src/libstore/outputs-query.cc index 531bcc8552..b7bd97d0ed 100644 --- a/src/libstore/outputs-query.cc +++ b/src/libstore/outputs-query.cc @@ -3,8 +3,26 @@ #include "nix/store/realisation.hh" #include "nix/util/util.hh" +#include + namespace nix { +namespace { + +/** + * Cache mapping a resolved derivation output to its realisation output path. + */ +using RealisationCache = boost::unordered_flat_map>; + +/* Forward declaration so resolveSingleDerivedPath can call it. */ +static std::optional deepQueryPartialDerivationOutputImpl( + Store & store, + const StorePath & drvPath, + const std::string & outputName, + Store * evalStore_, + QueryRealisationFun & queryRealisation, + RealisationCache & resCache); + /** * Resolve a `SingleDerivedPath` to a concrete store path. * @@ -15,16 +33,21 @@ namespace nix { * @param queryRealisation must already be initialized (not empty) */ static std::optional resolveSingleDerivedPath( - Store & store, const SingleDerivedPath & path, Store * evalStore_, QueryRealisationFun & queryRealisation) + Store & store, + const SingleDerivedPath & path, + Store * evalStore_, + QueryRealisationFun & queryRealisation, + RealisationCache & resCache) { return std::visit( overloaded{ [](const SingleDerivedPath::Opaque & opaque) -> std::optional { return opaque.path; }, [&](const SingleDerivedPath::Built & built) -> std::optional { - auto innerPath = resolveSingleDerivedPath(store, *built.drvPath, evalStore_, queryRealisation); + auto innerPath = resolveSingleDerivedPath(store, *built.drvPath, evalStore_, queryRealisation, resCache); if (!innerPath) return std::nullopt; - return deepQueryPartialDerivationOutput(store, *innerPath, built.output, evalStore_, queryRealisation); + return deepQueryPartialDerivationOutputImpl( + store, *innerPath, built.output, evalStore_, queryRealisation, resCache); }, }, path.raw()); @@ -35,8 +58,12 @@ static std::optional resolveSingleDerivedPath( * * @param queryRealisation must already be initialized (not empty) */ -static std::pair -resolveDerivation(Store & store, const StorePath & drvPath, Store * evalStore_, QueryRealisationFun & queryRealisation) +static std::pair resolveDerivation( + Store & store, + const StorePath & drvPath, + Store * evalStore_, + QueryRealisationFun & queryRealisation, + RealisationCache & resCache) { auto & evalStore = evalStore_ ? *evalStore_ : store; @@ -50,11 +77,12 @@ resolveDerivation(Store & store, const StorePath & drvPath, Store * evalStore_, store, [&](ref depDrvPath, const std::string & depOutputName) -> std::optional { - auto concreteDrvPath = resolveSingleDerivedPath(store, *depDrvPath, evalStore_, queryRealisation); + auto concreteDrvPath = + resolveSingleDerivedPath(store, *depDrvPath, evalStore_, queryRealisation, resCache); if (!concreteDrvPath) return std::nullopt; - return deepQueryPartialDerivationOutput( - store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation); + return deepQueryPartialDerivationOutputImpl( + store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation, resCache); }); if (resolvedDrv) drv = Derivation{*resolvedDrv}; @@ -69,21 +97,79 @@ void queryPartialDerivationOutputMapCA( const StorePath & drvPath, const BasicDerivation & drv, std::map> & outputs, - QueryRealisationFun queryRealisation) + QueryRealisationFun queryRealisation, + RealisationCache & resCache) { if (!queryRealisation) queryRealisation = [&store](const DrvOutput & o) { return store.queryRealisation(o); }; for (auto & [outputName, _] : drv.outputs) { - auto realisation = queryRealisation(DrvOutput{drvPath, outputName}); - if (realisation) { - outputs.insert_or_assign(outputName, realisation->outPath); + DrvOutput id{drvPath, outputName}; + auto it = resCache.find(id); + if (it != resCache.end()) { + outputs.insert_or_assign(outputName, it->second); + continue; + } + + auto realisation = queryRealisation(id); + std::optional outPath = realisation ? std::optional{realisation->outPath} : std::nullopt; + resCache.emplace(id, outPath); + + if (outPath) { + outputs.insert_or_assign(outputName, *outPath); } else { outputs.insert({outputName, std::nullopt}); } } } +/** + * Internal implementation of deepQueryPartialDerivationOutput that accepts a + * shared RealisationCache, allowing memoization of realisation queries across recursive calls. + */ +static std::optional deepQueryPartialDerivationOutputImpl( + Store & store, + const StorePath & drvPath, + const std::string & outputName, + Store * evalStore_, + QueryRealisationFun & queryRealisation, + RealisationCache & resCache) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + auto staticResult = evalStore.queryStaticPartialDerivationOutput(drvPath, outputName); + if (staticResult || !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + return staticResult; + + auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, resCache); + + if (drv.outputs.count(outputName) == 0) + throw Error("derivation '%s' does not have an output named '%s'", store.printStorePath(drvPath), outputName); + + DrvOutput id{resolvedDrvPath, outputName}; + auto it = resCache.find(id); + if (it != resCache.end()) + return it->second; + + auto realisation = queryRealisation(id); + std::optional outPath = realisation ? std::optional{realisation->outPath} : std::nullopt; + resCache.emplace(id, outPath); + return outPath; +} + +} // namespace + +void queryPartialDerivationOutputMapCA( + Store & store, + const StorePath & drvPath, + const BasicDerivation & drv, + std::map> & outputs, + QueryRealisationFun queryRealisation) +{ + RealisationCache resCache; + queryPartialDerivationOutputMapCA(store, drvPath, drv, outputs, queryRealisation, resCache); +} + std::map> deepQueryPartialDerivationOutputMap( Store & store, const StorePath & drvPath, Store * evalStore_, QueryRealisationFun queryRealisation) { @@ -97,8 +183,9 @@ std::map> deepQueryPartialDerivationOutput if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return outputs; - auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation); - queryPartialDerivationOutputMapCA(store, resolvedDrvPath, drv, outputs, queryRealisation); + RealisationCache resCache; + auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, resCache); + queryPartialDerivationOutputMapCA(store, resolvedDrvPath, drv, outputs, queryRealisation, resCache); return outputs; } @@ -123,22 +210,12 @@ std::optional deepQueryPartialDerivationOutput( Store * evalStore_, QueryRealisationFun queryRealisation) { - auto & evalStore = evalStore_ ? *evalStore_ : store; - if (!queryRealisation) queryRealisation = [&store](const DrvOutput & o) { return store.queryRealisation(o); }; - auto staticResult = evalStore.queryStaticPartialDerivationOutput(drvPath, outputName); - if (staticResult || !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) - return staticResult; - - auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation); - - if (drv.outputs.count(outputName) == 0) - throw Error("derivation '%s' does not have an output named '%s'", store.printStorePath(drvPath), outputName); - - auto realisation = queryRealisation(DrvOutput{resolvedDrvPath, outputName}); - return realisation ? std::optional{realisation->outPath} : std::nullopt; + RealisationCache resCache; + return deepQueryPartialDerivationOutputImpl( + store, drvPath, outputName, evalStore_, queryRealisation, resCache); } } // namespace nix From e094bbd1cfba091a8cf973d540122cb1a41ca1a5 Mon Sep 17 00:00:00 2001 From: Lennart Kolmodin Date: Fri, 1 May 2026 20:35:38 +0200 Subject: [PATCH 342/927] Fix exponential complexity in CA derivation output resolution By memoizing the structural resolution of derivations in ResolveCache, we restore linear O(N) performance for graph traversal. This prevents the O(Fib(N)) blowup previously seen in graphs with many overlapping dependencies. --- src/libstore/outputs-query.cc | 45 +++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/libstore/outputs-query.cc b/src/libstore/outputs-query.cc index b7bd97d0ed..6a5cce2223 100644 --- a/src/libstore/outputs-query.cc +++ b/src/libstore/outputs-query.cc @@ -9,6 +9,13 @@ namespace nix { namespace { +/** + * Cache mapping an unresolved drv path to its resolved (Derivation, StorePath) + * pair. Shared across a single top-level resolution call to prevent exponential + * re-traversal of the closure when many derivations share the same dependencies. + */ +using ResolveCache = boost::unordered_flat_map>; + /** * Cache mapping a resolved derivation output to its realisation output path. */ @@ -21,6 +28,7 @@ static std::optional deepQueryPartialDerivationOutputImpl( const std::string & outputName, Store * evalStore_, QueryRealisationFun & queryRealisation, + ResolveCache & cache, RealisationCache & resCache); /** @@ -37,24 +45,31 @@ static std::optional resolveSingleDerivedPath( const SingleDerivedPath & path, Store * evalStore_, QueryRealisationFun & queryRealisation, + ResolveCache & cache, RealisationCache & resCache) { return std::visit( overloaded{ [](const SingleDerivedPath::Opaque & opaque) -> std::optional { return opaque.path; }, [&](const SingleDerivedPath::Built & built) -> std::optional { - auto innerPath = resolveSingleDerivedPath(store, *built.drvPath, evalStore_, queryRealisation, resCache); + auto innerPath = + resolveSingleDerivedPath(store, *built.drvPath, evalStore_, queryRealisation, cache, resCache); if (!innerPath) return std::nullopt; return deepQueryPartialDerivationOutputImpl( - store, *innerPath, built.output, evalStore_, queryRealisation, resCache); + store, *innerPath, built.output, evalStore_, queryRealisation, cache, resCache); }, }, path.raw()); } /** - * Resolve a derivation and compute its store path. + * Resolve a derivation and compute its store path, with memoization. + * + * Results are stored in `cache` (keyed on the unresolved `drvPath`) so that + * each derivation in the closure is resolved at most once per top-level call, + * preventing the exponential re-traversal that would otherwise occur for + * content-addressed derivation closures. * * @param queryRealisation must already be initialized (not empty) */ @@ -63,8 +78,13 @@ static std::pair resolveDerivation( const StorePath & drvPath, Store * evalStore_, QueryRealisationFun & queryRealisation, + ResolveCache & cache, RealisationCache & resCache) { + auto it = cache.find(drvPath); + if (it != cache.end()) + return it->second; + auto & evalStore = evalStore_ ? *evalStore_ : store; Derivation drv = evalStore.readInvalidDerivation(drvPath); @@ -78,18 +98,20 @@ static std::pair resolveDerivation( [&](ref depDrvPath, const std::string & depOutputName) -> std::optional { auto concreteDrvPath = - resolveSingleDerivedPath(store, *depDrvPath, evalStore_, queryRealisation, resCache); + resolveSingleDerivedPath(store, *depDrvPath, evalStore_, queryRealisation, cache, resCache); if (!concreteDrvPath) return std::nullopt; return deepQueryPartialDerivationOutputImpl( - store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation, resCache); + store, *concreteDrvPath, depOutputName, evalStore_, queryRealisation, cache, resCache); }); if (resolvedDrv) drv = Derivation{*resolvedDrv}; } auto resolvedDrvPath = computeStorePath(store, drv); - return {std::move(drv), std::move(resolvedDrvPath)}; + auto result = std::make_pair(drv, resolvedDrvPath); + cache.emplace(drvPath, result); + return result; } void queryPartialDerivationOutputMapCA( @@ -125,7 +147,7 @@ void queryPartialDerivationOutputMapCA( /** * Internal implementation of deepQueryPartialDerivationOutput that accepts a - * shared RealisationCache, allowing memoization of realisation queries across recursive calls. + * shared ResolveCache and RealisationCache, allowing memoization across recursive calls. */ static std::optional deepQueryPartialDerivationOutputImpl( Store & store, @@ -133,6 +155,7 @@ static std::optional deepQueryPartialDerivationOutputImpl( const std::string & outputName, Store * evalStore_, QueryRealisationFun & queryRealisation, + ResolveCache & cache, RealisationCache & resCache) { auto & evalStore = evalStore_ ? *evalStore_ : store; @@ -141,7 +164,7 @@ static std::optional deepQueryPartialDerivationOutputImpl( if (staticResult || !experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return staticResult; - auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, resCache); + auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, cache, resCache); if (drv.outputs.count(outputName) == 0) throw Error("derivation '%s' does not have an output named '%s'", store.printStorePath(drvPath), outputName); @@ -183,8 +206,9 @@ std::map> deepQueryPartialDerivationOutput if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return outputs; + ResolveCache cache; RealisationCache resCache; - auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, resCache); + auto [drv, resolvedDrvPath] = resolveDerivation(store, drvPath, evalStore_, queryRealisation, cache, resCache); queryPartialDerivationOutputMapCA(store, resolvedDrvPath, drv, outputs, queryRealisation, resCache); return outputs; @@ -213,9 +237,10 @@ std::optional deepQueryPartialDerivationOutput( if (!queryRealisation) queryRealisation = [&store](const DrvOutput & o) { return store.queryRealisation(o); }; + ResolveCache cache; RealisationCache resCache; return deepQueryPartialDerivationOutputImpl( - store, drvPath, outputName, evalStore_, queryRealisation, resCache); + store, drvPath, outputName, evalStore_, queryRealisation, cache, resCache); } } // namespace nix From 94e24ddea9d2e8fb9099a95e8f8c0ea4f9d9bff7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 2 May 2026 23:50:00 +0300 Subject: [PATCH 343/927] libutil: Fix path traversal in unpackTarfile Fixes GHSA-gr92-w2r5-qw5p. The primary fix is .relative_path() calls, everything else is making sure we use native path handling in libarchive. --- src/libutil/tarfile.cc | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index eea0376637..b61710c7e0 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -5,6 +5,7 @@ #include "nix/util/serialise.hh" #include "nix/util/tarfile.hh" #include "nix/util/file-system.hh" +#include "nix/util/os-string.hh" namespace nix { @@ -123,6 +124,12 @@ TarArchive::~TarArchive() archive_read_free(this->archive); } +#ifndef _WIN32 +# define NIX_LIBARCHIVE_NATIVE_PATH_FUNC(func) func +#else +# define NIX_LIBARCHIVE_NATIVE_PATH_FUNC(func) func##_w +#endif + static void extract_archive(TarArchive & archive, const std::filesystem::path & destDir) { int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS | ARCHIVE_EXTRACT_SECURE_NODOTDOT; @@ -132,24 +139,34 @@ static void extract_archive(TarArchive & archive, const std::filesystem::path & int r = archive_read_next_header(archive.archive, &entry); if (r == ARCHIVE_EOF) break; - auto name = archive_entry_pathname(entry); - if (!name) - throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); - if (r == ARCHIVE_WARN) - warn("getting archive member '%1%': %2%", name, archive_error_string(archive.archive)); - else - archive.check(r); - archive_entry_copy_pathname(entry, (destDir / name).string().c_str()); + const auto relPath = [&]() -> std::filesystem::path { + /* Some archives might lack a pathname https://github.com/libarchive/libarchive/issues/2089. */ + auto * name = NIX_LIBARCHIVE_NATIVE_PATH_FUNC(archive_entry_pathname)(entry); + if (!name) + throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); + if (r == ARCHIVE_WARN) + warn( + "getting archive member '%1%': %2%", + os_string_to_string(OsStringView(name)), + archive_error_string(archive.archive)); + else + archive.check(r); + + return std::filesystem::path(name).relative_path(); + }(); + + NIX_LIBARCHIVE_NATIVE_PATH_FUNC(archive_entry_copy_pathname)(entry, (destDir / relPath).c_str()); // sources can and do contain dirs with no rx bits if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500) archive_entry_set_mode(entry, archive_entry_mode(entry) | 0500); // Patch hardlink path - const char * original_hardlink = archive_entry_hardlink(entry); - if (original_hardlink) { - archive_entry_copy_hardlink(entry, (destDir / original_hardlink).string().c_str()); + const auto * originalHardlink = NIX_LIBARCHIVE_NATIVE_PATH_FUNC(archive_entry_hardlink)(entry); + if (originalHardlink) { + auto hardlinkPath = std::filesystem::path(originalHardlink).relative_path(); + NIX_LIBARCHIVE_NATIVE_PATH_FUNC(archive_entry_copy_hardlink)(entry, (destDir / hardlinkPath).c_str()); } archive.check(archive_read_extract(archive.archive, entry, flags)); @@ -158,6 +175,8 @@ static void extract_archive(TarArchive & archive, const std::filesystem::path & archive.close(); } +#undef NIX_LIBARCHIVE_NATIVE_PATH_FUNC + void unpackTarfile(const std::filesystem::path & tarFile, const std::filesystem::path & destDir) { auto archive = TarArchive(tarFile); From e29ca2f114e21ffccaf0de523fe798b6930ee706 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 24 Apr 2025 11:51:10 -0400 Subject: [PATCH 344/927] Simplify the Meson now that upstream bug is fixed Upstream bug is https://github.com/mesonbuild/meson/issues/13584. This has been fixed in https://github.com/mesonbuild/meson/commit/fc9fd42899e1e2160a69ec245931c3aa79b0d267, which is available since Meson 1.8. We now have it with nixpkgs 25.11 and distros should already be caught up. Also does a minor spring clean of our meson accordingly. I noticed that distros have a tendency to patch out our unconditional nix-function-tests subproject from the top-level project, so this also adds an option to disable it. --- doc/manual/meson.build | 2 +- meson.build | 19 +++++++++---------- meson.options | 7 +++++++ src/clang-tidy-plugin/meson.build | 2 +- src/external-api-docs/meson.build | 2 +- src/internal-api-docs/meson.build | 2 +- src/json-schema-checks/meson.build | 2 +- src/libcmd/meson.build | 2 +- src/libexpr-c/meson.build | 2 +- src/libexpr-test-support/meson.build | 2 +- src/libexpr-tests/meson.build | 2 +- src/libexpr/meson.build | 2 +- src/libfetchers-c/meson.build | 2 +- src/libfetchers-tests/meson.build | 2 +- src/libfetchers/meson.build | 2 +- src/libflake-c/meson.build | 2 +- src/libflake-tests/meson.build | 2 +- src/libflake/meson.build | 2 +- src/libmain-c/meson.build | 2 +- src/libmain/meson.build | 2 +- src/libstore-c/meson.build | 2 +- src/libstore-test-support/meson.build | 2 +- src/libstore-tests/meson.build | 2 +- src/libstore/meson.build | 13 +++---------- src/libutil-c/meson.build | 2 +- src/libutil-test-support/meson.build | 2 +- src/libutil-tests/meson.build | 2 +- src/libutil/meson.build | 2 +- src/nix/meson.build | 7 +------ src/nswrapper/meson.build | 3 +-- 30 files changed, 46 insertions(+), 53 deletions(-) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 6fd841e80c..5f1fe2ed67 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -1,7 +1,7 @@ project( 'nix-manual', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/meson.build b/meson.build index b5d9434a1a..f2a062e396 100644 --- a/meson.build +++ b/meson.build @@ -1,15 +1,12 @@ -# This is just a stub project to include all the others as subprojects -# for development shell purposes +# This is just a top-level project to include all the others as subprojects +# for development shell purposes (when building via Nix) or for distro packaging purposes. project( - 'nix-dev-shell', + 'Nix', 'cpp', version : files('.version'), subproject_dir : 'src', - default_options : [ - 'localstatedir=/nix/var', - ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', ) # Internal Libraries @@ -45,8 +42,6 @@ subproject('libexpr-c') subproject('libflake-c') subproject('libmain-c') -asan_enabled = 'address' in get_option('b_sanitize') - # Testing if get_option('unit-tests') subproject('libutil-test-support') @@ -58,7 +53,11 @@ if get_option('unit-tests') subproject('libexpr-tests') subproject('libflake-tests') endif -subproject('nix-functional-tests') + +if get_option('functional-tests') + subproject('nix-functional-tests') +endif + if get_option('json-schema-checks') subproject('json-schema-checks') endif diff --git a/meson.options b/meson.options index 7b847beba8..2e0d873fae 100644 --- a/meson.options +++ b/meson.options @@ -14,6 +14,13 @@ option( description : 'Build unit tests', ) +option( + 'functional-tests', + type : 'boolean', + value : true, + description : 'Build functional (E2E) tests', +) + option( 'benchmarks', type : 'boolean', diff --git a/src/clang-tidy-plugin/meson.build b/src/clang-tidy-plugin/meson.build index 60cfd15149..7896bd0854 100644 --- a/src/clang-tidy-plugin/meson.build +++ b/src/clang-tidy-plugin/meson.build @@ -6,7 +6,7 @@ project( 'cpp_std=c++23', 'warning_level=2', ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build index 1903b36e58..d96da3863e 100644 --- a/src/external-api-docs/meson.build +++ b/src/external-api-docs/meson.build @@ -1,7 +1,7 @@ project( 'nix-external-api-docs', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build index 844cb262ee..3976e4c3f8 100644 --- a/src/internal-api-docs/meson.build +++ b/src/internal-api-docs/meson.build @@ -1,7 +1,7 @@ project( 'nix-internal-api-docs', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build index 8a0bde04b8..66dc9b758a 100644 --- a/src/json-schema-checks/meson.build +++ b/src/json-schema-checks/meson.build @@ -6,7 +6,7 @@ project( 'nix-json-schema-checks', version : files('.version'), - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index d970a8e4b0..9638b491f2 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index c47704ce41..c4ca08a34f 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index df28661b7e..4a87bb4545 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index 0b0a01c206..92e550a052 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 510b6d696e..d44a0965d9 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libfetchers-c/meson.build b/src/libfetchers-c/meson.build index db415d9173..58f0c26dbf 100644 --- a/src/libfetchers-c/meson.build +++ b/src/libfetchers-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index ba9774e956..b9664eaa16 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index d34dd4f434..ed52e56527 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index fddb39bdf9..01fc3e0f4f 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 3512be10bc..00b592195c 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 58916ecd9a..c06bf6ba54 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 36332fdb70..8b144c8aa9 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 2ac59924e5..0084643bda 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index c81235bf16..542b9a1a94 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 4d904cb1d0..ca451db4ab 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index ab03bb2d2a..a7353f1ec7 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 753c786876..0bd42969d8 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -9,7 +9,7 @@ project( 'errorlogs=true', # Please print logs for tests that fail 'localstatedir=/nix/var', ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) @@ -220,15 +220,14 @@ path_opts = [ 'libdir', 'includedir', 'libexecdir', + 'localstatedir', # Homecooked Nix directories. 'store-dir', - 'localstatedir', 'log-dir', ] # For your grepping pleasure, this loop sets the following variables that aren't mentioned # literally above: # store_dir -# localstatedir # log_dir # profile_dir foreach optname : path_opts @@ -399,13 +398,7 @@ libraries_private = [] extra_pkg_config_variables = { 'storedir' : get_option('store-dir'), + 'localstatedir' : get_option('localstatedir'), } -# Working around https://github.com/mesonbuild/meson/issues/13584 -if host_machine.system() != 'darwin' - extra_pkg_config_variables += { - 'localstatedir' : get_option('localstatedir'), - } -endif - subdir('nix-meson-build-support/export') diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index 1806dbb6f9..c454853c46 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index 64231107eb..7299cd65c3 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 6a86504ded..82322b9cc3 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/libutil/meson.build b/src/libutil/meson.build index d132ce67c7..4c801ffa67 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -8,7 +8,7 @@ project( 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) diff --git a/src/nix/meson.build b/src/nix/meson.build index 3327b846c9..9320016495 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -7,7 +7,6 @@ project( # TODO(Qyriad): increase the warning level 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail - 'localstatedir=/nix/var', ], meson_version : '>= 1.4', license : 'LGPL-2.1-or-later', @@ -262,11 +261,7 @@ custom_target( # TODO(Ericson3214): Doesn't yet work #meson.override_find_program(linkname, t) -localstatedir = nix_store.get_variable( - 'localstatedir', - default_value : get_option('localstatedir'), -) -assert(localstatedir == get_option('localstatedir')) +localstatedir = nix_store.get_variable('localstatedir') store_dir = nix_store.get_variable('storedir') subdir('scripts') subdir('misc') diff --git a/src/nswrapper/meson.build b/src/nswrapper/meson.build index 77b96d677e..1d1ccc6b21 100644 --- a/src/nswrapper/meson.build +++ b/src/nswrapper/meson.build @@ -7,9 +7,8 @@ project( # TODO(Qyriad): increase the warning level 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail - 'localstatedir=/nix/var', ], - meson_version : '>= 1.1', + meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', ) From 2585efd3aa42814b225680f72b33b8eca677ad3a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 3 May 2026 18:39:44 +0300 Subject: [PATCH 345/927] tests/functional: Migrate more repl tests into characterisation framework This is a much more maintainable and easy way of writing tests. I've dropped some filtering (like the number of variables loaded), since we want to test this too and the whole point of characterisation testing is to avoid the manual churn, so it's not an issue. Also added a way to add comments to the repl test without it affecting the output (special `# COM:` prefix in the input, so that tests can be documented). --- tests/functional/repl.sh | 220 +----------------- .../repl/add-lots-of-variables.expected | 32 +++ .../functional/repl/add-lots-of-variables.in | 2 + .../repl/add-overwritten-symbol-env.expected | 13 ++ .../repl/add-overwritten-symbol-env.in | 4 + .../repl/add-variable-with-spaces.expected | 12 + .../repl/add-variable-with-spaces.in | 3 + tests/functional/repl/attribute-set.nix | 6 + .../repl/doc-comment-curried-args.expected | 3 +- .../repl/doc-comment-formals.expected | 3 +- tests/functional/repl/doc-compact.expected | 3 +- tests/functional/repl/doc-constant.expected | 3 +- tests/functional/repl/doc-floatedIn.expected | 3 +- tests/functional/repl/doc-functor.expected | 3 +- .../repl/doc-lambda-flavors.expected | 3 +- .../functional/repl/doc-measurement.expected | 3 +- tests/functional/repl/doc-multiply.expected | 3 +- .../functional/repl/doc-unambiguous.expected | 3 +- .../functional/repl/dollar-escaping.expected | 5 + tests/functional/repl/dollar-escaping.in | 2 + tests/functional/repl/file-a.nix | 1 + tests/functional/repl/file-b.nix | 1 + .../repl/inherit-and-assignment.expected | 9 + .../functional/repl/inherit-and-assignment.in | 4 + .../repl/inherit-current-scope.expected | 9 + .../functional/repl/inherit-current-scope.in | 4 + .../repl/inherit-missing-shows-pos.expected | 13 ++ .../repl/inherit-missing-shows-pos.in | 4 + .../repl/inherit-multiple-attrs.expected | 9 + .../functional/repl/inherit-multiple-attrs.in | 4 + .../repl/inherit-with-semicolon.expected | 9 + .../functional/repl/inherit-with-semicolon.in | 4 + tests/functional/repl/inherit.expected | 9 + tests/functional/repl/inherit.in | 4 + .../repl/list-loaded-nothing-loaded.expected | 5 + .../repl/list-loaded-nothing-loaded.in | 1 + .../repl/multiple-bindings-same-line.expected | 7 + .../repl/multiple-bindings-same-line.in | 3 + .../functional/repl/nested-attr-path.expected | 7 + tests/functional/repl/nested-attr-path.in | 3 + .../repl/pretty-print-idempotent.expected | 3 +- tests/functional/repl/printing.expected | 59 +++++ tests/functional/repl/printing.in | 11 + .../reload-with-non-existent-file.expected | 24 ++ .../repl/reload-with-non-existent-file.in | 5 + 45 files changed, 315 insertions(+), 226 deletions(-) create mode 100644 tests/functional/repl/add-lots-of-variables.expected create mode 100644 tests/functional/repl/add-lots-of-variables.in create mode 100644 tests/functional/repl/add-overwritten-symbol-env.expected create mode 100644 tests/functional/repl/add-overwritten-symbol-env.in create mode 100644 tests/functional/repl/add-variable-with-spaces.expected create mode 100644 tests/functional/repl/add-variable-with-spaces.in create mode 100644 tests/functional/repl/attribute-set.nix create mode 100644 tests/functional/repl/dollar-escaping.expected create mode 100644 tests/functional/repl/dollar-escaping.in create mode 100644 tests/functional/repl/file-a.nix create mode 100644 tests/functional/repl/file-b.nix create mode 100644 tests/functional/repl/inherit-and-assignment.expected create mode 100644 tests/functional/repl/inherit-and-assignment.in create mode 100644 tests/functional/repl/inherit-current-scope.expected create mode 100644 tests/functional/repl/inherit-current-scope.in create mode 100644 tests/functional/repl/inherit-missing-shows-pos.expected create mode 100644 tests/functional/repl/inherit-missing-shows-pos.in create mode 100644 tests/functional/repl/inherit-multiple-attrs.expected create mode 100644 tests/functional/repl/inherit-multiple-attrs.in create mode 100644 tests/functional/repl/inherit-with-semicolon.expected create mode 100644 tests/functional/repl/inherit-with-semicolon.in create mode 100644 tests/functional/repl/inherit.expected create mode 100644 tests/functional/repl/inherit.in create mode 100644 tests/functional/repl/list-loaded-nothing-loaded.expected create mode 100644 tests/functional/repl/list-loaded-nothing-loaded.in create mode 100644 tests/functional/repl/multiple-bindings-same-line.expected create mode 100644 tests/functional/repl/multiple-bindings-same-line.in create mode 100644 tests/functional/repl/nested-attr-path.expected create mode 100644 tests/functional/repl/nested-attr-path.in create mode 100644 tests/functional/repl/printing.expected create mode 100644 tests/functional/repl/printing.in create mode 100644 tests/functional/repl/reload-with-non-existent-file.expected create mode 100644 tests/functional/repl/reload-with-non-existent-file.in diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 88d6e91cd9..9e752337f1 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -120,83 +120,6 @@ testReplResponseNoRegex () { testReplResponseGeneral --fixed-strings "$@" } -# :a uses the newest version of a symbol -# -# shellcheck disable=SC2016 -testReplResponse ' -:a { a = "1"; } -:a { a = "2"; } -"result: ${a}" -' "result: 2" - -# check dollar escaping https://github.com/NixOS/nix/issues/4909 -# note the escaped \, -# \\ -# because the second argument is a regex -# -# shellcheck disable=SC2016 -testReplResponseNoRegex ' -"$" + "{hi}" -' '"\${hi}"' - -# Test inherit statement support (issue #15053) -testReplResponseNoRegex ' -a = { b = 1; c = 2; } -inherit (a) b -b -' '1' - -# inherit multiple attributes -testReplResponseNoRegex ' -a = { x = 10; y = 20; } -inherit (a) x y -x + y -' '30' - -# inherit from current scope -testReplResponseNoRegex ' -foo = 42 -inherit foo -foo -' '42' - -# inherit with semicolon (also works) -testReplResponseNoRegex ' -a = { z = 99; } -inherit (a) z; -z -' '99' - -# multiple bindings on one line -testReplResponseNoRegex ' -a = 1; b = 2; -a + b -' '3' - -# nested attribute path -testReplResponseNoRegex ' -a.b.c = 1; -a.b -' '{ c = 1; }' - -# mixed bindings: inherit and assignment -testReplResponseNoRegex ' -x = { p = 10; } -inherit (x) p; q = 20; -p + q -' '30' - -# inherit error shows position (without spurious semicolon from retry) -testReplResponse ' -a = { x = 1; } -inherit (a) y -y -' "error: attribute 'y' missing -.*at .string.:1:13: -.*inherit (a) y -.* \\^ -.*Did you mean x" - testReplResponse ' drvPath ' '".*-simple.drv"' \ @@ -222,32 +145,6 @@ foo + baz ' "3" \ ./flake ./flake\#bar --experimental-features 'flakes' -testReplResponse $' -:a { a = 1; b = 2; longerName = 3; "with spaces" = 4; } -' 'Added 4 variables. -a, b, longerName, "with spaces" -' - -cat < attribute-set.nix -{ - a = 1; - b = 2; - longerName = 3; - "with spaces" = 4; -} -EOF -testReplResponse ' -:l ./attribute-set.nix -' 'Added 4 variables. -a, b, longerName, "with spaces" -' - -testReplResponseNoRegex $' -:a builtins.foldl\' (x: y: x // y) {} (map (x: { ${builtins.toString x} = x; }) (builtins.genList (x: x) 23)) -' 'Added 23 variables. -"0", "1", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "2", "20", "21", "22", "3", "4", "5", "6" -... and 3 more; view with :ll' - # Test the `:reload` mechansim with flakes: # - Eval `./flake#changingThing` # - Modify the flake @@ -313,22 +210,8 @@ EOF grep -q "afterChange" repl_output || fail ":reload didn't pick up git work tree change" fi -# Regression: a failed `:l` / `:lf` must not be remembered for `:reload`, +# Regression: a failed `:lf` must not be remembered for `:reload`, # and an error in one loaded file must not drop later ones from the reload list. -cat > reloadA.nix < reloadB.nix <@g" \ - -e "/Added [0-9]* variables/{s@ [0-9]* @ @;n;d}" \ - -e '/\.\.\. and [0-9]* more; view with :ll/d' \ | grep -vF $'warning: you don\'t have Internet access; disabling some network-dependent features' \ ; } @@ -500,7 +287,10 @@ for test in $(cd "$testDir/repl"; echo *.in); do read -r -a flags < "$testDir/repl/$test.flags" fi - (cd "$testDir/repl"; set +x; runRepl "${flags[@]}" 2>&1) < "$in" > "$actual" || { + # Allow putting comments (lines starting with `# COM:`) in the test for + # documentation purposes. Regular comments are not skipped, since those are + # also interpreted by the repl. + (cd "$testDir/repl"; set +x; runRepl "${flags[@]}" 2>&1) < <(grep -Ev '^[[:space:]]*#[[:space:]]*COM:' "$in") > "$actual" || { echo "FAIL: $test (exit code $?)" >&2 badExitCode=1 } diff --git a/tests/functional/repl/add-lots-of-variables.expected b/tests/functional/repl/add-lots-of-variables.expected new file mode 100644 index 0000000000..4ba282263b --- /dev/null +++ b/tests/functional/repl/add-lots-of-variables.expected @@ -0,0 +1,32 @@ +Nix +Type :? for help. + +nix-repl> :a builtins.foldl' (x: y: x // y) {} (map (x: { ${builtins.toString x} = x; }) (builtins.genList (x: x) 23)) +Added 23 variables. +"0", "1", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "2", "20", "21", "22", "3", "4", "5", "6" +... and 3 more; view with :ll + +nix-repl> :ll +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 diff --git a/tests/functional/repl/add-lots-of-variables.in b/tests/functional/repl/add-lots-of-variables.in new file mode 100644 index 0000000000..77796389df --- /dev/null +++ b/tests/functional/repl/add-lots-of-variables.in @@ -0,0 +1,2 @@ +:a builtins.foldl' (x: y: x // y) {} (map (x: { ${builtins.toString x} = x; }) (builtins.genList (x: x) 23)) +:ll diff --git a/tests/functional/repl/add-overwritten-symbol-env.expected b/tests/functional/repl/add-overwritten-symbol-env.expected new file mode 100644 index 0000000000..026b7cceb6 --- /dev/null +++ b/tests/functional/repl/add-overwritten-symbol-env.expected @@ -0,0 +1,13 @@ +Nix +Type :? for help. + +nix-repl> :a { a = "1"; } +Added 1 variables. +a + +nix-repl> :a { a = "2"; } +Added 1 variables. +a + +nix-repl> "result: ${a}" +"result: 2" diff --git a/tests/functional/repl/add-overwritten-symbol-env.in b/tests/functional/repl/add-overwritten-symbol-env.in new file mode 100644 index 0000000000..dbbf496772 --- /dev/null +++ b/tests/functional/repl/add-overwritten-symbol-env.in @@ -0,0 +1,4 @@ +# COM: :a uses the newest version of a symbol +:a { a = "1"; } +:a { a = "2"; } +"result: ${a}" diff --git a/tests/functional/repl/add-variable-with-spaces.expected b/tests/functional/repl/add-variable-with-spaces.expected new file mode 100644 index 0000000000..dff5a0cbd2 --- /dev/null +++ b/tests/functional/repl/add-variable-with-spaces.expected @@ -0,0 +1,12 @@ +Nix +Type :? for help. + +nix-repl> :a { a = 1; b = 2; longerName = 3; "with spaces" = 4; } +Added 4 variables. +a, b, longerName, "with spaces" + +nix-repl> :reload + +nix-repl> :l ./attribute-set.nix +Added 4 variables. +a, b, longerName, "with spaces" diff --git a/tests/functional/repl/add-variable-with-spaces.in b/tests/functional/repl/add-variable-with-spaces.in new file mode 100644 index 0000000000..763a619012 --- /dev/null +++ b/tests/functional/repl/add-variable-with-spaces.in @@ -0,0 +1,3 @@ +:a { a = 1; b = 2; longerName = 3; "with spaces" = 4; } +:reload +:l ./attribute-set.nix diff --git a/tests/functional/repl/attribute-set.nix b/tests/functional/repl/attribute-set.nix new file mode 100644 index 0000000000..7b2e71badd --- /dev/null +++ b/tests/functional/repl/attribute-set.nix @@ -0,0 +1,6 @@ +{ + a = 1; + b = 2; + longerName = 3; + "with spaces" = 4; +} diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected index d2a5bf3285..29ca40cee2 100644 --- a/tests/functional/repl/doc-comment-curried-args.expected +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc curriedArgs Function `curriedArgs`\ diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected index 357cf99868..65fbc2bd5d 100644 --- a/tests/functional/repl/doc-comment-formals.expected +++ b/tests/functional/repl/doc-comment-formals.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> "Note that this is not yet complete" "Note that this is not yet complete" diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected index 276de2e60b..9fcf6169a5 100644 --- a/tests/functional/repl/doc-compact.expected +++ b/tests/functional/repl/doc-compact.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc compact Function `compact`\ diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected index a68188b25a..bda53bbfa6 100644 --- a/tests/functional/repl/doc-constant.expected +++ b/tests/functional/repl/doc-constant.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc constant error: value does not have documentation diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected index 3bf1c40715..e01f8868ee 100644 --- a/tests/functional/repl/doc-floatedIn.expected +++ b/tests/functional/repl/doc-floatedIn.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc floatedIn Function `floatedIn`\ diff --git a/tests/functional/repl/doc-functor.expected b/tests/functional/repl/doc-functor.expected index 8b86fe9134..e5984098c3 100644 --- a/tests/functional/repl/doc-functor.expected +++ b/tests/functional/repl/doc-functor.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-functor.nix -Added variables. +Added 13 variables. +diverging, doubler, helper, helper2, helper3, lib, makeOverridable, makeVeryOverridable, multiplier, multiply, recursive, recursive2, square nix-repl> :doc multiplier Function `__functor`\ diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected index 437c09d2b3..676ef73135 100644 --- a/tests/functional/repl/doc-lambda-flavors.expected +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc nonStrict Function `nonStrict`\ diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected index 862697613b..ef176cea88 100644 --- a/tests/functional/repl/doc-measurement.expected +++ b/tests/functional/repl/doc-measurement.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc measurement Function `measurement`\ diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected index 21523e24c8..c2024b0949 100644 --- a/tests/functional/repl/doc-multiply.expected +++ b/tests/functional/repl/doc-multiply.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc multiply Function `multiply`\ diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected index 32ca9aef22..d2a452ba7b 100644 --- a/tests/functional/repl/doc-unambiguous.expected +++ b/tests/functional/repl/doc-unambiguous.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l doc-comments.nix -Added variables. +Added 13 variables. +compact, constant, curriedArgs, documentedFormals, floatedIn, lib, measurement, multiply, nonStrict, strict, strictPost, strictPre, unambiguous nix-repl> :doc unambiguous Function `unambiguous`\ diff --git a/tests/functional/repl/dollar-escaping.expected b/tests/functional/repl/dollar-escaping.expected new file mode 100644 index 0000000000..6e9d237f91 --- /dev/null +++ b/tests/functional/repl/dollar-escaping.expected @@ -0,0 +1,5 @@ +Nix +Type :? for help. + +nix-repl> "$" + "{hi}" +"\${hi}" diff --git a/tests/functional/repl/dollar-escaping.in b/tests/functional/repl/dollar-escaping.in new file mode 100644 index 0000000000..39734f24b8 --- /dev/null +++ b/tests/functional/repl/dollar-escaping.in @@ -0,0 +1,2 @@ +# COM: Check dollar escaping https://github.com/NixOS/nix/issues/4909 +"$" + "{hi}" diff --git a/tests/functional/repl/file-a.nix b/tests/functional/repl/file-a.nix new file mode 100644 index 0000000000..f113eebfd9 --- /dev/null +++ b/tests/functional/repl/file-a.nix @@ -0,0 +1 @@ +{ fromA = 1; } diff --git a/tests/functional/repl/file-b.nix b/tests/functional/repl/file-b.nix new file mode 100644 index 0000000000..ddde63c16c --- /dev/null +++ b/tests/functional/repl/file-b.nix @@ -0,0 +1 @@ +{ fromB = 2; } diff --git a/tests/functional/repl/inherit-and-assignment.expected b/tests/functional/repl/inherit-and-assignment.expected new file mode 100644 index 0000000000..43ad52b5ef --- /dev/null +++ b/tests/functional/repl/inherit-and-assignment.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> x = { p = 10; } + +nix-repl> inherit (x) p; q = 20; + +nix-repl> p + q +30 diff --git a/tests/functional/repl/inherit-and-assignment.in b/tests/functional/repl/inherit-and-assignment.in new file mode 100644 index 0000000000..583b9096b5 --- /dev/null +++ b/tests/functional/repl/inherit-and-assignment.in @@ -0,0 +1,4 @@ +# COM: mixed bindings: inherit and assignment +x = { p = 10; } +inherit (x) p; q = 20; +p + q diff --git a/tests/functional/repl/inherit-current-scope.expected b/tests/functional/repl/inherit-current-scope.expected new file mode 100644 index 0000000000..d2b2f39d22 --- /dev/null +++ b/tests/functional/repl/inherit-current-scope.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> foo = 42 + +nix-repl> inherit foo + +nix-repl> foo +42 diff --git a/tests/functional/repl/inherit-current-scope.in b/tests/functional/repl/inherit-current-scope.in new file mode 100644 index 0000000000..d7eb0047af --- /dev/null +++ b/tests/functional/repl/inherit-current-scope.in @@ -0,0 +1,4 @@ +# COM: inherit from current scope +foo = 42 +inherit foo +foo diff --git a/tests/functional/repl/inherit-missing-shows-pos.expected b/tests/functional/repl/inherit-missing-shows-pos.expected new file mode 100644 index 0000000000..dc02de2070 --- /dev/null +++ b/tests/functional/repl/inherit-missing-shows-pos.expected @@ -0,0 +1,13 @@ +Nix +Type :? for help. + +nix-repl> a = { x = 1; } + +nix-repl> inherit (a) y + +nix-repl> y +error: attribute 'y' missing + at «string»:1:13: + 1| inherit (a) y + | ^ + Did you mean x? diff --git a/tests/functional/repl/inherit-missing-shows-pos.in b/tests/functional/repl/inherit-missing-shows-pos.in new file mode 100644 index 0000000000..057b2e816e --- /dev/null +++ b/tests/functional/repl/inherit-missing-shows-pos.in @@ -0,0 +1,4 @@ +# COM: inherit error shows position (without spurious semicolon from retry) +a = { x = 1; } +inherit (a) y +y diff --git a/tests/functional/repl/inherit-multiple-attrs.expected b/tests/functional/repl/inherit-multiple-attrs.expected new file mode 100644 index 0000000000..e60df510a3 --- /dev/null +++ b/tests/functional/repl/inherit-multiple-attrs.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> a = { x = 10; y = 20; } + +nix-repl> inherit (a) x y + +nix-repl> x + y +30 diff --git a/tests/functional/repl/inherit-multiple-attrs.in b/tests/functional/repl/inherit-multiple-attrs.in new file mode 100644 index 0000000000..0da0815b2d --- /dev/null +++ b/tests/functional/repl/inherit-multiple-attrs.in @@ -0,0 +1,4 @@ +# COM: inherit multiple attributes +a = { x = 10; y = 20; } +inherit (a) x y +x + y diff --git a/tests/functional/repl/inherit-with-semicolon.expected b/tests/functional/repl/inherit-with-semicolon.expected new file mode 100644 index 0000000000..37d4020104 --- /dev/null +++ b/tests/functional/repl/inherit-with-semicolon.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> a = { z = 99; } + +nix-repl> inherit (a) z; + +nix-repl> z +99 diff --git a/tests/functional/repl/inherit-with-semicolon.in b/tests/functional/repl/inherit-with-semicolon.in new file mode 100644 index 0000000000..d7a4972a2b --- /dev/null +++ b/tests/functional/repl/inherit-with-semicolon.in @@ -0,0 +1,4 @@ +# COM: inherit with semicolon +a = { z = 99; } +inherit (a) z; +z diff --git a/tests/functional/repl/inherit.expected b/tests/functional/repl/inherit.expected new file mode 100644 index 0000000000..9fde67b8cf --- /dev/null +++ b/tests/functional/repl/inherit.expected @@ -0,0 +1,9 @@ +Nix +Type :? for help. + +nix-repl> a = { b = 1; c = 2; } + +nix-repl> inherit (a) b + +nix-repl> b +1 diff --git a/tests/functional/repl/inherit.in b/tests/functional/repl/inherit.in new file mode 100644 index 0000000000..804bbbec17 --- /dev/null +++ b/tests/functional/repl/inherit.in @@ -0,0 +1,4 @@ +# COM: Test inherit statement support (issue #15053) +a = { b = 1; c = 2; } +inherit (a) b +b diff --git a/tests/functional/repl/list-loaded-nothing-loaded.expected b/tests/functional/repl/list-loaded-nothing-loaded.expected new file mode 100644 index 0000000000..1f9f97c984 --- /dev/null +++ b/tests/functional/repl/list-loaded-nothing-loaded.expected @@ -0,0 +1,5 @@ +Nix +Type :? for help. + +nix-repl> :ll +error: nothing has been loaded yet diff --git a/tests/functional/repl/list-loaded-nothing-loaded.in b/tests/functional/repl/list-loaded-nothing-loaded.in new file mode 100644 index 0000000000..f47ea2c888 --- /dev/null +++ b/tests/functional/repl/list-loaded-nothing-loaded.in @@ -0,0 +1 @@ +:ll diff --git a/tests/functional/repl/multiple-bindings-same-line.expected b/tests/functional/repl/multiple-bindings-same-line.expected new file mode 100644 index 0000000000..77062af5fe --- /dev/null +++ b/tests/functional/repl/multiple-bindings-same-line.expected @@ -0,0 +1,7 @@ +Nix +Type :? for help. + +nix-repl> a = 1; b = 2; + +nix-repl> a + b +3 diff --git a/tests/functional/repl/multiple-bindings-same-line.in b/tests/functional/repl/multiple-bindings-same-line.in new file mode 100644 index 0000000000..6dd1d62a0e --- /dev/null +++ b/tests/functional/repl/multiple-bindings-same-line.in @@ -0,0 +1,3 @@ +# COM: multiple bindings on one line +a = 1; b = 2; +a + b diff --git a/tests/functional/repl/nested-attr-path.expected b/tests/functional/repl/nested-attr-path.expected new file mode 100644 index 0000000000..59164fede4 --- /dev/null +++ b/tests/functional/repl/nested-attr-path.expected @@ -0,0 +1,7 @@ +Nix +Type :? for help. + +nix-repl> a.b.c = 1; + +nix-repl> a.b +{ c = 1; } diff --git a/tests/functional/repl/nested-attr-path.in b/tests/functional/repl/nested-attr-path.in new file mode 100644 index 0000000000..4862c3c34b --- /dev/null +++ b/tests/functional/repl/nested-attr-path.in @@ -0,0 +1,3 @@ +# COM: nested attribute path +a.b.c = 1; +a.b diff --git a/tests/functional/repl/pretty-print-idempotent.expected b/tests/functional/repl/pretty-print-idempotent.expected index 311855dae3..83ce764357 100644 --- a/tests/functional/repl/pretty-print-idempotent.expected +++ b/tests/functional/repl/pretty-print-idempotent.expected @@ -2,7 +2,8 @@ Nix Type :? for help. nix-repl> :l pretty-print-idempotent.nix -Added variables. +Added 4 variables. +oneDeep, oneDeepList, twoDeep, twoDeepList nix-repl> oneDeep { homepage = "https://example.com"; } diff --git a/tests/functional/repl/printing.expected b/tests/functional/repl/printing.expected new file mode 100644 index 0000000000..f111939eb5 --- /dev/null +++ b/tests/functional/repl/printing.expected @@ -0,0 +1,59 @@ +Nix +Type :? for help. + +nix-repl> { a = { b = 2; }; l = [ 1 2 3 ]; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +{ + a = { ... }; + l = [ ... ]; + n = 1234; + s = "string"; + x = { ... }; +} + +nix-repl> [ 42 1 "thingy" ({ a = 1; }) ([ 1 2 3 ]) ] +[ + 42 + 1 + "thingy" + { ... } + [ ... ] +] + +nix-repl> let x = { y = { a = 1; }; inherit x; }; in x +{ + x = «repeated»; + y = { ... }; +} + +nix-repl> :p { a = { b = 2; }; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +{ + a = { b = 2; }; + n = 1234; + s = "string"; + x = { + y = { + z = { + y = «repeated»; + }; + }; + }; +} + +nix-repl> :p [ 42 1 "thingy" (rec { a = 1; b = { inherit a; inherit b; }; }) ([ 1 2 3 ]) ] +[ + 42 + 1 + "thingy" + { + a = 1; + b = { + a = 1; + b = «repeated»; + }; + } + [ + 1 + 2 + 3 + ] +] diff --git a/tests/functional/repl/printing.in b/tests/functional/repl/printing.in new file mode 100644 index 0000000000..f3ca2cc88f --- /dev/null +++ b/tests/functional/repl/printing.in @@ -0,0 +1,11 @@ +# COM: Test recursive printing and formatting +# COM: Normal output should print attributes in lexicographical order non-recursively +{ a = { b = 2; }; l = [ 1 2 3 ]; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +# COM: Same for lists, but order is preserved +[ 42 1 "thingy" ({ a = 1; }) ([ 1 2 3 ]) ] +# COM: Same for let expressions +let x = { y = { a = 1; }; inherit x; }; in x +# COM: The :p command should recursively print sets, but prevent infinite recursion +:p { a = { b = 2; }; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +# COM: Same for lists +:p [ 42 1 "thingy" (rec { a = 1; b = { inherit a; inherit b; }; }) ([ 1 2 3 ]) ] diff --git a/tests/functional/repl/reload-with-non-existent-file.expected b/tests/functional/repl/reload-with-non-existent-file.expected new file mode 100644 index 0000000000..e15be6e730 --- /dev/null +++ b/tests/functional/repl/reload-with-non-existent-file.expected @@ -0,0 +1,24 @@ +Nix +Type :? for help. + +nix-repl> :l file-a.nix +Added 1 variables. +fromA + +nix-repl> :l ./does-not-exist.nix +error: path '/path/to/tests/functional/repl/does-not-exist.nix' does not exist + +nix-repl> :l file-b.nix +Added 1 variables. +fromB + +nix-repl> :r +Loading "file-a.nix"... +Added 1 variables. +fromA +Loading "file-b.nix"... +Added 1 variables. +fromB + +nix-repl> fromA + fromB +3 diff --git a/tests/functional/repl/reload-with-non-existent-file.in b/tests/functional/repl/reload-with-non-existent-file.in new file mode 100644 index 0000000000..740b3be9a4 --- /dev/null +++ b/tests/functional/repl/reload-with-non-existent-file.in @@ -0,0 +1,5 @@ +:l file-a.nix +:l ./does-not-exist.nix +:l file-b.nix +:r +fromA + fromB From a953d61686126d5cd39e5430bbf89674e3662261 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 2 May 2026 17:33:36 +0300 Subject: [PATCH 346/927] derivaton-builder: Reap recursive-nix daemon worker threads early Not reaping threads early on leads to resource exhaustion when handling a lot of connection (like in nix-ninja). Ideally this would all just be async coroutine code so that we didn't have to spawn threads to handle connections, but it's a long-term goal. Fixes ENOMEM errors when building nix via nix-ninja. --- src/libstore/unix/build/derivation-builder.cc | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 3e9ad54347..9c0463791e 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -41,6 +41,8 @@ #include #include #include +#include +#include #include "nix/util/strings.hh" #include "nix/util/signals.hh" @@ -226,10 +228,16 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder */ std::thread daemonThread; + struct DaemonWorkerState + { + std::thread thread; + ref done; + }; + /** * The daemon worker threads. */ - std::vector daemonWorkerThreads; + std::list daemonWorkerThreads; const StorePathSet & originalPaths() override { @@ -1201,19 +1209,41 @@ void DerivationBuilderImpl::startDaemon() debug("received daemon connection"); - auto workerThread = std::thread([store, remote{std::move(remote)}]() { + auto doneFlag = make_ref(); + + auto workerThread = std::thread([doneFlag, store, remote{std::move(remote)}]() { try { daemon::processConnection( store, FdSource(remote.get()), FdSink(remote.get()), NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (const Interrupted &) { debug("interrupted daemon connection"); - } catch (SystemError &) { + } catch (...) { + /* Swallow all exceptions to avoid crashing the the process (exceptions that escape from the thread + * trigger std::terminate()). */ ignoreExceptionExceptInterrupt(); } + + doneFlag->test_and_set(std::memory_order_relaxed); }); - daemonWorkerThreads.push_back(std::move(workerThread)); + daemonWorkerThreads.push_back( + DaemonWorkerState{ + .thread = std::move(workerThread), + .done = std::move(doneFlag), + }); + + /* Prune threads eagerly to free up resources. Ideally we'd also limit the number of concurrent workers. */ + for (auto it = daemonWorkerThreads.begin(), end = daemonWorkerThreads.end(); it != end;) { + auto & state = *it; + auto & thread = state.thread; + if (state.done->test(std::memory_order_relaxed) && thread.joinable()) { + thread.join(); + it = daemonWorkerThreads.erase(it); + } else { + ++it; + } + } } debug("daemon shutting down"); @@ -1242,9 +1272,7 @@ void DerivationBuilderImpl::stopDaemon() if (daemonThread.joinable()) daemonThread.join(); - // FIXME: should prune worker threads more quickly. - // FIXME: shutdown the client socket to speed up worker termination. - for (auto & thread : daemonWorkerThreads) + for (auto & [thread, doneFlag] : daemonWorkerThreads) thread.join(); daemonWorkerThreads.clear(); From 2acb40b2aa1c272dff4eb698921554c7ef948d30 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 May 2026 14:51:32 +0200 Subject: [PATCH 347/927] Don't destroy _fileTransfer on shutdown This is responsible for a lot of crash reports in Sentry (presumably due to destructor ordering issues). Since there is no cleanup done by this class that we care about, just leak it. --- src/libstore/filetransfer.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 6d97fb4e3f..a07c92fe0c 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1183,14 +1183,16 @@ ref makeCurlFileTransfer(const FileTransferSettings & settings return make_ref(settings); } +static auto * const _fileTransfer = new Sync>; + ref getFileTransfer() { - static ref fileTransfer = makeCurlFileTransfer(); + auto fileTransfer(_fileTransfer->lock()); - if (fileTransfer->state_.lock()->isQuitting()) - fileTransfer = makeCurlFileTransfer(); + if (!*fileTransfer || (*fileTransfer)->state_.lock()->isQuitting()) + *fileTransfer = makeCurlFileTransfer().get_ptr(); - return fileTransfer; + return ref(*fileTransfer); } ref makeFileTransfer(const FileTransferSettings & settings) From 8f40805cc63d9e55e5e8409bdf53baad91be1850 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Apr 2026 21:29:39 +0100 Subject: [PATCH 348/927] feat(libutil): add `memo(f0)` memoization combinator Memoizes a `fun`. Useful because "call by need" has its use cases outside the evaluator too. Motivating use case: libfetchers lazy attributes. --- src/libutil-tests/memo.cc | 46 ++++++++++++++++++++++++ src/libutil-tests/meson.build | 1 + src/libutil/include/nix/util/memo.hh | 40 +++++++++++++++++++++ src/libutil/include/nix/util/meson.build | 1 + 4 files changed, 88 insertions(+) create mode 100644 src/libutil-tests/memo.cc create mode 100644 src/libutil/include/nix/util/memo.hh diff --git a/src/libutil-tests/memo.cc b/src/libutil-tests/memo.cc new file mode 100644 index 0000000000..76be652609 --- /dev/null +++ b/src/libutil-tests/memo.cc @@ -0,0 +1,46 @@ +#include +#include + +#include "nix/util/memo.hh" + +namespace nix { + +TEST(memo, computesOnce) +{ + int calls = 0; + fun f = memo([&calls]() -> int { + calls++; + return 42; + }); + EXPECT_EQ(f(), 42); + EXPECT_EQ(f(), 42); + EXPECT_EQ(f(), 42); + EXPECT_EQ(calls, 1); +} + +TEST(memo, copiesShareCache) +{ + int calls = 0; + fun f = memo([&calls]() -> int { + calls++; + return 7; + }); + auto g = f; + EXPECT_EQ(f(), 7); + EXPECT_EQ(g(), 7); + EXPECT_EQ(calls, 1); +} + +TEST(memo, worksWithString) +{ + int calls = 0; + fun f = memo([&calls]() -> std::string { + calls++; + return "hello"; + }); + EXPECT_EQ(f(), "hello"); + EXPECT_EQ(f(), "hello"); + EXPECT_EQ(calls, 1); +} + +} // namespace nix diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 6a86504ded..527d5e3fbf 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -79,6 +79,7 @@ sources = files( 'json-utils.cc', 'logging.cc', 'lru-cache.cc', + 'memo.cc', 'memory-source-accessor.cc', 'monitorfdhup.cc', 'nar-listing.cc', diff --git a/src/libutil/include/nix/util/memo.hh b/src/libutil/include/nix/util/memo.hh new file mode 100644 index 0000000000..ba09bac6eb --- /dev/null +++ b/src/libutil/include/nix/util/memo.hh @@ -0,0 +1,40 @@ +#pragma once +///@file + +#include "nix/util/fun.hh" + +#include +#include +#include + +namespace nix { + +/** + * Memoize a `fun`. + * + * Copies of the returned `fun` share the same cache. + * Thread-safe. + */ +template +fun memo(fun f) +{ + struct State + { + fun compute; + std::once_flag flag; + std::optional cached; + + explicit State(fun compute) + : compute(std::move(compute)) + { + } + }; + + auto state = std::shared_ptr(new State(std::move(f))); + return [state]() -> T { + std::call_once(state->flag, [&]() { state->cached.emplace(state->compute()); }); + return *state->cached; + }; +} + +} // namespace nix diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 0f7a40df7a..cea6d48f59 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -60,6 +60,7 @@ headers = [ config_pub_h ] + files( 'json-utils.hh', 'logging.hh', 'lru-cache.hh', + 'memo.hh', 'memory-source-accessor.hh', 'mounted-source-accessor.hh', 'muxable-pipe.hh', From ee78fe13ecc2e7d49d4763661760960b4c949362 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 21 Apr 2026 12:48:48 +0100 Subject: [PATCH 349/927] feat(libfetchers): add lazy attribute values Extend the Attr variant with a LazyAttr alternative: a deferred computation that is only evaluated when the attribute value is needed. This lets fetchers defer expensive work (like revCount) until an expression demands it. The existing getters and serialization transparently force any lazy attrs. --- src/libfetchers-tests/attrs.cc | 75 +++++++++++++++++++ src/libfetchers-tests/meson.build | 1 + src/libfetchers/attrs.cc | 35 ++++++--- src/libfetchers/include/nix/fetchers/attrs.hh | 26 ++++++- src/libflake/flake-primops.cc | 3 +- 5 files changed, 129 insertions(+), 11 deletions(-) create mode 100644 src/libfetchers-tests/attrs.cc diff --git a/src/libfetchers-tests/attrs.cc b/src/libfetchers-tests/attrs.cc new file mode 100644 index 0000000000..4d0cbbb9ef --- /dev/null +++ b/src/libfetchers-tests/attrs.cc @@ -0,0 +1,75 @@ +#include + +#include "nix/fetchers/attrs.hh" + +#include + +namespace nix::fetchers { + +TEST(LazyAttr, resolveToInt) +{ + Attrs attrs; + attrs.insert_or_assign( + "count", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return uint64_t(42); + }}))); + EXPECT_EQ(maybeGetIntAttr(attrs, "count"), 42); +} + +TEST(LazyAttr, resolveToString) +{ + Attrs attrs; + attrs.insert_or_assign( + "name", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return std::string("hello"); + }}))); + EXPECT_EQ(maybeGetStrAttr(attrs, "name"), "hello"); +} + +TEST(LazyAttr, resolveToBool) +{ + Attrs attrs; + attrs.insert_or_assign( + "flag", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return Explicit{true}; + }}))); + EXPECT_EQ(maybeGetBoolAttr(attrs, "flag"), true); +} + +TEST(LazyAttr, attrsToJSONForcesLazy) +{ + Attrs attrs; + attrs.insert_or_assign( + "x", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return uint64_t(99); + }}))); + auto json = attrsToJSON(attrs); + EXPECT_EQ(json["x"], 99); +} + +TEST(LazyAttr, attrsToQueryForcesLazy) +{ + Attrs attrs; + attrs.insert_or_assign( + "v", LazyAttr(make_ref(LazyAttrComputation{.compute = []() -> ResolvedAttr { + return std::string("val"); + }}))); + auto query = attrsToQuery(attrs); + EXPECT_EQ(query.at("v"), "val"); +} + +TEST(LazyAttr, notCalledUntilForced) +{ + int calls = 0; + Attrs attrs; + attrs.insert_or_assign( + "lazy", LazyAttr(make_ref(LazyAttrComputation{.compute = [&calls]() -> ResolvedAttr { + calls++; + return uint64_t(1); + }}))); + EXPECT_EQ(calls, 0); + maybeGetIntAttr(attrs, "lazy"); + EXPECT_EQ(calls, 1); +} + +} // namespace nix::fetchers diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index ba9774e956..8cbb42e213 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -40,6 +40,7 @@ subdir('nix-meson-build-support/common') sources = files( 'access-tokens.cc', + 'attrs.cc', 'git-utils.cc', 'git.cc', 'input.cc', diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index cc9e72af46..39e7a0fd5d 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -4,6 +4,18 @@ namespace nix::fetchers { +ResolvedAttr forceAttr(const Attr & attr) +{ + return std::visit( + overloaded{ + [](const LazyAttr & lazy) -> ResolvedAttr { return lazy->compute(); }, + [](const std::string & v) -> ResolvedAttr { return v; }, + [](uint64_t v) -> ResolvedAttr { return v; }, + [](const Explicit & v) -> ResolvedAttr { return v; }, + }, + attr); +} + Attrs jsonToAttrs(const nlohmann::json & json) { Attrs attrs; @@ -26,11 +38,12 @@ nlohmann::json attrsToJSON(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + auto resolved = forceAttr(attr.second); + if (auto v = std::get_if(&resolved)) { json[attr.first] = *v; - } else if (auto v = std::get_if(&attr.second)) { + } else if (auto v = std::get_if(&resolved)) { json[attr.first] = *v; - } else if (auto v = std::get_if>(&attr.second)) { + } else if (auto v = std::get_if>(&resolved)) { json[attr.first] = v->t; } else unreachable(); @@ -43,7 +56,8 @@ std::optional maybeGetStrAttr(const Attrs & attrs, const std::strin auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if(&resolved)) return *v; throw Error("input attribute '%s' is not a string %s", name, attrsToJSON(attrs).dump()); } @@ -61,7 +75,8 @@ std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if(&resolved)) return *v; throw Error("input attribute '%s' is not an integer", name); } @@ -79,7 +94,8 @@ std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & na auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if>(&i->second)) + auto resolved = forceAttr(i->second); + if (auto v = std::get_if>(&resolved)) return v->t; throw Error("input attribute '%s' is not a Boolean", name); } @@ -96,11 +112,12 @@ StringMap attrsToQuery(const Attrs & attrs) { StringMap query; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + auto resolved = forceAttr(attr.second); + if (auto v = std::get_if(&resolved)) { query.insert_or_assign(attr.first, fmt("%d", *v)); - } else if (auto v = std::get_if(&attr.second)) { + } else if (auto v = std::get_if(&resolved)) { query.insert_or_assign(attr.first, *v); - } else if (auto v = std::get_if>(&attr.second)) { + } else if (auto v = std::get_if>(&resolved)) { query.insert_or_assign(attr.first, v->t ? "1" : "0"); } else unreachable(); diff --git a/src/libfetchers/include/nix/fetchers/attrs.hh b/src/libfetchers/include/nix/fetchers/attrs.hh index 8a21b8ddbf..60730d65d3 100644 --- a/src/libfetchers/include/nix/fetchers/attrs.hh +++ b/src/libfetchers/include/nix/fetchers/attrs.hh @@ -3,6 +3,8 @@ #include "nix/util/types.hh" #include "nix/util/hash.hh" +#include "nix/util/ref.hh" +#include "nix/util/fun.hh" #include @@ -12,7 +14,24 @@ namespace nix::fetchers { -typedef std::variant> Attr; +/** + * The resolved (non-lazy) subset of attribute value types. + */ +using ResolvedAttr = std::variant>; + +/** + * A deferred attribute computation. Wrapping in `ref<>` gives + * pointer-identity equality/ordering, which is correct: two lazy + * attrs are equal iff they are the same computation. + */ +struct LazyAttrComputation +{ + fun compute; +}; + +using LazyAttr = ref; + +using Attr = std::variant, LazyAttr>; /** * An `Attrs` can be thought of a JSON object restricted or simplified @@ -21,6 +40,11 @@ typedef std::variant> Attr; */ typedef std::map Attrs; +/** + * Force a potentially lazy attribute to its resolved value. + */ +ResolvedAttr forceAttr(const Attr & attr); + Attrs jsonToAttrs(const nlohmann::json & json); nlohmann::json attrsToJSON(const Attrs & attrs); diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 66962a0148..5ee587c3e0 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -105,12 +105,13 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg for (const auto & [key, value] : attrs) { auto s = state.symbols.create(key); auto & vv = binds.alloc(s); + auto resolved = forceAttr(value); std::visit( overloaded{ [&vv, &state](const std::string & value) { vv.mkString(value, state.mem); }, [&vv](const uint64_t & value) { vv.mkInt(value); }, [&vv](const Explicit & value) { vv.mkBool(value.t); }}, - value); + resolved); } v.mkAttrs(binds); } From a08c14bf8b8dd91034a7fe7c1014b05aee12d772 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 21 Apr 2026 13:56:25 +0100 Subject: [PATCH 350/927] feat(libexpr): emit thunks for lazy fetcher attributes The "revCount" attribute is now emitted as a lazy attribute if libfetchers were to return it as such; as will be done in next commit. --- src/libexpr-tests/lazy-fetcher-attr.cc | 100 ++++++++++++++++++ src/libexpr-tests/meson.build | 1 + src/libexpr/include/nix/expr/fetch-tree.hh | 18 ++++ src/libexpr/include/nix/expr/meson.build | 1 + src/libexpr/primops/fetchTree.cc | 95 ++++++++++++++++- src/libfetchers/attrs.cc | 10 ++ src/libfetchers/include/nix/fetchers/attrs.hh | 5 + src/libflake/flake.cc | 1 + src/libflake/include/nix/flake/flake.hh | 8 -- 9 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 src/libexpr-tests/lazy-fetcher-attr.cc create mode 100644 src/libexpr/include/nix/expr/fetch-tree.hh diff --git a/src/libexpr-tests/lazy-fetcher-attr.cc b/src/libexpr-tests/lazy-fetcher-attr.cc new file mode 100644 index 0000000000..4c36424ecb --- /dev/null +++ b/src/libexpr-tests/lazy-fetcher-attr.cc @@ -0,0 +1,100 @@ +#include + +#include "nix/expr/fetch-tree.hh" +#include "nix/expr/tests/libexpr.hh" +#include "nix/fetchers/attrs.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/store/path.hh" + +namespace nix { + +class LazyFetcherAttrTest : public LibExprTest +{ +protected: + StorePath dummyPath() + { + return StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-test"}; + } +}; + +TEST_F(LazyFetcherAttrTest, nonLazyAttrProducesImmediateValue) +{ + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign("revCount", uint64_t(5)); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 5); +} + +TEST_F(LazyFetcherAttrTest, lazyAttrProducesThunk) +{ + int calls = 0; + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign( + "revCount", + fetchers::LazyAttr( + make_ref( + fetchers::LazyAttrComputation{.compute = [&calls]() -> fetchers::ResolvedAttr { + calls++; + return uint64_t(42); + }}))); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + + // Not yet forced, so the lazy function should not have been called + EXPECT_EQ(calls, 0); + + // Force the thunk + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 42); + EXPECT_EQ(calls, 1); +} + +TEST_F(LazyFetcherAttrTest, lazyFunctionOnlyCalledOnAccess) +{ + int calls = 0; + fetchers::Input input; + input.attrs.insert_or_assign("type", std::string("git")); + input.attrs.insert_or_assign("lastModified", uint64_t(1000)); + input.attrs.insert_or_assign( + "revCount", + fetchers::LazyAttr( + make_ref( + fetchers::LazyAttrComputation{.compute = [&calls]() -> fetchers::ResolvedAttr { + calls++; + return uint64_t(99); + }}))); + + Value v; + emitTreeAttrs(state, dummyPath(), input, v, false, false); + state.forceValue(v, noPos); + + // Access lastModified, so should not trigger lazy revCount + auto * lmAttr = v.attrs()->get(state.symbols.create("lastModified")); + ASSERT_NE(lmAttr, nullptr); + state.forceValue(*lmAttr->value, noPos); + EXPECT_EQ(lmAttr->value->integer().value, 1000); + EXPECT_EQ(calls, 0); + + // Now access revCount + auto * rcAttr = v.attrs()->get(state.symbols.create("revCount")); + ASSERT_NE(rcAttr, nullptr); + state.forceValue(*rcAttr->value, noPos); + EXPECT_EQ(rcAttr->value->integer().value, 99); + EXPECT_EQ(calls, 1); +} + +} // namespace nix diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index 0b0a01c206..d18d5a4830 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -51,6 +51,7 @@ sources = files( 'error_traces.cc', 'eval.cc', 'json.cc', + 'lazy-fetcher-attr.cc', 'main.cc', 'nix_api_expr.cc', 'nix_api_external.cc', diff --git a/src/libexpr/include/nix/expr/fetch-tree.hh b/src/libexpr/include/nix/expr/fetch-tree.hh new file mode 100644 index 0000000000..3eb8a01c0c --- /dev/null +++ b/src/libexpr/include/nix/expr/fetch-tree.hh @@ -0,0 +1,18 @@ +#pragma once + +#include "nix/expr/eval.hh" + +namespace nix { + +/** + * Convert a libfetchers `Input` to libexpr `Value`. + */ +void emitTreeAttrs( + EvalState & state, + const StorePath & storePath, + const fetchers::Input & input, + Value & v, + bool emptyRevFallback = false, + bool forceDirty = false); + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/meson.build b/src/libexpr/include/nix/expr/meson.build index 4213476fe7..7334b42f8f 100644 --- a/src/libexpr/include/nix/expr/meson.build +++ b/src/libexpr/include/nix/expr/meson.build @@ -20,6 +20,7 @@ headers = [ config_pub_h ] + files( 'eval-profiler.hh', 'eval-settings.hh', 'eval.hh', + 'fetch-tree.hh', 'function-trace.hh', 'gc-small-vector.hh', 'get-drvs.hh', diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index afd61e90fb..429009301d 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,7 +1,9 @@ +#include "nix/expr/value.hh" #include "nix/fetchers/attrs.hh" #include "nix/expr/primops.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" +#include "nix/expr/fetch-tree.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/fetchers.hh" #include "nix/store/filetransfer.hh" @@ -19,6 +21,95 @@ namespace nix { +/** + * Adapter for putting libfetchers data into a thunk closure. + * Used as the argument to prim_forceLazyFetcherAttr in a lazy apply thunk. + */ +class LazyFetcherAttr : public ExternalValueBase, public gc_cleanup +{ + fetchers::LazyAttr lazy; + +public: + LazyFetcherAttr(fetchers::LazyAttr lazy) + : lazy(std::move(lazy)) + { + } + + fetchers::ResolvedAttr force() + { + return lazy->compute(); + } + +protected: + std::ostream & print(std::ostream & str) const override + { + unreachable(); + } + +public: + std::string showType() const override + { + unreachable(); + } + + std::string typeOf() const override + { + unreachable(); + } +}; + +/** + * Initialize a `Value` from a resolved fetcher attribute. + */ +static void resolvedAttrToValue(EvalState & state, Value & v, const fetchers::ResolvedAttr & resolved) +{ + std::visit( + overloaded{ + [&](const std::string & s) { v.mkString(s, state.mem); }, + [&](uint64_t n) { v.mkInt(n); }, + [&](const Explicit & b) { v.mkBool(b.t); }, + }, + resolved); +} + +/** + * internal primop: Force a LazyFetcherAttr external value. + */ +static void prim_forceLazyFetcherAttr(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + Value & arg = *args[0]; + + state.forceValue(arg, pos); + // We only construct this primop with LazyFetcherAttr preapplied. + assert(arg.type() == nExternal); + auto * ext = dynamic_cast(args[0]->external()); + assert(ext); + + resolvedAttrToValue(state, v, ext->force()); +} + +/** + * Emit a lazy thunk for a LazyAttr: mkApp(primop, externalValue). + */ +static void emitLazyAttrThunk(EvalState & state, const fetchers::LazyAttr & lazyAttr, Value & dest) +{ + // not user-callable (unregistered, internal) + static PrimOp forcePrimOp{ + .name = "__forceLazyFetcherAttr", + .arity = 1, + .impl = prim_forceLazyFetcherAttr, + .internal = true, + }; + + auto * vExt = state.allocValue(); + vExt->mkExternal(new LazyFetcherAttr(lazyAttr)); + + auto * vPrimOp = state.allocValue(); + vPrimOp->mkPrimOp(&forcePrimOp); + + dest.mkApp(vPrimOp, vExt); +} + void emitTreeAttrs( EvalState & state, const StorePath & storePath, @@ -51,7 +142,9 @@ void emitTreeAttrs( attrs.alloc("shortRev").mkString(emptyHash.gitShortRev(), state.mem); } - if (auto revCount = input.getRevCount()) + if (auto revCount = maybeGetLazyAttr(input.attrs, "revCount")) + emitLazyAttrThunk(state, *revCount, attrs.alloc("revCount")); + else if (auto revCount = input.getRevCount()) attrs.alloc("revCount").mkInt(*revCount); else if (emptyRevFallback) attrs.alloc("revCount").mkInt(0); diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 39e7a0fd5d..f3ecc3a87b 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -51,6 +51,16 @@ nlohmann::json attrsToJSON(const Attrs & attrs) return json; } +std::optional maybeGetLazyAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) + return {}; + if (auto v = std::get_if(&i->second)) + return *v; + return {}; +} + std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) { auto i = attrs.find(name); diff --git a/src/libfetchers/include/nix/fetchers/attrs.hh b/src/libfetchers/include/nix/fetchers/attrs.hh index 60730d65d3..8eede58086 100644 --- a/src/libfetchers/include/nix/fetchers/attrs.hh +++ b/src/libfetchers/include/nix/fetchers/attrs.hh @@ -45,6 +45,11 @@ typedef std::map Attrs; */ ResolvedAttr forceAttr(const Attr & attr); +/** + * Retrieve an attr, but only if it's a LazyAttr. + */ +std::optional maybeGetLazyAttr(const Attrs & attrs, const std::string & name); + Attrs jsonToAttrs(const nlohmann::json & json); nlohmann::json attrsToJSON(const Attrs & attrs); diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index e8dbf42f54..03e68e1f99 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -38,6 +38,7 @@ #include "nix/fetchers/input-cache.hh" #include "nix/expr/attr-set.hh" #include "nix/expr/eval-error.hh" +#include "nix/expr/fetch-tree.hh" #include "nix/expr/nixexpr.hh" #include "nix/expr/symbol-table.hh" #include "nix/expr/value.hh" diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index fd52dbebac..aa063a08e3 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -233,14 +233,6 @@ ref openEvalCache(EvalState & state, ref Date: Tue, 21 Apr 2026 14:08:14 +0100 Subject: [PATCH 351/927] feat(git): make revCount lazy revCount requires an expensive walk of the entire commit graph. Now it's wrapped in a LazyAttr so the cost is only paid when the Nix expression actually accesses `.revCount`. This also fixes fetching from shallow clones with fetchGit. Previously the eager revCount computation failed the whole fetch, but now `.outPath` and some other attributes succeed, while `.revCount` access fails (as expected for shallow repos). --- src/libfetchers/git.cc | 46 +++++++++++++++++++---------- tests/functional/fetchGitShallow.sh | 8 +++-- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 447b4a3694..6baf7d2e52 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -13,6 +13,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/util/json-utils.hh" #include "nix/util/archive.hh" +#include "nix/util/memo.hh" #include "nix/util/mounted-source-accessor.hh" #include @@ -163,6 +164,13 @@ std::vector getPublicKeys(const Attrs & attrs) static const Hash nullRev{HashAlgorithm::SHA1}; +static LazyAttr makeLazyAttr(fun compute) +{ + return make_ref(LazyAttrComputation{ + .compute = memo(std::move(compute)), + }); +} + struct GitInputScheme : InputScheme { std::optional inputFromURL(const Settings & settings, const ParsedURL & url, bool requireTree) const override @@ -723,14 +731,12 @@ struct GitInputScheme : InputScheme } uint64_t getRevCount( - const Settings & settings, - const RepoInfo & repoInfo, - const std::filesystem::path & repoDir, - const Hash & rev) const + ref cache, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { - Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; + if (GitRepo::openRepo(repoDir, {})->isShallow()) + throw Error("'%s' is a shallow Git repository, so 'revCount' is not available", repoInfo.locationToArg()); - auto cache = settings.getCache(); + Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; if (auto revCountAttrs = cache->lookup(key)) return getIntAttr(*revCountAttrs, "revCount"); @@ -745,6 +751,18 @@ struct GitInputScheme : InputScheme return revCount; } + LazyAttr lazyRevCount( + const Settings & settings, + const RepoInfo & repoInfo, + const std::filesystem::path & repoDir, + const Hash & rev) const + { + auto cache = settings.getCache(); + return makeLazyAttr([this, cache, repoInfo, repoDir, rev]() -> ResolvedAttr { + return getRevCount(cache, repoInfo, repoDir, rev); + }); + } + std::string getDefaultRef(const Settings & settings, const RepoInfo & repoInfo, bool shallow) const { auto head = std::visit( @@ -891,13 +909,6 @@ struct GitInputScheme : InputScheme auto repo = GitRepo::openRepo(repoDir, {}); - auto isShallow = repo->isShallow(); - - if (isShallow && !getShallowAttr(input)) - throw Error( - "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", - repoInfo.locationToArg()); - // FIXME: check whether rev is an ancestor of ref? auto rev = *input.getRev(); @@ -911,7 +922,7 @@ struct GitInputScheme : InputScheme if (!getShallowAttr(input)) { /* Like lastModified, skip revCount if supplied by the caller. */ if (!input.attrs.contains("revCount")) - input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); + input.attrs.insert_or_assign("revCount", lazyRevCount(settings, repoInfo, repoDir, rev)); } printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); @@ -1033,8 +1044,11 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", rev.gitRev()); if (!getShallowAttr(input)) { - input.attrs.insert_or_assign( - "revCount", rev == nullRev ? 0 : getRevCount(settings, repoInfo, repoPath, rev)); + if (rev == nullRev) { + input.attrs.insert_or_assign("revCount", uint64_t(0)); + } else { + input.attrs.insert_or_assign("revCount", lazyRevCount(settings, repoInfo, repoPath, rev)); + } } verifyCommit(input, repo); diff --git a/tests/functional/fetchGitShallow.sh b/tests/functional/fetchGitShallow.sh index 6b91d60cd9..67a8e3d3bd 100644 --- a/tests/functional/fetchGitShallow.sh +++ b/tests/functional/fetchGitShallow.sh @@ -29,9 +29,11 @@ git -C "$TEST_ROOT/shallow-parent" commit -m "Branch commit" # Make a shallow clone (depth=1) git clone --depth 1 "file://$TEST_ROOT/shallow-parent" "$TEST_ROOT/shallow-clone" -# Test 1: Fetching a shallow repo shouldn't work by default, because we can't -# return a revCount. -(! nix eval --impure --raw --expr "(builtins.fetchGit { url = \"$TEST_ROOT/shallow-clone\"; ref = \"dev\"; }).outPath") +# Test 1: Fetching a shallow repo succeeds for outPath because revCount is lazy. +path1=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"$TEST_ROOT/shallow-clone\"; ref = \"dev\"; }).outPath") +[[ -d "$path1" ]] +# But accessing revCount on a shallow clone fails. +(! nix eval --impure --expr "(builtins.fetchGit { url = \"$TEST_ROOT/shallow-clone\"; ref = \"dev\"; }).revCount" 2>/dev/null) # Test 2: But you can request a shallow clone, which won't return a revCount. path=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow-clone\"; ref = \"dev\"; shallow = true; }).outPath") From 0c7b61dadd1b575bb8abc36e66a53e339988d200 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 30 Apr 2026 02:07:29 +0200 Subject: [PATCH 352/927] fix(flake): don't force revCount in fingerprint The flake eval cache fingerprint eagerly forced revCount via getRevCount(), which defeats lazy revCount on shallow clones. Since revCount is functionally determined by rev (already part of the fingerprint), we only need to include its *presence* in the fingerprint, not its value. Note that before these changes, Nix had a bug where it could produce a wrong revcount on shallow clones. It arguably should have inferred `shallow = true;` but it didn't and I'm not changing that behavior either. Leaving it at its default is more "pure" in a sense, but a case could certainly be made to let the CLI infer the value that works automagically. (Purity in the CLI is a spectrum; system attribute selection could be argued to be impure too, for instance.) --- src/libflake/flake.cc | 10 +++++++--- tests/functional/fetchGitShallow.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 03e68e1f99..c34fa2d0a1 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -991,9 +991,13 @@ std::optional LockedFlake::getFingerprint(Store & store, const fetc /* Include revCount and lastModified because they're not necessarily implied by the content fingerprint (e.g. for - tarball flakes) but can influence the evaluation result. */ - if (auto revCount = flake.lockedRef.input.getRevCount()) - *fingerprint += fmt(";revCount=%d", *revCount); + tarball flakes) but can influence the evaluation result. + For revCount, we only include its presence (not its value) + because the value is functionally determined by rev, which + is already part of the fingerprint. This avoids forcing a + lazy revCount computation. */ + if (flake.lockedRef.input.attrs.contains("revCount")) + *fingerprint += ";hasRevCount"; if (auto lastModified = flake.lockedRef.input.getLastModified()) *fingerprint += fmt(";lastModified=%d", *lastModified); diff --git a/tests/functional/fetchGitShallow.sh b/tests/functional/fetchGitShallow.sh index 67a8e3d3bd..0da36ee9ba 100644 --- a/tests/functional/fetchGitShallow.sh +++ b/tests/functional/fetchGitShallow.sh @@ -65,3 +65,32 @@ fi # Verify that we can shallow fetch the worktree git -C "$TEST_ROOT/shallow-worktree" rev-list --count HEAD >/dev/null nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$TEST_ROOT/shallow-worktree\"; shallow = true; }).rev" + +# Test 5: nix build --dry-run on a shallow clone must not force revCount. +# The flake fingerprint must not eagerly evaluate revCount, because that +# would fail (or produce wrong results) on shallow clones. +# Nor should it generate a complete lock file that serializes the root node. +# We remove the parent repo to ensure that a future improvement that +# tries to fetch missing history can't paper over the issue. +createGitRepo "$TEST_ROOT/shallow-build-parent" +echo "" > "$TEST_ROOT/shallow-build-parent/file.txt" +git -C "$TEST_ROOT/shallow-build-parent" add file.txt +git -C "$TEST_ROOT/shallow-build-parent" commit -m "first" +cat > "$TEST_ROOT/shallow-build-parent/flake.nix" < \\\$out" ]; + }; + }; +} +EOF +git -C "$TEST_ROOT/shallow-build-parent" add flake.nix +git -C "$TEST_ROOT/shallow-build-parent" commit -m "add flake" +git clone --depth 1 "file://$TEST_ROOT/shallow-build-parent" "$TEST_ROOT/shallow-build-clone" +rm -rf "$TEST_ROOT/shallow-build-parent" +nix build --dry-run "git+file://$TEST_ROOT/shallow-build-clone" From ad649e34a3e630c3a91bab5217e422439edcee08 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 5 May 2026 00:29:59 +0200 Subject: [PATCH 353/927] chore: remove redundant comment --- src/libutil/include/nix/util/fun.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libutil/include/nix/util/fun.hh b/src/libutil/include/nix/util/fun.hh index c480ffe717..59a7cc8793 100644 --- a/src/libutil/include/nix/util/fun.hh +++ b/src/libutil/include/nix/util/fun.hh @@ -1,6 +1,5 @@ #pragma once ///@file -// Tests in: src/libutil-tests/fun.cc #include #include From 7609abe60a6359dc349cf8ab645c910a5eae0293 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 13:11:19 +0200 Subject: [PATCH 354/927] Document how to use GC roots safely --- src/libstore/include/nix/store/store-api.hh | 41 +++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index bfd4ffce21..b6114cce65 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -816,6 +816,47 @@ public: /** * Add a store path as a temporary root of the garbage collector. * The root disappears as soon as we exit. + * Before exiting, if you want to avoid the path being GC'ed, you either have to make it a permanent root using + * `LocalFSStore::addPermRoot()`, or make sure it's reachable from a permanent root (e.g. by adding it as a + * reference of a reachable path). + * + * To avoid races, you should call either this function or `LocalFSStore::addPermRoot()` *before* creating and using + * a store path, e.g. + * + * ```c++ + * auto path = store.computeStorePath(...); + * store->addTempRoot(path); + * if (!store->isValidPath(path)) + * store->addToStore(...); + * ``` + * + * By contrast, registering a root just before *using* a path is not sufficient to prevent GC races. For + * instance, don't do this: + * + * ```c++ + * store->addTempRoot(path); + * auto drv = store->readDerivation(path); + * ``` + * + * since the path may be GC'ed just before the call to `addTempRoot()`. + * + * Note that `addToStore()` implicitly calls `addTempRoot()`, so you don't need to call it yourself if you're + * calling `addToStore()` unconditionally. + * + * It is generally the responsibility of the caller of Nix APIs and CLI tools to ensure that paths are reachable by + * the garbage collector. For example, `buildPath(drvPath)` does not need to register *drvPath* as a GC root, since + * that's the responsibility of the caller, and it would be too late for `buildPath()` to do so anyway. Thus, this + * can race: + * ```console + * drv=$(nix-instantiate foo.nix) + * nix-store -r $drv + * ``` + * whereas this is safe: + * ```console + * nix-instantiate foo.nix --add-root ./drv + * nix-store -r ./drv + * ``` + * */ virtual void addTempRoot(const StorePath & path) { From b87d9e810b759a703c7d72c0cbe04414b2edcf22 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 13:11:38 +0200 Subject: [PATCH 355/927] Add TODO item --- src/libfetchers/fetch-to-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 3af2d4c83e..09112a9655 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -82,6 +82,7 @@ std::pair fetchToStore2( auto [storePath, hash] = mode == FetchMode::DryRun ? [&]() { + // FIXME: we may have already computed this above. auto [storePath, hash] = store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2); debug( From e2490c72941e6eb34af2455e3bc459644be55755 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 19:42:50 +0200 Subject: [PATCH 356/927] StoreDirConfig::parseStorePath(): Don't crash on empty paths --- src/libstore/store-dir-config.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 61f82029d7..962a3830f1 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -8,6 +8,8 @@ namespace nix { StorePath StoreDirConfig::parseStorePath(std::string_view path) const { + if (path.empty()) + throw BadStorePath("empty path is not a valid store path"); // On Windows, `/nix/store` is not a canonical path. More broadly it // is unclear whether this function should be using the native // notion of a canonical path at all. For example, it makes to From 8f3d7023cb4c168fa45dedac120efdb46794ed10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 19:43:01 +0200 Subject: [PATCH 357/927] canonPath(): Don't crash on empty paths --- src/libutil-tests/file-system.cc | 2 +- src/libutil/file-system.cc | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index f290e11780..417622a47c 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -114,7 +114,7 @@ TEST(canonPath, requiresAbsolutePath) ASSERT_ANY_THROW(canonPath("."sv)); ASSERT_ANY_THROW(canonPath(".."sv)); ASSERT_ANY_THROW(canonPath("../"sv)); - ASSERT_DEATH({ canonPath(""sv); }, "!path.empty\\(\\)"); + ASSERT_ANY_THROW(canonPath(""sv)); } /* ---------------------------------------------------------------------------- diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index b3087700d7..b6ad1e1280 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -95,7 +95,8 @@ absPath(const std::filesystem::path & path0, const std::filesystem::path * dir, std::filesystem::path canonPath(const std::filesystem::path & path, bool resolveSymlinks) { - assert(!path.empty()); + if (path.empty()) + throw Error("cannot canonicalise an empty path"); if (!path.is_absolute()) throw Error("not an absolute path: %s", PathFmt(path)); From 1a4d5782ffd33f6069295b3b1488bb402e34a9c2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 6 May 2026 03:02:45 +0300 Subject: [PATCH 358/927] Clean up and deduplicate some Logger code Puts some logger classes in anonymous namespaces. This also appeases Wweak-vtables and avoids making those symbols visible in the DSO. Also marks those classes as final and deduplicates storePathToNameWithoutDrvSuffix in the progress bar code. --- src/libmain/progress-bar.cc | 35 ++++++++++++++++++++++------------- src/libutil/tee-logger.cc | 7 ++++++- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index a8044a240d..f81e90a06e 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -2,8 +2,10 @@ #include "nix/util/terminal.hh" #include "nix/util/sync.hh" #include "nix/util/signals.hh" -#include "nix/store/store-api.hh" +#include "nix/store/path.hh" +#include "nix/util/file-system.hh" #include "nix/store/names.hh" +#include "nix/util/util.hh" #include #include @@ -13,17 +15,19 @@ namespace nix { +namespace { + static std::string_view getS(const std::vector & fields, size_t n) { - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tString); + if (n >= fields.size() || fields[n].type != Logger::Field::tString) + throw Error("could not get expected log field of type 'string' at index %d", n); return fields[n].s; } static uint64_t getI(const std::vector & fields, size_t n) { - assert(n < fields.size()); - assert(fields[n].type == Logger::Field::tInt); + if (n >= fields.size() || fields[n].type != Logger::Field::tInt) + throw Error("could not get expected log field of type 'int' at index %d", n); return fields[n].i; } @@ -34,10 +38,17 @@ static std::string_view storePathToName(std::string_view path) return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1); } -class ProgressBar : public Logger +static std::string_view storePathToNameWithoutDrvSuffix(std::string_view path) { -private: + auto res = storePathToName(path); + if (hasSuffix(res, drvExtension)) + res.remove_suffix(drvExtension.size()); + return res; +} +class ProgressBar final : public Logger +{ +private: struct ActInfo { std::string s, lastLine, phase; @@ -223,9 +234,7 @@ class ProgressBar : public Logger state->activitiesByType[type].its.emplace(act, i); if (type == actBuild) { - std::string name(storePathToName(getS(fields, 0))); - if (hasSuffix(name, ".drv")) - name = name.substr(0, name.size() - 4); + auto name = storePathToNameWithoutDrvSuffix(getS(fields, 0)); i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); auto machineName = getS(fields, 1); if (machineName != "") @@ -250,9 +259,7 @@ class ProgressBar : public Logger } if (type == actPostBuildHook) { - auto name = storePathToName(getS(fields, 0)); - if (hasSuffix(name, ".drv")) - name = name.substr(0, name.size() - 4); + auto name = storePathToNameWithoutDrvSuffix(getS(fields, 0)); i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); i->name = DrvName(name).name; } @@ -687,6 +694,8 @@ class ProgressBar : public Logger } }; +} // namespace + std::unique_ptr makeProgressBar() { return std::make_unique(isTTY()); diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index 8433168a5a..8e27714e3f 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -2,10 +2,13 @@ namespace nix { -struct TeeLogger : Logger +namespace { + +class TeeLogger final : public Logger { std::vector> loggers; +public: TeeLogger(std::vector> && loggers) : loggers(std::move(loggers)) { @@ -94,6 +97,8 @@ struct TeeLogger : Logger } }; +} // namespace + std::unique_ptr makeTeeLogger(std::unique_ptr mainLogger, std::vector> && extraLoggers) { From a253942be956f946744c3da3f4e694934b56cc51 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 6 May 2026 03:07:07 +0300 Subject: [PATCH 359/927] libmain: Hide/unhide cursor in the progress bar Uses corresponding ANSI/VT escape sequences to hide/unhide the cursor when the output is a TTY. nom does the same AFAICT. --- src/libmain/progress-bar.cc | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index f81e90a06e..512ba4faad 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -107,6 +107,18 @@ class ProgressBar final : public Logger std::unique_ptr interruptCallback; + void hideCursorIfNeeded() const + { + if (isTTY) + writeToStderr("\e[?25l"); + } + + void unhideCursorIfNeeded() const + { + if (isTTY) + writeToStderr("\e[?25h"); + } + public: ProgressBar(bool isTTY) @@ -116,6 +128,7 @@ class ProgressBar final : public Logger redraw("\rshutting down\e[K"); })) { + hideCursorIfNeeded(); state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); @@ -142,6 +155,7 @@ class ProgressBar final : public Logger if (state->active) { state->active = false; clearProgressDisplay(); + unhideCursorIfNeeded(); updateCV.notify_one(); quitCV.notify_one(); } @@ -159,8 +173,10 @@ class ProgressBar final : public Logger return; } - if (state->active) + if (state->active) { clearProgressDisplay(); + unhideCursorIfNeeded(); + } } void resume() override @@ -173,8 +189,10 @@ class ProgressBar final : public Logger state->suspensions--; } if (state->suspensions == 0) { - if (state->active) + if (state->active) { clearProgressDisplay(); + hideCursorIfNeeded(); + } state->haveUpdate = true; updateCV.notify_one(); } @@ -681,7 +699,9 @@ class ProgressBar final : public Logger return {}; invalidateRedrawCache(); std::cerr << fmt("\r\e[K%s ", msg); + unhideCursorIfNeeded(); auto s = trim(readLine(getStandardInput(), true)); + hideCursorIfNeeded(); if (s.size() != 1) return {}; draw(*state); From 4fa3a8cad2b651ab0bb3a30d78d9499c8ae16216 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 7 May 2026 00:41:57 +0300 Subject: [PATCH 360/927] libstore: Use member fileTransfer instead of global getFileTransfer() in the s3 store Ideally we'd get rid of the global file transfer object altogether and this moves us in that direction. fileTransfer was already a member of the HttpBinaryCacheStore, so just use that instead. In practice it pointed to the same singleton object though (it's primarily used for tests now). --- src/libstore/s3-binary-cache-store.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f6d300c093..54bf1d0ad7 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -322,7 +322,7 @@ std::string S3BinaryCacheStore::createMultipartUpload( std::move(headers->begin(), headers->end(), std::back_inserter(req.headers)); } - auto result = getFileTransfer()->enqueueFileTransfer(req).get(); + auto result = fileTransfer->enqueueFileTransfer(req).get(); std::regex uploadIdRegex("([^<]+)"); std::smatch match; @@ -353,7 +353,7 @@ S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, req.data = {payload}; req.mimeType = "application/octet-stream"; - auto result = getFileTransfer()->enqueueFileTransfer(req).get(); + auto result = fileTransfer->enqueueFileTransfer(req).get(); if (result.etag.empty()) { throw Error("S3 UploadPart response missing ETag for part %d", partNumber); @@ -374,7 +374,7 @@ void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_ req.uri = VerbatimURL(url); req.method = HttpMethod::Delete; - getFileTransfer()->enqueueFileTransfer(req).get(); + fileTransfer->enqueueFileTransfer(req).get(); } catch (...) { ignoreExceptionInDestructor(); } @@ -407,7 +407,7 @@ void S3BinaryCacheStore::completeMultipartUpload( req.data = {payload}; req.mimeType = "text/xml"; - getFileTransfer()->enqueueFileTransfer(req).get(); + fileTransfer->enqueueFileTransfer(req).get(); debug("S3 multipart upload completed: %d parts uploaded for '%s'", partEtags.size(), key); } From 220ccc746d156b5ea62c45b42e91a4b6f380f6ef Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 7 May 2026 01:26:05 +0200 Subject: [PATCH 361/927] fix: Restore fingerprint value for untrustworthy revCount values It was fine for trustworthy revCounts, but technically the functional dependency described does not always hold up, or we can not trust it to. - buggy tarball providers - merge conflicts in lock files or other kinds of chaos in "user space" --- src/libflake/flake.cc | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index c34fa2d0a1..bd3ea4acbc 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -989,15 +989,24 @@ std::optional LockedFlake::getFingerprint(Store & store, const fetc *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile); - /* Include revCount and lastModified because they're not - necessarily implied by the content fingerprint (e.g. for - tarball flakes) but can influence the evaluation result. - For revCount, we only include its presence (not its value) - because the value is functionally determined by rev, which - is already part of the fingerprint. This avoids forcing a - lazy revCount computation. */ - if (flake.lockedRef.input.attrs.contains("revCount")) - *fingerprint += ";hasRevCount"; + if (auto revCount = get(flake.lockedRef.input.attrs, "revCount")) { + if (std::get_if(revCount)) { + /* A lazy revCount is computed by the fetcher, so its + value is functionally determined by `rev`. We only + need to record its presence, not force its value. + + This means a lazy and a concrete revCount that would + resolve to the same value produce different + fingerprints, sacrificing some cache hits to avoid + the cost of forcing. */ + *fingerprint += ";hasRevCount"; + } else if (auto n = flake.lockedRef.input.getRevCount()) { + /* A concrete revCount comes from a lockfile or explicit + user input. The fetcher passes it through as-is, so + it can affect evaluation and must be fingerprinted. */ + *fingerprint += fmt(";revCount=%d", *n); + } + } if (auto lastModified = flake.lockedRef.input.getLastModified()) *fingerprint += fmt(";lastModified=%d", *lastModified); From 2a42376bbe96340cf5f94b96d881225b5366bf59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 8 May 2026 10:54:10 +0000 Subject: [PATCH 362/927] tests/nixos: serve fetchers-substitute cache via nginx nix-serve depends on the perl bindings, which are slated for removal, and nix-serve-ng adds an extra dependency just to serve a static binary cache over HTTP. Replace it with plain nginx serving a file:// binary cache populated via 'nix copy', which is simpler and only relies on components already in nixpkgs. :house: Remote-Dev: homespace --- tests/nixos/fetchers-substitute.nix | 83 +++++++++-------------------- 1 file changed, 24 insertions(+), 59 deletions(-) diff --git a/tests/nixos/fetchers-substitute.nix b/tests/nixos/fetchers-substitute.nix index b61a3929af..0959512e1d 100644 --- a/tests/nixos/fetchers-substitute.nix +++ b/tests/nixos/fetchers-substitute.nix @@ -11,22 +11,18 @@ "fetch-tree" ]; - networking.firewall.allowedTCPPorts = [ 5000 ]; + networking.firewall.allowedTCPPorts = [ 80 ]; - # TODO stop using this, because it has to depend on an older version of Nix that still has the perl bindings. - services.nix-serve = { + systemd.tmpfiles.rules = [ "d /var/cache/binary-cache 0755 root root -" ]; + + services.nginx = { enable = true; - secretKeyFile = - let - key = pkgs.writeTextFile { - name = "secret-key"; - text = '' - substituter:SerxxAca5NEsYY0DwVo+subokk+OoHcD9m6JwuctzHgSQVfGHe6nCc+NReDjV3QdFYPMGix4FMg0+K/TM1B3aA== - ''; - }; - in - "${key}"; + virtualHosts."substituter".root = "/var/cache/binary-cache"; }; + + environment.etc."nix/secret-key".text = '' + substituter:SerxxAca5NEsYY0DwVo+subokk+OoHcD9m6JwuctzHgSQVfGHe6nCc+NReDjV3QdFYPMGix4FMg0+K/TM1B3aA== + ''; }; nodes.importer = @@ -39,13 +35,12 @@ "nix-command" "fetch-tree" ]; - substituters = lib.mkForce [ "http://substituter:5000" ]; + substituters = lib.mkForce [ "http://substituter" ]; trusted-public-keys = lib.mkForce [ "substituter:EkFXxh3upwnPjUXg41d0HRWDzBoseBTINPiv0zNQd2g=" ]; }; }; - testScript = - { nodes }: # python + testScript = # python '' import json import os @@ -53,11 +48,11 @@ start_all() substituter.wait_for_unit("multi-user.target") + importer.wait_for_unit("multi-user.target") - ########################################## - # Test 1: builtins.fetchurl with substitution - ########################################## + binary_cache = "file:///var/cache/binary-cache?secret-key=/etc/nix/secret-key" + # builtins.fetchurl is substituted missing_file = "/only-on-substituter.txt" substituter.succeed(f"echo 'this should only exist on the substituter' > {missing_file}") @@ -75,11 +70,8 @@ file_store_path = json.loads(file_store_path_json) - substituter.succeed(f"nix store sign --key-file ${nodes.substituter.services.nix-serve.secretKeyFile} {file_store_path}") - - importer.wait_for_unit("multi-user.target") + substituter.succeed(f"nix copy --to '{binary_cache}' {file_store_path}") - print("Testing fetchurl with substitution...") importer.succeed(f""" nix-instantiate -vvvvv --eval --json --read-write-mode --expr ' builtins.fetchurl {{ @@ -88,26 +80,19 @@ }} ' """) - print("✓ fetchurl substitution works!") - - ########################################## - # Test 2: builtins.fetchTarball with substitution - ########################################## + # builtins.fetchTarball is substituted missing_tarball = "/only-on-substituter.tar.gz" - # Create a directory with some content substituter.succeed(""" mkdir -p /tmp/test-tarball echo 'Hello from tarball!' > /tmp/test-tarball/hello.txt echo 'Another file' > /tmp/test-tarball/file2.txt """) - - # Create a tarball substituter.succeed(f"tar czf {missing_tarball} -C /tmp test-tarball") - # For fetchTarball, we need to first fetch it without hash to get the store path, - # then compute the NAR hash of that path + # Fetch once without a hash to learn the store path, then derive the + # hashes the importer needs. tarball_store_path_json = substituter.succeed(f""" nix-instantiate --eval --json --read-write-mode --expr ' builtins.fetchTarball {{ @@ -118,22 +103,14 @@ tarball_store_path = json.loads(tarball_store_path_json) - # Get the NAR hash of the unpacked tarball in SRI format path_info_json = substituter.succeed(f"nix path-info --json-format 2 --json {tarball_store_path}").strip() path_info_dict = json.loads(path_info_json)["info"] - # narHash is already in SRI format tarball_hash_sri = path_info_dict[os.path.basename(tarball_store_path)]["narHash"] - print(f"Tarball NAR hash (SRI): {tarball_hash_sri}") - # Also get the old format hash for fetchTarball (which uses sha256 parameter) tarball_hash = substituter.succeed(f"nix-store --query --hash {tarball_store_path}").strip() - # Sign the tarball's store path - substituter.succeed(f"nix store sign --recursive --key-file ${nodes.substituter.services.nix-serve.secretKeyFile} {tarball_store_path}") + substituter.succeed(f"nix copy --to '{binary_cache}' {tarball_store_path}") - # Now try to fetch the same tarball on the importer - # The file doesn't exist locally, so it should be substituted - print("Testing fetchTarball with substitution...") result = importer.succeed(f""" nix-instantiate -vvvvv --eval --json --read-write-mode --expr ' builtins.fetchTarball {{ @@ -144,23 +121,14 @@ """) result_path = json.loads(result) - print(f"✓ fetchTarball substitution works! Result: {result_path}") - # Verify the content is correct - # fetchTarball strips the top-level directory if there's only one content = importer.succeed(f"cat {result_path}/hello.txt").strip() assert content == "Hello from tarball!", f"Content mismatch: {content}" - print("✓ fetchTarball content verified!") - - ########################################## - # Test 3: Verify fetchTree does NOT substitute (preserves metadata) - ########################################## - - print("Testing that fetchTree without __final does NOT use substitution...") - # fetchTree with just narHash (not __final) should try to download, which will fail - # since the file doesn't exist on the importer - exit_code = importer.fail(f""" + # fetchTree does NOT substitute non-final inputs: without __final it + # must perform the real fetch (to preserve metadata like lastModified), + # so it fails since the file only exists on the substituter. + output = importer.fail(f""" nix-instantiate --eval --json --read-write-mode --expr ' builtins.fetchTree {{ type = "tarball"; @@ -170,9 +138,6 @@ ' 2>&1 """) - # Should fail with "does not exist" since it tries to download instead of substituting - assert "does not exist" in exit_code or "Couldn't open file" in exit_code, f"Expected download failure, got: {exit_code}" - print("✓ fetchTree correctly does NOT substitute non-final inputs!") - print(" (This preserves metadata like lastModified from the actual fetch)") + assert "does not exist" in output or "Couldn't open file" in output, f"Expected download failure, got: {output}" ''; } From 67e442bbbf0ea04b206305b290463ae210d515b4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 15:57:32 +0300 Subject: [PATCH 363/927] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'https://releases.nixos.org/nixos/25.11/nixos-25.11.6495.e764fc9a4058/nixexprs.tar.xz?narHash=sha256-jEA8WggGKtMFeNeCKq3NK8cLEjJmG6/RLUElYYbBZ0E%3D' (2026-02-24) → 'https://releases.nixos.org/nixos/25.11/nixos-25.11.10470.0c88e1f2bdb9/nixexprs.tar.xz?narHash=sha256-amc4Y3GF3%2BanUi7IJeLVzf7hVqLb3ZqCGzYtkVyp7Qw%3D' (2026-05-05) --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 1212049c3b..95b771a30c 100644 --- a/flake.lock +++ b/flake.lock @@ -60,11 +60,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1771903837, - "narHash": "sha256-jEA8WggGKtMFeNeCKq3NK8cLEjJmG6/RLUElYYbBZ0E=", - "rev": "e764fc9a405871f1f6ca3d1394fb422e0a0c3951", + "lastModified": 1778003029, + "narHash": "sha256-amc4Y3GF3+anUi7IJeLVzf7hVqLb3ZqCGzYtkVyp7Qw=", + "rev": "0c88e1f2bdb93d5999019e99cb0e61e1fe2af4c5", "type": "tarball", - "url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.6495.e764fc9a4058/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.10470.0c88e1f2bdb9/nixexprs.tar.xz" }, "original": { "type": "tarball", From 31f253b136d265bed12273f4a9de5388095936b4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 16:10:51 +0300 Subject: [PATCH 364/927] tests: Fix daemon compat tests for structured attrs --- tests/functional/structured-attrs.sh | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index 321cb61079..d0b4167096 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -51,15 +51,17 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet " hacky=$(nix-instantiate --expr "$hackyExpr") nix derivation show "$hacky" | jq --exit-status '.derivations."'"$(basename "$hacky")"'".structuredAttrs | . == {"a": 1}' -# Test warning for non-object exportReferencesGraph in structured attrs -# shellcheck disable=SC2016 -expectStderr 0 nix-build --no-out-link --expr ' - with import ./config.nix; - mkDerivation { - name = "export-graph-non-object"; - __structuredAttrs = true; - exportReferencesGraph = [ "foo" "bar" ]; - builder = "/bin/sh"; - args = ["-c" "echo foo > ${builtins.placeholder "out"}"]; - } -' | grepQuiet "warning:.*exportReferencesGraph.*not a JSON object" +if isDaemonNewer "2.34pre"; then + # Test warning for non-object exportReferencesGraph in structured attrs + # shellcheck disable=SC2016 + expectStderr 0 nix-build --no-out-link --expr ' + with import ./config.nix; + mkDerivation { + name = "export-graph-non-object"; + __structuredAttrs = true; + exportReferencesGraph = [ "foo" "bar" ]; + builder = "/bin/sh"; + args = ["-c" "echo foo > ${builtins.placeholder "out"}"]; + } + ' | grepQuiet "warning:.*exportReferencesGraph.*not a JSON object" +fi From acadae1ee65637bc6003813456b5cfe37c38fe05 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 16:19:42 +0300 Subject: [PATCH 365/927] packaging: Use libgit2 >= 1.9.3 It's been a while since libgit2 has been released. No notable features in this release though - just bugfixes. Unstable has been updated in [1]. [1]: https://github.com/NixOS/nixpkgs/pull/517329 --- packaging/dependencies.nix | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index b44f6ae46c..68e44a925f 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -62,6 +62,21 @@ scope: { useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); }; + libgit2 = + if lib.versionAtLeast pkgs.libgit2.version "1.9.3" then + pkgs.libgit2 + else + # Grab newer libgit2. + pkgs.libgit2.overrideAttrs rec { + version = "1.9.3"; + src = pkgs.fetchFromGitHub { + owner = "libgit2"; + repo = "libgit2"; + tag = "v${version}"; + hash = "sha256-nJrRdPs86oGNL4W2CJb16oSUgfzYr9A2i5sw9BAehME="; + }; + }; + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. boost = (pkgs.boost.override { From bd2d65991100386983d9dbdd776e1afdb811383b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 16:26:22 +0300 Subject: [PATCH 366/927] packaging: Use mimalloc >= 3.3.2 --- packaging/dependencies.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 68e44a925f..6a1873d9b3 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -17,16 +17,16 @@ scope: { inherit stdenv; mimalloc = - if lib.versionAtLeast pkgs.mimalloc.version "3.3.0" then + if lib.versionAtLeast pkgs.mimalloc.version "3.3.2" then pkgs.mimalloc else pkgs.mimalloc.overrideAttrs rec { - version = "3.3.0"; + version = "3.3.2"; src = pkgs.fetchFromGitHub { owner = "microsoft"; repo = "mimalloc"; tag = "v${version}"; - hash = "sha256-xy9gPihw3xvhnd6BrCYfMnnRp5dPSodynKRToYwxuzg="; + hash = "sha256-GZ37qQVDe9jgMb4Coe5oKvgaLTspZDlSkS5rdy1MfUU="; }; }; From c5aa96ef10ba7fffb581c5ea067a4f436ebe8283 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 10 May 2026 16:29:15 +0300 Subject: [PATCH 367/927] packaging: Remove .broken override for libcurl Out nixpkgs is new enough now. --- packaging/dependencies.nix | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 6a1873d9b3..f531b709ee 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -44,19 +44,13 @@ scope: { NIX_CFLAGS_COMPILE = "-DINITIAL_MARK_STACK_SIZE=1048576"; }); - curl = - (pkgs.curl.override { - http3Support = !pkgs.stdenv.hostPlatform.isWindows; - # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. - zstdSupport = true; - brotliSupport = true; - zlibSupport = true; - }).overrideAttrs - { - # TODO: Fix in nixpkgs. Static build with brotli is marked as broken, but it's not the case. - # Remove once https://github.com/NixOS/nixpkgs/pull/494111 lands in the 25.11 channel. - meta.broken = false; - }; + curl = pkgs.curl.override { + http3Support = !pkgs.stdenv.hostPlatform.isWindows; + # Make sure we enable all the dependencies for Content-Encoding/Transfer-Encoding decompression. + zstdSupport = true; + brotliSupport = true; + zlibSupport = true; + }; libblake3 = pkgs.libblake3.override { useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic); From 3fd52d9a32ab70251dbad20575cc42b985c8cd5b Mon Sep 17 00:00:00 2001 From: Felix Stupp Date: Fri, 8 May 2026 22:05:58 +0000 Subject: [PATCH 368/927] doc: add documentation to __addErrorContext primop This adds .args and .doc fields to the primop registration for __addErrorContext, making it appear in the builtins reference manual. --- src/libexpr/primops.cc | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 297b2bf234..08b4fd76ac 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1047,7 +1047,35 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value ** a static RegisterPrimOp primop_addErrorContext( PrimOp{ .name = "__addErrorContext", + .args = {"context", "value"}, .arity = 2, + .doc = R"( + Evaluate *context*, which can be coerced to a string, + and append it to any error or stack traces displayed while evaluating *value*. + Then return *value*. + + This function is useful for providing helpful context in complex Nix expressions + when the evaluation of *value* fails. + The additional context is applied when evaluating *value* itself fails, + not when attributes or elements of *value* are evaluated. + + For example, the module system from nixpkgs uses this to show + the relevant information about the options that were evaluating + when an error occurs. + + ```nix-repl + nix-repl> addErrorContext "while evaluating foo" (throw "bar") + error: + … while evaluating foo + + … while calling the 'throw' builtin + at «string»:1:56: + 1| with builtins; addErrorContext "while evaluating foo" (throw "bar") + | ^ + + error: bar + ``` + )", // The normal trace item is redundant .addTrace = false, .impl = prim_addErrorContext, From cabb895f190c33fda1871eb110547191f3481a06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 May 2026 22:12:25 +0000 Subject: [PATCH 369/927] build(deps): bump cachix/install-nix-action from 31.10.5 to 31.10.6 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 31.10.5 to 31.10.6. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Changelog](https://github.com/cachix/install-nix-action/blob/master/RELEASE.md) - [Commits](https://github.com/cachix/install-nix-action/compare/ab739621df7a23f52766f9ccc97f38da6b7af14f...8aa03977d8d733052d78f4e008a241fd1dbf36b3) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-version: 31.10.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49fff49b97..2f294211ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,7 +189,7 @@ jobs: - name: Looking up the installer tarball URL id: installer-tarball-url run: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5 + - uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6 if: ${{ !matrix.rust-installer }} with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} @@ -255,7 +255,7 @@ jobs: id: installer-tarball-url run: | echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@ab739621df7a23f52766f9ccc97f38da6b7af14f # v31.10.5 + - uses: cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3 # v31.10.6 with: install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} From fcbe6cc5bffb374482b56d7b3445e2040a3972db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 May 2026 22:12:30 +0000 Subject: [PATCH 370/927] build(deps): bump aws-actions/configure-aws-credentials Bumps [aws-actions/configure-aws-credentials](https://github.com/aws-actions/configure-aws-credentials) from 5.1.1 to 6.1.1. - [Release notes](https://github.com/aws-actions/configure-aws-credentials/releases) - [Changelog](https://github.com/aws-actions/configure-aws-credentials/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-actions/configure-aws-credentials/compare/61815dcd50bd041e203e49132bacad1fd04d2708...d979d5b3a71173a29b74b5b88418bfda9437d885) --- updated-dependencies: - dependency-name: aws-actions/configure-aws-credentials dependency-version: 6.1.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/upload-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml index f00dce4a5c..f7da2f6636 100644 --- a/.github/workflows/upload-release.yml +++ b/.github/workflows/upload-release.yml @@ -34,7 +34,7 @@ jobs: # get the same uberhack that nix-shell has to support it. echo "NIX_PATH=nixpkgs=$NIXPKGS_PATH" >> "$GITHUB_ENV" - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 + uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 with: role-to-assume: "arn:aws:iam::080433136561:role/nix-release" role-session-name: nix-release-oidc-${{ github.run_id }} From 5a870acfc9f5bccb9b50b2f6b1f541ecdfbfb1a6 Mon Sep 17 00:00:00 2001 From: znmz <267793835+znmz@users.noreply.github.com> Date: Mon, 11 May 2026 16:27:20 +0300 Subject: [PATCH 371/927] Fix typos in all .md, .sh, .nix files Also fixes one typo in each of the "remove_before_wrapper.py" "build-trace-entry-v3.yaml" "head.hbs" "sender.c" files --- doc/manual/remove_before_wrapper.py | 2 +- doc/manual/source/command-ref/env-common.md | 2 +- .../protocols/json/schema/build-trace-entry-v3.yaml | 2 +- doc/manual/source/release-notes/rl-2.30.md | 2 +- doc/manual/source/release-notes/rl-2.32.md | 2 +- doc/manual/source/store/file-system-object.md | 2 +- doc/manual/theme/head.hbs | 2 +- packaging/everything.nix | 2 +- scripts/create-darwin-volume.sh | 4 ++-- scripts/install-multi-user.sh | 4 ++-- tests/functional/binary-cache.sh | 2 +- tests/functional/build-delete.sh | 2 +- tests/functional/build.sh | 4 ++-- tests/functional/characterisation-test-infra.sh | 6 +++--- tests/functional/common/functions.sh | 2 +- tests/functional/dyn-drv/eval-outputOf.sh | 2 +- tests/functional/fetchTree-file.sh | 4 ++-- tests/functional/flakes/develop.sh | 10 +++++----- tests/functional/flakes/follow-paths.sh | 2 +- tests/functional/flakes/relative-paths.sh | 2 +- tests/functional/gc-closure.sh | 4 ++-- tests/functional/lang-gc.sh | 2 +- tests/functional/linux-sandbox.sh | 2 +- tests/functional/read-only-store.sh | 2 +- tests/functional/repl.sh | 2 +- tests/functional/simple.sh | 2 +- tests/functional/structured-attrs.sh | 2 +- tests/functional/suggestions.sh | 6 +++--- tests/nixos/ca-fd-leak/sender.c | 2 +- tests/nixos/fetch-git/test-cases/lfs/default.nix | 2 +- 30 files changed, 43 insertions(+), 43 deletions(-) diff --git a/doc/manual/remove_before_wrapper.py b/doc/manual/remove_before_wrapper.py index 6da4c19b0c..a0fcb6a557 100644 --- a/doc/manual/remove_before_wrapper.py +++ b/doc/manual/remove_before_wrapper.py @@ -22,7 +22,7 @@ def main(): shutil.rmtree(output, ignore_errors=True) shutil.rmtree(output_temp, ignore_errors=True) - # Execute nix command with `--write-to` tempary output + # Execute nix command with `--write-to` temporary output nix_command_write_to = nix_command + ['--write-to', output_temp] subprocess.run(nix_command_write_to, check=True) diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index 7ea1d0e5aa..cc7fe77eae 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -160,7 +160,7 @@ When [`use-xdg-base-directories`] is enabled, the configuration directory is res Likewise for the state and cache directories. -## Miscellanous environment variables +## Miscellaneous environment variables - [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml index c3a27d2a6e..3ff606672c 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml @@ -40,7 +40,7 @@ additionalProperties: false title: Build Trace Key description: | A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. - This is the "key" part, refering to a derivation and output. + This is the "key" part, referring to a derivation and output. type: object required: - drvPath diff --git a/doc/manual/source/release-notes/rl-2.30.md b/doc/manual/source/release-notes/rl-2.30.md index 34d3e5bab4..5a65ed99af 100644 --- a/doc/manual/source/release-notes/rl-2.30.md +++ b/doc/manual/source/release-notes/rl-2.30.md @@ -13,7 +13,7 @@ - Deprecate manually making structured attrs using the `__json` attribute [#13220](https://github.com/NixOS/nix/pull/13220) The proper way to create a derivation using [structured attrs] in the Nix language is by using `__structuredAttrs = true` with [`builtins.derivation`]. - However, by exploiting how structured attrs are implementated, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string. + However, by exploiting how structured attrs are implemented, it has also been possible to create them by setting the `__json` environment variable to a serialized JSON string. This sneaky alternative method is now deprecated, and may be disallowed in future versions of Nix. [structured attrs]: @docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs diff --git a/doc/manual/source/release-notes/rl-2.32.md b/doc/manual/source/release-notes/rl-2.32.md index 5d90da0c9e..c59ecd6c24 100644 --- a/doc/manual/source/release-notes/rl-2.32.md +++ b/doc/manual/source/release-notes/rl-2.32.md @@ -8,7 +8,7 @@ - Derivation JSON format now uses store path basenames only [#13570](https://github.com/NixOS/nix/issues/13570) [#13980](https://github.com/NixOS/nix/pull/13980) - Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra paramater (the store directory). + Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra parameter (the store directory). We ultimately want to rectify this issue with all JSON formats to the extent allowed by our stability promises. To start with, we are changing the JSON format for derivations because the `nix derivation` commands are — in addition to being formally unstable — less widely used than other unstable commands. diff --git a/doc/manual/source/store/file-system-object.md b/doc/manual/source/store/file-system-object.md index 60cb3e5720..2ffad5f0ec 100644 --- a/doc/manual/source/store/file-system-object.md +++ b/doc/manual/source/store/file-system-object.md @@ -19,7 +19,7 @@ Every file system object is one of the following: In general, Nix does not assign any semantics to symbolic links. Certain operations however, may make additional assumptions and attempt to use the target to find another file system object. - > See [the Wikpedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept. + > See [the Wikipedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept. File system objects and their children form a tree. A bare file or symlink can be a root file system object. diff --git a/doc/manual/theme/head.hbs b/doc/manual/theme/head.hbs index e514a99777..40bfef7d2f 100644 --- a/doc/manual/theme/head.hbs +++ b/doc/manual/theme/head.hbs @@ -11,5 +11,5 @@ MathJax = { } }; - + diff --git a/packaging/everything.nix b/packaging/everything.nix index df7d57a858..74629d6840 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -105,7 +105,7 @@ stdenv.mkDerivation (finalAttrs: { dontBuild = true; /** - `doCheck` controles whether tests are added as build gate for the combined package. + `doCheck` controls whether tests are added as build gate for the combined package. This includes both the unit tests and the functional tests, but not the integration tests that run in CI (the flake's `hydraJobs` and some of the `checks`). */ diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 7a61764d4f..538a5e74d5 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -832,8 +832,8 @@ EOF # TODO: should probably alert the user if this is disabled? _sudo "to launch the Nix volume mounter" \ launchctl bootstrap system "$NIX_VOLUME_MOUNTD_DEST" || true - # TODO: confirm whether kickstart is necessesary? - # I feel a little superstitous, but it can guard + # TODO: confirm whether kickstart is necessary? + # I feel a little superstitious, but it can guard # against multiple problems (doesn't start, old # version still running for some reason...) _sudo "to launch the Nix volume mounter" \ diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index d4ea88b5ea..ae6625e1bc 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -270,7 +270,7 @@ _diff() { printf -v CHANGED_GROUP_FORMAT "%b" "${GREEN}%>${RED}%<${ESC}" diff --changed-group-format="$CHANGED_GROUP_FORMAT" "$@" else - # simple colorized diff comatible w/ pre `--color` versions + # simple colorized diff compatible w/ pre `--color` versions diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format=" %L" "$@" fi } @@ -961,7 +961,7 @@ configure_shell_profile() { cert_in_store() { # in a subshell # - change into the cert-file dir - # - get the phyiscal pwd + # - get the physical pwd # and test if this path is in the Nix store [[ "$(cd -- "$(dirname "$NIX_SSL_CERT_FILE")" && exec pwd -P)" == "$NIX_ROOT/store/"* ]] } diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 6826345933..d5b81523b3 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -44,7 +44,7 @@ cacheDir2="$TEST_ROOT/binary+cache" nix copy --to "file://$cacheDir2" "$outPath" && [[ -d "$cacheDir2" ]] basicDownloadTests() { - # No uploading tests bcause upload with force HTTP doesn't work. + # No uploading tests because upload with force HTTP doesn't work. # By default, a binary cache doesn't support "nix-env -qas", but does # support installation. diff --git a/tests/functional/build-delete.sh b/tests/functional/build-delete.sh index 66b14fd143..e65615407a 100755 --- a/tests/functional/build-delete.sh +++ b/tests/functional/build-delete.sh @@ -44,7 +44,7 @@ issue_6572_dependent_outputs() { # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. if [[ -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then - # Resolved derivations interferre with the deletion + # Resolved derivations interfere with the deletion nix-store --delete "${NIX_STORE_DIR}"/*.drv fi nix-store --delete "$(jq -r <"$TEST_ROOT"/a.json .[0].outputs.second)" diff --git a/tests/functional/build.sh b/tests/functional/build.sh index 0e76f949f5..9f86f81003 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -53,7 +53,7 @@ nix build -f multiple-outputs.nix --json nothing-to-install --no-link | jq --exi (.outputs | keys == ["out"])) ' -# But not when it's overriden. +# But not when it's overridden. nix build -f multiple-outputs.nix --json e^a_a --no-link nix build -f multiple-outputs.nix --json e^a_a --no-link | jq --exit-status ' (.[0] | @@ -67,7 +67,7 @@ nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status ' (.outputs | keys == ["a_a", "b", "c"])) ' -# test buidling from non-drv attr path +# test building from non-drv attr path nix build -f multiple-outputs.nix --json 'e.a_a.outPath' --no-link | jq --exit-status ' (.[0] | diff --git a/tests/functional/characterisation-test-infra.sh b/tests/functional/characterisation-test-infra.sh index fecae29e80..9651b4f7b6 100755 --- a/tests/functional/characterisation-test-infra.sh +++ b/tests/functional/characterisation-test-infra.sh @@ -19,14 +19,14 @@ cp "$TEST_ROOT/got" "$TEST_ROOT/expected" (( "$badDiff" == 0 )) ) -# matches empty, non-existant file is the same as empty file +# matches empty, non-existent file is the same as empty file echo -n > "$TEST_ROOT/got" ( diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" (( "$badDiff" == 0 )) ) -# doesn't matches non-empty, non-existant file is the same as empty file +# doesn't matches non-empty, non-existent file is the same as empty file echo Hi! > "$TEST_ROOT/got" ( diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" @@ -64,7 +64,7 @@ echo Bye! > "$TEST_ROOT/expected" (( "$badDiff" == 0 )) ) -# _NIX_TEST_ACCEPT matches empty, non-existant file not created +# _NIX_TEST_ACCEPT matches empty, non-existent file not created echo -n > "$TEST_ROOT/got" ( _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exists" diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index 771bbca785..0e571cc27a 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -101,7 +101,7 @@ killDaemon() { die "killDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." fi - # Don't fail trying to stop a non-existant daemon twice. + # Don't fail trying to stop a non-existent daemon twice. if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then return fi diff --git a/tests/functional/dyn-drv/eval-outputOf.sh b/tests/functional/dyn-drv/eval-outputOf.sh index 3681bd0988..13f7e7ba4c 100644 --- a/tests/functional/dyn-drv/eval-outputOf.sh +++ b/tests/functional/dyn-drv/eval-outputOf.sh @@ -34,7 +34,7 @@ testStaticHello () { (assert a == b; null))' } -# Test with a regular old input-addresed derivation +# Test with a regular old input-addressed derivation # # `builtins.outputOf` works without ca-derivations and doesn't create a # placeholder but just returns the output path. diff --git a/tests/functional/fetchTree-file.sh b/tests/functional/fetchTree-file.sh index 66be928c7b..a7c614b8ec 100755 --- a/tests/functional/fetchTree-file.sh +++ b/tests/functional/fetchTree-file.sh @@ -76,12 +76,12 @@ EOF # For backwards compatibility, flake inputs that correspond to the # old 'tarball' fetcher should still have their type set to 'tarball' assert (nodes.tarball_default_unpack.locked.type == "tarball"); - # Unless explicitely specified, the 'unpack' parameter shouldn’t appear here + # Unless explicitly specified, the 'unpack' parameter shouldn’t appear here # because that would break older Nix versions assert (!nodes.tarball_default_unpack.locked ? unpack); assert (nodes.tarball_default_unpack.locked.narHash == "$input_directory_hash"); - # Explicitely passing the unpack parameter should enforce the desired behavior + # Explicitly passing the unpack parameter should enforce the desired behavior assert (nodes.no_ext_explicit_unpack.locked.narHash == nodes.tarball_default_unpack.locked.narHash); assert (nodes.tarball_explicit_no_unpack.locked.narHash == nodes.no_ext_default_no_unpack.locked.narHash); diff --git a/tests/functional/flakes/develop.sh b/tests/functional/flakes/develop.sh index 0248adf49e..bb88508027 100755 --- a/tests/functional/flakes/develop.sh +++ b/tests/functional/flakes/develop.sh @@ -66,7 +66,7 @@ echo "\$ENVVAR" EOF )" ]] -# Test wether `--keep-env-var` keeps the environment variable. +# Test whether `--keep-env-var` keeps the environment variable. ( expect='BAR' got="$(FOO='BAR' nix develop --ignore-env --keep-env-var FOO --no-write-lock-file .#hello <"$flakeFollowsA"/flake.nix < "$TEST_ROOT"/log grepQuietInverse 'error: renaming' "$TEST_ROOT"/log grepQuiet 'may not be deterministic' "$TEST_ROOT"/log diff --git a/tests/functional/read-only-store.sh b/tests/functional/read-only-store.sh index 8ccca2192a..8bcf42cfc4 100755 --- a/tests/functional/read-only-store.sh +++ b/tests/functional/read-only-store.sh @@ -4,7 +4,7 @@ source common.sh enableFeatures "read-only-local-store" -needLocalStore "cannot open store read-only when daemon has already opened it writeable" +needLocalStore "cannot open store read-only when daemon has already opened it writable" TODO_NixOS diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 9e752337f1..cb181fbb62 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -145,7 +145,7 @@ foo + baz ' "3" \ ./flake ./flake\#bar --experimental-features 'flakes' -# Test the `:reload` mechansim with flakes: +# Test the `:reload` mechanism with flakes: # - Eval `./flake#changingThing` # - Modify the flake # - Re-eval it diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh index f6507d7418..b7e8911930 100755 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -20,7 +20,7 @@ text=$(cat "$outPath/hello") TODO_NixOS # Directed delete: $outPath is not reachable from a root, so it should -# be deleteable. +# be deletable. nix-store --delete "$outPath" [[ ! -e $outPath/hello ]] diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index d0b4167096..32d647a772 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -26,7 +26,7 @@ TODO_NixOS # following line fails. # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` -# (which is an associative array in thsi case) being fine. +# (which is an associative array in this case) being fine. # shellcheck disable=SC2016 nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' diff --git a/tests/functional/suggestions.sh b/tests/functional/suggestions.sh index fbca93da85..e4324f2dfa 100755 --- a/tests/functional/suggestions.sh +++ b/tests/functional/suggestions.sh @@ -30,7 +30,7 @@ EOF # Probable typo in the requested attribute path. Suggest some close possibilities NIX_BUILD_STDERR_WITH_SUGGESTIONS=$(! nix build .\#fob 2>&1 1>/dev/null) [[ "$NIX_BUILD_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ - fail "The nix build stderr should suggest the three closest possiblities" + fail "The nix build stderr should suggest the three closest possibilities" # None of the possible attributes is close to `bar`, so shouldn’t suggest anything NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) @@ -39,8 +39,8 @@ NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'"$system"'.fob' 2>&1 1>/dev/null) [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ - fail "The evaluator should suggest the three closest possiblities" + fail "The evaluator should suggest the three closest possibilities" NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '({ foo }: foo) { foo = 1; fob = 2; }' 2>&1 1>/dev/null) [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean foo?" ]] || \ - fail "The evaluator should suggest the three closest possiblities" + fail "The evaluator should suggest the three closest possibilities" diff --git a/tests/nixos/ca-fd-leak/sender.c b/tests/nixos/ca-fd-leak/sender.c index 639b889002..f9fdd9405a 100644 --- a/tests/nixos/ca-fd-leak/sender.c +++ b/tests/nixos/ca-fd-leak/sender.c @@ -62,6 +62,6 @@ int main(int argc, char ** argv) int buf; // Wait for the server to close the socket, implying that it has - // received the commmand. + // received the command. recv(sock, (void *) &buf, sizeof(int), 0); } diff --git a/tests/nixos/fetch-git/test-cases/lfs/default.nix b/tests/nixos/fetch-git/test-cases/lfs/default.nix index 289c377091..09b6fe44ff 100644 --- a/tests/nixos/fetch-git/test-cases/lfs/default.nix +++ b/tests/nixos/fetch-git/test-cases/lfs/default.nix @@ -172,7 +172,7 @@ f"did not set lfs, yet lfs-enrolled file is {file_size_default}b (>= 1KiB), probably bad default value" with subtest("Use as flake input"): - # May seem reduntant, but this has minor differences compared to raw + # May seem redundant, but this has minor differences compared to raw # fetchGit which caused failures before with TemporaryDirectory() as tempdir: client.succeed(f"mkdir -p {tempdir}") From 25295886ce8c4a8fa5ffd629540cac14a9736498 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 18:18:57 +0200 Subject: [PATCH 372/927] parseString(): Fix out-of-bounds read If the string isn't terminated, parseString() returns a string of size std::string::npos, which then causes an out-of-bounds read later. Fixes: ==47978== Invalid read of size 1 ==47978== at 0x4BEF70A: nix::expect(nix::(anonymous namespace)::StringViewStream&, char) (../src/libstore/derivations.cc:232) ==47978== by 0x4BEE3CA: parseDerivationOutput (../src/libstore/derivations.cc:383) ==47978== by 0x4BEE3CA: nix::parseDerivation(nix::StoreDirConfig const&, std::__cxx11::basic_string, std::allocator >&&, std::basic_string_view >, nix::ExperimentalFeatureSettings const&) (???:492) ==47978== by 0x3F3803: nix::DerivationTest_UnterminatedString_Test::TestBody() (../src/libstore-tests/derivation/external-formats.cc:27) ==47978== by 0x52AD3DD: void testing::internal::HandleExceptionsInMethodIfSupported(testing::Test*, void (testing::Test::*)(), char const*) (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x5298E3D: testing::Test::Run() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x5298FCC: testing::TestInfo::Run() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x529920E: testing::TestSuite::Run() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x52A3996: testing::internal::UnitTestImpl::RunAllTests() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x52A3F74: testing::UnitTest::Run() (in /nix/store/qyg0071v3bf8vgcnccd6zi0gvc5abs3f-gtest-1.17.0/lib/libgtest.so.1.17.0) ==47978== by 0x49DD52: RUN_ALL_TESTS (gtest.h:2334) ==47978== by 0x49DD52: main (???:16) --- src/libstore-tests/derivation/external-formats.cc | 8 ++++++++ src/libstore/derivations.cc | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/libstore-tests/derivation/external-formats.cc b/src/libstore-tests/derivation/external-formats.cc index 6fee675e98..e31b3e8544 100644 --- a/src/libstore-tests/derivation/external-formats.cc +++ b/src/libstore-tests/derivation/external-formats.cc @@ -15,6 +15,14 @@ TEST_F(DerivationTest, BadATerm_version) parseDerivation(*store, readFile(goldenMaster("bad-version.drv")), "whatever", mockXpSettings), FormatError); } +TEST_F(DerivationTest, UnterminatedString) +{ + ASSERT_THROW( + parseDerivation( + *store, "Derive([(\"out\",\"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo", "bar", mockXpSettings), + FormatError); +} + TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { ASSERT_THROW( diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 8475532bca..37c89461de 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -223,6 +223,7 @@ static BackedStringView parseString(StringViewStream & str) size_t start = 0; size_t end = str.remaining.size(); const auto data = str.remaining.data(); + bool foundClose = false; while (start < end) { auto idx = str.remaining.find('"', start); if (idx == std::string_view::npos) { @@ -233,10 +234,13 @@ static BackedStringView parseString(StringViewStream & str) ; if ((idx - pos) % 2 == 0) { // even number of backslashes end = idx; + foundClose = true; break; } start = idx + 1; } + if (!foundClose) + throw FormatError("unterminated string in derivation"); start = 0; const auto content = str.remaining.substr(start, end); From e6dab09c2f4b87a62dc7b0022548fa7939f688cd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 May 2026 12:01:04 -0400 Subject: [PATCH 373/927] NAR listing: always serialize `executable` field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously `NarListing::Regular::to_json` only included `"executable"` when true, while `MemorySourceAccessor::File::Regular::to_json` always included it. This inconsistency makes it harder for other implementations to use a single JSON definition for both — e.g. with Rust's `#[serde(derive)]` there's no way to have `skip_serializing_if` vary by generic parameter. Always serializing the field can only help clients (they get more information), never hurt them (any correct client already handles both `true` and `false`). --- src/libutil-tests/data/nar-listing/deep.json | 1 + src/libutil/memory-source-accessor/json.cc | 3 +- tests/functional/binary-cache.sh | 22 +++- tests/functional/nar-access.sh | 110 +++++++++++++++++-- 4 files changed, 126 insertions(+), 10 deletions(-) diff --git a/src/libutil-tests/data/nar-listing/deep.json b/src/libutil-tests/data/nar-listing/deep.json index a7ed47c4c0..f58974f66e 100644 --- a/src/libutil-tests/data/nar-listing/deep.json +++ b/src/libutil-tests/data/nar-listing/deep.json @@ -15,6 +15,7 @@ "type": "directory" }, "foo": { + "executable": false, "size": 15, "type": "regular" } diff --git a/src/libutil/memory-source-accessor/json.cc b/src/libutil/memory-source-accessor/json.cc index 84fbb71bb2..ff3808d3c3 100644 --- a/src/libutil/memory-source-accessor/json.cc +++ b/src/libutil/memory-source-accessor/json.cc @@ -51,8 +51,7 @@ void adl_serializer::to_json(json & j, const NarListing::Re { if (r.contents.fileSize) j["size"] = *r.contents.fileSize; - if (r.executable) - j["executable"] = true; + j["executable"] = r.executable; if (r.contents.narOffset) j["narOffset"] = *r.contents.narOffset; } diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 6826345933..b76362bc26 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -276,7 +276,27 @@ nix copy --to "file://$cacheDir"?write-nar-listing=1 "$outPath" diff -u \ <(jq -S < "$cacheDir/$(basename "$outPath" | cut -c1-32).ls") \ - <(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S) + <(jq -S <<'EOF' +{ + "version": 1, + "root": { + "type": "directory", + "entries": { + "bar": { + "type": "regular", + "executable": false, + "size": 4, + "narOffset": 232 + }, + "link": { + "type": "symlink", + "target": "xyzzy" + } + } + } +} +EOF + ) # Test debug info index generation. diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index cd419b4eef..f15a56d688 100755 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -37,24 +37,120 @@ cp -r "$storePath" "$invalidPath" expect 1 nix store cat "$invalidPath/foo/baz" # Test --json. + +# Shallow listing of root (no --recursive) diff -u \ <(nix nar ls --json "$narFile" / | jq -S) \ - <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "directory", + "entries": { + "foo": {}, + "foo-x": {}, + "qux": {}, + "zyx": {} + } +} +EOF + ) + +# Recursive listing of /foo from NAR (includes narOffset) diff -u \ <(nix nar ls --json -R "$narFile" /foo | jq -S) \ - <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "directory", + "entries": { + "bar": { + "type": "regular", + "executable": false, + "size": 0, + "narOffset": 368 + }, + "baz": { + "type": "regular", + "executable": false, + "size": 0, + "narOffset": 552 + }, + "data": { + "type": "regular", + "executable": false, + "size": 58, + "narOffset": 736 + } + } +} +EOF + ) + +# Single file from NAR diff -u \ <(nix nar ls --json -R "$narFile" /foo/bar | jq -S) \ - <(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "regular", + "executable": false, + "size": 0, + "narOffset": 368 +} +EOF + ) + +# Shallow listing from store diff -u \ <(nix store ls --json "$storePath" | jq -S) \ - <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "directory", + "entries": { + "foo": {}, + "foo-x": {}, + "qux": {}, + "zyx": {} + } +} +EOF + ) + +# Recursive listing from store (no narOffset) diff -u \ <(nix store ls --json -R "$storePath/foo" | jq -S) \ - <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S) + <(jq -S <<'EOF' +{ + "type": "directory", + "entries": { + "bar": { + "type": "regular", + "executable": false, + "size": 0 + }, + "baz": { + "type": "regular", + "executable": false, + "size": 0 + }, + "data": { + "type": "regular", + "executable": false, + "size": 58 + } + } +} +EOF + ) + +# Single file from store diff -u \ - <(nix store ls --json -R "$storePath/foo/bar"| jq -S) \ - <(echo '{"type":"regular","size":0}' | jq -S) + <(nix store ls --json -R "$storePath/foo/bar" | jq -S) \ + <(jq -S <<'EOF' +{ + "type": "regular", + "executable": false, + "size": 0 +} +EOF + ) # Test missing files. expect 1 nix store ls --json -R "$storePath/xyzzy" 2>&1 | grep 'does not exist' From 8ecf74430e3e4035444cff133a5e86db355738b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 19:05:10 +0200 Subject: [PATCH 374/927] Fix posix_fallocate() error case On error, it returns errno directly, not -1. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0bc7b6e1b6..a963f2d3cd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -218,7 +218,7 @@ LocalStore::LocalStore(ref config) #if HAVE_POSIX_FALLOCATE res = posix_fallocate(fd.get(), 0, gcSettings.reservedSize); #endif - if (res == -1) { + if (res != 0) { writeFull(fd.get(), std::string(gcSettings.reservedSize, 'X')); [[gnu::unused]] auto res2 = From 033ac553ccd2959c053b292da8e89a991ef31cfd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 20:10:28 +0200 Subject: [PATCH 375/927] Avoid size_t overflow --- src/libutil/serialise.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 5345883ceb..0477cd2c9a 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -28,14 +28,14 @@ void BufferedSink::operator()(std::string_view data) while (!data.empty()) { /* Optimisation: bypass the buffer if the data exceeds the buffer size. */ - if (bufPos + data.size() >= bufSize) { + if (data.size() >= bufSize - bufPos) { flush(); writeUnbuffered(data); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer when it's full. */ - size_t n = bufPos + data.size() > bufSize ? bufSize - bufPos : data.size(); + size_t n = data.size() > bufSize - bufPos ? bufSize - bufPos : data.size(); memcpy(buffer.get() + bufPos, data.data(), n); data.remove_prefix(n); bufPos += n; From f61d9920d9206359ac12b0c080b2c8edca68cc7c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 20:10:59 +0200 Subject: [PATCH 376/927] Avoid calling lseek() with an unintended negative offset --- src/libutil/serialise.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 0477cd2c9a..d989f1d090 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -287,6 +287,8 @@ void FdSource::skip(size_t len) #ifndef _WIN32 /* If we can, seek forward in the file to skip the rest. */ if (isSeekable && len) { + if (len > static_cast(std::numeric_limits::max())) + throw Error("cannot skip %d bytes: exceeds maximum file offset", len); if (lseek(fd, len, SEEK_CUR) == -1) { if (errno == ESPIPE) isSeekable = false; From 7afb83f0b694b9036f245a7cd8fad722bceca972 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 May 2026 20:29:18 +0200 Subject: [PATCH 377/927] readError(): Replace assertions by exceptions We shouldn't crash on input from the other side. --- src/libutil/serialise.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index d989f1d090..7633732672 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -560,7 +560,8 @@ template StringSet readStrings(Source & source); Error readError(Source & source) { auto type = readString(source); - assert(type == "Error"); + if (type != "Error") + throw SerialisationError("unexpected error type '%s'", type); auto level = (Verbosity) readInt(source); [[maybe_unused]] auto name = readString(source); // removed auto msg = readString(source); @@ -569,11 +570,13 @@ Error readError(Source & source) .msg = HintFmt(msg), }; auto havePos = readNum(source); - assert(havePos == 0); + if (havePos != 0) + throw SerialisationError("deserializing error positions is not supported"); auto nrTraces = readNum(source); for (size_t i = 0; i < nrTraces; ++i) { havePos = readNum(source); - assert(havePos == 0); + if (havePos != 0) + throw SerialisationError("deserializing error positions is not supported"); info.traces.push_back(Trace{.hint = HintFmt(readString(source))}); } return Error(std::move(info)); From 22d1e6eef7eaaac25b322ac141bad07d25239357 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 30 Nov 2025 14:00:03 -0500 Subject: [PATCH 378/927] manual: More on stores, building and mounting in the file system Expand the manual on 1. Store paths: be more explicit about store path base names, introduce `{#digest}`, `{#name}`, and `{#base-name}` anchors, and add cross-links from `introduction.md`, `derivations.md`, `derivation-aterm.md`, `input-address.md`, and `rl-2.33.md`. 2. How store objects are exposed on the file system (new `os-file-system.md` page and new `{#exposing}` section of the store path page). 3. Building derivations: rewrite `building.md` with new sections on the build lifecycle, derivation resolution, file system setup (including `__structuredAttrs`/`passAsFile` file details), environment variables, and output processing. Try to be clearer about what *must* happen (spec, implementation agnostic), vs what Nix happens to do today. 4. Binary cache protocol: new `protocols/binary-cache/` section with an index page, a `.narinfo` format page (with field-by-field mapping to the JSON store object info schema), and move `nix-cache-info.md` into it. Link binary cache store types (HTTP, local, S3) back to the protocol page. 5. Store object metadata: new `{#metadata}` section in `store-object.md` linking to the JSON and narinfo formats. 6. Build trace: add `{#entry}` anchor, update JSON schema links. 7. Miscellaneous: fix typos in `os-file-system.md`, update `nix path-info --json-format` docs to cover v3, link `nix log` to store path base name and store object metadata, add `#store-directory` redirect, update stale links to match renamed anchors and moved pages. Co-authored-by: Eelco Dolstra Co-authored-by: Sergei Zimmerman --- doc/manual/redirects.json | 5 +- doc/manual/source/SUMMARY.md.in | 7 +- doc/manual/source/_redirects | 1 + doc/manual/source/command-ref/nix-hash.md | 20 +- .../source/command-ref/nix-prefetch-url.md | 2 +- doc/manual/source/glossary.md | 9 +- doc/manual/source/introduction.md | 2 +- .../source/language/advanced-attributes.md | 4 +- doc/manual/source/language/derivations.md | 2 +- .../binary-cache-substituter.md | 4 +- .../source/package-management/profiles.md | 3 +- .../source/protocols/binary-cache/index.md | 19 ++ .../source/protocols/binary-cache/narinfo.md | 42 +++ .../{ => binary-cache}/nix-cache-info.md | 4 +- .../source/protocols/derivation-aterm.md | 2 +- .../json/schema/build-trace-entry-v3.yaml | 6 +- .../json/schema/store-object-info-v3.yaml | 2 +- .../protocols/json/schema/store-path-v1.yaml | 2 +- doc/manual/source/protocols/nix32.md | 2 +- doc/manual/source/protocols/store-path.md | 6 +- doc/manual/source/release-notes/rl-2.33.md | 4 +- doc/manual/source/store/build-trace.md | 2 +- doc/manual/source/store/building.md | 246 +++++++++++++----- .../source/store/derivation/outputs/index.md | 2 +- .../store/derivation/outputs/input-address.md | 2 +- .../file-system-object/os-file-system.md | 40 +++ doc/manual/source/store/index.md | 31 ++- doc/manual/source/store/store-object.md | 6 + .../store/store-object/content-address.md | 2 +- doc/manual/source/store/store-path.md | 130 ++++++--- src/libstore/http-binary-cache-store.md | 2 +- .../include/nix/store/local-settings.hh | 2 +- src/libstore/local-binary-cache-store.md | 2 +- src/libstore/s3-binary-cache-store.md | 2 +- src/nix/flake.md | 4 +- src/nix/hash-convert.md | 4 +- src/nix/key-generate-secret.md | 2 +- src/nix/log.md | 14 +- src/nix/path-info.cc | 6 +- src/nix/path-info.md | 5 +- 40 files changed, 490 insertions(+), 162 deletions(-) create mode 100644 doc/manual/source/protocols/binary-cache/index.md create mode 100644 doc/manual/source/protocols/binary-cache/narinfo.md rename doc/manual/source/protocols/{ => binary-cache}/nix-cache-info.md (90%) create mode 100644 doc/manual/source/store/file-system-object/os-file-system.md diff --git a/doc/manual/redirects.json b/doc/manual/redirects.json index 0a6c715080..329560ce78 100644 --- a/doc/manual/redirects.json +++ b/doc/manual/redirects.json @@ -337,7 +337,7 @@ "string-literal": "string-literals.html" }, "language/derivations.html": { - "builder-execution": "../store/building.html#builder-execution" + "builder-execution": "../store/building.html" }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", @@ -363,6 +363,9 @@ "reverting": "contributing.html#reverting", "branches": "contributing.html#branches" }, + "store/store-path.html": { + "store-directory": "#store-directory-path" + }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", "package-attribute-set": "#package", diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 5a17426b90..3efe9a38de 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -19,9 +19,10 @@ - [Nix Store](store/index.md) - [File System Object](store/file-system-object.md) - [Content-Addressing File System Objects](store/file-system-object/content-address.md) + - [Exposing in OS File Systems](store/file-system-object/os-file-system.md) - [Store Object](store/store-object.md) - [Content-Addressing Store Objects](store/store-object/content-address.md) - - [Store Path](store/store-path.md) + - [Store Path and Store Directory](store/store-path.md) - [Store Derivation and Deriving Path](store/derivation/index.md) - [Derivation Outputs and Types of Derivations](store/derivation/outputs/index.md) - [Content-addressing derivation outputs](store/derivation/outputs/content-address.md) @@ -137,7 +138,9 @@ - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) - - [Nix Cache Info Format](protocols/nix-cache-info.md) + - [Binary Cache](protocols/binary-cache/index.md) + - [`nix-cache-info` Format](protocols/binary-cache/nix-cache-info.md) + - [`.narinfo` Format](protocols/binary-cache/narinfo.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Nix32 Encoding](protocols/nix32.md) - [C API](c-api.md) diff --git a/doc/manual/source/_redirects b/doc/manual/source/_redirects index 7e4557f7d5..be82fe872d 100644 --- a/doc/manual/source/_redirects +++ b/doc/manual/source/_redirects @@ -47,6 +47,7 @@ /package-management/package-management /package-management 301! /package-management/s3-substituter /store/types/s3-binary-cache-store 301! +/protocols/nix-cache-info /protocols/binary-cache/nix-cache-info 301! /protocols/protocols /protocols 301! /json/* /protocols/json/:splat 301! diff --git a/doc/manual/source/command-ref/nix-hash.md b/doc/manual/source/command-ref/nix-hash.md index 7c17ce9095..c1a4251b09 100644 --- a/doc/manual/source/command-ref/nix-hash.md +++ b/doc/manual/source/command-ref/nix-hash.md @@ -45,20 +45,20 @@ md5sum`. - `--base32` - Print the hash in a base-32 representation rather than hexadecimal. - This base-32 representation is more compact and can be used in Nix + Print the hash in [Nix32](@docroot@/protocols/nix32.md) representation rather than hexadecimal. + This representation is more compact and can be used in Nix expressions (such as in calls to `fetchurl`). - `--base64` - Similar to --base32, but print the hash in a base-64 representation, - which is more compact than the base-32 one. + Similar to `--base32`, but print the hash in a [Base64](https://en.wikipedia.org/wiki/Base64) representation, + which is more compact than the Nix32 one. - `--sri` - Print the hash in SRI format with base-64 encoding. + Print the hash in [SRI](@docroot@/glossary.md#gloss-sri) format with Base64 encoding. The type of hash algorithm will be prepended to the hash string, - followed by a hyphen (-) and the base-64 hash body. + followed by a hyphen (-) and the Base64 hash body. - `--truncate` @@ -71,18 +71,18 @@ md5sum`. - `--to-base16` - Don’t hash anything, but convert the base-32 hash representation + Don’t hash anything, but convert the [Nix32](@docroot@/protocols/nix32.md) hash representation *hash* to hexadecimal. - `--to-base32` Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-32. + *hash* to [Nix32](@docroot@/protocols/nix32.md). - `--to-base64` Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-64. + *hash* to Base64. - `--to-sri` @@ -134,7 +134,7 @@ $ nix-hash --type sha256 --flat test/world 5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 ``` -Converting between hexadecimal, base-32, base-64, and SRI: +Converting between hexadecimal, Nix32, Base64, and SRI: ```console $ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 diff --git a/doc/manual/source/command-ref/nix-prefetch-url.md b/doc/manual/source/command-ref/nix-prefetch-url.md index 8451778ad4..86c20b9e1d 100644 --- a/doc/manual/source/command-ref/nix-prefetch-url.md +++ b/doc/manual/source/command-ref/nix-prefetch-url.md @@ -32,7 +32,7 @@ Otherwise, the file is downloaded, and an error is signaled if the actual hash of the file does not match the specified hash. This command prints the hash on standard output. -The hash is printed using base-32 unless `--type md5` is specified, +The hash is printed using [Nix32](@docroot@/protocols/nix32.md) unless `--type md5` is specified, in which case it's printed using base-16. Additionally, if the option `--print-path` is used, the path of the downloaded file in the Nix store is also printed. diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index d7436a2052..112b0e3f8c 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -104,7 +104,7 @@ A derivation can be thought of as a [pure function](https://en.wikipedia.org/wiki/Pure_function) that produces new [store objects][store object] from existing store objects. - Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md#builder-execution). + Derivations are implemented as [operating system processes that run in a sandbox](@docroot@/store/building.md). This sandbox by default only allows reading from store objects specified as inputs, and only allows writing to designated [outputs][output] to be [captured as store objects](@docroot@/store/building.md#processing-outputs). A derivation is typically specified as a [derivation expression] in the [Nix language], and [instantiated][instantiate] to a [store derivation]. @@ -375,6 +375,13 @@ [path]: ./language/types.md#type-path [attribute name]: ./language/types.md#type-attrs +- [SRI]{#gloss-sri} + + [Subresource Integrity](https://www.w3.org/TR/SRI/) (SRI) is a [W3C specification](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) for integrity metadata. + Nix uses the SRI hash format (`-`) to specify content hashes in a way that is self-describing, since the hash algorithm is part of the format. + + [SRI]: #gloss-sri + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index 85de7982c9..33c8ba29fa 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -49,7 +49,7 @@ builds correctly on your system, this is because you specified the dependency explicitly. This takes care of the build-time dependencies. Once a package is built, runtime dependencies are found by scanning -binaries for the hash parts of Nix store paths (such as `r8vvq9kq…`). +binaries for the [hash parts](@docroot@/store/store-path.md#digest) of Nix store paths (such as `r8vvq9kq…`). This sounds risky, but it works extremely well. ## Multi-user support diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index 67612029c8..cc2743dc5c 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -337,8 +337,8 @@ Here is more information on the `output*` attributes, and what values they may b This will specify the output hash of the single output of a [fixed-output derivation]. - The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). - The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of base-32 encoding. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](@docroot@/glossary.md#gloss-sri). + The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of Base32 encoding. > **Note** > diff --git a/doc/manual/source/language/derivations.md b/doc/manual/source/language/derivations.md index 2403183fc2..50aa525acb 100644 --- a/doc/manual/source/language/derivations.md +++ b/doc/manual/source/language/derivations.md @@ -165,7 +165,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > > for an Autoconf-style package. - The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used. + The name of an output is combined with the name of the derivation to create the [name part](@docroot@/store/store-path.md#name) of the output's store path, unless it is `out`, in which case just the name of the derivation is used. > **Example** > diff --git a/doc/manual/source/package-management/binary-cache-substituter.md b/doc/manual/source/package-management/binary-cache-substituter.md index e6a772213d..bc2cdfb27a 100644 --- a/doc/manual/source/package-management/binary-cache-substituter.md +++ b/doc/manual/source/package-management/binary-cache-substituter.md @@ -19,7 +19,7 @@ whatever port you like: $ nix-serve -p 8080 ``` -To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) file on the client: +To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/binary-cache/nix-cache-info.md) file on the client: ```console $ curl http://avalon:8080/nix-cache-info @@ -28,7 +28,7 @@ WantMassQuery: 1 Priority: 30 ``` -When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) automatically if it doesn't exist. +When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/binary-cache/nix-cache-info.md) automatically if it doesn't exist. On the client side, you can tell Nix to use your binary cache using `--substituters`, e.g.: diff --git a/doc/manual/source/package-management/profiles.md b/doc/manual/source/package-management/profiles.md index 1d9e672a8d..53cf5061f8 100644 --- a/doc/manual/source/package-management/profiles.md +++ b/doc/manual/source/package-management/profiles.md @@ -11,8 +11,7 @@ in a directory another version might be stored in `/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2`. The long strings prefixed to the directory names are cryptographic hashes (to be -precise, 160-bit truncations of SHA-256 hashes encoded in a base-32 -notation) of *all* inputs involved in building the package — sources, +precise, 160-bit truncations of SHA-256 hashes encoded in [Nix32](@docroot@/protocols/nix32.md)) of *all* inputs involved in building the package — sources, dependencies, compiler flags, and so on. So if two packages differ in any way, they end up in different locations in the file system, so they don’t interfere with each other. Here is what a part of a typical Nix diff --git a/doc/manual/source/protocols/binary-cache/index.md b/doc/manual/source/protocols/binary-cache/index.md new file mode 100644 index 0000000000..d86d307ad9 --- /dev/null +++ b/doc/manual/source/protocols/binary-cache/index.md @@ -0,0 +1,19 @@ +# Binary Cache + +The binary cache format is an interface designed for exposing a store over HTTP. + +A binary cache consists of: + +- A [`nix-cache-info`](./nix-cache-info.md) file at the root with remote-side configuration. +- For each [store object](@docroot@/store/store-object.md): + - A [`.narinfo`](./narinfo.md) file containing the object's [metadata](@docroot@/store/store-object.md#metadata) and a (usually relative) URL to the corresponding compressed NAR. + - A possibly-compressed [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) containing the store object's file system data. +- For every entry in the [build trace](@docroot@/store/build-trace.md), a JSON file at `build-trace-v2//.doi`: + - the path encodes the [key](@docroot@/protocols/json/build-trace-entry.md#key) + - the contents are the [value](@docroot@/protocols/json/build-trace-entry.md#value). + +The following [store types](@docroot@/store/types/index.md) use the binary cache format: + +- [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md) — served over HTTP(S) +- [Local Binary Cache Store](@docroot@/store/types/local-binary-cache-store.md) — stored on the file system +- [S3 Binary Cache Store](@docroot@/store/types/s3-binary-cache-store.md) — stored in an AWS S3 bucket diff --git a/doc/manual/source/protocols/binary-cache/narinfo.md b/doc/manual/source/protocols/binary-cache/narinfo.md new file mode 100644 index 0000000000..e2e2efac0e --- /dev/null +++ b/doc/manual/source/protocols/binary-cache/narinfo.md @@ -0,0 +1,42 @@ +# `.narinfo` Format + +A `.narinfo` file contains the [metadata of a store object](@docroot@/store/store-object.md#metadata) in the [binary cache](@docroot@/protocols/binary-cache/index.md) format. +It is a simple line-oriented format where each line is a `Key: Value` pair. +Some keys (e.g. `Sig`) may appear multiple times. + +The file is named `.narinfo`, where `` is the [hash part](@docroot@/store/store-path.md#digest) of the store object's [store path](@docroot@/store/store-path.md). + +The fields correspond to those documented in the [store object info](@docroot@/protocols/json/store-object-info.md) JSON format: + +| `.narinfo` field | JSON field | Differences | +|---|---|---| +| `StorePath` | [`path`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_path) | Full [store path](@docroot@/store/store-path.md) rather than [store path base name](@docroot@/store/store-path.md#base-name) | +| `URL` | [`url`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_url) | | +| `Compression` | [`compression`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_compression) | Defaults to `bzip2` if omitted | +| `FileHash` | [`downloadHash`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_downloadHash) | String-encoded hash rather than structured | +| `FileSize` | [`downloadSize`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_downloadSize) | | +| `NarHash` | [`narHash`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_narHash) | String-encoded hash rather than structured | +| `NarSize` | [`narSize`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_narSize) | | +| `References` | [`references`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_references) | Space-separated [store path base names](@docroot@/store/store-path.md#base-name) rather than a JSON array | +| `Deriver` | [`deriver`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_deriver) | [Store path base name](@docroot@/store/store-path.md#base-name); `unknown-deriver` instead of `null` | +| `Sig` | [`signatures`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_signatures) | May appear multiple times rather than using an array | +| `CA` | [`ca`](@docroot@/protocols/json/store-object-info.md#oneOf_i2_ca) | String-encoded [content address](@docroot@/store/store-object/content-address.md) rather than structured | + +## Example + + + +``` +StorePath: /nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo +URL: nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz?sha256=1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3 +Compression: xz +FileHash: sha256:09ymwqf5i9q7d4dm7x4pjjcqqj0qrcp5lnznbh42gfsci5hcbqqm +FileSize: 4029176 +NarHash: sha256:09ymwqf5i9q7d4dm7x4pjjcqqj0qrcp5lnznbh42gfsci5hcbqqm +NarSize: 34878 +References: g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar n5wkd9frr45pa74if5gpz9j7mifg27fh-foo +Deriver: g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv +Sig: asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +Sig: qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA== +CA: fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh +``` diff --git a/doc/manual/source/protocols/nix-cache-info.md b/doc/manual/source/protocols/binary-cache/nix-cache-info.md similarity index 90% rename from doc/manual/source/protocols/nix-cache-info.md rename to doc/manual/source/protocols/binary-cache/nix-cache-info.md index e8351e1ceb..3859b2a10a 100644 --- a/doc/manual/source/protocols/nix-cache-info.md +++ b/doc/manual/source/protocols/binary-cache/nix-cache-info.md @@ -1,6 +1,6 @@ -# Nix Cache Info Format +# `nix-cache-info` Format -The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/package-management/binary-cache-substituter.md) (e.g., `https://cache.example.com/nix-cache-info`). +The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/protocols/binary-cache/index.md) (e.g., `https://cache.example.com/nix-cache-info`). MIME type: `text/x-nix-cache-info` diff --git a/doc/manual/source/protocols/derivation-aterm.md b/doc/manual/source/protocols/derivation-aterm.md index 523678e663..778614eb16 100644 --- a/doc/manual/source/protocols/derivation-aterm.md +++ b/doc/manual/source/protocols/derivation-aterm.md @@ -26,7 +26,7 @@ Derivations are serialised in one of the following formats: When derivation is encoded to a [store object] we make the following choices: -- The store path name is the derivation name with `.drv` suffixed at the end +- The store path [name](@docroot@/store/store-path.md#name) is the derivation name with `.drv` suffixed at the end Indeed, the ATerm format above does *not* contain the name of the derivation, on the assumption that a store path will also be provided out-of-band. diff --git a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml index 3ff606672c..2e825af6fb 100644 --- a/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml +++ b/doc/manual/source/protocols/json/schema/build-trace-entry-v3.yaml @@ -4,7 +4,7 @@ title: Build Trace Entry description: | A record of a successful build outcome for a specific derivation output. - This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md). + This schema describes the JSON representation of an [entry](@docroot@/store/build-trace.md#entry) in a [build trace](@docroot@/store/build-trace.md). > **Warning** > @@ -39,7 +39,7 @@ additionalProperties: false key: title: Build Trace Key description: | - A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. + A [build trace entry](@docroot@/store/build-trace.md#entry) is a key-value pair. This is the "key" part, referring to a derivation and output. type: object required: @@ -61,7 +61,7 @@ additionalProperties: false value: title: Build Trace Value description: | - A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair. + A [build trace entry](@docroot@/store/build-trace.md#entry) is a key-value pair. This is the "value" part, describing an output. type: object required: diff --git a/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml index e0be716ef7..e0c1ffef13 100644 --- a/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml +++ b/doc/manual/source/protocols/json/schema/store-object-info-v3.yaml @@ -109,7 +109,7 @@ $defs: type: string title: Store Directory description: | - The [store directory](@docroot@/store/store-path.md#store-directory) this store object belongs to (e.g. `/nix/store`). + The [path to the store directory](@docroot@/store/store-path.md#store-directory-path) this store object belongs within (e.g. `/nix/store`). additionalProperties: false impure: diff --git a/doc/manual/source/protocols/json/schema/store-path-v1.yaml b/doc/manual/source/protocols/json/schema/store-path-v1.yaml index f1f58c2bf1..b1251c7426 100644 --- a/doc/manual/source/protocols/json/schema/store-path-v1.yaml +++ b/doc/manual/source/protocols/json/schema/store-path-v1.yaml @@ -24,7 +24,7 @@ description: | The format follows this pattern: `${digest}-${name}` - - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) + - **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md) (20 hash bytes become 32 ASCII characters) - **name**: The package name and optional version/suffix information type: string diff --git a/doc/manual/source/protocols/nix32.md b/doc/manual/source/protocols/nix32.md index 72afe893ea..d8da1e9952 100644 --- a/doc/manual/source/protocols/nix32.md +++ b/doc/manual/source/protocols/nix32.md @@ -1,6 +1,6 @@ # Nix32 Encoding -Nix32 is Nix's variant of base-32 encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. +Nix32 is Nix's variant of [Base32](https://en.wikipedia.org/wiki/Base32) encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute. ## Alphabet diff --git a/doc/manual/source/protocols/store-path.md b/doc/manual/source/protocols/store-path.md index 1aa79615d1..bdce19d1d6 100644 --- a/doc/manual/source/protocols/store-path.md +++ b/doc/manual/source/protocols/store-path.md @@ -18,11 +18,9 @@ where - `name` = the name of the store object. -- `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory) +- `store-dir` = the [path of the store directory](@docroot@/store/store-path.md#store-directory-path) -- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. - - Nix uses a custom base-32 encoding called [Nix32](@docroot@/protocols/nix32.md). +- `digest` = [Nix32](@docroot@/protocols/nix32.md) representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`. For the definition of the hash compression algorithm, please refer to section 5.1 of the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf). diff --git a/doc/manual/source/release-notes/rl-2.33.md b/doc/manual/source/release-notes/rl-2.33.md index bed6970293..668230693b 100644 --- a/doc/manual/source/release-notes/rl-2.33.md +++ b/doc/manual/source/release-notes/rl-2.33.md @@ -149,9 +149,9 @@ The new structured format follows the [JSON guidelines](@docroot@/development/js } ``` - The map from store path base names to store object info is nested under the `info` field. + The map from [store path base names](@docroot@/store/store-path.md#base-name) to store object info is nested under the `info` field. -- **Store path base names instead of full paths**: +- **[Store path base names](@docroot@/store/store-path.md#base-name) instead of full paths**: Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths. Combined with `storeDir`, the full path can be reconstructed. diff --git a/doc/manual/source/store/build-trace.md b/doc/manual/source/store/build-trace.md index a879d37d20..cb9cb30996 100644 --- a/doc/manual/source/store/build-trace.md +++ b/doc/manual/source/store/build-trace.md @@ -8,7 +8,7 @@ The *build trace* is a [memoization table](https://en.wikipedia.org/wiki/Memoization) for builds. It maps the inputs of builds to the outputs of builds. -Concretely, that means it maps [derivations][derivation] to maps of [output] names to [store objects][store object]. +Each *[entry]{#entry}* in the build trace maps a [derivation][derivation] to a map of [output] names to [store objects][store object]. In general the derivations used as a key should be [*resolved*](./resolution.md). A build trace with all-resolved-derivation keys is also called a *base build trace* for extra clarity. diff --git a/doc/manual/source/store/building.md b/doc/manual/source/store/building.md index 32e8001293..0874134064 100644 --- a/doc/manual/source/store/building.md +++ b/doc/manual/source/store/building.md @@ -1,72 +1,178 @@ # Building -## Normalizing derivation inputs +As discussed in the [main page on derivations](./derivation/index.md): -- Each input must be [realised] prior to building the derivation in question. +> A derivation is a specification for running an executable on precisely defined input to produce one or more [store objects][store object]. + +This page describes *building* a derivation, which is to say following the instructions in the derivation to actually run the executable. +Some elements of derivations are self-explanatory. +For example, the arguments specified in the derivation really are the arguments passed to the executable. +In other cases, however, there is additional common steps performed by Nix for all derivations --- mostly for setting up the build environment and collecting the built outputs. + +The chief design consideration for the building process is *determinism*. +Conventional operating systems are typically not designed with determinism in mind. +But determinism is needed to make Nix's build caching a transparent abstraction. + +> **Explanation** +> +> For example, no one wants to slightly modify a derivation, and then find that it no longer builds for an unrelated reason, because the original derivation *also* doesn't build anymore, but the cache hit on the original derivation was hiding this. +> We want builds that succeed once to continue succeeding, to encourage fearless modification of old build recipes. +> Determinism is what enables things that once worked to keep working. + +The life cycle of a build can be broken down into 3 parts: + +1. Spawn the builder process with the proper environment, including the correct process arguments, environment variables, and file system state. + +2. Wait for the builder process to exit and collect its exit status. + Exit code 0 means success; anything else is a build failure. + (Strictly speaking, Nix detects process exit by waiting for the standard output and error streams to close. + If a builder explicitly closes these streams without exiting, Nix will kill it, and deem the build a failure. + Processes should therefore exit *without* explicitly closing those standard streams, and let the exiting of the process close them implicitly.) + + Nix also logs the standard output and error of the process, but this is just for human convenience and does not influence the behavior of the system. + (Builder processes have no idea what the consumer of their standard output and error does with the pseudo-terminal master, only that they are indeed consumed so buffers do not fill up etc. and writes to each output standard stream will continue to succeed. + In practice, Nix will store the log in `/nix/var/log/nix`) + +3. Processing the outputs after the builder has exited. + + The builder process on exit should have left behind files for each output the derivation is supposed to produce. + The files must be processed to turn them into bona fide store objects. + If the processing succeeds, those store objects are associated with the derivation as (the results of) a successful build. + +Step (3) is done by Nix externally to the build itself, which is just steps (1) and (2). +In step (3), just inert data is processed, since the builder process has exited or been killed by then. +Step (1) however is best described not from Nix's perspective, but from the build process's perspective. + +> **Explanation** +> +> Ultimately, what matters for determinism is what the build process can observe: what resources (files, networking, etc.) it can see, what syscalls succeed or fail, etc. +> Nix can achieve this through many different sandboxing strategies (namespaces, VMs, chroots, ...), but the process shouldn't be able to tell them apart. +> We therefore specify building from the process's perspective, not Nix's perspective, to focus on *what*, not *how*. + +## What derivations can be built + +Actually only some derivations are ready to be built. +In particular, only [*resolved*](./resolution.md) derivations can be built. +That is to say, a derivation that depends on other derivations is not ready yet to be built, because some of those other derivations might not have yet been built. +If the other derivations are indeed all built, we can witness this fact by resolving the derivation, and converting all the derivation's input references into plain store paths. + +> **Note** +> +> Note that [input-addressing](derivation/outputs/input-address.md) derivations are improperly resolved. +> As discussed on the linked page, the current input-addressing algorithm does not respect resolution-equivalence of derivations (\\(\\sim_\mathrm{Drv}\\)). +> That means that if Nix properly resolved an input-addressed derivation, the resolved derivation would have different input addresses, violating expectations. +> Nix therefore improperly resolves the derivation, keeping its original input-addressed output paths, creating an invalid derivation that is both resolved and instructed to create the outputs at the originally expected paths. + +## Environment of the builder process + +This section describes how the [`builder`](./derivation/index.md#builder) is executed. + +> **Implementation detail** +> +> Nix prevents multiple [Nix instances][Nix instance] from performing the same build at the same time, for example by acquiring exclusive file locks. + +### File system + +The builder should have access to a limited file system where only certain objects are available. +The most important exposed files are the inputs (other store objects) of the (resolved) derivation. +Additionally, some other files are exposed. + +#### Store inputs + +The builder will be run against a file system in which the [store directory][store directory path] contains the [closure] of the inputs. +In particular, consider a store that just contains this closure. +That store is exposed to the file system according to the rules specified in the [Exposing Store Objects in OS File Systems](./store-path.md#exposing) documentation. +This precisely defines the file system layout of the store that should be visible to the builder process. + +> **Note** +> +> Historically, Nix exposed *at least* the following store contents to the builder, but also arbitrarily other store objects, due to limitations around operating systems' file system virtualization capabilities, and wanting to avoid copying or moving files. +> It still can do this in so-called *unsandboxed* builds. +> +> Such builds should be considered discouraged, but one that works less badly against non-mischievous derivations than might be expected. +> This is because store paths are relatively unpredictable, so a well-behaved program is unlikely to stumble upon a store object it wasn't supposed to know about. +> +> As operating systems developed better file system primitives, the need for disabling sandboxing has lessened greatly over the years, and this trend should continue into the future. + +The outputs are expected to be created in that store directory as if they were valid store objects. +(They are just files during builder execution, but during [processing outputs](#processing-outputs) they will be turned into proper store objects.) +The [environment variables](#env-vars) for each output indicate where the builder should write them; +Nix ensures that those paths do not yet exist when the builder is run. + +> **Note** +> +> In sandboxed builds, ensuring that the outputs do not exist in the store directory is trivial. +> In unsandboxed builds, it is harder in general. +> In the worst case, the derivation is in fact rewritten so different output paths are used instead, and then the outputs are rewritten back to the intended output paths after. +> In the content-addressing case rewriting would be needed either way, but in the input-addressing case, this is a significant degradation, as the point of input addressing is to avoid rewrites by knowing output paths in advance. [realised]: @docroot@/glossary.md#gloss-realise +[closure]: @docroot@/glossary.md#gloss-closure +[store directory path]: ./store-path.md#store-directory-path + +### Other file system state + +- The current working directory of the builder process will be a fresh temporary directory. + It is initially empty when the process starts except for a few input files: -- Once this is done, the derivation is *normalized*, replacing each input deriving path with its store path, which we now know from realising the input. + - If [`__structuredAttrs`](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs) is enabled: `.attrs.json` (the derivation attributes as JSON) and `.attrs.sh` (a Bash-compatible rendering of the same). + The environment variables `NIX_ATTRS_JSON_FILE` and `NIX_ATTRS_SH_FILE` point to these files, respectively. -## Builder Execution {#builder-execution} + - If [`passAsFile`](@docroot@/language/advanced-attributes.md#adv-attr-passAsFile) is used (only without `__structuredAttrs`): for each attribute name listed, a file `.attr-` where `` is the [Nix32](@docroot@/protocols/nix32.md)-encoded SHA-256 hash of the attribute name. + The environment variable `Path` points to the file containing the attribute's value. -The [`builder`](./derivation/index.md#builder) is executed as follows: + In sandboxed builds, this directory is at a deterministic path inside the sandbox (controlled by the [`sandbox-build-dir`](@docroot@/command-ref/conf-file.md#conf-sandbox-build-dir) setting, default `/build`). + See also the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for the host-side location. -- A temporary directory is created where the build will take place. The - current directory is changed to this directory. +- Basic device nodes for essential operations (null device, random number generation, standard streams as a pseudo terminal) - See the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for more information. + (A pseudo terminal would not be strictly necessary since the standard streams are passively logging, not there to facilitate interaction. + But it is still useful to entice programs to do nicer logging with e.g. colors etc.) -- The environment is cleared and set to the derivation attributes, as - specified above. +- On Linux: Process information via `/proc` -- In addition, the following variables are set: +- Minimal user and group identity information - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. +- A loopback-only network configuration with hostname set to `localhost` - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. +> **Note** +> +> Fixed-output derivations have access to additional operating system state to facilitate communication with the outside world, such as network name resolution and TLS certificate verification. +> This is necessary because these derivations are allowed to access the network, unlike regular derivations which are fully sandboxed. - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. +### Environment variables {#env-vars} - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. +The environment is cleared and set to the derivation attributes, as +specified above. - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). +For most derivations types this must contain at least: - - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` - is set to `true` for the derivation. A detailed explanation of this - behavior can be found in the - [section about structured attrs](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs). +- For each output declared in `outputs`, the corresponding environment variable is set to point to the intended path in the Nix store for that output. + Each output path is a concatenation of the cryptographic hash of all build inputs, the `name` attribute and the output name. + (The output name is omitted if it's `out`.) - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) +In addition, the following variables are set: -- If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple [Nix instances][Nix instance] from performing the same - build at the same time. +- `NIX_BUILD_TOP` contains the path of the temporary directory for this build. -- A log of the combined standard output and error is written to - `/nix/var/log/nix`. +- Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the temporary directory. + This is to prevent the builder from accidentally writing temporary files anywhere else. + Doing so might cause interference by other processes. -- The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. +- `PATH` is set to `/path-not-set` to prevent shells from initialising it to their built-in default value. -- The temporary directory is removed (unless the `-K` option was - specified). +- `HOME` is set to `/homeless-shelter`. + (Without sandboxing, this discourages programs from using `/etc/passwd` or the like to find the user's home directory, which could cause impurity.) + Usually, when `HOME` is set, it is used as the location of the home directory, even if it points to a non-existent path. + +- `NIX_STORE` is set to the path of the top-level Nix [store directory path] (typically, `/nix/store`). + +- `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` is set to `true` for the derivation. + A detailed explanation of this behavior can be found in the [section about structured attrs](@docroot@/language/advanced-attributes.md#adv-attr-structuredAttrs). + +### Arguments + +The builder is passed the arguments specified by the derivation attribute `args`. ## Processing outputs @@ -74,28 +180,40 @@ If the builder exited successfully, the following steps happen in order to turn - **Normalize the file permissions** - Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Any possible `setuid` and `setgid` - bits are cleared. - - > **Note** - > - > Setuid and setgid programs are not currently supported by Nix. - > This is because the Nix archives used in deployment have no concept of ownership information, - > and because it makes the build result dependent on the user performing the build. + The files must conform to the model described in the [Exposing in OS file systems](./file-system-object/os-file-system.md) section. + For example, timestamps and permissions are canonicalised. - **Calculate the references** - Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. + Nix scans each output path for [references] to input store objects by looking for the [digest][store path digest] of each input. + (The name part and the [store directory path] are ignored when scanning; an input's hash part that is neither followed by a `-` nor proceeded by a `/` still scans as a reference.) + Since these are potential runtime dependencies, Nix will register them as references of the output store object they occur in. + + Nix also scans for references from one output to another in the same way, because outputs are allowed to refer to each other. + + The outputs' references must form a [directed acyclic graph](@docroot@/glossary.md#gloss-directed-acyclic-graph). + (This is not a special restriction for outputs; it is true for the references of all store objects in general.) + + In the case of derivations with output paths that are fixed in advance (i.e. [input-addressing] derivations, or [fixed content-addressing] derivations), the actual final store path to each output is used during the build if possible. + For [floating content-addressing] derivations, however, the final store path is not known in advance by definition. + Scratch store paths must therefore be used instead. + Scanning will use those scratch paths, but then any output-to-be that contains such a scanned scratch path must be rewritten to instead use the final (content-addressed) path of the output in question. - Nix also scans for references to other outputs' paths in the same way, because outputs are allowed to refer to each other. - If the outputs' references to each other form a cycle, this is an error, because the references of store objects much be acyclic. +At this point, the file system data is in the proper form, and the valid acyclic reference data for each output is also calculated, so the outputs are added to the store as proper store objects. +Additionally, those store objects (at least in the case that they are [content-addressed][content-addressing]) can be associated with the derivation in the [build trace] in the record for a successful build. +> **Implementation detail** +> +> Nix will normally clean up and remove the temporary build directory after every build, successful or unsuccessful. +> The builder doesn't know whether Nix does or not, however, as it will have exited before the build directory is cleaned up, and it will not see any old build directory if (after a failed build) it is run again. +> The [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) option can be specified to keep the build directory in the case of a failing build. +[references]: ./store-object.md#references +[store path digest]: ./store-path.md#digest +[store object]: ./store-object.md [Nix instance]: @docroot@/glossary.md#gloss-nix-instance +[content-addressing]: ./derivation/outputs/content-address.md +[input-addressing]: ./derivation/outputs/input-address.md +[fixed content-addressing]: ./derivation/outputs/content-address.md#fixed +[floating content-addressing]: ./derivation/outputs/content-address.md#floating +[build trace]: ./build-trace.md diff --git a/doc/manual/source/store/derivation/outputs/index.md b/doc/manual/source/store/derivation/outputs/index.md index ca2ce6665b..9b46405cdf 100644 --- a/doc/manual/source/store/derivation/outputs/index.md +++ b/doc/manual/source/store/derivation/outputs/index.md @@ -9,7 +9,7 @@ The outputs specification is a map, from names to specifications for individual ## Output Names {#outputs} -Output names can be any string which is also a valid [store path](@docroot@/store/store-path.md) name. +Output names can be any string which is also a valid [store path name](@docroot@/store/store-path.md#name). The name mapped to each output specification is not actually the name of the output. In the general case, the output store object has name `derivationName + "-" + outputSpecName`, not any other metadata about it. However, an output spec named "out" describes and output store object whose name is just the derivation name. diff --git a/doc/manual/source/store/derivation/outputs/input-address.md b/doc/manual/source/store/derivation/outputs/input-address.md index 3fd20f17d7..6df9b94961 100644 --- a/doc/manual/source/store/derivation/outputs/input-address.md +++ b/doc/manual/source/store/derivation/outputs/input-address.md @@ -16,7 +16,7 @@ Concretely, this would cause a "mass rebuild" whenever any fetching detail chang To solve this problem, we compute output hashes differently, so that certain output hashes become identical. We call this concept quotient hashing, in reference to quotient types or sets. -So how do we compute the hash part of the output paths of an input-addressed derivation? +So how do we compute the [hash part](@docroot@/store/store-path.md#digest) of the output paths of an input-addressed derivation? This is done by the function `hashQuotientDerivation`, shown below. First, a word on inputs. diff --git a/doc/manual/source/store/file-system-object/os-file-system.md b/doc/manual/source/store/file-system-object/os-file-system.md new file mode 100644 index 0000000000..6ce21f7788 --- /dev/null +++ b/doc/manual/source/store/file-system-object/os-file-system.md @@ -0,0 +1,40 @@ +# Exposing File System Objects in real operating system file systems + +Nix's [file system object] data model is minimal. +All the various other bits and pieces of real world filesystem interfaces, such as [extended file attributes](https://en.wikipedia.org/wiki/Extended_file_attributes), are specifically ignored to reduce our interface surface and the reproducibility issues associated with a larger interface. +In the view of Nix's developers, the types of simple, fine-grained batch jobs (typically, building software) that Nix specializes in simply don't benefit enough from that extra complexity for it to be worth the costs of supporting it. + +But to actually be used by software, file system objects need to be made available through the operating system's file system. +This is sometimes called "mounting" or "exposing" the file system object, though do note it may or may not be implemented with what the operating system calls "mounting". + +[file system object]: ../file-system-object.md + +## Metadata normalization + +File systems typically contain other metadata that is outside Nix's data model. +To avoid this other metadata being a side channel and source of nondeterminism, Nix is careful to normalize to fixed values. +For example, on Unix, the following metadata normalization occurs: + +- The creation and last modification timestamps on all files are set to Unix Epoch 1s (00:00:01 1/1/1970 UTC) + +- The group is set to the [default group](@docroot@/command-ref/conf-file.md#conf-build-users-group) + +- The Unix mode of the file to 0444 or 0555 (i.e., read-only, with execute permission enabled if the file was originally executable). + +- Any possible `setuid` and `setgid` bits are cleared. + + > **Note** + > + > `setuid` and `setgid` programs are not currently supported by Nix. + > These special file system permissions are in general a security footgun, and with data owned by different users in different stores, it would especially be a hazard when copying store objects between stores. + > + > This restriction has not proved to be onerous in practice. + > For example, NixOS uses so called setuid-wrappers which are outside the store. + +> **Explanation** +> +> As discussed before, Nix essentially shares its file system object data model with other tools like Git. +> But those tools tend to ignore this metadata in both directions --- when reading files, like Nix, but when writing files, timestamps are set organically, and the user is free to set other special permissions (`setuid`, `setgid`, sticky, etc.) however they like. +> Normalizing, and not just ignoring, this metadata is therefore what distinguishes Nix from these other tools more than the file system object data model itself. +> +> Nix's approach is motivated by deterministic building. Whereas Git can assume that humans running commands will simply ignore timestamps etc. as appropriate, understanding they are local and ephemeral, Nix aims to run software that was not necessarily designed with Nix in mind, and is unaware of whatever sandboxing/virtualization is in place. diff --git a/doc/manual/source/store/index.md b/doc/manual/source/store/index.md index f1e8f14029..d063fc4fdc 100644 --- a/doc/manual/source/store/index.md +++ b/doc/manual/source/store/index.md @@ -2,4 +2,33 @@ The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. -There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md). +Concretely, albeit using concepts that are only defined in the rest of the chapter, a store consists of: + +- A set of [store objects][store object], the immutable file system data. + + This can also be looked at as a map from [store paths][store path] to store objects. + +- A set of [derivations][derivation], instructions for building store objects. + + This can also be looked at as a map from [store paths][store path] to derivations. + Since store paths to derivations always end in `.drv`, and store paths to other store objects never do, the two maps can also be combined into one. + Derivations can also be encoded as store objects too. + +- A [build trace], a record of which derivations have been built and what they produced. + + > **Warning** + > + > The concept of a build trace is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) + > and subject to change. + +There are [multiple types of Nix stores][store type] with different capabilities, such as the default one on the [local file system][local store] (`/nix/store`) or [binary caches][binary cache]. + +[store object]: ./store-object.md +[store path]: ./store-path.md +[derivation]: ./derivation/index.md +[build trace]: ./build-trace.md + +[store type]: ./types/index.md +[local store]: ./types/local-store.md +[binary cache]: ./types/http-binary-cache-store.md diff --git a/doc/manual/source/store/store-object.md b/doc/manual/source/store/store-object.md index 170d6246cf..eb84ab8437 100644 --- a/doc/manual/source/store/store-object.md +++ b/doc/manual/source/store/store-object.md @@ -66,3 +66,9 @@ A store can only contain a store object if it also contains all the store object > > The "closure property" isn't meant to prohibit, for example, [lazy loading](https://en.wikipedia.org/wiki/Lazy_loading) of store objects. > However, the "closure property" and immutability in conjunction imply that any such lazy loading ought to be deterministic. + +### Store Object Metadata {#metadata} + +[Store implementations](@docroot@/store/types/index.md) currently associate more information than described above with a store object. +Quite arguably some of this information doesn't belong here, because it conflates concerns. +For details see the [store object info](@docroot@/protocols/json/store-object-info.md) JSON format or the [narinfo](@docroot@/protocols/binary-cache/narinfo.md) format. diff --git a/doc/manual/source/store/store-object/content-address.md b/doc/manual/source/store/store-object/content-address.md index 94d94ec6d4..282b7545a2 100644 --- a/doc/manual/source/store/store-object/content-address.md +++ b/doc/manual/source/store/store-object/content-address.md @@ -9,7 +9,7 @@ In particular, the content-addressing scheme will ensure that the digest of the - file system object graph (the root one and its children, if it has any) - references -- [store directory](../store-path.md#store-directory) +- [store directory path](../store-path.md#store-directory-path) - name of the store object, and not any other information, which would not be an intrinsic property of that store object. diff --git a/doc/manual/source/store/store-path.md b/doc/manual/source/store/store-path.md index 04bdfec004..43037f7baa 100644 --- a/doc/manual/source/store/store-path.md +++ b/doc/manual/source/store/store-path.md @@ -1,72 +1,136 @@ -# Store Path +# Store Path and Store Directory -> **Example** -> -> `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1` -> -> A rendered store path +Nix's [store object] and [file system object] data models are minimal and abstract. +But to actually be used by software, store objects need to be made available through the operating system's file system. + +This is done by exposing all the store objects in a single *[store directory][store directory path]*. +Every entry in that directory is a *[store path base name]* pointing to a store object. +Store objects exposed in this way can then be referenced by *[store paths][store path]*. + +[store object]: ./store-object.md +[file system object]: ./file-system-object.md +[store path]: #store-path +[store path base name]: #base-name +[store directory path]: #store-directory-path + +## Store Path Base Name {#base-name} -Nix implements references to [store objects](./store-object.md) as *store paths*. +Nix implements references to store objects as *store path base names*. -Think of a store path as an [opaque], [unique identifier]: -The only way to obtain store path is by adding or building store objects. -A store path will always reference exactly one store object. +Think of a store path base name as an [opaque], [unique identifier]: +The only way to obtain a store path base name is by adding or building store objects. +A store path base name will always reference exactly one store object. [opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type [unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier -Store paths are pairs of +Store path base names are pairs of -- A 20-byte digest for identification -- A symbolic name for people to read +- A 20-byte [digest]{#digest} for identification +- A symbolic [name]{#name} for people to read > **Example** > > - Digest: `q06x3jll2yfzckz2bzqak089p43ixkkq` > - Name: `firefox-33.1` -To make store objects accessible to operating system processes, stores have to expose store objects through the file system. +A store path base name is rendered to a string as the concatenation of -A store path is rendered to a file system path as the concatenation of - -- [Store directory](#store-directory) (typically `/nix/store`) -- Path separator (`/`) -- Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters) +- Digest rendered in [Nix32], a variant of [Base32] (20 hash bytes become 32 ASCII characters) - Hyphen (`-`) - Name > **Example** > > ``` -> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 -> |--------| |------------------------------| |----------| -> store directory digest name +> q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 +> |------------------------------| |----------| +> digest name > ``` -Exactly how the digest is calculated depends on the type of store path. +[Nix32]: @docroot@/protocols/nix32.md +[Base32]: https://en.wikipedia.org/wiki/Base32 + +Exactly how the digest is calculated depends on the type of store object being referenced. Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). -## Store Directory - -Every [Nix store](./index.md) has a store directory. +## Store Directory Path -Not every store can be accessed through the file system. -But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path). +Every [Nix store] has a store directory path. +This is an absolute, lexically canonical (not containing any `..`, `.`, or similar) path which points to the directory where all store objects are to be found. -[file system objects]: ./file-system-object.md - -This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. +[Nix store]: ./index.md > **Note** > > The store directory defaults to `/nix/store`, but is in principle arbitrary. -It is important which store a given store object belongs to: +## Store Path + +A store path is the pair of a store directory path and a [store path base name]. +It is rendered to a file system path as the concatenation of + +- [Store directory path] (typically `/nix/store`) +- Path separator (`/`) +- The [store path base name] + +> **Example** +> +> ``` +> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1 +> |--------| |------------------------------| |----------| +> store directory digest name +> ``` + +When we have fixed a given store, or given store directory path (that all the stores in use share), the abstract syntax for store paths and the abstract syntax for store path base names coincide: the store directory path is known from context, so only the other two fields vary from one store path to the next. + +## Exposing Store Objects in OS File Systems {#exposing} + +Not every store can be accessed through the file system. +But if the store has a file system representation, the following should be true: + +- The store directory path is canonical: no prefix of the path (i.e. path of the first *n* path segments) points to a symlink. + In other words, the store directory can be looked up from the store directory path without following any symlinks. + (This condition is a separate condition in addition to the "lexical canonicity" described above, which is a property of just the path itself. + This (regular) "canonicity" is a property about the path and the filesystem it navigates jointly.) + + > **Note** + > + > The [`allow-symlinked-store`](@docroot@/command-ref/conf-file.md#conf-allow-symlinked-store) setting can be used to relax this requirement. + +- The store directory path in fact points to a directory. + +- The store directory contains, for every store object in the store, the [file system object] of that store object at the (rendered) [store path base name]. + The permissions and other metadata for these files in the store directory is in the normal form described in [Exposing in OS file systems](./file-system-object/os-file-system.md). + +The above properties mean that the following file accesses will work. +Suppose we have a store available on the file system per the above rules, and `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1` is the store path base name of a store object in that store. + +- Suppose that the store directory (path) is `/foo/bar`. + Then, `/foo/bar/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1` exists and is the file system object of that store object. + +- Suppose that we don't know what the store directory path of the store is, but we do have a capability `storeDir` to the store directory on the file system. + (This would be a "file descriptor" on Unix, or a "file handle" on Windows.) + Then (using the Unix notation for this): + ``` + openat(storeDir, "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1", O_NOFOLLOW) + ``` + will succeed (so long as the file system object is not a symlink), and the yielded capability will point to the file system object of that store object. + + (The behavior for symlinks is harder to specify because of limitations in POSIX.) + +## Relocating store objects + +The inclusion of the store directory path in the full rendered store path means that the full rendered store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. +(And actually, all of the currently-supported ways of computing the digest of a store path also depend on the store directory path, as described in the [specification of store path digests](@docroot@/protocols/store-path.md). +So this is also true even just for store path base names, in general.) + +It is therefore important to consider which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. Nix can only guarantee referential integrity if store paths do not cross store boundaries. -Therefore one can only copy store objects to a different store if +One can only copy store objects to a different store if - The source and target stores' directories match diff --git a/src/libstore/http-binary-cache-store.md b/src/libstore/http-binary-cache-store.md index 20c26d0c2c..03dd350ec5 100644 --- a/src/libstore/http-binary-cache-store.md +++ b/src/libstore/http-binary-cache-store.md @@ -2,7 +2,7 @@ R"( **Store URL format**: `http://...`, `https://...` -This store allows a binary cache to be accessed via the HTTP +This store allows a [binary cache](@docroot@/protocols/binary-cache/index.md) to be accessed via the HTTP protocol. )" diff --git a/src/libstore/include/nix/store/local-settings.hh b/src/libstore/include/nix/store/local-settings.hh index 7381b5b8e7..dc4322753d 100644 --- a/src/libstore/include/nix/store/local-settings.hh +++ b/src/libstore/include/nix/store/local-settings.hh @@ -196,7 +196,7 @@ struct LocalSettings : public virtual Config, public GCSettings, public AutoAllo 0, "cores", R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#builder-execution) of a derivation. + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/store/building.md#env-vars) of a derivation. The `builder` executable can use this variable to control its own maximum amount of parallelism. - From 87c3c3251c4a1ac39bac6772342bca00cceb18f7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 16 Jun 2026 21:48:22 +0300 Subject: [PATCH 513/927] Fix truncated store hash part in the manual introduction, document FreeBSD support --- doc/manual/source/introduction.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index 85de7982c9..51f9fb45d2 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -10,7 +10,7 @@ as /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1/ -where `b6gvzjyb2pg0…` is a unique identifier for the package that +where `q06x3jll2yfz…` is a unique identifier for the package that captures all its dependencies (it’s a cryptographic hash of the package’s build dependency graph). This enables many powerful features. @@ -174,7 +174,7 @@ the package: ## Portability -Nix runs on Linux and macOS. +Nix runs on Linux, macOS and FreeBSD. ## NixOS From 3d445e7bc597cd78d28a3c0f74fc9bc5b7e8ba03 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 16 Jun 2026 21:48:52 +0300 Subject: [PATCH 514/927] Bring prerequisites-source.md up to date This has bitrotten a lot and needed a lot of updates since the meson migration. I've removed version bounds that we don't check in the build system, which have had likely bitrotten too. Also start to follow SEMBR in these docs. --- .../installation/prerequisites-source.md | 126 +++++++----------- 1 file changed, 49 insertions(+), 77 deletions(-) diff --git a/doc/manual/source/installation/prerequisites-source.md b/doc/manual/source/installation/prerequisites-source.md index 640032c517..e98067348e 100644 --- a/doc/manual/source/installation/prerequisites-source.md +++ b/doc/manual/source/installation/prerequisites-source.md @@ -1,80 +1,52 @@ # Prerequisites - - GNU Autoconf () and the - autoconf-archive macro collection - (). These are - needed to run the bootstrap script. - - - GNU Make. - - - Bash Shell. The `./configure` script relies on bashisms, so Bash is - required. - - - A version of GCC or Clang that supports C++23. - - - `pkg-config` to locate dependencies. If your distribution does not - provide it, you can get it from - . - - - The OpenSSL library to calculate cryptographic hashes. If your - distribution does not provide it, you can get it from - . - - - The `libbrotlienc` and `libbrotlidec` libraries to provide - implementation of the Brotli compression algorithm. They are - available for download from the official repository - . - - - cURL and its library. If your distribution does not provide it, you - can get it from . - - - The SQLite embedded database library, version 3.6.19 or higher. If - your distribution does not provide it, please install it from - . - - - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce - the evaluator’s memory consumption (optional). - - To enable it, install - `pkgconfig` and the Boehm garbage collector, and pass the flag - `--enable-gc` to `configure`. - - - The `boost` library of version 1.66.0 or higher. It can be obtained - from the official web site . - - - The `editline` library of version 1.14.0 or higher. It can be - obtained from the its repository - . - - - The `libsodium` library for verifying cryptographic signatures - of contents fetched from binary caches. - It can be obtained from the official web site - . - - - Recent versions of Bison and Flex to build the parser. (This is - because Nix needs GLR support in Bison and reentrancy support in - Flex.) For Bison, you need version 2.6, which can be obtained from - the [GNU FTP server](ftp://alpha.gnu.org/pub/gnu/bison). For Flex, - you need version 2.5.35, which is available on - [SourceForge](http://lex.sourceforge.net/). Slightly older versions - may also work, but ancient versions like the ubiquitous 2.5.4a - won't. - - - The `libseccomp` is used to provide syscall filtering on Linux. This - is an optional dependency and can be disabled passing a - `--disable-seccomp-sandboxing` option to the `configure` script (Not - recommended unless your system doesn't support `libseccomp`). To get - the library, visit . - - - On 64-bit x86 machines only, `libcpuid` library - is used to determine which microarchitecture levels are supported + This list and lower version bounds are maintained on best-effort basis. When in doubt, check the `meson.build` files. + + - Meson build system (). + + - Ninja (). + + - A version of GCC or Clang that supports C++23 (anything newer than Clang 19 or GCC 14 is likely to work). + + - `pkg-config` to locate dependencies. + If your distribution does not provide it, you can get it from . + + - The OpenSSL library to calculate cryptographic hashes. + If your distribution does not provide it, you can get it from . + + - The `libbrotlienc` and `libbrotlidec` libraries to provide implementation of the Brotli compression algorithm. + They are available for download from the official repository . + + - cURL library. + If your distribution does not provide it, you can get it from . + + - The SQLite embedded database library, version 3.6.19 or higher. + If your distribution does not provide it, please install it from . + + - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce the evaluator’s memory consumption (optional). + To enable it, install `pkgconfig` and the Boehm garbage collector, and pass the option `-Dlibexpr:gc=enabled` to `meson setup`. + + - The `boost` library of version 1.87.0 or higher. + It can be obtained from the official web site . + + - The `editline` library of version 1.14.0 or higher. + It can be obtained from the its repository . + + - The `libsodium` library for verifying cryptographic signatures of contents fetched from binary caches. + It can be obtained from the official web site . + + - Recent versions of Bison and Flex to build the parser. + (This is because Nix needs C++ template support in Bison and reentrancy support in Flex.) + + - The `libseccomp` is used to provide syscall filtering on Linux. + This is an optional dependency and can be disabled passing a `-Dlibstore:seccomp-sandboxing=disabled` option to the `meson setup` command + (Not recommended unless your system doesn't support `libseccomp`). + To get the library, visit . + + - On 64-bit x86 machines only, `libcpuid` library is used to determine which microarchitecture levels are supported (e.g., as whether to have `x86_64-v2-linux` among additional system types). - The library is available from its homepage - . - This is an optional dependency and can be disabled - by providing a `--disable-cpuid` to the `configure` script. - - - Unless `meson setup build -Dunit-tests=false` is specified, GoogleTest (GTest) and - RapidCheck are required, which are available at - and - respectively. + The library is available from its homepage . + This is an optional dependency and can be disabled by providing a `-Dlibutil:cpuid=disabled` option to `meson setup` script. + + - Unless `meson setup build -Dunit-tests=false` is specified, GoogleTest (GTest) and RapidCheck are required, which are available at + and respectively. From 5d2600a01dfc0e722ddbb66d9ae3b4b65cec41ae Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Wed, 17 Jun 2026 13:49:11 -0400 Subject: [PATCH 515/927] Fix exceptions not being caught in nix::Pid::~Pid Using function try blocks with destructors causes the exception to be rethrown instead of properly caught. --- src/libutil/unix/processes.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 7accf2e175..32d5953588 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -50,11 +50,13 @@ Pid::Pid(pid_t pid) } Pid::~Pid() -try { - if (pid != -1) - kill(/*allowInterrupts=*/false); -} catch (...) { - ignoreExceptionInDestructor(); +{ + try { + if (pid != -1) + kill(/*allowInterrupts=*/false); + } catch (...) { + ignoreExceptionInDestructor(); + } } void Pid::operator=(pid_t pid) From c37e17c280934a6ca667984151516b0bb6212de4 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 16 Jun 2026 20:35:42 +0300 Subject: [PATCH 516/927] Increase the sourceToSink/sinkToSource coroutine stack sizes We had too little headroom previously, since the default stack size if 128KiB and half of that is typically consumed by a 64KiB buffer for copying between Source/Sink. 512KiB should be more than enough for the limit that we have (64 levels in NARs , which usually also bounds the recursion depth with some constant factor). --- src/libutil/archive.cc | 3 +-- src/libutil/serialise.cc | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 8d9b833c29..56172bf32b 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -36,8 +36,7 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings); /* Maximum directory nesting depth for dumpPath()/parseDump(). Bounds stack usage so deep trees cannot overflow the (possibly coroutine) - stack these run on. Chosen to fit comfortably in the default 128 KiB - boost coroutine stack. */ + stack these run on. */ static constexpr size_t narMaxDepth = 64; PathFilter defaultPathFilter = [](const std::string &) { return true; }; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 0529c2ca2f..eb12098431 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -369,6 +369,13 @@ void StringSource::skip(size_t len) pos += len; } +/* 512KiB is a conservative estimate for deeply nested NARs, which are limited + to 64 levels. We also tend to allocate rather large buffers on the stack, so + we should leave plenty of headroom. Note that no evaluation is supposed to + happen on sourceToSink/sinkToSource coroutine stacks (for Boehm GC reasons), + which requires much more stack space. */ +static constexpr size_t defaultCoroutineStackSize = 512 * 1024; + std::unique_ptr sourceToSink(fun reader) { struct SourceToSink : FinishSink @@ -392,8 +399,9 @@ std::unique_ptr sourceToSink(fun reader) cur = in; if (!coro) { - coro = - coro_t::push_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::pull_type & yield) { + coro = coro_t::push_type( + boost::coroutines2::protected_fixedsize_stack(defaultCoroutineStackSize), + [&](coro_t::pull_type & yield) { LambdaSource source([&](char * out, size_t out_len) { if (cur.empty()) { yield(); @@ -450,8 +458,9 @@ std::unique_ptr sinkToSource(fun writer, fun eof) { bool hasCoro = coro.has_value(); if (!hasCoro) { - coro = - coro_t::pull_type(boost::coroutines2::protected_fixedsize_stack(), [&](coro_t::push_type & yield) { + coro = coro_t::pull_type( + boost::coroutines2::protected_fixedsize_stack(defaultCoroutineStackSize), + [&](coro_t::push_type & yield) { /* Feed the consumer in chunks, instead of on each write to avoid excessive context switching. parseDump does lots of small writes to the sink, which we should From 9586b80af2c75d9ffefc01402a4e15c63847ebbd Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 17 Jun 2026 21:04:34 +0300 Subject: [PATCH 517/927] Print attributes in repl commands (:env or :st) in lexicographic order Depending on how nix is built, the order of variables in the environments can differ [1], due to static initialisation order. Technically, in non-unity builds the order is fully undefined due to the static initialisation order fiasco. I recently added tests for this functionality and they started failing in nixpkgs, because it doesn't use unity builds. [1]: https://github.com/NixOS/nixpkgs/pull/532575 --- src/libexpr/eval.cc | 27 ++++++++++++------- .../repl/debugger-fail-throw-env-1.expected | 2 +- .../repl/debugger-fail-throw-env-2.expected | 2 +- .../repl/debugger-okay-ignore-try.expected | 8 +++--- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 81ed77c905..069d7e2c51 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -667,27 +667,34 @@ std::optional EvalState::getDoc(Value & v) return {}; } +static StaticEnv::Vars lexicographicOrder(const SymbolTable & st, StaticEnv::Vars vars) +{ + std::ranges::sort(vars, [&st](const auto & lhs, const auto & rhs) { + return std::string_view(st[lhs.first]) < std::string_view(st[rhs.first]); + }); + return vars; +} + // just for the current level of StaticEnv, not the whole chain. -void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) +static void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) { std::cout << ANSI_MAGENTA; - for (auto & i : se.vars) - std::cout << st[i.first] << " "; + for (auto & [name, displacement] : lexicographicOrder(st, se.vars)) + std::cout << st[name] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; } // just for the current level of Env, not the whole chain. -void printWithBindings(const SymbolTable & st, const Env & env) +static void printWithBindings(const SymbolTable & st, const Env & env) { if (!env.values[0]->isThunk()) { std::cout << "with: "; std::cout << ANSI_MAGENTA; - auto j = env.values[0]->attrs()->begin(); - while (j != env.values[0]->attrs()->end()) { - std::cout << st[j->name] << " "; - ++j; - } + auto * bindings = env.values[0]->attrs(); + /* TODO: Don't print the whole attribute set, since it can be quite large. */ + for (const Attr * attr : bindings->lexicographicOrder(st)) + std::cout << st[attr->name] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; } @@ -708,7 +715,7 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::cout << ANSI_MAGENTA; // for the top level, don't print the double underscore ones; // they are in builtins. - for (auto & i : se.vars) + for (auto & i : lexicographicOrder(st, se.vars)) if (!hasPrefix(st[i.first], "__")) std::cout << st[i.first] << " "; std::cout << ANSI_NORMAL; diff --git a/tests/functional/repl/debugger-fail-throw-env-1.expected b/tests/functional/repl/debugger-fail-throw-env-1.expected index 865aa58eaf..921395b8f9 100644 --- a/tests/functional/repl/debugger-fail-throw-env-1.expected +++ b/tests/functional/repl/debugger-fail-throw-env-1.expected @@ -8,7 +8,7 @@ Env level 0 static: _ Env level 1 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :quit error: diff --git a/tests/functional/repl/debugger-fail-throw-env-2.expected b/tests/functional/repl/debugger-fail-throw-env-2.expected index 251da649f3..3eb47bdf53 100644 --- a/tests/functional/repl/debugger-fail-throw-env-2.expected +++ b/tests/functional/repl/debugger-fail-throw-env-2.expected @@ -15,7 +15,7 @@ Env level 2 static: x Env level 3 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :bt diff --git a/tests/functional/repl/debugger-okay-ignore-try.expected b/tests/functional/repl/debugger-okay-ignore-try.expected index 23a5f476ac..7d4e72fdae 100644 --- a/tests/functional/repl/debugger-okay-ignore-try.expected +++ b/tests/functional/repl/debugger-okay-ignore-try.expected @@ -29,7 +29,7 @@ Env level 0 static: someFailingExpr tried Env level 1 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :p bricked error: undefined variable 'bricked' @@ -54,7 +54,7 @@ Env level 1 static: someFailingExpr tried Env level 2 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :p bricked error: infinite recursion encountered @@ -85,7 +85,7 @@ Env level 1 static: someFailingExpr tried Env level 2 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :p bricked error: infinite recursion encountered @@ -110,7 +110,7 @@ Env level 1 static: someFailingExpr tried Env level 2 -builtins true false null scopedImport import isNull break abort throw derivationStrict placeholder baseNameOf dirOf removeAttrs map toString fetchMercurial fetchTree fetchTarball fetchGit fromTOML derivation +abort baseNameOf break builtins derivation derivationStrict dirOf false fetchGit fetchMercurial fetchTarball fetchTree fromTOML import isNull map null placeholder removeAttrs scopedImport throw toString true nix-repl> :p bricked error: infinite recursion encountered From 1ba3bc047c6d0494f13c4138268bca8392c1ad4c Mon Sep 17 00:00:00 2001 From: Adam Dinwoodie Date: Wed, 17 Jun 2026 20:12:52 +0100 Subject: [PATCH 518/927] worker-settings: speed factor is non-integer The value of the speed factor when specifying a remote builder can be any positive number. Correct the comments / documentation to reflect that fact. --- src/libstore/include/nix/store/worker-settings.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/include/nix/store/worker-settings.hh b/src/libstore/include/nix/store/worker-settings.hh index 32c2b36842..d78ca420f7 100644 --- a/src/libstore/include/nix/store/worker-settings.hh +++ b/src/libstore/include/nix/store/worker-settings.hh @@ -185,7 +185,7 @@ public: 4. The maximum number of builds that Nix executes in parallel on the machine. Typically this should be equal to the number of CPU cores. - 5. The “speed factor”, indicating the relative speed of the machine as a positive integer. + 5. The “speed factor”, indicating the relative speed of the machine as a positive integer or decimal number. If there are multiple machines of the right type, Nix prefers the fastest, taking load into account. 6. A comma-separated list of supported [system features](#conf-system-features). From f88c306340939391bdec8e3cd9d753c4f6f5ccf2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 16 Jun 2026 23:09:28 +0300 Subject: [PATCH 519/927] Unbreak the formatting of the builtins in the manual 2a71fbf41017b4b9ad87f7c6fa5506b80743e1a7 broke the manual. Also the documentation was quite strange in some places (and slightly inaccurate). Using inline documentation is much more readable and doesn't have the formatting footguns. --- src/libexpr/primops.cc | 199 ++++++----------------------------------- 1 file changed, 28 insertions(+), 171 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 951c6e7460..1a837a1b96 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3090,11 +3090,7 @@ static RegisterPrimOp primop_attrNames({ alphabetically sorted list. For instance, `builtins.attrNames { y = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`. - # Time Complexity - - - O(n log n), where: - - n = number of attributes in the set + Has `O(n log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_attrNames, }); @@ -3128,11 +3124,7 @@ static RegisterPrimOp primop_attrValues({ Return the values of the attributes in the set *set* in the order corresponding to the sorted attribute names. - # Time Complexity - - - O(n log n), where: - - n = number of attributes in the set + Has `O(n log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_attrValues, }); @@ -3159,9 +3151,7 @@ static RegisterPrimOp primop_getAttr({ the `.` operator, since *s* is an expression rather than an identifier. - # Time Complexity - - O(log n) where n = number of attributes in the set + Has `O(log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_getAttr, }); @@ -3251,9 +3241,7 @@ static RegisterPrimOp primop_hasAttr({ `false` otherwise. This is a dynamic version of the `?` operator, since *s* is an expression rather than an identifier. - # Time Complexity - - O(log n) where n = number of attributes in the set + Has `O(log n)` time complexity, where `n` is number of attributes in the *set*. )", .impl = prim_hasAttr, }); @@ -3314,12 +3302,7 @@ static RegisterPrimOp primop_removeAttrs({ evaluates to `{ y = 2; }`. - # Time Complexity - - O(n + k log k) where: - - n = number of attributes in input set - k = number of attribute names to remove + Has `O(n + k log k)` time complexity, where `n` is number of attributes in the *set* and `k` is the size of *list*. )", .impl = prim_removeAttrs, }); @@ -3408,9 +3391,7 @@ static RegisterPrimOp primop_listToAttrs({ { foo = 123; bar = 456; } ``` - # Time Complexity - - O(n log n) where n = number of list elements + Has `O(n log n)` time complexity, where `n` is size of the list. )", .impl = prim_listToAttrs, }); @@ -3487,12 +3468,7 @@ static RegisterPrimOp primop_intersectAttrs({ Return a set consisting of the attributes in the set *e2* which have the same name as some attribute in *e1*. - # Time Complexity - - O(n * log m) where: - - n = number of attributes in the smaller set - m = number of attributes in the larger set + Has `O(n log m)` time complexity, where `n` and `m` are the sizes of the smallest and largest set respectively. )", .impl = prim_intersectAttrs, }); @@ -3533,12 +3509,7 @@ static RegisterPrimOp primop_catAttrs({ evaluates to `[1 2]`. - # Time Complexity - - O(n * log m) where: - - n = list length - m = number of attributes per set + Has `O(n)` time complexity, where `n` is the size of the *list*. )", .impl = prim_catAttrs, }); @@ -3583,9 +3554,7 @@ static RegisterPrimOp primop_functionArgs({ the function. Plain lambdas are not included, e.g. `functionArgs (x: ...) = { }`. - # Time Complexity - - O(n) where n = number of formal arguments + Has constant time complexity. )", .impl = prim_functionArgs, }); @@ -3619,13 +3588,9 @@ static RegisterPrimOp primop_mapAttrs({ evaluates to `{ a = 10; b = 20; }`. - # Time Complexity - - O(n) where: - - n = number of attributes - - Calls to `f` are performed afterwards, when needed. + Has `O(n)` time complexity, where `n` is the size of the *attrset*. + Note that no calls to *f* are performed by the builtin. + The function *f* is called on demand when a resulting attribute value is evaluated. )", .impl = prim_mapAttrs, }); @@ -3714,12 +3679,7 @@ static RegisterPrimOp primop_zipAttrsWith({ } ``` - # Time Complexity - - O(N * log k) where: - - N = total attributes across all sets - k = number of unique keys across all sets + Has `O(n log n)` time complexity, where `n` is the number of attributes across all sets. )", .impl = prim_zipAttrsWith, }); @@ -3787,9 +3747,7 @@ static RegisterPrimOp primop_head({ isn’t a list or is an empty list. You can test whether a list is empty by comparing it with `[]`. - # Time Complexity - - O(1) + Has constant time complexity. )", .impl = prim_head, }); @@ -3821,10 +3779,6 @@ static RegisterPrimOp primop_tail({ > This function should generally be avoided since it's inefficient: > unlike Haskell's `tail`, it takes O(n) time, so recursing over a > list by repeatedly calling `tail` takes O(n^2) time. - - # Time Complexity - - O(n) where n = list length (copies n-1 elements) )", .impl = prim_tail, }); @@ -3860,13 +3814,9 @@ static RegisterPrimOp primop_map({ evaluates to `[ "foobar" "foobla" "fooabc" ]`. - # Time Complexity - - O(n) where: - - n = list length - - Calls to `f` are performed afterwards when needed. + Has `O(n)` time complexity, where `n` is the size of the *list*. + Note that no calls to *f* are performed by the builtin, but *f* itself is evaluated and its type is checked eagerly. + The function *f* is called on demand when a resulting list element is evaluated. )", .impl = prim_map, }); @@ -3916,13 +3866,7 @@ static RegisterPrimOp primop_filter({ .doc = R"( Return a list consisting of the elements of *list* for which the function *f* returns `true`. - - # Time Complexity - - O(n * T_f) (eager; predicate is forced) where: - - n = list length - T_f = predicate evaluation time + Has linear time complexity in the size of the input *list*. )", .impl = prim_filter, }); @@ -3946,15 +3890,7 @@ static RegisterPrimOp primop_elem({ .doc = R"( Return `true` if a value equal to *x* occurs in the list *xs*, and `false` otherwise. - - # Time Complexity - - O(n * T) (worst case) where: - - n = list length - T = time to compare two elements - - returns early if the elements is found + Short-circuits and does not evaluate elements that occur in the list after the first match. )", .impl = prim_elem, }); @@ -3972,12 +3908,6 @@ static RegisterPrimOp primop_concatLists({ .args = {"lists"}, .doc = R"( Concatenate a list of lists into a single list. - - # Time Complexity - - O(N) where: - - N = total number of elements across all lists )", .impl = prim_concatLists, }); @@ -3994,10 +3924,6 @@ static RegisterPrimOp primop_length({ .args = {"e"}, .doc = R"( Return the length of the list *e*. - - # Time Complexity - - O(1) )", .impl = prim_length, }); @@ -4063,12 +3989,7 @@ static RegisterPrimOp primop_foldlStrict({ but lacks these benefits. See also [Nixpkgs `lib.foldl`](https://nixos.org/manual/nixpkgs/unstable/#function-library-lib.lists.foldl). - # Time Complexity - - O(n * T_op) where: - - n = list length - T_op = `op` call evaluation time + Has linear time complexity in the size of the list. )", .impl = prim_foldlStrict, }); @@ -4107,15 +4028,7 @@ static RegisterPrimOp primop_any({ .doc = R"( Return `true` if the function *pred* returns `true` for at least one element of *list*, and `false` otherwise. - - # Time Complexity - - O(n * T_pred) where: - - - n = `list` length - - T_pred = `pred` call evaluation time - - returns early when `pred` returns `true` + Short-circuits and does not evaluate elements that appear later in the list if `pred` evaluates to `true`. )", .impl = prim_any, }); @@ -4131,15 +4044,7 @@ static RegisterPrimOp primop_all({ .doc = R"( Return `true` if the function *pred* returns `true` for all elements of *list*, and `false` otherwise. - - # Time Complexity - - O(n * T_f) where: - - - n = list length - - T_f = predicate evaluation time - - returns early when `pred` returns `false` + Short-circuits and does not evaluate elements that appear later in the list if `pred` evaluates to `false`. )", .impl = prim_all, }); @@ -4179,16 +4084,7 @@ static RegisterPrimOp primop_genList({ returns the list `[ 0 1 4 9 16 ]`. - # Time Complexity - - Complexity of `genList generator n`: O(n) - - Complexity of `deepSeq (genList generator n)`: O(n * T_f) - - where: - - n = requested length - T_f = `generator` call evaluation time + Has linear time complexity. )", .impl = prim_genList, }); @@ -4300,15 +4196,8 @@ static RegisterPrimOp primop_sort({ If the *comparator* violates any of these properties, then `builtins.sort` reorders elements in an unspecified manner. - # Time Complexity - - O(n log n * T_cmp), where: - - n = `list` length - T_cmp = `comparator` call evaluation time - - Uses an adaptive sort that exploits existing sorted runs in the input, - down to O(n * T_cmp) when the list is already sorted. + Runs in `O(n log n)` time on average, where `n` is the size of the *list*. + Uses an adaptive sort that exploits existing sorted runs in the input, down to `O(n)` when the list is already sorted. )", .impl = prim_sort, }); @@ -4371,12 +4260,7 @@ static RegisterPrimOp primop_partition({ { right = [ 23 42 ]; wrong = [ 1 9 3 ]; } ``` - # Time Complexity - - O(n * T_pred) where: - - n = list length - T_pred = `pred` call evaluation time + Runs in linear time in the size of the *list*. )", .impl = prim_partition, }); @@ -4431,13 +4315,7 @@ static RegisterPrimOp primop_groupBy({ { b = [ "bar" "baz" ]; f = [ "foo" ]; } ``` - # Time Complexity - - O(N * T_f + N * log k) where: - - N = number of `list` elements - T_f = `f` call evaluation time - k = number of unique groups + Has `O(n log n)` time complexity, where `n` is the size of the input *list*. )", .impl = prim_groupBy, }); @@ -4480,14 +4358,6 @@ static RegisterPrimOp primop_concatMap({ .doc = R"( This function is equivalent to `builtins.concatLists (map f list)` but is more efficient. - - # Time Complexity - - O(k * T_f + N) where: - - k = length of input list - T_f = time to call `f` on an element - N = total number of elements returned by `f` calls )", .impl = prim_concatMap, }); @@ -5191,13 +5061,6 @@ static RegisterPrimOp primop_concatStringsSep({ Concatenate a list of strings with a separator between each element, e.g. `concatStringsSep "/" ["usr" "local" "bin"] == "usr/local/bin"`. - - # Time Complexity - - O(n + m) (amortized) where: - - n = number of list elements - m = total length of output string )", .impl = prim_concatStringsSep, }); @@ -5283,13 +5146,7 @@ static RegisterPrimOp primop_replaceStrings({ evaluates to `"fabir"`. - # Time Complexity - - O(n * k * c) (worst case) where: - - n = length of input string - k = number of replacement patterns - c = average length of patterns in 'from' list + Has `O(n k)` time complexity, where `n` is the length of *s* and `k` is the number of replacements. )", .impl = prim_replaceStrings, }); From b308d8cea8f79fb2d0cee2c5732524f3c68ba4ea Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 17 Jun 2026 22:50:23 +0200 Subject: [PATCH 520/927] doc: Generalize the JSON guidelines to "Data Modeling Guidelines" They were kind of the same thing and we've been applying them more broadly on occasion. --- doc/manual/source/SUMMARY.md.in | 2 +- doc/manual/source/_redirects | 3 ++- .../{json-guideline.md => data-modeling.md} | 11 ++++++++--- doc/manual/source/release-notes/rl-2.23.md | 4 ++-- doc/manual/source/release-notes/rl-2.33.md | 2 +- src/libmain/include/nix/main/common-args.hh | 4 ++-- 6 files changed, 16 insertions(+), 10 deletions(-) rename doc/manual/source/development/{json-guideline.md => data-modeling.md} (86%) diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 5a17426b90..a73446a990 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -149,7 +149,7 @@ - [Debugging](development/debugging.md) - [Documentation](development/documentation.md) - [CLI guideline](development/cli-guideline.md) - - [JSON guideline](development/json-guideline.md) + - [Data Modeling Guidelines](development/data-modeling.md) - [C++ style guide](development/cxx.md) - [Static Analysis](development/static-analysis.md) - [Experimental Features](development/experimental-features.md) diff --git a/doc/manual/source/_redirects b/doc/manual/source/_redirects index 7e4557f7d5..17aba776a3 100644 --- a/doc/manual/source/_redirects +++ b/doc/manual/source/_redirects @@ -27,7 +27,8 @@ /contributing/documentation /development/documentation 301! /contributing/experimental-features /development/experimental-features 301! /contributing/cli-guideline /development/cli-guideline 301! -/contributing/json-guideline /development/json-guideline 301! +/contributing/json-guideline /development/data-modeling 301! +/development/json-guideline /development/data-modeling 301! /contributing/cxx /development/cxx 301! /expressions/expression-language /language/ 301! diff --git a/doc/manual/source/development/json-guideline.md b/doc/manual/source/development/data-modeling.md similarity index 86% rename from doc/manual/source/development/json-guideline.md rename to doc/manual/source/development/data-modeling.md index 309b4b3a06..a9d1311475 100644 --- a/doc/manual/source/development/json-guideline.md +++ b/doc/manual/source/development/data-modeling.md @@ -1,7 +1,12 @@ -# JSON guideline +# Data Modeling Guidelines -Nix consumes and produces JSON in a variety of contexts. -These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. +Nix consumes and produces JSON and attribute sets in a variety of contexts. +These guidelines ensure consistent practices for our interfaces, for ease of use, and so that experience in one part carries over to another. + +For these guidelines, we will use JSON terminology, but they apply equally well to new attribute set interfaces (primops, etc.). +Note that these are guidelines first and foremost. Exceptions include: +- Feature testing: e.g., it is OK to do `builtins?frobnicate`. +- Compatibility: we generally do not change stable interfaces just to make them comply. New replacements can be added with care. ## Extensibility diff --git a/doc/manual/source/release-notes/rl-2.23.md b/doc/manual/source/release-notes/rl-2.23.md index b358a0fdc3..92e5f45994 100644 --- a/doc/manual/source/release-notes/rl-2.23.md +++ b/doc/manual/source/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/json-guideline.md). + The JSON format for derivations has been slightly revised to better conform to our [data modeling guidelines](@docroot@/development/data-modeling.md). In particular, the hash algorithm and content addressing method of content-addressed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. @@ -89,7 +89,7 @@ This makes records of this sort more self-describing, and easier to consume programmatically. We will follow this design principle going forward; - the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + the [data modeling guidelines](@docroot@/development/data-modeling.md) in the contributing section have been updated accordingly. - Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) diff --git a/doc/manual/source/release-notes/rl-2.33.md b/doc/manual/source/release-notes/rl-2.33.md index bed6970293..cc5781f373 100644 --- a/doc/manual/source/release-notes/rl-2.33.md +++ b/doc/manual/source/release-notes/rl-2.33.md @@ -135,7 +135,7 @@ This is the legacy format, preserved for backwards compatibility: ### Version 2 (`--json-format 2`) -The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes: +The new structured format follows the [data modeling guidelines](@docroot@/development/data-modeling.md) with the following changes: - **Nested structure with top-level metadata**: diff --git a/src/libmain/include/nix/main/common-args.hh b/src/libmain/include/nix/main/common-args.hh index d67fc2ad0c..b20df3a99e 100644 --- a/src/libmain/include/nix/main/common-args.hh +++ b/src/libmain/include/nix/main/common-args.hh @@ -81,8 +81,8 @@ struct MixPrintJSON : virtual Args * This is a template to avoid accidental coercions from `string` to `json` in the caller, * to avoid mistakenly passing an already serialized JSON to this function. * - * It is not recommended to print a JSON string - see the JSON guidelines - * about extensibility, https://nix.dev/manual/nix/development/development/json-guideline.html - + * It is not recommended to print a JSON string - see the data modeling guidelines + * about extensibility, https://nix.dev/manual/nix/development/development/data-modeling.html - * but you _can_ print a sole JSON string by explicitly coercing it to * `nlohmann::json` first. */ From 5a3d3986e6af6a6b43027ef0c767f3d39d855505 Mon Sep 17 00:00:00 2001 From: Tom Hunze Date: Sat, 4 Apr 2026 12:18:03 +0200 Subject: [PATCH 521/927] nix develop: use `runPhase` to run phases, fall back to `runHook` Using `runHook`, `nix develop --phase ` still executes the generic `Phase` when `Phase` is defined when calling `stdenv.mkDerivation` [1]. `runPhase` was specifically added to avoid this problem and to be more consistent with `genericBuild` behavior [2]. [1] https://github.com/NixOS/nix/issues/6202 [2] https://github.com/NixOS/nixpkgs/pull/230874 --- src/nix/develop.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 50b248bcb8..3105efa825 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -605,7 +605,13 @@ struct CmdDevelop : Common, MixEnvironment // FIXME: foundMakefile is set by buildPhase, need to get // rid of that. script += fmt("foundMakefile=1\n"); - script += fmt("runHook %1%Phase\n", *phase); + script += + fmt("if declare -f runPhase >/dev/null; then\n" + " runPhase %1%Phase\n" + "else\n" + " runHook %1%Phase\n" + "fi\n", + *phase); } else if (!command.empty()) { From 2662dc95aff7b9f77ddf00421d9354de2c57d284 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 18 Jun 2026 23:52:40 +0300 Subject: [PATCH 522/927] Remove UnkeyedValidPathInfo::id This doesn't seem to be used anywhere now. It got added initially in 762cee72ccd860e72c7b639a1dd542ac0f298bb2, where it was used in registerValidPaths. But nowadays we do queryValidPathId for that use-case unconditionally. That can be improved to avoid some unnecessary SQLite queries in case we know the primary id, but it can be done in a much more local way that's not exposed in the interface. --- src/libstore/include/nix/store/path-info.hh | 8 -------- src/libstore/local-store.cc | 4 +--- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index 7ba30359b5..0dd19b81ec 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -90,14 +90,6 @@ struct UnkeyedValidPathInfo */ uint64_t narSize = 0; - /** - * internal use only: SQL primary key for on-disk store objects with - * `LocalStore`. - * - * @todo Remove, layer violation - */ - uint64_t id = 0; - /** * Whether the path is ultimately trusted, that is, it's a * derivation output that was built locally. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e4a4ffc988..dd5d325239 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -814,8 +814,6 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s auto info = std::make_shared(path, UnkeyedValidPathInfo(*this, narHash)); - info->id = id; - info->registrationTime = useQueryPathInfo.getInt(2); auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3); @@ -836,7 +834,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s info->ca = ContentAddress::parseOpt(s); /* Get the references. */ - auto useQueryReferences(state.stmts->QueryReferences.use().apply(info->id)); + auto useQueryReferences(state.stmts->QueryReferences.use().apply(id)); while (useQueryReferences.next()) info->references.insert(parseStorePath(useQueryReferences.getStr(0))); From f0f0d950795ef2ea2ac5989ab75b000364c2016d Mon Sep 17 00:00:00 2001 From: Michael Wang <41721295+zwang20@users.noreply.github.com> Date: Mon, 8 Jun 2026 21:13:49 +1000 Subject: [PATCH 523/927] Fix formatting in documentation --- doc/manual/source/command-ref/nix-collect-garbage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/command-ref/nix-collect-garbage.md b/doc/manual/source/command-ref/nix-collect-garbage.md index 763179b8ee..07229255e7 100644 --- a/doc/manual/source/command-ref/nix-collect-garbage.md +++ b/doc/manual/source/command-ref/nix-collect-garbage.md @@ -62,9 +62,9 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. See the documentation of that command for additional information about the *period* argument. - - [`--max-freed`](#opt-max-freed) *bytes* +- [`--max-freed`](#opt-max-freed) *bytes* - + Keep deleting paths until at least *bytes* bytes have been deleted, then stop. The argument *bytes* can be followed by the From 09c8a1b9aac0b041136dd178be6db6666d6fdd75 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Mon, 22 Jun 2026 11:13:27 -0500 Subject: [PATCH 524/927] Fix boost format error when diff hook fails Signed-off-by: Lisanna Dettwyler --- src/libstore/unix/build/derivation-builder.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 94bff9c2bc..3b02dfc4f1 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -137,8 +137,7 @@ static void handleDiffHook( .gid = gid, .chdir = "/"}); if (!statusOk(diffRes.first)) - throw ExecError( - diffRes.first, "diff-hook program %s %2%", PathFmt(diffHook), statusToString(diffRes.first)); + throw ExecError(diffRes.first, "diff-hook program %s %s", PathFmt(diffHook), statusToString(diffRes.first)); if (diffRes.second != "") printError(chomp(diffRes.second)); From 4df375be378db8ebf5f25ef9549e2b76a1db83e1 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 20 Jun 2026 01:41:36 +0300 Subject: [PATCH 525/927] Add test for broken positions on repl :load-file This is my SNAFU from a091a8100a8. I neglected the fact that :l and a bit of other things also nuke the cache. The following commit will have the fix. (cherry picked from commit acfc7d845176c7c6a514ff8527be7266850d495c) --- tests/functional/repl/file-b.nix | 9 ++++++- .../functional/repl/load-and-reload.expected | 21 ++++++++++++++++ ...on-existent-file.in => load-and-reload.in} | 6 +++-- .../reload-with-non-existent-file.expected | 24 ------------------- 4 files changed, 33 insertions(+), 27 deletions(-) create mode 100644 tests/functional/repl/load-and-reload.expected rename tests/functional/repl/{reload-with-non-existent-file.in => load-and-reload.in} (70%) delete mode 100644 tests/functional/repl/reload-with-non-existent-file.expected diff --git a/tests/functional/repl/file-b.nix b/tests/functional/repl/file-b.nix index ddde63c16c..7416216c9f 100644 --- a/tests/functional/repl/file-b.nix +++ b/tests/functional/repl/file-b.nix @@ -1 +1,8 @@ -{ fromB = 2; } +{ + fromBFails = throw "b"; + fromB = 2; + /** + Some documentation. + */ + funcFromB = x: x; +} diff --git a/tests/functional/repl/load-and-reload.expected b/tests/functional/repl/load-and-reload.expected new file mode 100644 index 0000000000..1642987fe6 --- /dev/null +++ b/tests/functional/repl/load-and-reload.expected @@ -0,0 +1,21 @@ +Nix +Type :? for help. + +nix-repl> :l file-b.nix +Added 3 variables. +fromB, fromBFails, funcFromB + +nix-repl> :l ./does-not-exist.nix +error: path '/path/to/tests/functional/repl/does-not-exist.nix' does not exist + +nix-repl> :p fromBFails +error: + … while calling the 'throw' builtin + at «string»:1:18: + 1| fromBFails + | ^ + + error: b + +nix-repl> :doc funcFromB +error: basic_string::substr: __pos (which is 3) > this->size() (which is 0) diff --git a/tests/functional/repl/reload-with-non-existent-file.in b/tests/functional/repl/load-and-reload.in similarity index 70% rename from tests/functional/repl/reload-with-non-existent-file.in rename to tests/functional/repl/load-and-reload.in index 740b3be9a4..a8b94a265b 100644 --- a/tests/functional/repl/reload-with-non-existent-file.in +++ b/tests/functional/repl/load-and-reload.in @@ -1,5 +1,7 @@ -:l file-a.nix -:l ./does-not-exist.nix :l file-b.nix +:l ./does-not-exist.nix +:p fromBFails +:doc funcFromB +:l file-a.nix :r fromA + fromB diff --git a/tests/functional/repl/reload-with-non-existent-file.expected b/tests/functional/repl/reload-with-non-existent-file.expected deleted file mode 100644 index e15be6e730..0000000000 --- a/tests/functional/repl/reload-with-non-existent-file.expected +++ /dev/null @@ -1,24 +0,0 @@ -Nix -Type :? for help. - -nix-repl> :l file-a.nix -Added 1 variables. -fromA - -nix-repl> :l ./does-not-exist.nix -error: path '/path/to/tests/functional/repl/does-not-exist.nix' does not exist - -nix-repl> :l file-b.nix -Added 1 variables. -fromB - -nix-repl> :r -Loading "file-a.nix"... -Added 1 variables. -fromA -Loading "file-b.nix"... -Added 1 variables. -fromB - -nix-repl> fromA + fromB -3 From dbcc91a1b39f65621da5fe502cc08a0036c3f0b3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 23 Jun 2026 00:27:06 +0300 Subject: [PATCH 526/927] Revert "libexpr: Clear PosTable contents in EvalState::resetFileCache" This reverts commit a091a8100a8587185e579d4cff04381e8e074f12. (cherry picked from commit e6cf0d2dcab3d8b675b817399f715a3fd5337cb4) --- src/libexpr/eval.cc | 1 - .../functional/repl/load-and-reload.expected | 29 ++++++++++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 069d7e2c51..29ecec69ee 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1189,7 +1189,6 @@ void EvalState::resetFileCache() fileEvalCache->clear(); inputCache->clear(); lookupPathResolved->clear(); - positions.clear(); rootFS->invalidateCache(); } diff --git a/tests/functional/repl/load-and-reload.expected b/tests/functional/repl/load-and-reload.expected index 1642987fe6..7f953e2861 100644 --- a/tests/functional/repl/load-and-reload.expected +++ b/tests/functional/repl/load-and-reload.expected @@ -11,11 +11,32 @@ error: path '/path/to/tests/functional/repl/does-not-exist.nix' does not exist nix-repl> :p fromBFails error: … while calling the 'throw' builtin - at «string»:1:18: - 1| fromBFails - | ^ + at /path/to/tests/functional/repl/file-b.nix:2:16: + 1| { + 2| fromBFails = throw "b"; + | ^ + 3| fromB = 2; error: b nix-repl> :doc funcFromB -error: basic_string::substr: __pos (which is 3) > this->size() (which is 0) +Function `funcFromB`\ + … defined at /path/to/tests/functional/repl/file-b.nix:7:15 + + +Some documentation. + +nix-repl> :l file-a.nix +Added 1 variables. +fromA + +nix-repl> :r +Loading "file-b.nix"... +Added 3 variables. +fromB, fromBFails, funcFromB +Loading "file-a.nix"... +Added 1 variables. +fromA + +nix-repl> fromA + fromB +3 From 1c7a37c6d70285f415c96a85484dce9188d3a522 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 23 Jun 2026 01:01:55 +0300 Subject: [PATCH 527/927] Align readline/editline final prompt behavior Aligning these different behaviours seems fraught, so this is like an easier solution. See: https://hydra.nixos.org/build/331681294/log (cherry picked from commit d8e330749a23511794b6310d076086d754b21c78) --- src/libcmd/include/nix/cmd/repl-interacter.hh | 4 +++ src/libcmd/repl-interacter.cc | 28 ++++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/libcmd/include/nix/cmd/repl-interacter.hh b/src/libcmd/include/nix/cmd/repl-interacter.hh index 7cba481059..9b9a03c3e4 100644 --- a/src/libcmd/include/nix/cmd/repl-interacter.hh +++ b/src/libcmd/include/nix/cmd/repl-interacter.hh @@ -1,8 +1,10 @@ #pragma once /// @file +#include "nix/util/file-descriptor.hh" #include "nix/util/finally.hh" #include "nix/util/fun.hh" +#include "nix/util/terminal.hh" #include "nix/util/types.hh" #include #include @@ -39,6 +41,8 @@ public: class ReadlineLikeInteracter : public virtual ReplInteracter { std::filesystem::path historyFile; + bool isInteractive = nix::isTTY(getStandardInput()); + public: ReadlineLikeInteracter(std::filesystem::path historyFile) : historyFile(std::move(historyFile)) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 81240af7f5..5f2417dc13 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -203,8 +203,25 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT setupSignals(); #endif - char * s = readline(promptForType(promptType)); - Finally doFree([&]() { free(s); }); + + /* Buffer for the non-interactive input. */ + std::string buffer; + const char * s = nullptr; + char * rl = nullptr; + + /* Use plain std::getline for non-interactive mode, which we also use for + testing purposes. readline/editline seem to disagree too much about how + to handle final prompts etc., so it's easier to bypass those. The tests + are mostly about testing the core repl logic, not input handling. */ + if (isInteractive) { + rl = ::readline(promptForType(promptType)); + s = rl; + } else { + s = std::getline(std::cin, buffer) ? buffer.c_str() : nullptr; + } + + Finally doFree([&]() { ::free(rl); }); + #ifndef _WIN32 // TODO use more signals.hh for this restoreSignals(); #endif @@ -215,15 +232,12 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return true; } - // editline doesn't echo the input to the output when non-interactive, unlike readline - // this results in a different behavior when running tests. The echoing is - // quite useful for reading the test output, so we add it here. + /* Echo the prompt into the output if run in non-interactive mode, somewhat + for the purposes of characterisation tests. */ if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") { -#if !USE_READLINE // This is probably not right for multi-line input, but we don't use that // in the characterisation tests, so it's fine. std::cout << promptForType(promptType) << s << std::endl; -#endif } if (!s) From 07ffc599aa22780040173c9cc1368effbc24731b Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 23 Jun 2026 21:08:37 +0300 Subject: [PATCH 528/927] Fix #15916 The core of the issue is that the trampoline goal reported an arbitrary exitCode, which without --keep-going ends up cancelling goals and we end up dying with an assert in Goal::amDone(). See: https://github.com/NixOS/nix/pull/16042#issuecomment-4760250433 (cherry picked from commit f2c1d45dfc54a263198613c9499c3715aaef4edc) --- .../build/derivation-trampoline-goal.cc | 79 ++++++++++++++++--- tests/functional/meson.build | 1 + .../multiple-outputs-substitute-failure.sh | 28 +++++++ 3 files changed, 97 insertions(+), 11 deletions(-) create mode 100755 tests/functional/multiple-outputs-substitute-failure.sh diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index edf8d1e86e..14864ed052 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -2,6 +2,9 @@ #include "nix/store/build/worker.hh" #include "nix/store/derivations.hh" +#include +#include + namespace nix { DerivationTrampolineGoal::DerivationTrampolineGoal( @@ -144,11 +147,14 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation }, wantedOutputs.raw); + /* Must have at least one wanted output. This is assumed below. */ + assert(!resolvedWantedOutputs.empty()); + Goals concreteDrvGoals; /* Build this step! */ - auto sharedDrv = make_ref(std::move(drv)); + auto sharedDrv = make_ref(std::move(drv)); for (auto & output : resolvedWantedOutputs) { auto g = upcast_goal(worker.makeDerivationGoal(drvPath, sharedDrv, output, buildMode, false)); @@ -157,20 +163,71 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation concreteDrvGoals.insert(std::move(g)); } - // Copy on purpose - co_await await(Goals(concreteDrvGoals)); + co_await await(concreteDrvGoals); trace("outer build done"); - auto & g = *concreteDrvGoals.begin(); - buildResult = g->buildResult; - if (auto * successP = buildResult.tryGetSuccess()) - for (auto & g2 : concreteDrvGoals) - if (auto * successP2 = g2->buildResult.tryGetSuccess()) - for (auto && [x, y] : successP2->builtOutputs) - successP->builtOutputs.insert_or_assign(x, y); + if (nrFailed != 0) { + auto gi = std::ranges::find_if(concreteDrvGoals, [](const GoalPtr & goal) -> bool { + auto exitCode = goal->exitCode; + /* Note that without --keep-going waitees might be cancelled before + we are woken up. */ + return exitCode != ecBusy && exitCode != ecSuccess; + }); + + const Goal * g = gi->get(); + assert(gi != concreteDrvGoals.end() && "expected a failing goal"); + auto exitCode = g->exitCode; + const auto * failure = g->buildResult.tryGetFailure(); + assert(failure && "failing goal does not report a failed build result"); + + /* Report the exit status of *some* failing goal. This might not be strictly + correct, since multiple subgoals can fail independently, but this should be + a good enough heuristic without --keep-going. */ + co_return doneFailure(exitCode, *failure); + } + + SingleDrvOutputs outputs; + + auto successes = std::views::transform(concreteDrvGoals, [](const GoalPtr & a) -> const BuildResult::Success & { + auto * success = a->buildResult.tryGetSuccess(); + assert(success && "goal succeeded, but some waitees do not report a successful status"); + return *success; + }); + + for (const auto & success : successes) + std::ranges::copy(success.builtOutputs, std::inserter(outputs, outputs.end())); + + auto statuses = successes | std::views::transform(&BuildResult::Success::status); + + /* Aggregate the status code. If some outputs we already valid, but we had + to build/substitute the other ones, report it as the smallest common + denominator. */ + auto compareSuccesses = [](auto a, auto b) { + /* This is technically an identity mapping of the underlying values, but + it would be worse to rely on the enum ordering here. */ + auto toPriority = [](auto st) { + using enum BuildResult::Success::Status; + switch (st) { + case Built: + return 0; + case Substituted: + return 1; + case AlreadyValid: + return 2; + case ResolvesToAlreadyValid: + return 3; + default: + unreachable(); + } + }; + return toPriority(a) < toPriority(b); + }; - co_return amDone(g->exitCode); + co_return doneSuccess({ + .status = std::ranges::min(statuses, compareSuccesses), + .builtOutputs = std::move(outputs), + }); } } // namespace nix diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 7d5110ebc4..8fdaf4ffe0 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -129,6 +129,7 @@ suites = [ 'logging.sh', 'make-content-addressed.sh', 'misc.sh', + 'multiple-outputs-substitute-failure.sh', 'multiple-outputs.sh', 'nar-access.sh', 'nars.sh', diff --git a/tests/functional/multiple-outputs-substitute-failure.sh b/tests/functional/multiple-outputs-substitute-failure.sh new file mode 100755 index 0000000000..fd4a7ec50b --- /dev/null +++ b/tests/functional/multiple-outputs-substitute-failure.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# See https://github.com/NixOS/nix/issues/15916 + +source common.sh + +TODO_NixOS # Requires substituting from a local binary cache, enable when we sign paths in NixOS tests +needLocalStore "'--no-require-sigs' can’t be used with the daemon" + +BINARY_CACHE=file://$cacheDir + +readarray -t outPaths < <(nix build -f multiple-outputs.nix 'independent^*' --no-link --print-out-paths) +[[ ${#outPaths[@]} -eq 2 ]] +nix copy --to "$BINARY_CACHE" "${outPaths[@]}" +for p in "${outPaths[@]}"; do + [[ $p == *-second ]] && secondOut=$p +done + +# Corrupt the second output, so that the substitution partially succeeds. +secondNarInfoFile="$cacheDir/$(basename "$secondOut" | cut -c1-32).narinfo" +sed -i 's|^NarHash:.*|NarHash: sha256:0000000000000000000000000000000000000000000000000000|' "$secondNarInfoFile" + +clearStore +clearCacheCache + +# Note that using "^*" matters here. We want all wanted outputs for the same goal. +expect 1 nix build -j 0 -f multiple-outputs.nix "independent^*" --no-link \ + --substituters "$BINARY_CACHE" --no-require-sigs --substitute 2>&1 | grepQuiet "hash mismatch" From 05d02b88ad73c806a1182a9f2238db081a08a011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 23 Jun 2026 10:01:53 +0000 Subject: [PATCH 529/927] nix-store --register-validity: fix ENOENT under chroot stores With --store local?root=, registerValidity() passed the logical store path to canonicalisePathMetaData(), which then tried to lstat a path that does not exist on the host filesystem and failed with ENOENT. Use toRealPath() so the chroot-prefixed real path is canonicalised instead. ensureLocalStore() is already required at the end of this function, so hoisting the downcast does not narrow the set of accepted stores. (cherry picked from commit d4248c87a4aac4df91282190abe235d55d80b21b) --- src/nix/nix-store/nix-store.cc | 5 +++-- tests/functional/chroot-store.sh | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index e90c7768d9..2bb56eb51b 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -570,6 +570,7 @@ static void opDumpDB(Strings opFlags, Strings opArgs) static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) { + auto localStore = ensureLocalStore(); ValidPathInfos infos; while (1) { @@ -587,7 +588,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) /* !!! races */ if (canonicalise) canonicalisePathMetaData( - store->printStorePath(info->path), + localStore->toRealPath(info->path), {NIX_WHEN_SUPPORT_ACLS(settings.getLocalSettings().ignoredAcls)}); if (!hashGiven) { HashResult hash = hashPath( @@ -601,7 +602,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) } } - ensureLocalStore()->registerValidPaths(infos); + localStore->registerValidPaths(infos); } static void opLoadDB(Strings opFlags, Strings opArgs) diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index cbb80c8710..d0f41ee872 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -60,6 +60,15 @@ PATH7=$(nix path-info --store "local://$TEST_ROOT/x%2Bchroot" "$CORRECT_PATH") # Path gets decoded. [[ ! -d "$TEST_ROOT/x%2Bchroot" ]] +# Regression test: `nix-store --register-validity` against a chroot store must +# canonicalise the real (chroot-prefixed) path, not the logical store path +# (which does not exist on the host filesystem). +regPath=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-reg +touch "$TEST_ROOT/x/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-reg" +[[ ! -e "$regPath" ]] +(echo "$regPath" && echo && echo 0) | nix-store --store "local?root=$TEST_ROOT/x" --register-validity +nix-store --store "$TEST_ROOT/x" --check-validity "$regPath" + # Ensure store info trusted works with local store nix --store "$TEST_ROOT/x" store info --json | jq -e '.trusted' From 779ed6e72fd821077a5c36f59946eedbc524da30 Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Tue, 23 Jun 2026 17:07:26 -0500 Subject: [PATCH 530/927] Set MADV_DONTDUMP for bump allocator This should drastically reduce the time needed to generate a coredump. Resolves #16057 Signed-off-by: Lisanna Dettwyler (cherry picked from commit b06d8e6e8b0db0016fe74938c871cb42e1b2f4a3) --- src/libutil/bump-memory-resource.cc | 10 +++++++++- src/libutil/include/nix/util/bump-memory-resource.hh | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libutil/bump-memory-resource.cc b/src/libutil/bump-memory-resource.cc index c892b1652d..938bd2b84e 100644 --- a/src/libutil/bump-memory-resource.cc +++ b/src/libutil/bump-memory-resource.cc @@ -1,4 +1,6 @@ #include "nix/util/bump-memory-resource.hh" +#include "nix/util/environment-variables.hh" +#include "nix/util/error.hh" #include "nix/util/file-system.hh" #include "nix/util/alignment.hh" #include "nix/util/logging.hh" @@ -57,7 +59,7 @@ static bool canOvercommit(std::size_t reserveSize) #endif // _WIN32 -BumpMemoryResource::BumpMemoryResource(std::size_t reserveSize, std::pmr::memory_resource * upstream) +BumpMemoryResource::BumpMemoryResource(std::size_t reserveSize, std::pmr::memory_resource * upstream, bool dontDump) : upstreamResource(upstream) { #ifndef _WIN32 @@ -83,6 +85,12 @@ BumpMemoryResource::BumpMemoryResource(std::size_t reserveSize, std::pmr::memory base = p; capacity = reserveSize; + +# ifdef MADV_DONTDUMP + static const bool dumpEverything = getEnv("_NIX_CORE_DUMP_EVERYTHING").value_or("0") == "1"; + if (!dumpEverything && dontDump && ::madvise(p, reserveSize, MADV_DONTDUMP)) + throw SysError("calling madvise"); +# endif #endif } diff --git a/src/libutil/include/nix/util/bump-memory-resource.hh b/src/libutil/include/nix/util/bump-memory-resource.hh index 15b160f35f..9ae6be0485 100644 --- a/src/libutil/include/nix/util/bump-memory-resource.hh +++ b/src/libutil/include/nix/util/bump-memory-resource.hh @@ -39,7 +39,8 @@ public: explicit BumpMemoryResource( std::size_t reserveSize = defaultReserveSize, - std::pmr::memory_resource * upstream = std::pmr::new_delete_resource()); + std::pmr::memory_resource * upstream = std::pmr::new_delete_resource(), + bool dontDump = true); BumpMemoryResource(BumpMemoryResource &&) = delete; BumpMemoryResource(const BumpMemoryResource &) = delete; From 08a209269a27fe087f1aef731e179043d8310089 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2026 22:13:04 +0000 Subject: [PATCH 531/927] build(deps): bump nixpkgs in the flake-inputs group Bumps the flake-inputs group with 1 update: [nixpkgs](https://github.com/NixOS/nixpkgs). Updates `nixpkgs` from `bd0ff2d` to `714a5f8` - [Commits](https://github.com/NixOS/nixpkgs/commits) --- updated-dependencies: - dependency-name: nixpkgs dependency-version: 714a5f8c4ead6b31148d829288440ed033ccc041 dependency-type: direct:production dependency-group: flake-inputs ... Signed-off-by: dependabot[bot] (cherry picked from commit d19f68cf337e1e0666c8a3ecd4b678a484a83ae1) --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 85432d06e2..f1e2fad802 100644 --- a/flake.lock +++ b/flake.lock @@ -60,11 +60,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1780902259, - "narHash": "sha256-YMnBf9lk/LYgvqfmSSJuOGigtRs5Lsy26pJHVlR9yMY=", - "rev": "bd0ff2d3eac24699c3664d5966b9ef36f388e2ca", + "lastModified": 1782535326, + "narHash": "sha256-r4TA57SL7nvj1R+GY/FCLwFU48w9IixTQlzBTIYkt8E=", + "rev": "714a5f8c4ead6b31148d829288440ed033ccc041", "type": "tarball", - "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.1550.bd0ff2d3eac2/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixos/26.05/nixos-26.05.3494.714a5f8c4ead/nixexprs.tar.xz" }, "original": { "type": "tarball", From 96b8708e6ce4eda78b0c5b29898d25f57d18410e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Apr 2026 20:18:30 +0200 Subject: [PATCH 532/927] AttrPath::to_string(): Escape attribute names properly --- src/libexpr/attr-path.cc | 8 +++++++- tests/functional/flakes/dubious-query.sh | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 63197e4634..0a8c9f2c2c 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -50,7 +50,13 @@ AttrPath AttrPath::fromStrings(EvalState & state, const std::vector std::string AttrPath::to_string(EvalState & state) const { - return dropEmptyInitThenConcatStringsSep(".", state.symbols.resolve({*this})); + std::ostringstream ss; + for (const auto & [idx, attr] : enumerate(state.symbols.resolve({*this}))) { + if (idx) + ss << '.'; + printAttributeName(ss, attr); + } + return ss.str(); } std::vector AttrPath::resolve(EvalState & state) const diff --git a/tests/functional/flakes/dubious-query.sh b/tests/functional/flakes/dubious-query.sh index 114def179a..bda5b8bf50 100644 --- a/tests/functional/flakes/dubious-query.sh +++ b/tests/functional/flakes/dubious-query.sh @@ -17,7 +17,7 @@ expectStderr 0 nix --offline build --dry-run "git+file://$repoDir?bar#foo" \ # Check that the anchor (#) is taken as a whole, not split, and throws an error. expectStderr 1 nix --offline build --dry-run "git+file://$repoDir#foo?bar" \ - | grepQuiet "error: flake 'git+file://$repoDir' does not provide attribute 'packages.$system.foo?bar', 'defaultPackage.$system.foo?bar', 'legacyPackages.$system.foo?bar' or 'foo?bar'" + | grepQuiet "error: flake 'git+file://$repoDir' does not provide attribute 'packages.$system.\"foo?bar\"', 'defaultPackage.$system.\"foo?bar\"', 'legacyPackages.$system.\"foo?bar\"' or '\"foo?bar\"'" # Check that a literal `?` in the query doesn't print dubious query warning. expectStderr 0 nix --offline build --dry-run "git+file://$repoDir?#foo" \ From 35fa531cbc7bb91fa7c3b5fd6824a3f73646fb29 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Apr 2026 20:19:00 +0200 Subject: [PATCH 533/927] WIP --- src/libexpr/primops/fakeDerivation.cc | 136 ++++++++++++++++++++++++++ src/libexpr/primops/meson.build | 1 + src/libstore/derivations.cc | 3 + src/nix/flake-bake.cc | 129 ++++++++++++++++++++++++ src/nix/flake-bake.md | 7 ++ src/nix/meson.build | 1 + 6 files changed, 277 insertions(+) create mode 100644 src/libexpr/primops/fakeDerivation.cc create mode 100644 src/nix/flake-bake.cc create mode 100644 src/nix/flake-bake.md diff --git a/src/libexpr/primops/fakeDerivation.cc b/src/libexpr/primops/fakeDerivation.cc new file mode 100644 index 0000000000..046c8db194 --- /dev/null +++ b/src/libexpr/primops/fakeDerivation.cc @@ -0,0 +1,136 @@ +#include "nix/expr/primops.hh" +#include "nix/expr/eval-inline.hh" +#include "nix/store/derivations.hh" +#include "nix/store/store-api.hh" +#include "nix/util/hash.hh" + +namespace nix { + +static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fakeDerivation"); + + std::optional name; + + struct FakeDerivationOutput + { + StorePath path; + std::optional narHash; + }; + + std::map outputs; + + for (auto & attr : *args[0]->attrs()) { + std::string_view attrName = state.symbols[attr.name]; + auto attrHint = fmt("while evaluating the attribute '%s' passed to builtins.fakeDerivation", attrName); + + if (attrName == "name") { + name = state.forceStringNoCtx(*attr.value, attr.pos, attrHint); + } + + else if (attrName == "outputs") { + state.forceAttrs(*attr.value, attr.pos, attrHint); + for (auto & outAttr : *attr.value->attrs()) { + std::string_view outName = state.symbols[outAttr.name]; + + state.forceAttrs( + *outAttr.value, + outAttr.pos, + fmt("while evaluating the output '%s' passed to builtins.fakeDerivation", outName)); + + std::optional path; + std::optional narHash; + + for (auto & outField : *outAttr.value->attrs()) { + std::string_view fieldName = state.symbols[outField.name]; + auto fieldHint = + fmt("while evaluating the attribute '%s' of output '%s' passed to builtins.fakeDerivation", + fieldName, + outName); + + if (fieldName == "path") { + auto s = state.forceStringNoCtx(*outField.value, outField.pos, fieldHint); + path = state.store->parseStorePath(s); + } + + else if (fieldName == "narHash") { + auto s = state.forceStringNoCtx(*outField.value, outField.pos, fieldHint); + narHash = Hash::parseAny(s, std::nullopt); + } + + else + state + .error( + "attribute '%s' isn't supported in an output passed to 'builtins.fakeDerivation'", + fieldName) + .atPos(outField.pos) + .debugThrow(); + } + + if (!path) + state + .error( + "attribute 'path' is missing in output '%s' passed to 'builtins.fakeDerivation'", outName) + .atPos(outAttr.pos) + .debugThrow(); + + outputs.emplace(std::string(outName), FakeDerivationOutput{std::move(*path), std::move(narHash)}); + } + } + + else + state.error("attribute '%s' isn't supported in call to 'builtins.fakeDerivation'", attrName) + .atPos(attr.pos) + .debugThrow(); + } + + if (!name) + state.error("attribute 'name' is missing in call to 'builtins.fakeDerivation'") + .atPos(pos) + .debugThrow(); + + if (outputs.empty()) + state.error("attribute 'outputs' is missing or empty in call to 'builtins.fakeDerivation'") + .atPos(pos) + .debugThrow(); + + Derivation drv; + drv.name = *name; + drv.platform = "builtin"; + drv.builder = "builtin:fake-derivation"; + + for (auto & [outName, out] : outputs) { +#if 0 + if (out.narHash) + drv.outputs.insert_or_assign( + outName, + DerivationOutput::CAFixed{ + .ca = + ContentAddress{ + .method = ContentAddressMethod::Raw::NixArchive, + .hash = *out.narHash, + }, + }); + else +#endif + drv.outputs.insert_or_assign(outName, DerivationOutput::InputAddressed{.path = out.path}); + } + + auto drvPath = writeDerivation(*state.store, *state.asyncPathWriter, drv, state.repair); + + // FIXME + state.waitForPath(drvPath); + + state.mkStorePathString(drvPath, v); +} + +static RegisterPrimOp primop_fakeDerivation({ + .name = "__fakeDerivation", + .args = {"attrs"}, + .doc = R"( + Placeholder. + )", + .fun = prim_fakeDerivation, +}); + +} // namespace nix diff --git a/src/libexpr/primops/meson.build b/src/libexpr/primops/meson.build index d62b6df4ea..78f40d331e 100644 --- a/src/libexpr/primops/meson.build +++ b/src/libexpr/primops/meson.build @@ -5,6 +5,7 @@ generated_headers += gen_header.process( sources += files( 'context.cc', + 'fakeDerivation.cc', 'fetchClosure.cc', 'fetchMercurial.cc', 'fetchTree.cc', diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index bfea6e992c..485eea0eb8 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1331,6 +1331,7 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string auto outPath = store.makeOutputPath(outputName, *h, drvName); if constexpr (std::is_same_v) { +#if 0 if (outputVariant.path == outPath) { return; // Correct case } @@ -1340,6 +1341,8 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string "derivation has incorrect output '%s', should be '%s'", store.printStorePath(outputVariant.path), store.printStorePath(outPath)); +#endif + return; // FIXME } else if constexpr (std::is_same_v) { if constexpr (fillIn) /* Fill in output path for Deferred diff --git a/src/nix/flake-bake.cc b/src/nix/flake-bake.cc new file mode 100644 index 0000000000..fd6f905fb6 --- /dev/null +++ b/src/nix/flake-bake.cc @@ -0,0 +1,129 @@ +#include "flake-command.hh" +#include "nix/cmd/flake-schemas.hh" +#include "nix/expr/parallel-eval.hh" +#include "nix/store/derivations.hh" + +using namespace nix; +using namespace nix::flake; + +struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas +{ + std::string description() override + { + return "bake a flake"; + } + + std::string doc() override + { + return +#include "flake-bake.md" + ; + } + + void run(nix::ref store) override + { + auto state = getEvalState(); + auto evalStore = getEvalStore(); + auto flake = make_ref(lockFlake()); + + auto cache = flake_schemas::call(*state, flake, getDefaultFlakeSchemas()); + + auto inventory = cache->getRoot()->getAttr("inventory"); + auto outputs = cache->getRoot()->getAttr("outputs"); + + FutureVector futures(*state->executor); + + // FIXME: generate an AST. + Sync result; + + *result.lock() = R""( +{ + outputs = + { self }: + { +)""; + + auto visit = [&](this auto & visit, ref node) -> void { + flake_schemas::visit( + {"x86_64-linux"}, // FIXME + true, + node, + flake->flake.provenance, + + [&](const flake_schemas::Leaf & leaf) { + printError("AT %s", leaf.node->getAttrPathStr()); + + auto drv = leaf.derivation(outputs); + if (!drv) + // FIXME: emit attrs that throw a useful error? + return; + + // FIXME: do we need to force the derivation? + auto drvPath = drv->forceDerivation(); + auto storeDrv = evalStore->derivationFromPath(drvPath); + std::map outputs; + for (auto & i : storeDrv.outputsAndOptPaths(*store)) { + if (auto outPath = i.second.second) + outputs.emplace(i.first, *outPath); + else + // FIXME: we could build the derivation to get the outputs. + return; + } + + auto name = drv->getAttr(state->s.name)->getString(); + + auto r(result.lock()); + *r += " " + leaf.node->getAttrPathStr() + " = {\n"; + *r += " drvPath = builtins.fakeDerivation {\n"; + *r += " name = \"" + name + "\";\n"; + for (auto & i : outputs) + *r += " outputs." + i.first + ".path = \"" + store->printStorePath(i.second) + "\";\n"; + *r += " };\n"; + *r += " type = \"derivation\";\n"; + *r += " name = \"" + name + "\";\n"; + *r += " system = \"x86_64-linux\";\n"; // FIXME + // meta.description = "Docker image with Nix for x86_64-linux"; + *r += " };\n\n"; + }, + + [&](std::function forEachChild) { + forEachChild([&](Symbol attrName, ref node, bool isLast) { + state->spawn(futures, 1, [&visit, node]() { + try { + visit(node); + } catch (EvalError & e) { + // FIXME: make it a flake schema attribute whether to ignore evaluation errors. + if (node->root->state.symbols[node->getAttrPath()[0]] != "legacyPackages") + throw; + } + }); + }); + }, + + [&](ref node, const std::vector & systems) {}, + + [&](ref node) {}); + }; + + flake_schemas::forEachOutput( + inventory, + [&](Symbol outputName, + std::shared_ptr output, + const std::string & doc, + bool isLast) { + if (output) + state->spawn(futures, 1, [&visit, output]() { visit(ref(output)); }); + }); + + futures.finishAll(); + + *result.lock() += R""( + }; +} +)""; + + logger->cout(*result.lock()); + } +}; + +static auto rCmdFlakeBake = registerCommand2({"flake", "bake"}); diff --git a/src/nix/flake-bake.md b/src/nix/flake-bake.md new file mode 100644 index 0000000000..e0a6d5872e --- /dev/null +++ b/src/nix/flake-bake.md @@ -0,0 +1,7 @@ +R""( + +# Description + +Bake a flake. + +)"" diff --git a/src/nix/meson.build b/src/nix/meson.build index b7ddcc8eec..d0a0d8c6eb 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -92,6 +92,7 @@ nix_sources = [ config_priv_h ] + files( 'edit.cc', 'env.cc', 'eval.cc', + 'flake-bake.cc', 'flake-prefetch-inputs.cc', 'flake-prefetch-inputs.cc', 'flake.cc', From 380294aa7b5390f8fedd62753705aa9cb1b2b52b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Apr 2026 16:39:46 +0200 Subject: [PATCH 534/927] Move most of `nix flake show` into libcmd --- src/libcmd/flake-schemas.cc | 111 ++++++++++++++++++ src/libcmd/include/nix/cmd/flake-schemas.hh | 20 ++++ src/nix/flake.cc | 119 ++------------------ 3 files changed, 140 insertions(+), 110 deletions(-) diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index c8e2992be7..1771c47ae1 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -4,6 +4,10 @@ #include "nix/util/memory-source-accessor.hh" #include "nix/util/mounted-source-accessor.hh" #include "nix/flake/provenance.hh" +#include "nix/expr/parallel-eval.hh" +#include "nix/store/globals.hh" + +#include namespace nix::flake_schemas { @@ -360,6 +364,113 @@ Schemas getSchemas(ref inventory) return schemas; } +nlohmann::json getFlakeInventory( + EvalState & state, + Store & evalStore, + LockedFlake & flake, + ref cache, + const FlakeInventoryOptions & options) +{ + auto inventory = cache->getRoot()->getAttr("inventory"); + auto outputs = cache->getRoot()->getAttr("outputs"); + + auto localSystem = std::string(settings.thisSystem.get()); + + FutureVector futures(*state.executor); + + std::function node, nlohmann::json & obj)> visit; + + visit = [&](ref node, nlohmann::json & obj) { + flake_schemas::visit( + options.showAllSystems ? std::optional() : localSystem, + options.showLegacy, + node, + flake.flake.provenance, + + [&](const flake_schemas::Leaf & leaf) { + if (auto what = leaf.what()) + obj.emplace("what", *what); + + if (auto shortDescription = leaf.shortDescription()) + obj.emplace("shortDescription", *shortDescription); + + if (auto drv = leaf.derivation(outputs)) { + auto drvObj = nlohmann::json::object(); + + if (options.showDrvNames) + drvObj.emplace("name", drv->getAttr(state.s.name)->getString()); + + if (options.showDrvPaths) { + auto drvPath = drv->forceDerivation(); + drvObj.emplace("path", state.store->printStorePath(drvPath)); + } + + if (options.showOutputPaths) { + auto outputs = nlohmann::json::object(); + auto drvPath = drv->forceDerivation(); + auto drv = evalStore.derivationFromPath(drvPath); + for (auto & i : drv.outputsAndOptPaths(*state.store)) { + if (auto outPath = i.second.second) + outputs.emplace(i.first, state.store->printStorePath(*outPath)); + else + outputs.emplace(i.first, nullptr); + } + drvObj.emplace("outputs", std::move(outputs)); + } + + obj.emplace("derivation", std::move(drvObj)); + } + + if (auto forSystems = leaf.forSystems()) + obj.emplace("forSystems", *forSystems); + }, + + [&](std::function forEachChild) { + auto children = nlohmann::json::object(); + forEachChild([&](Symbol attrName, ref node, bool isLast) { + auto & j = children.emplace(state.symbols[attrName], nlohmann::json::object()).first.value(); + state.spawn(futures, 1, [&visit, &j, node]() { + try { + visit(node, j); + } catch (EvalError & e) { + // FIXME: make it a flake schema attribute whether to ignore evaluation errors. + if (node->root->state.symbols[node->getAttrPath()[0]] == "legacyPackages") + j.emplace("failed", true); + else + throw; + } + }); + }); + obj.emplace("children", std::move(children)); + }, + + [&](ref node, const std::vector & systems) { + obj.emplace("filtered", true); + }, + + [&](ref node) { obj.emplace("isLegacy", true); }); + }; + + auto inv = nlohmann::json::object(); + + flake_schemas::forEachOutput( + inventory, + [&](Symbol outputName, std::shared_ptr output, const std::string & doc, bool isLast) { + auto & j = inv.emplace(state.symbols[outputName], nlohmann::json::object()).first.value(); + + if (output) { + j.emplace("doc", doc); + auto & j2 = j.emplace("output", nlohmann::json::object()).first.value(); + state.spawn(futures, 1, [&visit, output, &j2]() { visit(ref(output), j2); }); + } else + j.emplace("unknown", true); + }); + + futures.finishAll(); + + return inv; +} + } // namespace nix::flake_schemas namespace nix { diff --git a/src/libcmd/include/nix/cmd/flake-schemas.hh b/src/libcmd/include/nix/cmd/flake-schemas.hh index bfc2a38cde..2084ab5036 100644 --- a/src/libcmd/include/nix/cmd/flake-schemas.hh +++ b/src/libcmd/include/nix/cmd/flake-schemas.hh @@ -4,6 +4,10 @@ #include "nix/flake/flake.hh" #include "nix/cmd/command.hh" +namespace nix::flake { +struct LockedFlake; +} + namespace nix::flake_schemas { using namespace eval_cache; @@ -93,4 +97,20 @@ using Schemas = std::map; Schemas getSchemas(ref root); +struct FlakeInventoryOptions +{ + bool showLegacy = false; + bool showAllSystems = false; + bool showOutputPaths = false; + bool showDrvPaths = false; + bool showDrvNames = false; +}; + +nlohmann::json getFlakeInventory( + EvalState & state, + Store & evalStore, + flake::LockedFlake & flake, + ref cache, + const FlakeInventoryOptions & options); + } // namespace nix::flake_schemas diff --git a/src/nix/flake.cc b/src/nix/flake.cc index f50a2b1bc3..57f4977dac 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -840,38 +840,34 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs struct CmdFlakeShow : FlakeCommand, MixJSON, MixFlakeSchemas { - bool showLegacy = false; - bool showAllSystems = false; - bool showOutputPaths = false; - bool showDrvPaths = false; - bool showDrvNames = false; + flake_schemas::FlakeInventoryOptions options; CmdFlakeShow() { addFlag({ .longName = "legacy", .description = "Show the contents of the `legacyPackages` output.", - .handler = {&showLegacy, true}, + .handler = {&options.showLegacy, true}, }); addFlag({ .longName = "all-systems", .description = "Show the contents of outputs for all systems.", - .handler = {&showAllSystems, true}, + .handler = {&options.showAllSystems, true}, }); addFlag({ .longName = "output-paths", .description = "Include the store paths of derivation outputs in the JSON output.", - .handler = {&showOutputPaths, true}, + .handler = {&options.showOutputPaths, true}, }); addFlag({ .longName = "drv-paths", .description = "Include the store paths of derivations in the JSON output.", - .handler = {&showDrvPaths, true}, + .handler = {&options.showDrvPaths, true}, }); addFlag({ .longName = "drv-names", .description = "Show the names and versions of derivations.", - .handler = {&showDrvNames, true}, + .handler = {&options.showDrvNames, true}, }); } @@ -889,115 +885,18 @@ struct CmdFlakeShow : FlakeCommand, MixJSON, MixFlakeSchemas void run(nix::ref store) override { - if (showOutputPaths && !json) + if (options.showOutputPaths && !json) throw UsageError("The '--output-paths' flag requires '--json'."); - if (showDrvPaths && !json) + if (options.showDrvPaths && !json) throw UsageError("The '--drv-paths' flag requires '--json'."); auto state = getEvalState(); auto flake = make_ref(lockFlake()); - auto localSystem = std::string(settings.thisSystem.get()); auto cache = flake_schemas::call(*state, flake, getDefaultFlakeSchemas()); - auto inventory = cache->getRoot()->getAttr("inventory"); - auto outputs = cache->getRoot()->getAttr("outputs"); - - FutureVector futures(*state->executor); - - std::function node, nlohmann::json & obj)> visit; - - visit = [&](ref node, nlohmann::json & obj) { - flake_schemas::visit( - showAllSystems ? std::optional() : localSystem, - showLegacy, - node, - flake->flake.provenance, - - [&](const flake_schemas::Leaf & leaf) { - if (auto what = leaf.what()) - obj.emplace("what", *what); - - if (auto shortDescription = leaf.shortDescription()) - obj.emplace("shortDescription", *shortDescription); - - if (auto drv = leaf.derivation(outputs)) { - auto drvObj = nlohmann::json::object(); - - if (json || showDrvNames) - drvObj.emplace("name", drv->getAttr(state->s.name)->getString()); - - if (showDrvPaths) { - auto drvPath = drv->forceDerivation(); - drvObj.emplace("path", store->printStorePath(drvPath)); - } - - if (showOutputPaths) { - auto outputs = nlohmann::json::object(); - auto drvPath = drv->forceDerivation(); - auto drv = getEvalStore()->derivationFromPath(drvPath); - for (auto & i : drv.outputsAndOptPaths(*store)) { - if (auto outPath = i.second.second) - outputs.emplace(i.first, store->printStorePath(*outPath)); - else - outputs.emplace(i.first, nullptr); - } - drvObj.emplace("outputs", std::move(outputs)); - } - - obj.emplace("derivation", std::move(drvObj)); - } - - if (auto forSystems = leaf.forSystems()) - obj.emplace("forSystems", *forSystems); - }, - - [&](std::function forEachChild) { - auto children = nlohmann::json::object(); - forEachChild([&](Symbol attrName, ref node, bool isLast) { - auto & j = children.emplace(state->symbols[attrName], nlohmann::json::object()).first.value(); - state->spawn(futures, 1, [&visit, &j, node]() { - try { - visit(node, j); - } catch (EvalError & e) { - // FIXME: make it a flake schema attribute whether to ignore evaluation errors. - if (node->root->state.symbols[node->getAttrPath()[0]] == "legacyPackages") - j.emplace("failed", true); - else - throw; - } - }); - }); - obj.emplace("children", std::move(children)); - }, - - [&](ref node, const std::vector & systems) { - obj.emplace("filtered", true); - }, - - [&](ref node) { obj.emplace("isLegacy", true); }); - }; - - auto inv = nlohmann::json::object(); - - flake_schemas::forEachOutput( - inventory, - [&](Symbol outputName, - std::shared_ptr output, - const std::string & doc, - bool isLast) { - auto & j = inv.emplace(state->symbols[outputName], nlohmann::json::object()).first.value(); - - if (output) { - j.emplace("doc", doc); - auto & j2 = j.emplace("output", nlohmann::json::object()).first.value(); - state->spawn(futures, 1, [&visit, output, &j2]() { visit(ref(output), j2); }); - } else - j.emplace("unknown", true); - }); - - futures.finishAll(); + auto inv = flake_schemas::getFlakeInventory(*state, *getEvalStore(), *flake, cache, options); if (json) { auto res = nlohmann::json{{"version", 2}, {"inventory", std::move(inv)}}; From 0844e56a5115beaa9f9a7692b7a64ff26ca5e0af Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 16:06:31 +0200 Subject: [PATCH 535/927] WIP --- src/libexpr/primops/fakeDerivation.cc | 4 +- src/nix/baked-flake.nix | 32 +++++++ src/nix/flake-bake.cc | 127 ++++++-------------------- src/nix/meson.build | 1 + tests/functional/meson.build | 1 + 5 files changed, 66 insertions(+), 99 deletions(-) create mode 100644 src/nix/baked-flake.nix diff --git a/src/libexpr/primops/fakeDerivation.cc b/src/libexpr/primops/fakeDerivation.cc index 046c8db194..03d1d96e0d 100644 --- a/src/libexpr/primops/fakeDerivation.cc +++ b/src/libexpr/primops/fakeDerivation.cc @@ -116,7 +116,7 @@ static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** ar drv.outputs.insert_or_assign(outName, DerivationOutput::InputAddressed{.path = out.path}); } - auto drvPath = writeDerivation(*state.store, *state.asyncPathWriter, drv, state.repair); + auto drvPath = state.store->writeDerivation(*state.asyncPathWriter, drv, state.repair); // FIXME state.waitForPath(drvPath); @@ -130,7 +130,7 @@ static RegisterPrimOp primop_fakeDerivation({ .doc = R"( Placeholder. )", - .fun = prim_fakeDerivation, + .impl = prim_fakeDerivation, }); } // namespace nix diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix new file mode 100644 index 0000000000..2f5344543e --- /dev/null +++ b/src/nix/baked-flake.nix @@ -0,0 +1,32 @@ +{ + outputs = { self }: + let + data = builtins.fromJSON (builtins.readFile ./outputs.json); + + convert = output: + if output ? children then + builtins.mapAttrs (childName: child: + convert child + ) output.children + else if output ? "derivation" then + { + type = "derivation"; + name = output.derivation.name; + system = builtins.head output.forSystems; # FIXME + meta.description = output.shortDescription; + drvPath = builtins.fakeDerivation { + name = output.derivation.name; + outputs = builtins.mapAttrs (outputName: output: + { path = output; } + ) output.derivation.outputs; + } ; + outPath = output.derivation.outputs.out; # FIXME + outputName = "out"; # FIXME + } + else + throw "Output not supported in a baked flake."; + in + builtins.mapAttrs (outputName: output: + convert output.output + ) data; +} diff --git a/src/nix/flake-bake.cc b/src/nix/flake-bake.cc index fd6f905fb6..51cb18082a 100644 --- a/src/nix/flake-bake.cc +++ b/src/nix/flake-bake.cc @@ -1,13 +1,28 @@ #include "flake-command.hh" #include "nix/cmd/flake-schemas.hh" -#include "nix/expr/parallel-eval.hh" -#include "nix/store/derivations.hh" +#include "nix/util/file-system.hh" + +#include using namespace nix; using namespace nix::flake; struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas { + std::filesystem::path destDir; + + CmdFlakeBake() + { + addFlag({ + .longName = "dest-dir", + .description = "Directory in which to write the baked `flake.nix`.", + .labels = {"path"}, + .handler = {&destDir}, + .completer = completePath, + .required = true, + }); + } + std::string description() override { return "bake a flake"; @@ -28,101 +43,19 @@ struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas auto cache = flake_schemas::call(*state, flake, getDefaultFlakeSchemas()); - auto inventory = cache->getRoot()->getAttr("inventory"); - auto outputs = cache->getRoot()->getAttr("outputs"); - - FutureVector futures(*state->executor); - - // FIXME: generate an AST. - Sync result; - - *result.lock() = R""( -{ - outputs = - { self }: - { -)""; - - auto visit = [&](this auto & visit, ref node) -> void { - flake_schemas::visit( - {"x86_64-linux"}, // FIXME - true, - node, - flake->flake.provenance, - - [&](const flake_schemas::Leaf & leaf) { - printError("AT %s", leaf.node->getAttrPathStr()); - - auto drv = leaf.derivation(outputs); - if (!drv) - // FIXME: emit attrs that throw a useful error? - return; - - // FIXME: do we need to force the derivation? - auto drvPath = drv->forceDerivation(); - auto storeDrv = evalStore->derivationFromPath(drvPath); - std::map outputs; - for (auto & i : storeDrv.outputsAndOptPaths(*store)) { - if (auto outPath = i.second.second) - outputs.emplace(i.first, *outPath); - else - // FIXME: we could build the derivation to get the outputs. - return; - } - - auto name = drv->getAttr(state->s.name)->getString(); - - auto r(result.lock()); - *r += " " + leaf.node->getAttrPathStr() + " = {\n"; - *r += " drvPath = builtins.fakeDerivation {\n"; - *r += " name = \"" + name + "\";\n"; - for (auto & i : outputs) - *r += " outputs." + i.first + ".path = \"" + store->printStorePath(i.second) + "\";\n"; - *r += " };\n"; - *r += " type = \"derivation\";\n"; - *r += " name = \"" + name + "\";\n"; - *r += " system = \"x86_64-linux\";\n"; // FIXME - // meta.description = "Docker image with Nix for x86_64-linux"; - *r += " };\n\n"; - }, - - [&](std::function forEachChild) { - forEachChild([&](Symbol attrName, ref node, bool isLast) { - state->spawn(futures, 1, [&visit, node]() { - try { - visit(node); - } catch (EvalError & e) { - // FIXME: make it a flake schema attribute whether to ignore evaluation errors. - if (node->root->state.symbols[node->getAttrPath()[0]] != "legacyPackages") - throw; - } - }); - }); - }, - - [&](ref node, const std::vector & systems) {}, - - [&](ref node) {}); - }; - - flake_schemas::forEachOutput( - inventory, - [&](Symbol outputName, - std::shared_ptr output, - const std::string & doc, - bool isLast) { - if (output) - state->spawn(futures, 1, [&visit, output]() { visit(ref(output)); }); - }); - - futures.finishAll(); - - *result.lock() += R""( - }; -} -)""; - - logger->cout(*result.lock()); + auto inv = flake_schemas::getFlakeInventory( + *state, + *getEvalStore(), + *flake, + cache, + {.showLegacy = true, .showOutputPaths = true, .showDrvNames = true}); + + std::filesystem::create_directories(destDir); + writeFile(destDir / "outputs.json", inv.dump()); + writeFile( + destDir / "flake.nix", +#include "baked-flake.nix.gen.hh" + ); } }; diff --git a/src/nix/meson.build b/src/nix/meson.build index d0a0d8c6eb..876a61dd9a 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -146,6 +146,7 @@ nix_sources += [ gen_header.process('get-env.sh'), gen_header.process('profiles.md'), gen_header.process('help-stores.md'), + gen_header.process('baked-flake.nix'), ] # The rest of the subdirectories aren't separate components, diff --git a/tests/functional/meson.build b/tests/functional/meson.build index ccde6e12a4..a0fcc14e78 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -98,6 +98,7 @@ suites = [ 'restricted.sh', 'fetchGitSubmodules.sh', 'fetchGitVerification.sh', + 'fetchGitLegacySubmodulesEmptyDir.sh', 'readfile-context.sh', 'nix-channel.sh', 'recursive.sh', From 952269caa7eb46017b5fe5ece0fe2132385916af Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 16:25:17 +0200 Subject: [PATCH 536/927] Formatting --- src/nix/baked-flake.nix | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix index 2f5344543e..c3563761a2 100644 --- a/src/nix/baked-flake.nix +++ b/src/nix/baked-flake.nix @@ -1,13 +1,13 @@ { - outputs = { self }: + outputs = + { self }: let data = builtins.fromJSON (builtins.readFile ./outputs.json); - convert = output: + convert = + output: if output ? children then - builtins.mapAttrs (childName: child: - convert child - ) output.children + builtins.mapAttrs (childName: child: convert child) output.children else if output ? "derivation" then { type = "derivation"; @@ -16,17 +16,13 @@ meta.description = output.shortDescription; drvPath = builtins.fakeDerivation { name = output.derivation.name; - outputs = builtins.mapAttrs (outputName: output: - { path = output; } - ) output.derivation.outputs; - } ; + outputs = builtins.mapAttrs (outputName: output: { path = output; }) output.derivation.outputs; + }; outPath = output.derivation.outputs.out; # FIXME outputName = "out"; # FIXME } else throw "Output not supported in a baked flake."; in - builtins.mapAttrs (outputName: output: - convert output.output - ) data; + builtins.mapAttrs (outputName: output: convert output.output) data; } From 4db60d39ee0ff746a9ac22bbd016863bdc28790d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 16:25:23 +0200 Subject: [PATCH 537/927] Short-circuit fake derivations --- .../build/derivation-trampoline-goal.cc | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index 7621268c15..39ed4ec5b8 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -151,6 +151,53 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation /* Must have at least one wanted output. This is assumed below. */ assert(!resolvedWantedOutputs.empty()); + /* Short-circuit fake derivations. Since these are never actually built, just create substitution goals for the + * outputs. */ + if (drv.builder == "builtin:fake-derivation") { + std::vector> outputGoals; + Goals waitees; + for (auto & outputName : resolvedWantedOutputs) { + auto i = drv.outputs.find(outputName); + if (i == drv.outputs.end()) + throw Error( + "derivation '%s' does not have wanted output '%s'", + worker.store.printStorePath(drvPath), + outputName); + auto outPath = i->second.path(worker.store, drv.name, outputName); + if (!outPath) + throw Error( + "output '%s' of derivation '%s' has no store path", + outputName, + worker.store.printStorePath(drvPath)); + auto g = upcast_goal(worker.makePathSubstitutionGoal(*outPath)); + outputGoals.emplace_back(*outPath, g); + waitees.insert(g); + } + + co_await await(std::move(waitees)); + + if (nrFailed != 0) { + StringSet failedPaths; + for (auto & [path, g] : outputGoals) + if (g->exitCode != ecSuccess) + failedPaths.insert(worker.store.printStorePath(path)); + co_return doneFailure( + ecFailed, + BuildResult::Failure{{ + .status = BuildResult::Failure::DependencyFailed, + .msg = HintFmt( + "failed to substitute outputs of '%s': %s", + worker.store.printStorePath(drvPath), + concatStringsSep(", ", quoteStrings(failedPaths))), + }}); + } + + co_return doneSuccess( + BuildResult::Success{ + .status = BuildResult::Success::Substituted, + }); + } + Goals concreteDrvGoals; /* Build this step! */ From 80f2b3789ddafd90f742396075a4c7596605d258 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 16:27:46 +0200 Subject: [PATCH 538/927] builtin:fake-derivation -> builtin:substitute --- src/libexpr/primops/fakeDerivation.cc | 2 +- src/libstore/build/derivation-trampoline-goal.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/fakeDerivation.cc b/src/libexpr/primops/fakeDerivation.cc index 03d1d96e0d..40190ceadc 100644 --- a/src/libexpr/primops/fakeDerivation.cc +++ b/src/libexpr/primops/fakeDerivation.cc @@ -97,7 +97,7 @@ static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** ar Derivation drv; drv.name = *name; drv.platform = "builtin"; - drv.builder = "builtin:fake-derivation"; + drv.builder = "builtin:substitute"; for (auto & [outName, out] : outputs) { #if 0 diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index 39ed4ec5b8..e7d85f0712 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -151,9 +151,9 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation /* Must have at least one wanted output. This is assumed below. */ assert(!resolvedWantedOutputs.empty()); - /* Short-circuit fake derivations. Since these are never actually built, just create substitution goals for the + /* Short-circuit `builtin:substitute`. Since these are never actually built, just create substitution goals for the * outputs. */ - if (drv.builder == "builtin:fake-derivation") { + if (drv.builder == "builtin:substitute") { std::vector> outputGoals; Goals waitees; for (auto & outputName : resolvedWantedOutputs) { From d1b46ea0fbc7c410b2017a294deb1782326966f2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 19:25:17 +0200 Subject: [PATCH 539/927] Introduce a "substituted" derivation type --- src/libexpr/primops/fakeDerivation.cc | 4 +-- .../build/derivation-trampoline-goal.cc | 2 +- src/libstore/derivations.cc | 32 ++++++++++++++++--- src/libstore/include/nix/store/derivations.hh | 13 ++++++-- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/libexpr/primops/fakeDerivation.cc b/src/libexpr/primops/fakeDerivation.cc index 40190ceadc..a5b7134b92 100644 --- a/src/libexpr/primops/fakeDerivation.cc +++ b/src/libexpr/primops/fakeDerivation.cc @@ -100,7 +100,6 @@ static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** ar drv.builder = "builtin:substitute"; for (auto & [outName, out] : outputs) { -#if 0 if (out.narHash) drv.outputs.insert_or_assign( outName, @@ -112,8 +111,7 @@ static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** ar }, }); else -#endif - drv.outputs.insert_or_assign(outName, DerivationOutput::InputAddressed{.path = out.path}); + drv.outputs.insert_or_assign(outName, DerivationOutput::InputAddressed{.path = out.path}); } auto drvPath = state.store->writeDerivation(*state.asyncPathWriter, drv, state.repair); diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index e7d85f0712..3144178853 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -153,7 +153,7 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation /* Short-circuit `builtin:substitute`. Since these are never actually built, just create substitution goals for the * outputs. */ - if (drv.builder == "builtin:substitute") { + if (drv.type() == DerivationType::Substituted{}) { std::vector> outputGoals; Goals waitees; for (auto & outputName : resolvedWantedOutputs) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 485eea0eb8..4e9441da3f 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -54,6 +54,7 @@ bool DerivationType::isCA() const [](const InputAddressed & ia) { return false; }, [](const ContentAddressed & ca) { return true; }, [](const Impure &) { return true; }, + [](const Substituted &) { return false; }, }, raw); } @@ -65,6 +66,7 @@ bool DerivationType::isFixed() const [](const InputAddressed & ia) { return false; }, [](const ContentAddressed & ca) { return ca.fixed; }, [](const Impure &) { return false; }, + [](const Substituted &) { return false; }, }, raw); } @@ -76,6 +78,7 @@ bool DerivationType::hasKnownOutputPaths() const [](const InputAddressed & ia) { return !ia.deferred; }, [](const ContentAddressed & ca) { return ca.fixed; }, [](const Impure &) { return false; }, + [](const Substituted &) { return true; }, }, raw); } @@ -87,6 +90,7 @@ bool DerivationType::isSandboxed() const [](const InputAddressed & ia) { return true; }, [](const ContentAddressed & ca) { return ca.sandboxed; }, [](const Impure &) { return false; }, + [](const Substituted &) { return true; }, }, raw); } @@ -98,6 +102,7 @@ bool DerivationType::isImpure() const [](const InputAddressed & ia) { return false; }, [](const ContentAddressed & ca) { return false; }, [](const Impure &) { return true; }, + [](const Substituted &) { return false; }, }, raw); } @@ -870,6 +875,12 @@ DerivationType BasicDerivation::type() const if (!ty) throw Error("must have at least one output"); + if (builder == "builtin:substitute") { + if (!std::holds_alternative(ty.value().raw)) + throw Error("'builtin:substitute' derivation must have input-addressed outputs"); + return DerivationType::Substituted{}; + } + return ty.value(); } @@ -934,14 +945,15 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut auto kind = std::visit( overloaded{ [](const DerivationType::InputAddressed & ia) { - /* This might be a "pesimistically" deferred output, so we don't + /* This might be a "pessimistically" deferred output, so we don't "taint" the kind yet. */ return DrvHash::Kind::Regular; }, [](const DerivationType::ContentAddressed & ca) { return ca.fixed ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred; }, - [](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }}, + [](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }, + [](const DerivationType::Substituted &) -> DrvHash::Kind { return DrvHash::Kind::Regular; }}, drv.type().raw); DerivedPathMap::ChildNode::Map inputs2; @@ -1165,6 +1177,7 @@ bool Derivation::shouldResolve() const : true; }, [&](const DerivationType::Impure &) { return true; }, + [&](const DerivationType::Substituted &) { return false; }, }, drvType.raw); @@ -1282,6 +1295,18 @@ std::optional Derivation::tryResolve( template static void processDerivationOutputPaths(Store & store, auto && drv, std::string_view drvName) { + if (drv.builder == "builtin:substitute") { + if (drv.platform != "builtin") + throw Error("'builtin:substitute' derivation must be a builtin"); + if (!drv.args.empty()) + throw Error("'builtin:substitute' derivation must have no arguments"); + if (!drv.env.empty()) + throw Error("'builtin:substitute' derivation must have no environment variables"); + if (!drv.inputSrcs.empty()) + throw Error("'builtin:substitute' derivation must have no inputs"); + return; + } + std::optional hashesModulo; for (auto & [outputName, output] : drv.outputs) { @@ -1331,7 +1356,6 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string auto outPath = store.makeOutputPath(outputName, *h, drvName); if constexpr (std::is_same_v) { -#if 0 if (outputVariant.path == outPath) { return; // Correct case } @@ -1341,8 +1365,6 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string "derivation has incorrect output '%s', should be '%s'", store.printStorePath(outputVariant.path), store.printStorePath(outPath)); -#endif - return; // FIXME } else if constexpr (std::is_same_v) { if constexpr (fillIn) /* Fill in output path for Deferred diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 88bfb6e49b..e61660063a 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -200,7 +200,7 @@ struct DerivationType /** * Impure derivation type * - * This is similar at build-time to the content addressed, not standboxed, not fixed + * This is similar at build-time to the content addressed, not sandboxed, not fixed * type, but has some restrictions on its usage. */ struct Impure @@ -209,7 +209,16 @@ struct DerivationType auto operator<=>(const Impure &) const = default; }; - typedef std::variant Raw; + /** + * A builtin:substitute derivation. + */ + struct Substituted + { + bool operator==(const Substituted &) const = default; + auto operator<=>(const Substituted &) const = default; + }; + + typedef std::variant Raw; Raw raw; From 9c7ca4a0db87d1b902aa478797d59ff9fb83ec7e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 19:42:50 +0200 Subject: [PATCH 540/927] StoreDirConfig::parseStorePath(): Don't crash on empty paths --- src/libstore/store-dir-config.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 61f82029d7..962a3830f1 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -8,6 +8,8 @@ namespace nix { StorePath StoreDirConfig::parseStorePath(std::string_view path) const { + if (path.empty()) + throw BadStorePath("empty path is not a valid store path"); // On Windows, `/nix/store` is not a canonical path. More broadly it // is unclear whether this function should be using the native // notion of a canonical path at all. For example, it makes to From b8bf9467168bd3c81a460c8292adb8933f028bb6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2026 19:43:01 +0200 Subject: [PATCH 541/927] canonPath(): Don't crash on empty paths --- src/libutil-tests/file-system.cc | 2 +- src/libutil/file-system.cc | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 95bfa16b14..da50d82bfe 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -121,7 +121,7 @@ TEST(canonPath, requiresAbsolutePath) ASSERT_ANY_THROW(canonPath("."sv)); ASSERT_ANY_THROW(canonPath(".."sv)); ASSERT_ANY_THROW(canonPath("../"sv)); - ASSERT_DEATH({ canonPath(""sv); }, "!path.empty\\(\\)"); + ASSERT_ANY_THROW(canonPath(""sv)); } /* ---------------------------------------------------------------------------- diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index e1efca0ddf..97bbfbb323 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -96,7 +96,8 @@ absPath(const std::filesystem::path & path0, const std::filesystem::path * dir, std::filesystem::path canonPath(const std::filesystem::path & path, bool resolveSymlinks) { - assert(!path.empty()); + if (path.empty()) + throw Error("cannot canonicalise an empty path"); if (!path.is_absolute()) throw Error("not an absolute path: %s", PathFmt(path)); From cfe08c83a3c75d235feb6b265ce74ac663a599c8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 May 2026 21:55:17 +0200 Subject: [PATCH 542/927] Doh --- tests/functional/meson.build | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/meson.build b/tests/functional/meson.build index a0fcc14e78..ccde6e12a4 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -98,7 +98,6 @@ suites = [ 'restricted.sh', 'fetchGitSubmodules.sh', 'fetchGitVerification.sh', - 'fetchGitLegacySubmodulesEmptyDir.sh', 'readfile-context.sh', 'nix-channel.sh', 'recursive.sh', From 2157805cd08a374f87e5e9feee78313b6c7cd3db Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 May 2026 22:43:03 +0200 Subject: [PATCH 543/927] baked-flake.nix: Don't barf on unsupported/skipped outputs --- src/nix/baked-flake.nix | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix index c3563761a2..70649e717b 100644 --- a/src/nix/baked-flake.nix +++ b/src/nix/baked-flake.nix @@ -4,10 +4,12 @@ let data = builtins.fromJSON (builtins.readFile ./outputs.json); + cleanup = builtins.filterAttrs (name: value: value != { }); + convert = output: if output ? children then - builtins.mapAttrs (childName: child: convert child) output.children + cleanup (builtins.mapAttrs (childName: child: convert child) output.children) else if output ? "derivation" then { type = "derivation"; @@ -22,7 +24,8 @@ outputName = "out"; # FIXME } else - throw "Output not supported in a baked flake."; + { + }; in - builtins.mapAttrs (outputName: output: convert output.output) data; + cleanup (builtins.mapAttrs (outputName: output: convert output.output) data); } From 543610326211b3f68c76fc96bc8ad2e90830928f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 May 2026 22:56:23 +0200 Subject: [PATCH 544/927] Fix test --- src/nix/flake.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 57f4977dac..ec4092cab3 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -891,6 +891,9 @@ struct CmdFlakeShow : FlakeCommand, MixJSON, MixFlakeSchemas if (options.showDrvPaths && !json) throw UsageError("The '--drv-paths' flag requires '--json'."); + if (json) + options.showDrvNames = true; + auto state = getEvalState(); auto flake = make_ref(lockFlake()); From 41804e630e972f5bb9ce091c4745649c538e3b75 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 May 2026 00:04:38 +0200 Subject: [PATCH 545/927] Add file to package.nix --- src/nix/package.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/package.nix b/src/nix/package.nix index b62feeec3e..ece95ff2cd 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -59,6 +59,7 @@ mkMesonExecutable (finalAttrs: { ../../doc/manual/source/store/types/index.md.in ./profiles.md ../../doc/manual/source/command-ref/files/profiles.md + ./baked-flake.nix # Files ] From 9dc4d7654a2f0dc9e97e8f490b6d40fde65c6a4e Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 26 Jun 2026 15:03:05 +0300 Subject: [PATCH 546/927] installer: Bail out early if running on macOS < 14.0 (cherry picked from commit 9205295fbd46e741c598acb4fced5de0920e9067) --- scripts/install-nix-from-tarball.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/install-nix-from-tarball.sh b/scripts/install-nix-from-tarball.sh index f17e4c2af3..73f389b6ff 100644 --- a/scripts/install-nix-from-tarball.sh +++ b/scripts/install-nix-from-tarball.sh @@ -28,14 +28,15 @@ fi OS="$(uname -s)" -# macOS support for 10.12.6 or higher +# Since nixpkgs 25.11 the minimum deployment target is macOS 14.0 if [ "$OS" = "Darwin" ]; then + # shellcheck disable=SC2034 IFS='.' read -r macos_major macos_minor macos_patch << EOF $(sw_vers -productVersion) EOF - if [ "$macos_major" -lt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -lt 12 ]; } || { [ "$macos_minor" -eq 12 ] && [ "$macos_patch" -lt 6 ]; }; then + if [ "$macos_major" -lt 14 ]; then # patch may not be present; command substitution for simplicity - echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" + echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 14.0 or higher" exit 1 fi fi From 128ce54597975dd8b5a48892fe0edeae0dfaf947 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jun 2026 17:09:58 +0200 Subject: [PATCH 547/927] SourceAccessor::readFile(sink): Remove default implementation This was defined in terms of the non-virtual string variant of readFile(), which is defined in terms of the sink variant. So this could never work. (cherry picked from commit 0db6bc69c1a38bcbbe24a9ff09b963701ea5c917) --- src/libutil/include/nix/util/source-accessor.hh | 3 ++- src/libutil/source-accessor.cc | 7 ------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 0c713fcdb5..0a325de168 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -81,7 +81,8 @@ public: * @note subclasses of `SourceAccessor` need to implement at least * one of the `readFile()` variants. */ - virtual void readFile(const CanonPath & path, Sink & sink, fun sizeCallback = [](uint64_t size) {}); + virtual void + readFile(const CanonPath & path, Sink & sink, fun sizeCallback = [](uint64_t size) {}) = 0; virtual bool pathExists(const CanonPath & path); diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 8ae9143754..a7cc160a6d 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -71,13 +71,6 @@ std::string SourceAccessor::readFile(const CanonPath & path) return std::move(sink.s); } -void SourceAccessor::readFile(const CanonPath & path, Sink & sink, fun sizeCallback) -{ - auto s = readFile(path); - sizeCallback(s.size()); - sink(s); -} - Hash SourceAccessor::hashPath(const CanonPath & path, PathFilter & filter, HashAlgorithm ha) { HashSink sink(ha); From 90823254bdefba32e8737f97da7acec2cb1d23d5 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 29 Jun 2026 02:39:57 +0300 Subject: [PATCH 548/927] libexpr: Handle lazy paths in builtins.storePath better MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When building nix.dev manual I ran into an issue with the current handling of lazy paths: … while calling the 'import' builtin at /nix/store/8kpx53qi52yhjai1vdw8zpa95iqa61bv-source/default.nix:167:19: 166| 167| flake = import (outPath + "/flake.nix"); | ^ 168| … while realising the context of a path … while calling the 'storePath' builtin at /nix/store/8kpx53qi52yhjai1vdw8zpa95iqa61bv-source/default.nix:115:15: 114| # If it's already a store path, don't copy it again. 115| builtins.storePath src | ^ 116| else error: path '/nix/store/5fn5lshxlh5zb8w1a747apqlqlbi3j0x-source' is required, but there is no substituter that can build it This also has the benefit of not relying on nix::canonPath(), which is one of the bits of I/O not funneled through the rootFS accessor. The responsible code is in flake-compat [1]. With a path we lack the correct context but it points to a "lazy" path in the store. Funneling I/O into the accessor seems like the correct solution here. [1]: https://github.com/NixOS/flake-compat/blob/5edf11c44bc78a0d334f6334cdaf7d60d732daab/default.nix#L147-L156 (cherry picked from commit 933f3140b1e73de2b909902b0e251c7da4031607) --- src/libexpr/primops.cc | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1a837a1b96..46ef49a367 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1961,25 +1961,21 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V .debugThrow(); NixStringContext context; - auto path = - state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'") - .path; - /* Here we are leaving the realm of the rootFS accessor and must actually fetch to the store. - TODO: This could probably get optimised to avoid the fetching altogether to short-circuit when the path - is already mounted on storeFS. */ - state.ensureLazyPathsCopied(context); + SourcePath sourcePath = state.coerceToPath( + pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'"); + /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ - if (!state.store->isStorePath(path.abs())) - path = CanonPath(canonPath(path.abs(), true).string()); - if (!state.store->isInStore(path.abs())) - state.error("path '%1%' is not in the Nix store", path).atPos(pos).debugThrow(); - auto path2 = state.store->toStorePath(path.abs()).first; - if (!settings.readOnlyMode) - state.store->ensurePath(path2); - context.insert(NixStringContextElem::Opaque{.path = path2}); - v.mkString(path.abs(), context, state.mem); + if (!state.store->isStorePath(sourcePath.path.abs())) + sourcePath = sourcePath.resolveSymlinks(SymlinkResolution::Full); + if (!state.store->isInStore(sourcePath.path.abs())) + state.error("path '%1%' is not in the Nix store", sourcePath).atPos(pos).debugThrow(); + auto storePath = state.store->toStorePath(sourcePath.path.abs()).first; + if (!state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath))) && !settings.readOnlyMode) + state.store->ensurePath(storePath); + context.insert(NixStringContextElem::Opaque{.path = storePath}); + v.mkString(sourcePath.path.abs(), context, state.mem); } static RegisterPrimOp primop_storePath({ From 199ad670792f9a887e0215ea92b07fec833abd96 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 26 Jun 2026 22:36:12 +0300 Subject: [PATCH 549/927] Fix LocalStoreAccessor::maybeLstat Paths that are not valid store path names (i.e. `foo` or `bar`) can't ever exist in the store, and the correct semantics for it is to return `std::nullopt` instead of throwing. Also noticed a lot of bugs in RemoteFSAccessor, but that's for a later patch. There are also lots of missing overrides of pathExists in SourceAccessor implementations, so I wonder whether that should just be made non-virtual. The evaluator doesn't benefit from potentially optimised pathExists, because MountedSourceAccessor and various other combinators don't propagate them. Plus we should typically still query the lstat to do positive caching (CachingSourceAccessor). See https://github.com/NixOS/nix/issues/16017. (cherry picked from commit 25dbbaf41697511ff09428b89191b2316fb21f59) --- src/libstore/dummy-store.cc | 3 ++ .../include/nix/store/remote-fs-accessor.hh | 2 + src/libstore/local-fs-store.cc | 37 +++++++++++++++---- src/libstore/remote-fs-accessor.cc | 2 + .../include/nix/util/source-accessor.hh | 7 ++++ tests/functional/flakes/follow-paths.sh | 2 +- tests/functional/lang.sh | 2 +- .../functional/lang/eval-okay-pathexists.nix | 3 ++ 8 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index b364c90e16..6238c88904 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -69,6 +69,8 @@ class WholeStoreViewAccessor : public SourceAccessor }); if (!res) + /* The accessor is truly empty, i.e. without any file at root so + any subsequent operation with it will fail. */ res = &emptyAccessor; return callback(*res, path); @@ -107,6 +109,7 @@ class WholeStoreViewAccessor : public SourceAccessor DirEntries readDirectory(const CanonPath & path) override { + /* FIXME: Special-case the root directory to read the whole store, not just an empty root. */ return callWithAccessorForPath( path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.readDirectory(path); }); } diff --git a/src/libstore/include/nix/store/remote-fs-accessor.hh b/src/libstore/include/nix/store/remote-fs-accessor.hh index fa7f5fc280..26fae0d63d 100644 --- a/src/libstore/include/nix/store/remote-fs-accessor.hh +++ b/src/libstore/include/nix/store/remote-fs-accessor.hh @@ -33,6 +33,8 @@ public: /** * @return nullptr if the store does not contain any object at that path. + * + * @todo This actually doesn't return nullptr, but throws on invalid paths. */ std::shared_ptr accessObject(const StorePath & path); diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 5be0e5b067..a6fd24bf34 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -49,13 +49,33 @@ struct LocalStoreAccessor : SourceAccessor { } - void requireStoreObject(const CanonPath & path) + void requireStoreObject(const StorePath & storePath) { - auto [storePath, rest] = store->toStorePath(store->storeDir + path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); } + static StorePath getStoreObjectPath(const CanonPath & path) + { + /* See special handling of isRoot() in maybeLstat. */ + if (path.isRoot()) + throw BadStorePath("path '%1%' is not a valid store path", path); + return StorePath(*path.begin()); + } + + static std::optional maybeGetStoreObjectPath(const CanonPath & path) + try { + return getStoreObjectPath(path); + } catch (BadStorePath &) { + /* FIXME: Stop using exceptions for control flow. */ + return std::nullopt; + } + + void requireStoreObject(const CanonPath & path) + { + requireStoreObject(getStoreObjectPath(path)); + } + std::optional maybeLstat(const CanonPath & path) override { /* Also allow `path` to point to the entire store, which is @@ -63,7 +83,13 @@ struct LocalStoreAccessor : SourceAccessor if (path.isRoot()) return Stat{.type = tDirectory}; - requireStoreObject(path); + /* Querying existence should not fail for things like + `/nix/store/foo.nix`. The store cannot contain such files (unless + some weird impurities sneak in, but that's UB from nix's PoV). */ + auto maybeStorePath = maybeGetStoreObjectPath(path); + if (!maybeStorePath) + return std::nullopt; + requireStoreObject(*maybeStorePath); return accessor->maybeLstat(path); } @@ -123,11 +149,6 @@ struct LocalStoreAccessor : SourceAccessor { return accessor->getLastModified(); } - - bool pathExists(const CanonPath & path) override - { - return accessor->pathExists(path); - } }; } // namespace diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 0129be46eb..acf6a4680d 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -39,6 +39,8 @@ std::optional RemoteFSAccessor::maybeLstat(const CanonPath { if (path.isRoot()) return Stat{.type = tDirectory}; + /* FIXME: Correctly handle invalid names (return nullopt) and don't fail on + non-existent paths. */ auto res = fetch(path); return res.first->maybeLstat(res.second); } diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 0a325de168..cc904a720d 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -84,6 +84,13 @@ public: virtual void readFile(const CanonPath & path, Sink & sink, fun sizeCallback = [](uint64_t size) {}) = 0; + /** + * @brief Check whether a file exists at @p path. + * + * @todo Consider making this non-virtual, since the evaluator uses + * maybeLstat as an indication that a file exists always (for positive + * caching purposes). + */ virtual bool pathExists(const CanonPath & path); enum Type { diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 143d0c2255..39e87348f1 100755 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -131,7 +131,7 @@ EOF git -C "$flakeFollowsA" add flake.nix expect 1 nix flake lock "$flakeFollowsA" 2>&1 | grep '/flakeB.*is forbidden in pure evaluation mode' -expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep "'flakeB' is too short to be a valid store path" +expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep '/flakeB.*does not exist' # Test relative non-flake inputs. cat > "$flakeFollowsA"/flake.nix <&1 | grepQuiet Hello diff --git a/tests/functional/lang/eval-okay-pathexists.nix b/tests/functional/lang/eval-okay-pathexists.nix index 022b22feae..a5a3875da1 100644 --- a/tests/functional/lang/eval-okay-pathexists.nix +++ b/tests/functional/lang/eval-okay-pathexists.nix @@ -32,3 +32,6 @@ builtins.pathExists (./lib.nix) && builtins.pathExists ./symlink-resolution/foo/overlays/overlay.nix && builtins.pathExists ./symlink-resolution/broken && builtins.pathExists (builtins.toString ./symlink-resolution/foo/overlays + "/.") +&& builtins.pathExists "${builtins.storeDir}" +&& !builtins.pathExists "${builtins.storeDir}/foo" +&& !builtins.pathExists "${builtins.storeDir}/foo/bar" From 18ed6c101d0d652d44b952f646f6dce790fc5799 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 1 Jul 2026 04:08:22 +0300 Subject: [PATCH 550/927] libstore: Fix libcurl thread wakeup with curl >= 8.21 Since https://github.com/curl/curl/commit/2a2104f3cff44bb28bb570a093be52bbeeed8f23 libcurl now swallows events in curl_multi_perform, so there's a chance that we miss a wakeup. This is somewhat reproducible on my machine by doing nix flake prefetch https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz. Another option is to bring back our own wakeup pipe, but that's less portable. Ideally this regression would be fixed in libcurl too... (cherry picked from commit 0d0c3335045ebc7383312265082e93928cd7f77a) --- src/libstore/filetransfer.cc | 63 ++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index d03af296b9..c58751e96b 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -989,6 +989,8 @@ struct curlFileTransfer : public FileTransfer private: bool quitting = false; public: + bool work = false; + void quit() { quitting = true; @@ -1036,12 +1038,14 @@ struct curlFileTransfer : public FileTransfer void stopWorkerThread() { /* Signal the worker thread to exit. */ - state_.lock()->quit(); - wakeupMulti(); + auto state(state_.lock()); + state->quit(); + wakeupMulti(*state); } - void wakeupMulti() + void wakeupMulti(State & state) { + state.work = true; if (auto ec = ::curl_multi_wakeup(curlm.get())) throw curlMultiError(ec); } @@ -1091,25 +1095,12 @@ struct curlFileTransfer : public FileTransfer } } - /* Wait for activity, including wakeup events. */ - long maxSleepTimeMs = items.empty() ? 10000 : 100; - auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() - ? std::max( - 0, - (int) std::chrono::duration_cast( - nextWakeup - std::chrono::steady_clock::now()) - .count()) - : maxSleepTimeMs; - - int numfds = 0; - mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds); - if (mc != CURLM_OK) - throw curlMultiError(mc); - nextWakeup = std::chrono::steady_clock::time_point(); std::vector> incoming; + std::vector> unpause; auto now = std::chrono::steady_clock::now(); + bool haveWork; { auto state(state_.lock()); @@ -1131,7 +1122,9 @@ struct curlFileTransfer : public FileTransfer break; } } + unpause = std::exchange(state->unpause, {}); quit = state->isQuitting(); + haveWork = std::exchange(state->work, false); } for (auto & item : incoming) { @@ -1142,14 +1135,10 @@ struct curlFileTransfer : public FileTransfer items[item->req] = item; } - /* NOTE: Unpausing may invoke callbacks to flush all buffers. */ - auto unpause = [&]() { - auto state(state_.lock()); - auto res = state->unpause; - state->unpause.clear(); - return res; - }(); + if (quit) + break; + /* NOTE: Unpausing may invoke callbacks to flush all buffers. */ for (auto & item : unpause) { /* The transfer might have completed (failed) between it getting enqueued for unpause and by the time the worker thread picked @@ -1159,6 +1148,26 @@ struct curlFileTransfer : public FileTransfer continue; static_cast(*ptr).unpause(); } + + /* Wait for activity, including wakeup events. */ + long maxSleepTimeMs = items.empty() ? 10000 : 100; + auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() + ? std::max( + 0, + (int) std::chrono::duration_cast( + nextWakeup - std::chrono::steady_clock::now()) + .count()) + : maxSleepTimeMs; + + /* Since https://github.com/curl/curl/commit/2a2104f3cff44bb28bb570a093be52bbeeed8f23 (8.21), + curl_multi_perform seems to swallow queued up events ¯\_(ツ)_/¯. */ + if (haveWork) + sleepTimeMs = 0; + + int numfds = 0; + mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds); + if (mc != CURLM_OK) + throw curlMultiError(mc); } debug("download thread shutting down"); @@ -1195,9 +1204,9 @@ struct curlFileTransfer : public FileTransfer throw nix::Error("cannot enqueue download request because the download thread is shutting down"); state->incoming.push(item); item->enqueued = true; /* Now any exceptions should be reported via the callback. */ + wakeupMulti(*state); } - wakeupMulti(); return ItemHandle(item.get_ptr()); } @@ -1217,7 +1226,7 @@ struct curlFileTransfer : public FileTransfer { auto state(state_.lock()); state->unpause.push_back(std::move(item)); - wakeupMulti(); + wakeupMulti(*state); } void unpauseTransfer(ItemHandle handle) override From 99b98e5b55f1d697dac0bebe9f2097e1d15c765a Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Mon, 23 Mar 2026 17:04:25 -0400 Subject: [PATCH 551/927] cli: error when remounting store without a private mount namespace Previously, when unshare(CLONE_NEWNS) failed and the store was read-only, Nix warned but still remounted the store writable on the host mount table. This silently affected other processes sharing the namespace. Now it throws an error, since proceeding would mutate shared state. (cherry picked from commit f218c7e8ee1b488921e78e0309eee1a22872366b) --- src/libstore/local-store.cc | 36 +--------- .../include/nix/util/linux-namespaces.hh | 22 +++--- src/libutil/linux/linux-namespaces.cc | 67 ++++++++++++++++++- src/nix/main.cc | 11 +-- 4 files changed, 83 insertions(+), 53 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index dd5d325239..5a1c161c07 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -37,9 +37,7 @@ #endif #ifdef __linux__ -# include -# include -# include +# include "nix/util/linux-namespaces.hh" #endif #ifdef __CYGWIN__ @@ -649,37 +647,7 @@ void LocalStore::makeStoreWritable() #ifdef __linux__ if (!isRootUser()) return; - /* Check if /nix/store is on a read-only mount. */ - struct statvfs stat; - if (statvfs(config->realStoreDir.get().c_str(), &stat) != 0) - throw SysError("getting info about the Nix store mount point"); - - if (stat.f_flag & ST_RDONLY) { - /* In a user namespace, mount flags like `nodev` and `nosuid` are - locked and dropping them causes `EPERM`, so here we translate each - `statvfs` flag to the corresponding `mount` flag individually. */ - unsigned long flags = MS_REMOUNT | MS_BIND; - if (stat.f_flag & ST_NODEV) - flags |= MS_NODEV; - if (stat.f_flag & ST_NOSUID) - flags |= MS_NOSUID; - if (stat.f_flag & ST_NOEXEC) - flags |= MS_NOEXEC; - if (stat.f_flag & ST_NOATIME) - flags |= MS_NOATIME; - if (stat.f_flag & ST_NODIRATIME) - flags |= MS_NODIRATIME; - if (stat.f_flag & ST_RELATIME) - flags |= MS_RELATIME; - if (stat.f_flag & ST_SYNCHRONOUS) - flags |= MS_SYNCHRONOUS; -# ifdef ST_NOSYMFOLLOW - if (stat.f_flag & ST_NOSYMFOLLOW) - flags |= MS_NOSYMFOLLOW; -# endif - if (mount(0, config->realStoreDir.get().c_str(), "none", flags, 0) == -1) - throw SysError("remounting %s writable", PathFmt(config->realStoreDir.get())); - } + remountReadOnlyWritable(config->realStoreDir.get()); #endif } diff --git a/src/libutil/linux/include/nix/util/linux-namespaces.hh b/src/libutil/linux/include/nix/util/linux-namespaces.hh index 8f7ffa8df4..0b6bef7bca 100644 --- a/src/libutil/linux/include/nix/util/linux-namespaces.hh +++ b/src/libutil/linux/include/nix/util/linux-namespaces.hh @@ -1,21 +1,27 @@ #pragma once ///@file -#include - -#include "nix/util/types.hh" +#include namespace nix { /** - * Save the current mount namespace. Ignored if called more than - * once. + * Save the parent mount namespace and enter a private one via + * `unshare(CLONE_NEWNS)`. + */ +void tryEnterPrivateMountNamespace(); + +/** + * Remount `path` writable if its mount is read-only, leaving + * already-writable mounts untouched. This throws if we aren't in a + * private mount namespace, since remounting would leak into the + * host mount table. */ -void saveMountNamespace(); +void remountReadOnlyWritable(const std::filesystem::path & path); /** - * Restore the mount namespace saved by saveMountNamespace(). Ignored - * if saveMountNamespace() was never called. + * Restore the parent mount namespace saved when we entered a private + * one. Ignored if `tryEnterPrivateMountNamespace()` never succeeded. */ void restoreMountNamespace(); diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index 26a7479050..f0416307d1 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -7,6 +7,7 @@ #include #include +#include namespace nix { @@ -90,8 +91,11 @@ bool mountAndPidNamespacesSupported() static AutoCloseFD fdSavedMountNamespace; static AutoCloseFD fdSavedRoot; +static bool havePrivateMountNs = false; -void saveMountNamespace() +/* Save the current mount namespace so restoreMountNamespace() can return + to it later. Ignored if called more than once. */ +static void saveMountNamespace() { static std::once_flag done; std::call_once(done, []() { @@ -103,12 +107,69 @@ void saveMountNamespace() }); } +void tryEnterPrivateMountNamespace() +{ + try { + saveMountNamespace(); + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + havePrivateMountNs = true; + } catch (Error & e) { + debug("failed to set up a private mount namespace: %s", e.message()); + } +} + +void remountReadOnlyWritable(const std::filesystem::path & path) +{ + struct statvfs stat; + if (statvfs(path.c_str(), &stat) != 0) + throw SysError("getting mount info for %s", PathFmt(path)); + + if (!(stat.f_flag & ST_RDONLY)) + return; + + if (!havePrivateMountNs) + throw Error( + "cannot remount %s writable: not in a private mount namespace, " + "so the remount would affect the host mount table. " + "This usually happens inside containers or user namespaces where unshare(CLONE_NEWNS) is not permitted", + PathFmt(path)); + + /* In a user namespace, mount flags like `nodev` and `nosuid` are + locked and dropping them causes `EPERM`, so here we translate each + `statvfs` flag to the corresponding `mount` flag individually. */ + unsigned long flags = MS_REMOUNT | MS_BIND; + if (stat.f_flag & ST_NODEV) + flags |= MS_NODEV; + if (stat.f_flag & ST_NOSUID) + flags |= MS_NOSUID; + if (stat.f_flag & ST_NOEXEC) + flags |= MS_NOEXEC; + if (stat.f_flag & ST_NOATIME) + flags |= MS_NOATIME; + if (stat.f_flag & ST_NODIRATIME) + flags |= MS_NODIRATIME; + if (stat.f_flag & ST_RELATIME) + flags |= MS_RELATIME; + if (stat.f_flag & ST_SYNCHRONOUS) + flags |= MS_SYNCHRONOUS; +#ifdef ST_NOSYMFOLLOW + if (stat.f_flag & ST_NOSYMFOLLOW) + flags |= MS_NOSYMFOLLOW; +#endif + if (mount(0, path.c_str(), "none", flags, 0) == -1) + throw SysError("remounting %s writable", PathFmt(path)); +} + void restoreMountNamespace() { + if (!havePrivateMountNs) + return; + try { auto savedCwd = std::filesystem::current_path(); - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + if (setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); if (fdSavedRoot) { @@ -120,6 +181,8 @@ void restoreMountNamespace() if (chdir(savedCwd.c_str()) == -1) throw SysError("restoring cwd"); + + havePrivateMountNs = false; } catch (Error & e) { debug(e.msg()); } diff --git a/src/nix/main.cc b/src/nix/main.cc index df54a14ccb..963bc07ce2 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -398,15 +398,8 @@ void mainWrapped(int argc, char ** argv) flakeSettings.configureEvalSettings(evalSettings); #ifdef __linux__ - if (isRootUser()) { - try { - saveMountNamespace(); - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - } catch (Error & e) { - warn("failed to set up a private mount namespace: %s", e.msg()); - } - } + if (isRootUser()) + tryEnterPrivateMountNamespace(); #endif Finally f([] { logger->stop(); }); From 3ba59c1b138cba0262648ea20b287038ce2b28d8 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 2 Jul 2026 23:20:25 +0300 Subject: [PATCH 552/927] libstore: Make FileTransfer throw Interrupted on interrupted transfers We are now issuing many more requests in queryMissing concurrently, without doing checkInterrupt() when enqueueing downloads (arguably we shouldn't). The exception ignoring behavior of querySubstitutablePathInfosAsync, where it swallows and prints exceptions derived from Error & doesn't help - we end up with a wall of text like: error: download of 'https://cache.nixos.org/b2s9lqzdi2pg80j0lawgw8ndian3k9mn.narinfo' was interrupted error: download of 'https://cache.nixos.org/5h2h4brfrgr89gvh1l93mh63mm0h5r6s.narinfo' was interrupted error: download of 'https://cache.nixos.org/azz5rixma8na4h9s75406w94yjsk2b4k.narinfo' was interrupted error: download of 'https://cache.nixos.org/xdlfnymy64jxfhnj19sjvdvmbbmyn468.narinfo' was interrupted error: download of 'https://cache.nixos.org/q8sra2q3n8gvalzbfq48jvv18xjsg2gb.narinfo' was interrupted error: download of 'https://cache.nixos.org/pzmm2bczb5gb1nvgv5xp6ims9iqcc4af.narinfo' was interrupted error: download of 'https://cache.nixos.org/s3slcivizcjkppyvf5drlbx0b8dxafg9.narinfo' was interrupted Rethrowing Interrupted instead of FileTransferError on proper interrupts seems correct to me. And since it doesn't inherit from Error (but rather BaseError) it escapes out of catch (Error &) blocks and properly unwinds the stack eagerly. Also add a new `Cancelled` exception type to reflect cases (not very useful for now) to indicate that a filetransfer was cancelled without a global interrupt. In the future we should add per-transfer cancellation mechanisms for this purpose. (cherry picked from commit 8c9426502817a493d23ce6e112965544fcb21aed) --- src/libstore/filetransfer.cc | 34 ++++++++++++------- .../include/nix/store/filetransfer.hh | 2 +- src/libutil/include/nix/util/signals.hh | 1 + src/libutil/unix/signals.cc | 2 ++ src/libutil/util.cc | 4 +++ src/libutil/windows/signals.cc | 2 ++ 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index c58751e96b..6e9f2256ea 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -294,12 +294,7 @@ struct curlFileTransfer : public FileTransfer } try { if (!done && enqueued) - fail(FileTransferError( - Interrupted, - {}, - "%s of '%s' was interrupted", - Uncolored(request.noun()), - request.displayUri())); + failInterruptedOrCancelled(); } catch (...) { ignoreExceptionInDestructor(); } @@ -328,6 +323,18 @@ struct curlFileTransfer : public FileTransfer failEx(std::make_exception_ptr(std::forward(e))); } + void failInterruptedOrCancelled() + { + HintFmt fmt("%s of '%s' was interrupted", Uncolored(request.noun()), request.displayUri()); + + /* Technically, we don't really have per-transfer cancellation currently, + but it's nice to distinguish between the two in the future. */ + if (getInterrupted()) + fail(nix::Interrupted(std::move(fmt))); + else + fail(nix::Cancelled(std::move(fmt))); + } + LambdaSink finalSink; std::optional errorSink; @@ -872,13 +879,14 @@ struct curlFileTransfer : public FileTransfer std::optional response; if (errorSink) response = std::move(errorSink->s); - auto exc = code == CURLE_ABORTED_BY_CALLBACK && getInterrupted() ? FileTransferError( - Interrupted, - std::move(response), - "%s of '%s' was interrupted", - Uncolored(request.noun()), - request.displayUri()) - : httpStatus != 0 + + /* TODO: Also support per-transfer cancellations. */ + if (code == CURLE_ABORTED_BY_CALLBACK && getInterrupted()) { + failInterruptedOrCancelled(); + return; + } + + auto exc = httpStatus != 0 ? FileTransferError( err, std::move(response), diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 04980c9ac4..774a4e1403 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -470,7 +470,7 @@ public: void download(FileTransferRequest && request, Sink & sink, std::function resultCallback = {}); - enum Error { NotFound, Unauthorized, Forbidden, Misc, Transient, Interrupted }; + enum Error { NotFound, Unauthorized, Forbidden, Misc, Transient }; }; /** diff --git a/src/libutil/include/nix/util/signals.hh b/src/libutil/include/nix/util/signals.hh index 4f9d9516bb..5124ebacca 100644 --- a/src/libutil/include/nix/util/signals.hh +++ b/src/libutil/include/nix/util/signals.hh @@ -43,6 +43,7 @@ inline void checkInterrupt(); * @note Never will happen on Windows */ MakeError(Interrupted, BaseError); +MakeError(Cancelled, BaseError); struct InterruptCallback { diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index d1de7842aa..38030a77dd 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -11,6 +11,8 @@ namespace nix { void Interrupted::anchor() {} +void Cancelled::anchor() {} + std::atomic unix::_isInterrupted = false; thread_local std::function unix::interruptCheck; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 4f5f427734..e6b2fffa25 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -249,6 +249,10 @@ void ignoreExceptionExceptInterrupt(Verbosity lvl) throw; } catch (const Interrupted & e) { throw; + } catch (const Cancelled & e) { + /* Morally the same as Interrupted, just not triggered by a user but some other + cancellation. */ + throw; } catch (Error & e) { printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.info().msg); } catch (std::exception & e) { diff --git a/src/libutil/windows/signals.cc b/src/libutil/windows/signals.cc index ca4900313f..40fe5eeb63 100644 --- a/src/libutil/windows/signals.cc +++ b/src/libutil/windows/signals.cc @@ -4,4 +4,6 @@ namespace nix { void Interrupted::anchor() {} +void Cancelled::anchor() {} + } // namespace nix From 1b6f4eae679e423a2dad8133a055a4ed592c1a12 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 2 Jul 2026 23:20:36 +0300 Subject: [PATCH 553/927] libutil: Special-case forEachAsync for one awaitable, document current lack of cancellation We don't need to do the whole async_initiate dance for just one awaitable. (cherry picked from commit dc8fac0b68168a4c9eb9d491f7abe92205335703) --- src/libutil/include/nix/util/async.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libutil/include/nix/util/async.hh b/src/libutil/include/nix/util/async.hh index 3f6be9b48c..952cdb5610 100644 --- a/src/libutil/include/nix/util/async.hh +++ b/src/libutil/include/nix/util/async.hh @@ -95,10 +95,14 @@ asio::awaitable forEachAsync(Range && range, const F & f) auto pending = std::ranges::size(range); if (pending == 0) co_return; + else if (pending == 1) + co_return co_await f(*range.begin()); auto executor = co_await asio::this_coro::executor; std::exception_ptr err; + /* TODO: Handle cancellation on first error. Not very useful for now since we + do all-or-nothing cancellation typically. */ co_await asio::async_initiate( [&](auto handler) { auto h = std::make_shared(std::move(handler)); From 50a3b06cd5d28d9c6d2f47c74a21fd85e19e4c18 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 4 Jul 2026 18:39:19 +0300 Subject: [PATCH 554/927] release notes: 2.35.0 Co-authored-by: Lisanna Dettwyler (cherry picked from commit db08619e33d7f1cc8a0fe0004b084adbd9a3769d) --- doc/manual/rl-next/async-post-build-hook.md | 9 - .../aws-sts-webidentity-region-fallback.md | 11 - doc/manual/rl-next/beta-installer | 29 -- doc/manual/rl-next/build-trace-rework.md | 96 ---- doc/manual/rl-next/closure-gc.md | 8 - doc/manual/rl-next/delete-keep.md | 8 - .../rl-next/filetransfer-retry-backoff.md | 27 -- doc/manual/rl-next/fix-primop-eval-state.md | 10 - doc/manual/rl-next/flake-check-out-links.md | 7 - .../rl-next/flake-check-print-output-paths.md | 7 - doc/manual/rl-next/freebsd-sandboxing.md | 10 - doc/manual/rl-next/getflake-path.md | 6 - doc/manual/rl-next/git-url-scp.md | 30 -- .../github-fetcher-param-validation.md | 7 - .../rl-next/hash-self-reference-positions.md | 12 - doc/manual/rl-next/http3.md | 25 -- doc/manual/rl-next/mimalloc.md | 15 - .../s3-credential-chain-web-identity.md | 26 -- doc/manual/rl-next/seccomp-block-listxattr.md | 10 - .../rl-next/store-config-get-state-dir.md | 36 -- doc/manual/rl-next/zstd-multiframe.md | 18 - doc/manual/source/SUMMARY.md.in | 1 + doc/manual/source/release-notes/rl-2.35.md | 414 ++++++++++++++++++ 23 files changed, 415 insertions(+), 407 deletions(-) delete mode 100644 doc/manual/rl-next/async-post-build-hook.md delete mode 100644 doc/manual/rl-next/aws-sts-webidentity-region-fallback.md delete mode 100644 doc/manual/rl-next/beta-installer delete mode 100644 doc/manual/rl-next/build-trace-rework.md delete mode 100644 doc/manual/rl-next/closure-gc.md delete mode 100644 doc/manual/rl-next/delete-keep.md delete mode 100644 doc/manual/rl-next/filetransfer-retry-backoff.md delete mode 100644 doc/manual/rl-next/fix-primop-eval-state.md delete mode 100644 doc/manual/rl-next/flake-check-out-links.md delete mode 100644 doc/manual/rl-next/flake-check-print-output-paths.md delete mode 100644 doc/manual/rl-next/freebsd-sandboxing.md delete mode 100644 doc/manual/rl-next/getflake-path.md delete mode 100644 doc/manual/rl-next/git-url-scp.md delete mode 100644 doc/manual/rl-next/github-fetcher-param-validation.md delete mode 100644 doc/manual/rl-next/hash-self-reference-positions.md delete mode 100644 doc/manual/rl-next/http3.md delete mode 100644 doc/manual/rl-next/mimalloc.md delete mode 100644 doc/manual/rl-next/s3-credential-chain-web-identity.md delete mode 100644 doc/manual/rl-next/seccomp-block-listxattr.md delete mode 100644 doc/manual/rl-next/store-config-get-state-dir.md delete mode 100644 doc/manual/rl-next/zstd-multiframe.md create mode 100644 doc/manual/source/release-notes/rl-2.35.md diff --git a/doc/manual/rl-next/async-post-build-hook.md b/doc/manual/rl-next/async-post-build-hook.md deleted file mode 100644 index ea061f14ae..0000000000 --- a/doc/manual/rl-next/async-post-build-hook.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: Make post-build-hook asynchronous -prs: [15451] -issues: [15406] ---- - -This change makes the `post-build-hook` run asynchronously but still as part of the goal. -This retains the current behavior that a waiting goal will not start until the `post-build-hook` of the goal it is waiting on completes. -However, multiple `post-build-hook`s can now run concurrently just as multiple goals can run concurrently. diff --git a/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md b/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md deleted file mode 100644 index 5034c65438..0000000000 --- a/doc/manual/rl-next/aws-sts-webidentity-region-fallback.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -synopsis: S3 substituters fall back to the URL's region for STS WebIdentity auth -prs: [15594] ---- - -When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, -GitHub Actions OIDC), Nix now uses the `?region=` parameter from the S3 URL -as a fallback for the STS endpoint region if neither `AWS_REGION` nor -`AWS_DEFAULT_REGION` is set. Previously, IRSA setups that exported -`AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` but no region would fail -with a misleading "IMDS provider" error. diff --git a/doc/manual/rl-next/beta-installer b/doc/manual/rl-next/beta-installer deleted file mode 100644 index b02564d95d..0000000000 --- a/doc/manual/rl-next/beta-installer +++ /dev/null @@ -1,29 +0,0 @@ ---- -synopsis: "Rust nix-installer in beta" -prs: [] ---- - -The Rust-based rewrite of the Nix installer is now in beta. -We'd love help testing it out! - -To test out the new installer, run: -``` -curl -sSfL https://artifacts.nixos.org/nix-installer | sh -s -- install -``` - -This installer can be run even when you have an existing, script-based Nix installation without any adjustments. - -This new installer also comes with the ability to uninstall your Nix installation; run: -``` -/nix/nix-installer uninstall -``` - -This will get rid of your entire Nix installation (even if you installed over an existing, script-based installation). - -This installer is a modified version of the [Determinate Nix Installer](https://github.com/DeterminateSystems/nix-installer) by Determinate Systems. -Thanks to Determinate Systems for all the investment they've put into the installer. - -Source for the installer is in https://github.com/NixOS/nix-installer. -Report any issues in that repo. - -For CI usage, a GitHub Action to install Nix using this installer is available at https://github.com/NixOS/nix-installer-action. diff --git a/doc/manual/rl-next/build-trace-rework.md b/doc/manual/rl-next/build-trace-rework.md deleted file mode 100644 index 61b6bc51b1..0000000000 --- a/doc/manual/rl-next/build-trace-rework.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -synopsis: "Content-addressed derivations: realisations keyed by store path instead of hash modulo" -issues: [11897] -prs: [12464] ---- - -The experimental content-addressed (CA) derivation feature has undergone a significant change to how build traces (formerly called "realisations") are identified. This affects the **binary cache protocol** and the **wire protocols**. - -### What changed - -#### Build trace format - -Previously, a build trace entry (realisation) was keyed by the **hash modulo** of the derivation. -A SHA-256 hash computed via the complex "derivation hash modulo" algorithm. -This required implementations to understand ATerm serialisation and the full derivation hashing scheme just to look up or store build results. - -Now, build trace entries are keyed by the **regular derivation store path** plus the output name. For example, instead of: - -``` -sha256:ba7816bf8f01...!out -``` - -The key is now: - -``` -/nix/store/abc...-foo.drv^out -``` - -This is simpler, more intuitive, and means that third-party tools implementing CA derivation support (e.g., Hydra) -no longer need to implement the derivation hash modulo algorithm. - -#### Build trace usage - -Previously the build trace contained entries for both unresolved and [resolved](@docroot@/store/resolution.md) derivations. -Now, it only contains entries for resolved derivations. -For now, unresolved derivations will be resolved from these underlying build trace entries. -This is slower, but avoids a bunch of correctness issues. - -### Binary cache protocol - -- The directory for build traces moved from `realisations/` to `build-trace-v2/`. -- File paths changed from `realisations/!.doi` to `build-trace-v2//.doi`. -- The JSON format of build trace entries is now split into `key` and `value` objects: - ```json - { - "key": { - "drvPath": "abc...-foo.drv", - "outputName": "out" - }, - "value": { - "outPath": "xyz...-foo", - "signatures": [{ "keyName": "cache.example.com-1", "sig": "..." }] - } - } - ``` - Previously, these were flat objects with a string `id` field like `"sha256:...!out"`. -- The deprecated `dependentRealisations` field has been removed. - -Existing binary caches will need to be re-populated with the new format for CA derivation build traces. -Old build traces at the previous URLs are simply abandoned. -Non-CA builds are unaffected. - -### Wire protocols - -- **Worker protocol**: - A new feature flag `realisation-with-path-not-hash` is negotiated during the handshake. - Clients and daemons that both support this feature use the new binary serialisation for `DrvOutput`, `UnkeyedRealisation`, and related types. - Fallback to older protocol versions gracefully degrades (realisations are unavailable). -- **Serve protocol**: - Bumped from 2.7 to 2.8 with native serialisers for the new types. - Fallback to older protocol versions gracefully degrades in the same way. - -Stable code paths do use the realization fields (`BuildResult::Success::builtOutputs`), but only the output name and outpath parts of that. -For older protocols, we can fake enough of the realisation format to provide those two parts forthat map, which keeps operations like `--print-output-paths` working. - -### Local Store SQLite schema - -The build trace entries no longer have any foreign key store objects in the store. -This is because we will need to remember the build trace entries for resolved derivations we may have deleted, otherwise we will effectively forget outputs resolved derivations we do have on disk. -GC for build trace will be implemented later --- there is no single correct choice (there is no closure property) so it will be a question of what policies users want. - -### Structured signatures - -[Signatures](@docroot@/protocols/json/signature.md) in JSON formats are now represented as structured objects with `keyName` and `sig` fields, rather than colon-separated strings. -`nix path-info --json --json-format 3` opts into the new version for this command. -JSON parsing accepts both the old string format and new structured format for backwards compatibility. - -### Impact - -- **Non-CA derivation users**: No impact. This only affects the experimental `ca-derivations` feature. -- **Binary cache operators**: - Binary caches serving CA derivation build traces will need to be repopulated. - Existing NARs and narinfo files are unaffected. -- **Tool authors**: - Implementations interfacing with the CA derivations protocol are simplified. - The derivation hash modulo algorithm is no longer required to form build trace keys. diff --git a/doc/manual/rl-next/closure-gc.md b/doc/manual/rl-next/closure-gc.md deleted file mode 100644 index 945a78dc9f..0000000000 --- a/doc/manual/rl-next/closure-gc.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -synopsis: "Added `--skip-alive` option to `nix store delete` for collecting garbage within a closure" -issues: 7239 -prs: [15236, 15727] ---- - -`nix store delete --recursive --skip-alive` can be used to collect garbage within a closure, in which case it will only collect the dead paths that are part of the closure of its arguments. -The additional option `--also-referrers` is added to support this mode, which allows referrers of paths in the closure to also be deleted. diff --git a/doc/manual/rl-next/delete-keep.md b/doc/manual/rl-next/delete-keep.md deleted file mode 100644 index 0332e0e393..0000000000 --- a/doc/manual/rl-next/delete-keep.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -synopsis: "Fixed a bug where keep-outputs and keep-derivations can interfere with delete commands" -prs: [15776] ---- - -Setting `keep-derivations = true` and trying to delete a derivation with realised outputs would previously fail. -Same with `keep-outputs = true` and trying to delete an output that still has derivers. -These options no longer affect the deletion commands, and are now documented as such. diff --git a/doc/manual/rl-next/filetransfer-retry-backoff.md b/doc/manual/rl-next/filetransfer-retry-backoff.md deleted file mode 100644 index 0e26ccd54a..0000000000 --- a/doc/manual/rl-next/filetransfer-retry-backoff.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -synopsis: Configurable file-transfer retry backoff with full jitter and Retry-After support -issues: [15419, 15023] -prs: [15449] ---- - -File transfer retries (downloads and uploads) now use AWS-style "full jitter" -exponential backoff, treat HTTP 503 as rate-limited (same longer delay as 429), -and honor the `Retry-After` response header. Retry timing is configurable via -new `nix.conf` settings: - -- `filetransfer-retry-delay` (default 100ms): base delay for transient errors -- `filetransfer-retry-delay-rate-limited` (default 5000ms): base delay for 429/503 -- `filetransfer-retry-max-delay` (default 60000ms): per-attempt delay ceiling -- `filetransfer-retry-jitter` (default true): enable full jitter - -The existing `download-attempts` setting has been renamed to -`filetransfer-retry-attempts` to reflect that it applies to uploads as well as -downloads. The old name remains as an alias for backwards compatibility. - -Per-substituter overrides are available as store URL parameters -(`retry-delay`, `retry-delay-rate-limited`, `retry-max-delay`, -`retry-attempts`), e.g. `s3://my-cache?retry-attempts=8`. - -This addresses thundering-herd scenarios where many CI jobs hit the same -S3 prefix and receive 503 SlowDown; previously the retry window for 503 -was only ~4 seconds. diff --git a/doc/manual/rl-next/fix-primop-eval-state.md b/doc/manual/rl-next/fix-primop-eval-state.md deleted file mode 100644 index 1a3bc28753..0000000000 --- a/doc/manual/rl-next/fix-primop-eval-state.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: "C API: Fix `EvalState` pointer passed to primop callbacks" -prs: [15300, 15383] ---- - -The `EvalState *` passed to C API primop callbacks was incorrectly pointing to -the internal `nix::EvalState` rather than the C API wrapper struct. This caused -a segfault when the callback used the pointer with C API functions such as -`nix_alloc_value()`. The same issue affected `printValueAsJSON` and -`printValueAsXML` callbacks on external values. diff --git a/doc/manual/rl-next/flake-check-out-links.md b/doc/manual/rl-next/flake-check-out-links.md deleted file mode 100644 index eb369a629c..0000000000 --- a/doc/manual/rl-next/flake-check-out-links.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: nix flake check now supports --out-link -prs: [15476] -issues: [13470] ---- - -`nix flake check` now supports the flag `--out-link`, defaulting to not creating out links if the flag is not specified. diff --git a/doc/manual/rl-next/flake-check-print-output-paths.md b/doc/manual/rl-next/flake-check-print-output-paths.md deleted file mode 100644 index 5e0fd35b67..0000000000 --- a/doc/manual/rl-next/flake-check-print-output-paths.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: nix flake check now supports --print-out-paths -prs: [15476] -issues: [13470] ---- - -`nix flake check` now supports the flag `--print-out-paths`. diff --git a/doc/manual/rl-next/freebsd-sandboxing.md b/doc/manual/rl-next/freebsd-sandboxing.md deleted file mode 100644 index 2c2d214494..0000000000 --- a/doc/manual/rl-next/freebsd-sandboxing.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: Enable FreeBSD sandboxing, add `x86_64-freebsd` to installer -prs: [15673, 13281, 9968] ---- - -A FreeBSD build has been added to the traditional installer script, with sandboxing enabled. -The beta installer is not yet supported. - -FreeBSD support is not as well-tested as Linux or macOS, but is fully capable of building packages -and performing other tasks expected of Nix on Linux. diff --git a/doc/manual/rl-next/getflake-path.md b/doc/manual/rl-next/getflake-path.md deleted file mode 100644 index 2360fe7693..0000000000 --- a/doc/manual/rl-next/getflake-path.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -synopsis: "`builtins.getFlake` now supports path values" -prs: [15290] ---- - -`builtins.getFlake` now accepts path values in addition to flakerefs, allowing you to write `builtins.getFlake ./subflake` instead of having to use ugly workarounds to construct a pure flakeref. diff --git a/doc/manual/rl-next/git-url-scp.md b/doc/manual/rl-next/git-url-scp.md deleted file mode 100644 index b1125f7b32..0000000000 --- a/doc/manual/rl-next/git-url-scp.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -synopsis: Support SCP-like URLs in fetchGit and type = "git" flake inputs -prs: [14863] -issues: [14852, 14867] ---- - -Nix now (once again) recognizes [SCP-like syntax for Git URLs](https://git-scm.com/docs/git-clone#_git_urls). This partially -restores compatibility with Nix 2.3 for `fetchGit`. The following syntax is once again supported: - -```nix -builtins.fetchGit "host:/absolute/path/to/repo" -``` - -Nix also passes through the tilde (for home directories) verbatim: - -```nix -builtins.fetchGit "host:~/relative/to/home" -``` - -IPv6 addresses also supported when bracketed: - -```nix -builtins.fetchGit "user@[::1]:~/relative/to/home" -``` - -`builtins.fetchTree` also supports this syntax now: - -```nix -builtins.fetchTree { type = "git"; url = "host:/path/to/repo"; } -``` diff --git a/doc/manual/rl-next/github-fetcher-param-validation.md b/doc/manual/rl-next/github-fetcher-param-validation.md deleted file mode 100644 index 2fb430ae4b..0000000000 --- a/doc/manual/rl-next/github-fetcher-param-validation.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: GitHub fetcher now validates URL parameters -prs: [15331] -issues: [15304] ---- - -The `github:` fetcher now validates URL parameters, and will error if an invalid parameter like `tag` is provided. diff --git a/doc/manual/rl-next/hash-self-reference-positions.md b/doc/manual/rl-next/hash-self-reference-positions.md deleted file mode 100644 index 0a7193b3a4..0000000000 --- a/doc/manual/rl-next/hash-self-reference-positions.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -synopsis: "Fix hash collision between store paths with self-references and their zeroed-out equivalents" -issues: [15837] -prs: [15931] ---- - -When computing the hash of a NAR with self-references, Nix zeroes out the self-references but also hashes their positions. -The latter was accidentally lost in Nix 2.17.0, which meant a NAR with self-references could hash to the same store path as an otherwise-identical NAR in which some of the self-references had been zeroed out. - -This release restores hashing the positions of self-references. -As a consequence, content-addressed store paths derived from self-referential NARs will differ from those produced by Nix 2.17 through 2.34. -This affects users of the experimental `ca-derivations` features, as well as users of `nix store make-content-addressed`. diff --git a/doc/manual/rl-next/http3.md b/doc/manual/rl-next/http3.md deleted file mode 100644 index 1f0a31227e..0000000000 --- a/doc/manual/rl-next/http3.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -synopsis: HTTP/3 (QUIC) support ---- - -Nix can now fetch from binary caches and other HTTP(S) sources over HTTP/3 -(QUIC), controlled by a new -[`http3`](@docroot@/command-ref/conf-file.md#conf-http3) setting (disabled by -default). When enabled, Nix requests HTTP/3 and transparently falls back to -HTTP/2 or HTTP/1.1 for servers that do not advertise QUIC. The setting only -takes effect when linked against a libcurl built with HTTP/3 support, otherwise -it is ignored and Nix keeps using HTTP/2 without warning or error. - -Enable it with: - -``` -nix.conf: http3 = true -CLI: --http3 -``` - -Or disable with: - -``` -nix.conf: http3 = false -CLI: --no-http3 -``` diff --git a/doc/manual/rl-next/mimalloc.md b/doc/manual/rl-next/mimalloc.md deleted file mode 100644 index dfcde0cab0..0000000000 --- a/doc/manual/rl-next/mimalloc.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -synopsis: "Link mimalloc for faster evaluation" -prs: [15596] ---- - -The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) -by default on non-Windows platforms, replacing glibc's malloc for all -non-GC allocations. - -This yields a **5–12% wall-clock improvement** on evaluation workloads, -ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS -configurations. - -The allocator can be disabled at build time with `-Dmimalloc=disabled` -or by passing `withMimalloc = false` to the Nix package. diff --git a/doc/manual/rl-next/s3-credential-chain-web-identity.md b/doc/manual/rl-next/s3-credential-chain-web-identity.md deleted file mode 100644 index 4dfece0e3b..0000000000 --- a/doc/manual/rl-next/s3-credential-chain-web-identity.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -synopsis: "S3: restore STS WebIdentity and ECS container credential providers" -prs: [15507] ---- - -Nix 2.33 replaced the S3 backend's `aws-sdk-cpp` credential chain with a -custom chain built on `aws-c-auth`. That chain omitted two providers, -breaking S3 binary cache access in container workloads: - -- **STS WebIdentity** (`AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_ROLE_ARN`, - `AWS_ROLE_SESSION_NAME`) — used by EKS IRSA, GitHub Actions OIDC, and - any `sts:AssumeRoleWithWebIdentity` federation. -- **ECS container metadata** (`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, - `AWS_CONTAINER_CREDENTIALS_FULL_URI`) — used by ECS tasks and EKS Pod - Identity. - -The typical symptom was a misleading IMDS error -(`Valid credentials could not be sourced by the IMDS provider`), because -IMDS is the last provider tried after the correct one was skipped. - -Both providers are now part of the chain, ordered to match the -pre-2.33 `DefaultAWSCredentialsProviderChain`: -`Environment → SSO → Profile → STS WebIdentity → (ECS | IMDS)`. -As in both the old and new AWS SDK default chains, ECS and IMDS are -mutually exclusive: when container credential environment variables are -set, IMDS is skipped. diff --git a/doc/manual/rl-next/seccomp-block-listxattr.md b/doc/manual/rl-next/seccomp-block-listxattr.md deleted file mode 100644 index 2455ed4f82..0000000000 --- a/doc/manual/rl-next/seccomp-block-listxattr.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: "Linux sandbox: also block `listxattr` syscalls" -prs: [15743] ---- - -The Linux sandbox now also returns `ENOTSUP` for `listxattr`, -`llistxattr` and `flistxattr`, matching the existing treatment of -`getxattr`/`setxattr`/`removexattr`. This prevents host xattrs (e.g. -`security.selinux`) from leaking into builds and fixes tools such as -`mkfs.ubifs` that probe xattr support via `listxattr`. diff --git a/doc/manual/rl-next/store-config-get-state-dir.md b/doc/manual/rl-next/store-config-get-state-dir.md deleted file mode 100644 index a789bdf835..0000000000 --- a/doc/manual/rl-next/store-config-get-state-dir.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -synopsis: "Improve daemon socket path logic for chroot stores" -prs: [15429] ---- - -The default daemon socket path now uses the per-store [`state`](@docroot@/store/types/local-store.md#store-local-store-state) directory whenever one is defined, rather than always using the global [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR). -This means [local chroot stores](@docroot@/store/types/local-store.md#chroot) each get their own socket path automatically. - -Example: - -```bash -nix-daemon --store /foo/bar -``` - -will now use a socket at: -``` -/foo/bar/nix/var/nix/daemon-socket/socket -``` -instead of -``` -$NIX_STATE_DIR/daemon-socket/socket -``` - -Users who wish to serve or connect to a chroot store at the old location will have to force the socket location: - -- When serving (running a daemon), use the new [`--socket-path`](@docroot@/command-ref/new-cli/nix3-daemon.md#opt-socket-path) flag: - - ```bash - nix daemon --socket-path "$NIX_STATE_DIR/daemon-socket/socket" - ``` - -- When connecting as a client put the path in the [store URL](@docroot@/store/types/local-daemon-store.md): - - ``` - unix://$NIX_STATE_DIR/daemon-socket/socket - ``` diff --git a/doc/manual/rl-next/zstd-multiframe.md b/doc/manual/rl-next/zstd-multiframe.md deleted file mode 100644 index 6f5d9875df..0000000000 --- a/doc/manual/rl-next/zstd-multiframe.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -synopsis: zstd compression now emits multi-frame output and uses less memory -prs: [15550] ---- - -zstd-compressed NARs are now written as a sequence of independent 16 MiB -frames instead of a single large frame. This lays the groundwork for -parallel decompression in a future release without requiring caches to be -repopulated, and significantly lowers peak memory use during compression -(e.g. from ~600 MiB to ~100 MiB for a 1 GiB store path). - -The output remains standard zstd and is decoded unchanged by existing Nix -binaries and the `zstd` CLI; compression ratio is effectively unchanged. - -Per-frame compression now uses up to 4 worker threads. For zstd this is the -new default: the `parallel-compression` store setting defaults to `true` when -`compression=zstd` (it remains `false` for `xz`). Set -`?parallel-compression=false` to opt out. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index a12e84becc..ae77759f55 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -159,6 +159,7 @@ - [Contributing](development/contributing.md) - [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.35 (2026-06-22)](release-notes/rl-2.35.md) - [Release 2.34 (2026-02-27)](release-notes/rl-2.34.md) - [Release 2.33 (2025-12-09)](release-notes/rl-2.33.md) - [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md) diff --git a/doc/manual/source/release-notes/rl-2.35.md b/doc/manual/source/release-notes/rl-2.35.md new file mode 100644 index 0000000000..a712b63856 --- /dev/null +++ b/doc/manual/source/release-notes/rl-2.35.md @@ -0,0 +1,414 @@ +# Release 2.35.0 (2026-06-22) + +## Highlights + +- Sources are copied to the store more lazily [#3121](https://github.com/NixOS/nix/issues/3121) [#15711](https://github.com/NixOS/nix/pull/15711) [#15920](https://github.com/NixOS/nix/pull/15920) + + Historically, flakes source trees have been eagerly fetched to and evaluated from the Nix store to ensure deterministic and hermetic evaluation, even if the resulting store object is not used as a derivation input. This made the implementation simpler, yet made flakes unusable in large repositories and performed unnecessary writes to the store on each change to the source tree. + + Since Nix 2.32, all I/O (excluding `path:` and `hg+:`-style inputs) for reading sources during evaluation has been funneled to their original filesystem location (or to the `~/.cache/nix/tarball-cache-v2` bare git repository for tarball-based inputs). However, the source tree was still fetched to the store -- primarily for computing the resulting content-addressed store path. In most cases, (such as importing the `nixpkgs` package set) this is not necessary. + + Touching (and hashing the NAR serialisation of) the whole source tree is unavoidable, since: + + - In case of flake inputs, `narHash` integrity must be checked eagerly. + - The `outPath` attribute of a flake must be known in advance, and for backwards compatibility must be a content-addressed store path string with [constant string context](@docroot@/language/string-context.md#string-context-constant) representing the flake source tree. + + Even within the constraints imposed by backwards compatibility requirements, there are several improvements that are achievable. To reduce the number of copies performed, Nix now hashes the input without copying first, assuming that the `.outPath` will not end up in a derivation attribute and thus would never have to be actually fetched to the store. This comes at the slight cost of doing more work in case the assumption is wrong, but results in less work in typical use cases. The evaluator continues to behave as if the copy was performed: + + - Flakes are still evaluated from the store, from the evaluator's point of view. + - `toString ./.` continues to produce a content-addressed store path string without context. + - Path resolution crossing trees located in the filesystem and in Nix's view of it (with "virtual" overlays on top) continues to work. For example, the flake source tree can contain a relative symlink pointing outside its corresponding store object (though such usage is discouraged and makes further improvements to laziness intractable). + - Reading files from the flake's `outPath` continues to work. For example, such code is well-formed and is not considered [IFD](@docroot@/language/import-from-derivation.md): + + ```nix + builtins.readFile ( /. + (builtins.unsafeDiscardStringContext self.outPath) + "/flake.nix" ) + ``` + + Similar treatment has been applied to `builtins.fetchTarball`, which no longer eagerly copies paths to the store. + `builtins.storePath` now also short-circuits on "lazy-ish" store paths and doesn't substitute unless necessary. + + This change is expected to significantly reduce disk usage required for typical evaluations and results in ~2x speedup for fetching and unpacking a nixpkgs tarball (either via `fetchTree`/flakes or via `fetchTarball`). + +- Support FreeBSD `libjail` based sandboxing, add `x86_64-freebsd` to installer [#9968](https://github.com/NixOS/nix/pull/9968) [#13281](https://github.com/NixOS/nix/pull/13281) [#15673](https://github.com/NixOS/nix/pull/15673) + + The FreeBSD build of Nix now supports build sandboxing via FreeBSD jails and is enabled by default. + A FreeBSD build has been added to the traditional installer script. The beta rust-based installer is not yet supported. + FreeBSD support is not as well-tested as Linux or macOS, but is fully capable of building packages and performing other tasks expected of Nix on Linux. + +## Improvements + +- HTTP/3 (QUIC) support [#15961](https://github.com/NixOS/nix/pull/15961) + + Nix can now fetch from binary caches and other HTTP(S) sources over HTTP/3 (QUIC), controlled by a new [`http3`](@docroot@/command-ref/conf-file.md#conf-http3) setting (disabled by default). + When enabled, Nix requests HTTP/3 and transparently falls back to HTTP/2 or HTTP/1.1 for servers that do not advertise QUIC. + The setting only takes effect when linked against a `libcurl` built with HTTP/3 support, otherwise it is ignored and Nix keeps using HTTP/2 without warning or error. + + Enable it with: + + ``` + nix.conf: http3 = true + CLI: --http3 + ``` + + Or disable with: + + ``` + nix.conf: http3 = false + CLI: --no-http3 + ``` + +- Link mimalloc for faster evaluation [#15596](https://github.com/NixOS/nix/pull/15596) + + The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) by default, replacing glibc's malloc for all non-GC allocations. + This yields a **5–12% wall-clock improvement** on evaluation workloads, ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS configurations. + The allocator can be disabled at build time with `-Dmimalloc=disabled`. + +- The `revCount` attribute of the Git fetchers is now lazily computed and passed-through as-is when explicitly specified [#15772](https://github.com/NixOS/nix/pull/15772) [#14596](https://github.com/NixOS/nix/pull/14596) + + `revCount` and `lastModified` attributes passed to the Git fetcher are no longer eagerly validated when explicitly specified. + + When not explicitly specified, `revCount` is now also a thunk value and not computed eagerly. This delays this (potentially) expensive computation until the value is actually required. + +- Configurable file-transfer retry backoff with full jitter and `Retry-After` support [#15023](https://github.com/NixOS/nix/issues/15023) [#15419](https://github.com/NixOS/nix/issues/15419) [#15449](https://github.com/NixOS/nix/pull/15449) + + File transfer retries (downloads and uploads) now use AWS-style "full jitter" exponential backoff, treat HTTP 503 as rate-limited (same longer delay as 429), + and honor the `Retry-After` response header. + + Retry timing is configurable via new `nix.conf` settings: + + - [`filetransfer-retry-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay): base delay for transient errors + - [`filetransfer-retry-delay-rate-limited`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-delay-rate-limited): base delay for 429/503 + - [`filetransfer-retry-max-delay`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-max-delay): per-attempt delay ceiling + - [`filetransfer-retry-jitter`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-jitter): enable full jitter + + The existing `download-attempts` setting has been renamed to [`filetransfer-retry-attempts`](@docroot@/command-ref/conf-file.md#conf-filetransfer-retry-attempts) to reflect that it applies to uploads as well as downloads. + The old name remains as an alias for backwards compatibility. + + Per-substituter overrides are available as store reference parameters ([`retry-delay`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-delay), [`retry-delay-rate-limited`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-delay-rate-limited), [`retry-max-delay`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-max-delay), [`retry-attempts`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-retry-attempts)), e.g. `s3://my-cache?retry-attempts=8`. + +- Improve daemon socket path logic for chroot stores [#15190](https://github.com/NixOS/nix/pull/15190) + + The default daemon socket path now uses the per-store [`state`](@docroot@/store/types/local-store.md#store-local-store-state) directory whenever one is defined, rather than always using the global [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR). + This means [local chroot stores](@docroot@/store/types/local-store.md#chroot) each get their own socket path automatically. + + Example: + + ```bash + nix-daemon --store /foo/bar + ``` + + will now use a socket at: + ``` + /foo/bar/nix/var/nix/daemon-socket/socket + ``` + instead of + ``` + $NIX_STATE_DIR/daemon-socket/socket + ``` + + Users who wish to serve or connect to a chroot store at the old location will have to force the socket location: + + - When serving (running a daemon), use the new [`--socket-path`](@docroot@/command-ref/new-cli/nix3-daemon.md#opt-socket-path) flag: + + ```bash + nix daemon --socket-path "$NIX_STATE_DIR/daemon-socket/socket" + ``` + + - When connecting as a client, put the path in the [store URL](@docroot@/store/types/local-daemon-store.md): + + ``` + unix://$NIX_STATE_DIR/daemon-socket/socket + ``` + +- Linux sandbox: also block `listxattr` syscalls [#15743](https://github.com/NixOS/nix/pull/15743) + + The Linux sandbox now also returns `ENOTSUP` for `listxattr`, `llistxattr` and `flistxattr`, matching the existing treatment of `getxattr`/`setxattr`/`removexattr`. + This prevents host xattrs (e.g. `security.selinux`) from leaking into builds and fixes tools such as `mkfs.ubifs` that probe xattr support via `listxattr`. + +- Support SCP-like URLs in fetchGit and type = "git" flake inputs [#14852](https://github.com/NixOS/nix/issues/14852) [#14867](https://github.com/NixOS/nix/issues/14867) [#14863](https://github.com/NixOS/nix/pull/14863) + + Nix now (once again) recognizes [SCP-like syntax for Git URLs](https://git-scm.com/docs/git-clone#_git_urls). This partially + restores compatibility with Nix 2.3 for `fetchGit`. The following syntax is once again supported: + + ```nix + builtins.fetchGit "host:/absolute/path/to/repo" + ``` + + Nix also passes through the tilde (for home directories) verbatim: + + ```nix + builtins.fetchGit "host:~/relative/to/home" + ``` + + IPv6 addresses also supported when bracketed: + + ```nix + builtins.fetchGit "user@[::1]:~/relative/to/home" + ``` + + `builtins.fetchTree` also supports this syntax now: + + ```nix + builtins.fetchTree { type = "git"; url = "host:/path/to/repo"; } + ``` + +- `nix flake check` now supports `--print-out-paths` [#13470](https://github.com/NixOS/nix/issues/13470) [#15476](https://github.com/NixOS/nix/pull/15476) and `--out-link` [#13470](https://github.com/NixOS/nix/issues/13470) [#15476](https://github.com/NixOS/nix/pull/15476) defaulting to not creating out links if the flag is not specified. + +- Added `--skip-alive` (and `--skip-live` alias for compatibility with Lix users) option to `nix store delete` for collecting garbage within a closure [#7239](https://github.com/NixOS/nix/issues/7239) [#15236](https://github.com/NixOS/nix/pull/15236) [#15727](https://github.com/NixOS/nix/pull/15727) + + `nix store delete --recursive --skip-alive` can be used to collect garbage within a closure, in which case it will only collect the dead paths that are part of the closure of its arguments. + The additional option `--also-referrers` is added to support this mode, which allows referrers of paths in the closure to also be deleted. + +- `builtins.getFlake` now supports path values [#15290](https://github.com/NixOS/nix/pull/15290) + + `builtins.getFlake` now accepts path values in addition to flakerefs. This improves the usability of relative flakes, allowing you to write `builtins.getFlake ./subflake`. + This change does not allow specifying paths that are not already in the store (though they do not have valid store objects, i.e. this will not force a copy if the flake has only been hashed -- and not copied to the store). This may change in a future release. + +- `nix-profile.fish` and `nix-profile-daemon.fish` now use `$NIX_LINK` for computing the value of `NIX_PROFILE` instead of `$HOME/.nix-profile` [#14293](https://github.com/NixOS/nix/pull/14293) + +- `nix` binary now exports symbols from C bindings [#15696](https://github.com/NixOS/nix/pull/15696) + + This allows Nix plugins written against the C API to look up symbols dynamically without linking to corresponding `libnix*c.so` libraries. + +- The computed Git LFS endpoint URLs have been fixed to follow the spec [#15891](https://github.com/NixOS/nix/pull/15891) and memory usage of LFS fetches has been decreased [#15912](https://github.com/NixOS/nix/pull/15912) + +- We now verify that fetched Git LFS objects have the same OID as requested [#15845](https://github.com/NixOS/nix/pull/15845) + +- Primop documentation now includes time complexity information [#14554](https://github.com/NixOS/nix/pull/14554) + +- Improved documentation on store paths and derivation building [#14699](https://github.com/NixOS/nix/pull/14699) + +- The [build hook](@docroot@/command-ref/conf-file.md#conf-build-hook) is now killed with `SIGTERM` instead of `SIGKILL` [#15105](https://github.com/NixOS/nix/pull/15105) + +- Download/upload logs strip `userinfo` URL components [#15715](https://github.com/NixOS/nix/pull/15715) + +## Content-addressed derivations changes + + The experimental content-addressed (CA) derivation feature has undergone a significant change to how build traces (formerly called "realisations") are identified. + This changes the binary cache endpoints for realisations and the daemon/nix-serve protocol (gated behind a daemon protocol feature flag). + +- Realisations keyed by store path instead of hash modulo [#11897](https://github.com/NixOS/nix/issues/11897) [#12464](https://github.com/NixOS/nix/pull/12464) + + Previously, a build trace entry (realisation) was keyed by the hash modulo of the derivation. In simpler terms, derivations transitively depending on distinct fixed-output derivations with the same `outPath` would share a realisation. + + Now, build trace entries are keyed by the regular derivation store path (`.drvPath`) plus the output name. For example, instead of: + + ``` + sha256:ba7816bf8f01...!out + ``` + + The key is now: + + ``` + /nix/store/abc...-foo.drv^out + ``` + +- Removed support for "deep" realisations [#15289](https://github.com/NixOS/nix/pull/15289) + + Previously the build trace (set of "realisations") contained entries for both unresolved and [resolved](@docroot@/store/resolution.md) derivations. + Now, it contains entries exclusively for resolved derivations. + For now, unresolved derivations will be resolved from these underlying build trace entries. + This is slower, but has the benefit of making build trace entries stateless and self-describing --- making sharing realisations easier between stores. + + This change necessitates changes to the binary cache format: + + - The directory for build traces moved from `realisations/` to `build-trace-v2/`. + - File paths changed from `realisations/!.doi` to `build-trace-v2//.doi`. + - The JSON format of build trace entries is now split into `key` and `value` objects: + ```json + { + "key": { + "drvPath": "abc...-foo.drv", + "outputName": "out" + }, + "value": { + "outPath": "xyz...-foo", + "signatures": [{ "keyName": "cache.example.com-1", "sig": "..." }] + } + } + ``` + Previously, these were flat objects with a string `id` field like `"sha256:...!out"`. + - The deprecated `dependentRealisations` field has been removed. + + The build trace entries stored in the local SQLite database no longer have any foreign key references to store objects. + This is because the build trace entries for resolved derivations that may have been deleted need to be preserved, otherwise the outputs of other unresolved derivations will be effectively forgotten. + GC for the build trace is not yet implemented due to the lack of a clear default policy. + +- Structured signature for realisations and `path-info` [#15009](https://github.com/NixOS/nix/pull/15009) + + [Signatures](@docroot@/protocols/json/signature.md) in JSON formats are now represented as structured objects with `keyName` and `sig` fields, rather than colon-separated strings. + `nix path-info --json --json-format 3` opts into the new version for this command. + JSON parsing accepts both the old string format and new structured format for backwards compatibility. + + This format is also used for the build trace entries in binary caches. + +- `nix realisation` command has been renamed to `nix store build-trace` [#16000](https://github.com/NixOS/nix/pull/16000) [#15948](https://github.com/NixOS/nix/pull/15948) + +## Build performance improvements + +- Make post-build-hook asynchronous [#15406](https://github.com/NixOS/nix/issues/15406) [#15451](https://github.com/NixOS/nix/pull/15451) + + The [`post-build-hook`](@docroot@/command-ref/conf-file.md#conf-post-build-hook) now runs asynchronously, without blocking the build event loop. + Dependent builds are not started until the hook finishes, but multiple hook instances are now launched concurrently -- up to the [`max-jobs`](@docroot@/command-ref/conf-file.md#conf-max-jobs) limit. + +- zstd compression now emits multi-frame output and uses less memory [#15550](https://github.com/NixOS/nix/pull/15550) + + zstd-compressed NARs are now written as a sequence of independent 16 MiB frames instead of a single large frame. + This lays the groundwork for parallel decompression in a future release without requiring caches to be repopulated, and significantly lowers peak memory use during compression + (e.g. from ~600 MiB to ~100 MiB for a 1 GiB store path). + + The output remains standard zstd and is decoded unchanged by existing Nix binaries and the `zstd` CLI; compression ratio is effectively unchanged. + + Per-frame compression now uses up to 4 worker threads. For zstd this is the new default: the [`parallel-compression`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-parallel-compression) store setting defaults to `true` when `compression=zstd` (it remains `false` for other compression algorithms like `xz`). + Set `?parallel-compression=false` to opt out. + +- More parallelism for binary cache uploads [#15957](https://github.com/NixOS/nix/pull/15957) + + Uploads of NARs now start without waiting for all references to be uploaded. + Also, NARs are now uploaded in order of descending (decompressed) NAR size. + The closure invariant is still maintained by copying `.narinfo` in a topologically sorted order. + +- The derivation build scheduler memory usage reduction and performance improvements [#15611](https://github.com/NixOS/nix/pull/15611) [#15695](https://github.com/NixOS/nix/pull/15695) + + Memory usage of the derivation build scheduler has been improved to allow more state sharing. + Inefficiencies leading to quadratic complexity of scheduling build/substitution jobs have been addressed. + Scheduling resources are allocated more sparingly and freed earlier to reduce peak consumption. + + These improvements amount to ~2-8x less `nix-daemon` memory usage for typical workloads and more in larger derivation graphs, not accounting for short-lived allocations used during substitution. + + Notably, the current architecture of the build scheduler gets proportionally slower on Linux with larger heaps as derivation "builder" processes are `fork`-ed directly from the Nix process, which blocks the builder event loop for the duration of the `fork`. Thus, smaller heap of `nix-daemon` translates into faster build startups. + +- Concurrent path substitutions and eval-time fetches of the same inputs now run only once [#15555](https://github.com/NixOS/nix/pull/15555) [#15644](https://github.com/NixOS/nix/pull/15644) + + This avoids redundant work in case multiple Nix processes try to substitute/download the same resource concurrently. + +- `.narinfo` lookups in binary caches are more concurrent + + Querying the existence and path metadata in binary caches is now more asynchronous. Operations like `nix path-info` on large closures are faster and more efficient. + The build scheduler event loop now doesn't block on `.narinfo` queries, which improves performance with passthru binary caches. + +## Bug fixes + +- Fix hash collision between store paths with self-references and their zeroed-out equivalents [#15837](https://github.com/NixOS/nix/issues/15837) [#15931](https://github.com/NixOS/nix/pull/15931) + + When computing the hash of a NAR with self-references, Nix zeroes out the self-references but also hashes their positions. + The latter was accidentally lost in Nix 2.17.0, which meant a NAR with self-references could hash to the same store path as an otherwise-identical NAR in which some of the self-references had been zeroed out. + + This release restores hashing the positions of self-references. + As a consequence, content-addressed store paths derived from self-referential NARs will differ from those produced by Nix 2.17 through 2.34. + This affects users of the experimental `ca-derivations` features, as well as users of `nix store make-content-addressed`. + +- C API: Fix `EvalState` pointer passed to primop callbacks [#15300](https://github.com/NixOS/nix/pull/15300) [#15383](https://github.com/NixOS/nix/pull/15383) + + The `EvalState *` passed to C API primop callbacks was incorrectly pointing to the internal `nix::EvalState` rather than the C API wrapper struct. + This caused a segfault when the callback used the pointer with C API functions such as `nix_alloc_value()`. + The same issue affected `printValueAsJSON` and `printValueAsXML` callbacks on external values. + +- GitHub fetcher now validates URL parameters [#15304](https://github.com/NixOS/nix/issues/15304) [#15331](https://github.com/NixOS/nix/pull/15331) + + The `github:` fetcher now validates URL parameters, and will error if an invalid parameter like `tag` is provided. + +- Fixed a bug where keep-outputs and keep-derivations can interfere with delete commands [#15776](https://github.com/NixOS/nix/pull/15776) + + Setting [`keep-derivations`](@docroot@/command-ref/conf-file.md#conf-keep-derivations) to `true` and trying to delete a derivation with realised outputs would previously fail. + Same with [`keep-outputs`](@docroot@/command-ref/conf-file.md#conf-keep-outputs) and trying to delete an output that still has derivers. + These options no longer affect the deletion commands, and are now documented as such. + +- S3 substituters fall back to the URL's region for STS WebIdentity auth [#15594](https://github.com/NixOS/nix/pull/15594) + + When authenticating to an S3 binary cache via STS WebIdentity (EKS IRSA, GitHub Actions OIDC), Nix now uses the `?region=` parameter from the S3 URL as a fallback for the STS endpoint region if neither `AWS_REGION` nor `AWS_DEFAULT_REGION` is set. + Previously, IRSA setups that exported `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` but no region would fail with a misleading "IMDS provider" error. + +- S3: restore STS WebIdentity and ECS container credential providers [#15507](https://github.com/NixOS/nix/pull/15507) + + Nix 2.33 replaced the S3 backend's `aws-sdk-cpp` credential chain with a custom chain built on `aws-c-auth`. + That chain omitted two providers, breaking S3 binary cache access in container workloads: + + - **STS WebIdentity** (`AWS_WEB_IDENTITY_TOKEN_FILE`, `AWS_ROLE_ARN`, `AWS_ROLE_SESSION_NAME`) -- used by EKS IRSA, GitHub Actions OIDC, and any `sts:AssumeRoleWithWebIdentity` federation. + - **ECS container metadata** (`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, `AWS_CONTAINER_CREDENTIALS_FULL_URI`) -- used by ECS tasks and EKS Pod Identity. + + The typical symptom was a misleading IMDS error (`Valid credentials could not be sourced by the IMDS provider`), because IMDS is the last provider tried after the correct one was skipped. + + Both providers are now part of the chain, ordered to match the pre-2.33 behaviour. + As in both the old and new AWS SDK default chains, ECS and IMDS are mutually exclusive: when container credential environment variables are set, IMDS is skipped. + +- HTTP 401 and 407 responses from binary caches are no longer treated as missing files [#15877](https://github.com/NixOS/nix/pull/15877) + + Nix no longer treats `Unauthorized` and `Proxy Authentication Required` HTTP codes as an indication of a missing file. This used to be the case because AWS S3 returns 403 `Forbidden` for missing objects in unlistable buckets. 401/407 were accidentally included and this workaround is now tightly scoped to 403 responses. + +- Fixed `nixbld` gid in `/etc/group` in the Linux build sandbox when user namespaces are not supported [#15131](https://github.com/NixOS/nix/pull/15131) + +- Store garbage collection is now more robust [#15992](https://github.com/NixOS/nix/pull/15992) [#15720](https://github.com/NixOS/nix/pull/15720) [#15616](https://github.com/NixOS/nix/pull/15616) + +- Fixed deadlock for hash-mismatching fixed-output derivations [#15874](https://github.com/NixOS/nix/pull/15874) + +- `nix-copy-closure` no longer ignores `--include-outputs` flag [#15896](https://github.com/NixOS/nix/pull/15896) + +- Fixes to `recursive-nix` experimental feature + + Prior to this release, internal datastructures used to implement this feature were not used in a thread-safe manner. + Threads handling daemon connections are now reaped promptly, fixing resource leaks. + +## Contributors + +This release was made possible by the following 59 contributors: + +- Michael Wang [**(@zwang20)**](https://github.com/zwang20) +- Amaan Qureshi [**(@amaanq)**](https://github.com/amaanq) +- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium) +- Reuben Gardos Reid [**(@ReubenJ)**](https://github.com/ReubenJ) +- StepBroBD [**(@stepbrobd)**](https://github.com/stepbrobd) +- dram [**(@dramforever)**](https://github.com/dramforever) +- Tom [**(@thunze)**](https://github.com/thunze) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- steveoliphant [**(@steveoliphant)**](https://github.com/steveoliphant) +- espes [**(@espes)**](https://github.com/espes) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Artemis Tosini [**(@artemist)**](https://github.com/artemist) +- sander [**(@sandydoo)**](https://github.com/sandydoo) +- Erik Jensen [**(@rkjnsn)**](https://github.com/rkjnsn) +- Cameron Will [**(@cwill747)**](https://github.com/cwill747) +- Maciej Krüger [**(@mkg20001)**](https://github.com/mkg20001) +- Dror Speiser [**(@drorspei)**](https://github.com/drorspei) +- Eveeifyeve [**(@Eveeifyeve)**](https://github.com/Eveeifyeve) +- Audrey Dutcher [**(@rhelmot)**](https://github.com/rhelmot) +- Lisanna Dettwyler [**(@lisanna-dettwyler)**](https://github.com/lisanna-dettwyler) +- TyIsI [**(@TyIsI)**](https://github.com/TyIsI) +- Adam Kliś [**(@BonusPlay)**](https://github.com/BonusPlay) +- Domen Kožar [**(@domenkozar)**](https://github.com/domenkozar) +- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii) +- ryota2357 [**(@ryota2357)**](https://github.com/ryota2357) +- LIN, Jian [**(@jian-lin)**](https://github.com/jian-lin) +- znmz [**(@znmz)**](https://github.com/znmz) +- Felix Stupp [**(@Zocker1999NET)**](https://github.com/Zocker1999NET) +- Johannes Kirschbauer [**(@hsjobeki)**](https://github.com/hsjobeki) +- Antonio Nuno Monteiro [**(@anmonteiro)**](https://github.com/anmonteiro) +- tomberek [**(@tomberek)**](https://github.com/tomberek) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- adisbladis [**(@adisbladis)**](https://github.com/adisbladis) +- Luna Nova [**(@LunNova)**](https://github.com/LunNova) +- Riccardo Mazzarini [**(@noib3)**](https://github.com/noib3) +- Bouke van der Bijl [**(@bouk)**](https://github.com/bouk) +- Dario [**(@dve00)**](https://github.com/dve00) +- Michael Hoang [**(@Enzime)**](https://github.com/Enzime) +- Paul Sbarra [**(@tones111)**](https://github.com/tones111) +- edef [**(@edef1c)**](https://github.com/edef1c) +- Adam Dinwoodie [**(@me-and)**](https://github.com/me-and) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Jeremy Fleischman [**(@jfly)**](https://github.com/jfly) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Alex Ionescu [**(@aionescu)**](https://github.com/aionescu) +- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy) +- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault) +- Pierre Penninckx [**(@ibizaman)**](https://github.com/ibizaman) +- Leonard Sheng Sheng Lee [**(@sheeeng)**](https://github.com/sheeeng) +- rszyma [**(@rszyma)**](https://github.com/rszyma) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Lennart Kolmodin [**(@kolmodin)**](https://github.com/kolmodin) +- zowoq [**(@zowoq)**](https://github.com/zowoq) +- Peter Collingbourne [**(@pcc)**](https://github.com/pcc) +- Simon Žlender [**(@szlend)**](https://github.com/szlend) +- Lily Foster [**(@lilyinstarlight)**](https://github.com/lilyinstarlight) +- randomizedcoder [**(@randomizedcoder)**](https://github.com/randomizedcoder) +- Krish Jaiswal From 7b99db2a09d65da80a23f9f40f45dd122703151a Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 8 Jul 2026 23:56:21 +0300 Subject: [PATCH 555/927] Once again follow final symlinks in GitRepoImpl::getAccessor This used to be the case before 02e4f4ad75fda707a594ffd8ad511691fc1a311f, and this restores that compatibility. Ideally we'd check that access to that path is allowed, but libfetchers does this now pretty inconsistently. Also test a lot more things. (cherry picked from commit 3aff4dc5edf30998d64eec024de186ac2d6fb5ea) --- src/libfetchers/git-utils.cc | 16 +++++++++------- tests/functional/git/meson.build | 5 ++++- tests/functional/git/symlinked-repo.sh | 17 +++++++++++++++++ 3 files changed, 30 insertions(+), 8 deletions(-) create mode 100644 tests/functional/git/symlinked-repo.sh diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index e035032808..cd04bfee77 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1449,13 +1449,15 @@ ref GitRepoImpl::getAccessor( const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) { auto self = ref(shared_from_this()); - ref fileAccessor = AllowListSourceAccessor::create( - makeFSSourceAccessor(path), - /*allowedPrefixes=*/wd.files, - // Always allow access to the root, but not its children. - /*allowedPaths=*/{CanonPath::root}, - std::move(makeNotAllowedError)) - .cast(); + ref fileAccessor = + AllowListSourceAccessor::create( + // Follow the final symlink to the repo. Older nix versions used to do this (maybe somewhat accidentally). + makeFSSourceAccessor(path, /*trackLastModified=*/false, FinalSymlink::Follow), + /*allowedPrefixes=*/wd.files, + // Always allow access to the root, but not its children. + /*allowedPaths=*/{CanonPath::root}, + std::move(makeNotAllowedError)) + .cast(); if (options.exportIgnore) fileAccessor = make_ref(self, fileAccessor, std::nullopt); return fileAccessor; diff --git a/tests/functional/git/meson.build b/tests/functional/git/meson.build index af6882698b..73c4bc7828 100644 --- a/tests/functional/git/meson.build +++ b/tests/functional/git/meson.build @@ -1,6 +1,9 @@ suites += { 'name' : 'git', 'deps' : [], - 'tests' : [ 'packed-refs-no-cache.sh' ], + 'tests' : [ + 'packed-refs-no-cache.sh', + 'symlinked-repo.sh', + ], 'workdir' : meson.current_source_dir(), } diff --git a/tests/functional/git/symlinked-repo.sh b/tests/functional/git/symlinked-repo.sh new file mode 100644 index 0000000000..e30b0b3c0d --- /dev/null +++ b/tests/functional/git/symlinked-repo.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +source ../common.sh + +requireGit + +repo=$TEST_ROOT/repo +link=$TEST_ROOT/link + +createGitRepo "$repo" +ln -s "$repo" "$link" + +# Dereferencing final symlink component should work and follow it to the directory. +# Also test various cases of symlink trickery in case that ever changes. +for path in {repo,link}{,/,/.} {repo,link}/.././repo{,/,/.}; do + [ "$(nix-instantiate --eval --expr "builtins.readFileType (builtins.fetchTree \"git+file://$TEST_ROOT/$path\").outPath")" == '"directory"' ] +done From 808f22dd25946c016cceeb9210ea78516f5debd8 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 11 Jul 2026 00:26:17 +0300 Subject: [PATCH 556/927] libutil: Don't trample parent's havePrivateMountNs in restoreMountNamespace This is only a band-aid, see the comment for the explanation why the current code sucks in an indescribable way. A much better solution would be to get rid of vfork() entirely (or pay the incredible cost of making it safe for each libc we target) by replacing that with a zygote process. That's a huge change though, so let's cross our fingers and pray that this sucky code won't lead to a security issue. (cherry picked from commit 9640ab1fa4a88def1c5be31e45cd819caa24a638) --- src/libutil/linux/linux-namespaces.cc | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/libutil/linux/linux-namespaces.cc b/src/libutil/linux/linux-namespaces.cc index f0416307d1..5ad9bfc9de 100644 --- a/src/libutil/linux/linux-namespaces.cc +++ b/src/libutil/linux/linux-namespaces.cc @@ -161,12 +161,25 @@ void remountReadOnlyWritable(const std::filesystem::path & path) throw SysError("remounting %s writable", PathFmt(path)); } +/* This code runs in a (possibly) vfork-ed child, so technically everything you see below is beyond + broken because vfork()-ed child: + + * Must not trample parent's memory in any way shape or form. That includes (but not limited to) + * Throwing any exceptions (because that would unwind into the parent stack frame and do who knows what). + * Modify any state - obviously that includes global state. + * Not allocate any memory, since that can also lead to a deadlock if some thread in the (now stopped) parent + holds a lock while we are running. That's because *all* of the parent tasks are suspended for the duration + of the vfork. + + As it stands now, this code should be considered incredibly fragile and slated for a complete rework. + */ void restoreMountNamespace() { if (!havePrivateMountNs) return; try { + /* FIXME: Allocation in a possibly vforked child. */ auto savedCwd = std::filesystem::current_path(); if (setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) @@ -182,7 +195,8 @@ void restoreMountNamespace() if (chdir(savedCwd.c_str()) == -1) throw SysError("restoring cwd"); - havePrivateMountNs = false; + /* Do not reset havePrivateMountNs! This code can run in a vfork-ed child and we absolutely + must not trample any of the parent's state. */ } catch (Error & e) { debug(e.msg()); } From 8ab6055b9980befdca4855d3e39d0f472a535d58 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 13 Jul 2026 20:23:01 +0300 Subject: [PATCH 557/927] Make official release --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index ac2bd55112..67532424ee 100644 --- a/flake.nix +++ b/flake.nix @@ -31,7 +31,7 @@ let inherit (nixpkgs) lib; - officialRelease = false; + officialRelease = true; linux32BitSystems = [ "i686-linux" ]; linux64BitSystems = [ From 022e8aa6cb5bbd9f86dae9db04b556d6539bf337 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sun, 12 Jul 2026 15:29:17 +0300 Subject: [PATCH 558/927] packaging: Fix sqlite on mingw tcl is broken on mingw in nixpkgs and so is sqlite. This makes it build now at least. (cherry picked from commit 526e916abd12032141f7bd45d67d2d9f252b1be3) --- packaging/dependencies.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 17071fb0c4..5777dd5649 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -62,6 +62,17 @@ scope: { ); }; + sqlite = + if !stdenv.hostPlatform.isWindows then + pkgs.sqlite + else + pkgs.sqlite.overrideAttrs (prevAttrs: { + nativeBuildInputs = lib.filter (x: !(x.pname == "tcl")) prevAttrs.nativeBuildInputs or [ ]; + configureFlags = (lib.filter (x: !(lib.hasPrefix "--with-tcl" x)) prevAttrs.configureFlags) ++ [ + "--disable-tcl" + ]; + }); + libgit2 = if lib.versionAtLeast pkgs.libgit2.version "1.9.4" then pkgs.libgit2 From 370bd13240d7688198b10cf3a30839fc4f079fbc Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Jul 2026 01:46:38 +0300 Subject: [PATCH 559/927] Bump pinned nix in install-nix-action composite action (cherry picked from commit f553163d4cbad6d437b8ee12d660cb5a96df1e99) --- .github/actions/install-nix-action/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-nix-action/action.yaml b/.github/actions/install-nix-action/action.yaml index f889944b2d..2fb433b741 100644 --- a/.github/actions/install-nix-action/action.yaml +++ b/.github/actions/install-nix-action/action.yaml @@ -9,7 +9,7 @@ inputs: install_url: description: "URL of the Nix installer" required: false - default: "https://releases.nixos.org/nix/nix-2.32.1/install" + default: "https://releases.nixos.org/nix/nix-2.34.8/install" github_token: description: "Github token" required: true From 6c985da02e17633b75f54de97ccd4e7e59a17f53 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Jul 2026 01:51:24 +0300 Subject: [PATCH 560/927] Use stable nix for PR CI --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f294211ea..75637c096b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ on: dogfood: description: 'Use dogfood Nix build' required: false - default: true + default: false type: boolean concurrency: @@ -29,7 +29,7 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/install-nix-action with: - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} + dogfood: false extra_nix_config: experimental-features = nix-command flakes github_token: ${{ secrets.GITHUB_TOKEN }} @@ -43,7 +43,7 @@ jobs: - uses: actions/checkout@v6 - uses: ./.github/actions/install-nix-action with: - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} + dogfood: false extra_nix_config: experimental-features = nix-command flakes github_token: ${{ secrets.GITHUB_TOKEN }} - run: ./ci/gha/tests/pre-commit-checks @@ -93,7 +93,7 @@ jobs: - uses: ./.github/actions/install-nix-action with: github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} + dogfood: false # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: @@ -150,7 +150,7 @@ jobs: - uses: ./.github/actions/install-nix-action with: github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} + dogfood: false - name: Run Windows unit tests run: | nix build --file ci/gha/tests/windows.nix unitTests.nix-util-tests -L @@ -223,7 +223,7 @@ jobs: - uses: ./.github/actions/install-nix-action with: github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} + dogfood: false extra_nix_config: "sandbox = true" - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Run clang-tidy @@ -278,7 +278,7 @@ jobs: - uses: ./.github/actions/install-nix-action with: github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} + dogfood: false extra_nix_config: | experimental-features = flakes nix-command ca-derivations impure-derivations max-jobs = 1 From 3d4e186068fafcaf984c469642b548ba2f5c8271 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Jul 2026 00:36:16 +0300 Subject: [PATCH 561/927] Don't openStore in nix-env --version This has caused dealocks after the fixes to acquire an exclusive lock for performing store DB migrations, because upgrade-nix.cc runs a nix-env --version to check that the command works at all. Since the old code would try to open a store, we'd deadlock. Fixes #16136 (cherry picked from commit 5063db70674d5db052af50d59135aa9ae03c16ba) --- src/libmain/include/nix/main/shared.hh | 2 +- src/nix/nix-env/nix-env.cc | 28 ++++++++++++++------------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/libmain/include/nix/main/shared.hh b/src/libmain/include/nix/main/shared.hh index f9e771205b..7b0e7a54d4 100644 --- a/src/libmain/include/nix/main/shared.hh +++ b/src/libmain/include/nix/main/shared.hh @@ -26,7 +26,7 @@ void parseCmdLine( const Strings & args, fun parseArg); -void printVersion(const std::string & programName); +[[noreturn]] void printVersion(const std::string & programName); /** * Ugh. No better place to put this. diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index abe04d9634..7fc518d215 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -1391,7 +1391,7 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr } } -static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) +[[noreturn]] static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) { printVersion("nix-env"); } @@ -1508,23 +1508,25 @@ static int main_nix_env(int argc, char ** argv) if (!op) throw UsageError("no operation specified"); - auto store = openStore(); + if (op != opVersion) { + auto store = openStore(); - globals.state = - std::shared_ptr(new EvalState(myArgs.lookupPath, store, fetchSettings, evalSettings)); - globals.state->repair = myArgs.repair; + globals.state = + std::shared_ptr(new EvalState(myArgs.lookupPath, store, fetchSettings, evalSettings)); + globals.state->repair = myArgs.repair; - globals.instSource.nixExprPath = std::make_shared( - file != "" ? lookupFileArg(*globals.state, file) - : globals.state->rootPath(CanonPath(nixExprPath.string()))); + globals.instSource.nixExprPath = std::make_shared( + file != "" ? lookupFileArg(*globals.state, file) + : globals.state->rootPath(CanonPath(nixExprPath.string()))); - globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); + globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); - if (globals.profile == "") - globals.profile = getEnv("NIX_PROFILE").value_or(""); + if (globals.profile == "") + globals.profile = getEnv("NIX_PROFILE").value_or(""); - if (globals.profile == "") - globals.profile = getDefaultProfile(settings.getProfileDirsOptions()).string(); + if (globals.profile == "") + globals.profile = getDefaultProfile(settings.getProfileDirsOptions()).string(); + } op(globals, std::move(opFlags), std::move(opArgs)); From 8a0b159b5a92800bec61f5ca552794f69fd9ac1b Mon Sep 17 00:00:00 2001 From: Lisanna Dettwyler Date: Mon, 13 Jul 2026 18:18:10 -0400 Subject: [PATCH 562/927] Revert temproots path name change This is causing breakages when downgrading from nix 2.35.0, since it expects the filename to have a parseable integer. Signed-off-by: Lisanna Dettwyler (cherry picked from commit c1cc521d2ffbc0b9e0a2cf3e3aa22c9d04f037ef) --- src/libstore/gc.cc | 4 ++-- src/libstore/local-store.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index e399dd949a..8e997c8e24 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -61,8 +61,8 @@ void LocalStore::createTempRootsFile() while (1) { if (pathExists(fnTempRoots)) - /* The file is stale since each LocalStore instance - uses a unique filename (pid + counter). */ + /* It *must* be stale, since there can be no two + processes with the same pid. */ tryUnlink(fnTempRoots); *fdTempRoots = openLockFile(fnTempRoots, true); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5a1c161c07..909f22369d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -129,7 +129,7 @@ LocalStore::LocalStore(ref config) , reservedPath(dbDir / "reserved") , schemaPath(dbDir / "schema") , tempRootsDir(config->stateDir.get() / "temproots") - , fnTempRoots(makeTempPath(tempRootsDir, "temproots")) + , fnTempRoots(tempRootsDir / std::to_string(getpid())) { auto state(_state->lock()); state->stmts = std::make_unique(); From 85855aacbf5659dd85a8a290c2532d5dc6196555 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Jul 2026 02:07:05 +0300 Subject: [PATCH 563/927] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index aa5388f637..d07233cc93 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.35.0 +2.35.1 From aa2613fa02ca6199fe0ebf61a0f7f6d781d32a47 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 14 Jul 2026 03:10:36 +0300 Subject: [PATCH 564/927] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index d07233cc93..c0ac27fe01 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.35.1 +2.35.2 From 822e0702823878e9a156b2e3fefc98b625225c3f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Jul 2026 23:27:45 +0300 Subject: [PATCH 565/927] libutil: Use O_PATH when opening parent dirFds Also extend the symlinked-home functional NixOS VM test to catch such issues. (cherry picked from commit a58925d4e41150e2ebde6e27579c556d9514e099) --- src/libutil/posix-source-accessor.cc | 16 ++++++++++++++-- tests/nixos/functional/symlinked-home.nix | 14 ++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index e426c43b34..55fa6342f4 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -383,8 +383,20 @@ std::pair> PosixDirectorySourceAccessor } try { - AutoCloseFD parentFdOwning = - openFileEnsureBeneathNoSymlinks(startFd, relPath, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, std::move(cb)); + AutoCloseFD parentFdOwning = openFileEnsureBeneathNoSymlinks( + startFd, + relPath, +# ifdef O_PATH + /* As to not require read permissions on the directory. */ + O_PATH | +# else + /* Sadly this will require read permissison for path resolution, + but without O_PATH that's unavoidable. */ + O_RDONLY | +# endif + O_DIRECTORY | O_CLOEXEC, + 0, + std::move(cb)); return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; } catch (SymlinkNotAllowed & e) { /* Need to fixup the error message to include the actual path relative to the (possibly) cached fd. */ diff --git a/tests/nixos/functional/symlinked-home.nix b/tests/nixos/functional/symlinked-home.nix index 900543d0cf..09780339da 100644 --- a/tests/nixos/functional/symlinked-home.nix +++ b/tests/nixos/functional/symlinked-home.nix @@ -1,9 +1,11 @@ /** This test runs the functional tests on a NixOS system where the home directory - is symlinked to another location. + is symlinked to another location that also happens to reside in parent directory + that we don't have read permissions for (only execute). The purpose of this test is to find cases where Nix uses low-level operations - that don't support symlinks on paths that include them. + that don't support symlinks on paths that include them or requires excessive + permissions for path resolution. It is not a substitute for more intricate, use case-specific tests, but helps catch common issues. @@ -27,8 +29,12 @@ machine.succeed(""" ( set -x - mv /home/alice /home/alice.real - ln -s alice.real /home/alice + mkdir -p /home/alice.parent + chown alice:users /home/alice.parent + # Make the parent unreadable for good measure + chmod 0110 /home/alice.parent + mv /home/alice /home/alice.parent/alice.real + ln -s alice.parent/alice.real /home/alice ) 1>&2 """) machine.succeed(""" From 65646c2b0b6eea5fcbc31f305e857b6a3a34aa43 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jul 2026 16:48:19 +0200 Subject: [PATCH 566/927] Fix building without wasmtime --- src/libstore/unix/build/wasi-derivation-builder.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/wasi-derivation-builder.cc b/src/libstore/unix/build/wasi-derivation-builder.cc index 9334fb0a70..6fb7c41c22 100644 --- a/src/libstore/unix/build/wasi-derivation-builder.cc +++ b/src/libstore/unix/build/wasi-derivation-builder.cc @@ -1,6 +1,10 @@ -#include "derivation-builder-impl.hh" +#include "store-config-private.hh" -#include +#if NIX_USE_WASMTIME + +# include "derivation-builder-impl.hh" + +# include namespace nix { @@ -90,3 +94,5 @@ DerivationBuilderUnique makeWasiDerivationBuilder( } } // namespace nix + +#endif // NIX_USE_WASMTIME From 7d803ff947dd9b10712d76ce63b3eb18ea8b3906 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jul 2026 17:07:48 +0200 Subject: [PATCH 567/927] Disable mimalloc in the static build on aarch64-darwin This fails with a duplicate symbol '_reallocarray' error between mimalloc and lowdown. --- src/nix/package.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nix/package.nix b/src/nix/package.nix index 3752f370e8..be9a253e6f 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -30,7 +30,12 @@ # workloads (~10-15% on large evaluations). # mimalloc is disabled on FreeBSD due to a crash in nixpkgs 25.11. # Once the nixpkgs flake is updated, mimalloc can be enabled again. - withMimalloc ? !stdenv.hostPlatform.isWindows && !stdenv.hostPlatform.isFreeBSD, + # It's also disabled on static aarch64-darwin because of a duplicate + # `reallocarray` symbol in libmimalloc.a and lowdown's compats.o. + withMimalloc ? + !stdenv.hostPlatform.isWindows + && !stdenv.hostPlatform.isFreeBSD + && !(stdenv.hostPlatform.isStatic && stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64), # Whether to embed the public C API into the `nix` executable so plugins can # resolve those symbols without linking Nix libraries directly. From ea2a649a58595751841436b75de8e4a00763a5f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jul 2026 17:33:26 +0200 Subject: [PATCH 568/927] Shut up compiler warning --- src/libutil-tests/compression.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libutil-tests/compression.cc b/src/libutil-tests/compression.cc index 7978a008fe..60d2cd29ed 100644 --- a/src/libutil-tests/compression.cc +++ b/src/libutil-tests/compression.cc @@ -29,8 +29,9 @@ TEST_P(CompressionDecompressionTest, roundtrip) TEST_P(CompressionDecompressionTest, empty) { auto compressed = compress(GetParam(), ""); - if (GetParam() != CompressionAlgo::none) + if (GetParam() != CompressionAlgo::none) { ASSERT_FALSE(compressed.empty()); + } auto o = decompress(GetParam(), compressed); ASSERT_EQ(o, ""); } From dbbc5e1f47a6430c11cd03c5d744f91a4c8580a8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jul 2026 19:36:50 +0200 Subject: [PATCH 569/927] Fix use of readOnlyMode --- src/libexpr/paths.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 84caa1ca5a..6d644fe961 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -29,7 +29,7 @@ StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites) fetchSettings, *store, SourcePath{ref(mount)}, - settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + settings.isReadOnly() ? FetchMode::DryRun : FetchMode::Copy, path.name()); assert(storePath.name() == path.name()); if (rewrites) From 26c2df11d2ca56cfb54f41c4d6eedfcc1b377b51 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jul 2026 19:58:35 +0200 Subject: [PATCH 570/927] Fix linkcheck --- .../protocols/json/schema/derivation-resolved-v4.yaml | 6 ------ doc/manual/source/protocols/json/schema/signature-v2.yaml | 6 ------ 2 files changed, 12 deletions(-) diff --git a/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml b/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml index 781f07fb37..705156eb84 100644 --- a/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml +++ b/doc/manual/source/protocols/json/schema/derivation-resolved-v4.yaml @@ -12,12 +12,6 @@ description: | [primary (not necessarily resolved) JSON schema](@docroot@/protocols/json/derivation/index.md), but actually it is the first version for this format. - > **Warning** - > - > This JSON format is currently - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) - > and subject to change. - type: object required: - name diff --git a/doc/manual/source/protocols/json/schema/signature-v2.yaml b/doc/manual/source/protocols/json/schema/signature-v2.yaml index e2b25dbc6f..d94798902f 100644 --- a/doc/manual/source/protocols/json/schema/signature-v2.yaml +++ b/doc/manual/source/protocols/json/schema/signature-v2.yaml @@ -6,12 +6,6 @@ description: | This schema describes the JSON representation of signatures as used in various Nix JSON APIs. - > **Warning** - > - > This JSON format is currently - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) - > and subject to change. - ## Version History - Version 1: Colon-separated string in the format `:` From f4b866b4b867a0129099bb96528f92f683f15a8f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jul 2026 12:08:11 +0200 Subject: [PATCH 571/927] LocalStore::addValidPath(): Use a SQLite upsert Instead of checking isValidPath_() and then doing either an insert or an update, registerValidPaths() now unconditionally executes a single "insert ... on conflict (path) do update ... returning id" statement. The update clause only touches the columns that updatePathInfo() used to update (hash, narSize, ultimate, sigs, ca), so registrationTime, deriver and provenance are preserved for already-valid paths. To distinguish an insert from an update (in which case the derivation outputs are already registered and we can return early), we clear the last-inserted rowid before executing the statement and compare it against the returned id afterwards. Also, registerValidPaths() now reuses the path ids returned by addValidPath() instead of looking them up again with queryValidPathId(). This reduces the number of SQLite calls instantiating my NixOS config on an empty store from 379086 to 352955. Assisted-by: Claude Fable 5 --- src/libstore/include/nix/store/sqlite.hh | 8 ++++++ src/libstore/local-store.cc | 35 +++++++++++++++++------- src/libstore/sqlite.cc | 5 ++++ 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src/libstore/include/nix/store/sqlite.hh b/src/libstore/include/nix/store/sqlite.hh index d11515b296..294bb372d6 100644 --- a/src/libstore/include/nix/store/sqlite.hh +++ b/src/libstore/include/nix/store/sqlite.hh @@ -77,6 +77,14 @@ struct SQLite void exec(const std::string & stmt); uint64_t getLastInsertedRowId(); + + /** + * Set the value returned by `getLastInsertedRowId()`. Since only + * successful inserts update the last-inserted rowid, this can be + * used to detect whether an upsert statement performed an insert + * or an update. + */ + void setLastInsertedRowId(uint64_t id); }; /** diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index fd85f90cfb..997f30c1f4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -355,7 +355,9 @@ LocalStore::LocalStore(ref config) /* Prepare SQL statements. */ state->stmts->RegisterValidPath.create( state->db, - fmt("insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca%s) values (?, ?, ?, ?, ?, ?, ?, ?%s);", + fmt("insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca%s) values (?, ?, ?, ?, ?, ?, ?, ?%s) " + "on conflict (path) do update set hash = excluded.hash, narSize = excluded.narSize, ultimate = excluded.ultimate, sigs = excluded.sigs, ca = excluded.ca " + "returning id;", experimentalFeatureSettings.isEnabled(Xp::Provenance) ? ", provenance" : "", experimentalFeatureSettings.isEnabled(Xp::Provenance) ? ", ?" : "")); state->stmts->UpdatePathInfo.create( @@ -734,8 +736,18 @@ uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info) .apply(renderContentAddress(info.ca), (bool) info.ca); if (experimentalFeatureSettings.isEnabled(Xp::Provenance)) query.apply(info.provenance ? info.provenance->to_json_str() : "", (bool) info.provenance); - query.exec(); - uint64_t id = state.db.getLastInsertedRowId(); + /* Valid ids start at 1, so a last-inserted rowid of 0 signals + that the upsert did an update rather than an insert. */ + state.db.setLastInsertedRowId(0); + if (!query.next()) + throw Error("registering path '%s' in the Nix database did not return an id", printStorePath(info.path)); + uint64_t id = query.getInt(0); + + /* If the path was already valid (i.e. the upsert did an update + rather than an insert), the derivation outputs have already + been registered, so we're done. */ + if (state.db.getLastInsertedRowId() != id) + return id; /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -990,20 +1002,23 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) SQLiteTxn txn(state->db); StorePathSet paths; + std::unordered_map ids; for (auto & [_, i] : infos) { assert(i.narHash.algo == HashAlgorithm::SHA256); - if (isValidPath_(*state, i.path)) - updatePathInfo(*state, i); - else - addValidPath(*state, i); + ids.insert_or_assign(i.path, addValidPath(*state, i)); paths.insert(i.path); } for (auto & [_, i] : infos) { - auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) - state->stmts->AddReference.use().apply(referrer).apply(queryValidPathId(*state, j)).exec(); + auto referrer = ids.at(i.path); + for (auto & j : i.references) { + auto k = ids.find(j); + state->stmts->AddReference.use() + .apply(referrer) + .apply(k != ids.end() ? k->second : queryValidPathId(*state, j)) + .exec(); + } } /* Do a topological sort of the paths. This will throw an diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 5bd3a4975c..5e62d96ebb 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -144,6 +144,11 @@ uint64_t SQLite::getLastInsertedRowId() return sqlite3_last_insert_rowid(db); } +void SQLite::setLastInsertedRowId(uint64_t id) +{ + sqlite3_set_last_insert_rowid(db, id); +} + void SQLiteStmt::create(sqlite3 * db, const std::string & sql) { checkInterrupt(); From d968ac31917670392a1deb1b76ccea4ec8b0e51a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jul 2026 12:18:51 +0200 Subject: [PATCH 572/927] LocalStore::queryValidPathId(): Use a dedicated SQLite statement queryValidPathId() and isValidPath_() used the QueryPathInfo statement, which returns a lot of columns (hash, deriver, sigs, ca, ...) that they don't need. Add a QueryValidPathId statement that only returns the id. Assisted-by: Claude Fable 5 --- src/libstore/local-store.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 997f30c1f4..5124e62f51 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -110,6 +110,7 @@ struct LocalStore::State::Stmts SQLiteStmt UpdatePathInfo; SQLiteStmt AddReference; SQLiteStmt QueryPathInfo; + SQLiteStmt QueryValidPathId; SQLiteStmt QueryReferences; SQLiteStmt QueryReferrers; SQLiteStmt InvalidatePath; @@ -367,6 +368,7 @@ LocalStore::LocalStore(ref config) state->db, fmt("select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca%s from ValidPaths where path = ?;", experimentalFeatureSettings.isEnabled(Xp::Provenance) ? ", provenance" : "")); + state->stmts->QueryValidPathId.create(state->db, "select id from ValidPaths where path = ?;"); state->stmts->QueryReferences.create( state->db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); state->stmts->QueryReferrers.create( @@ -857,7 +859,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) { - auto use(state.stmts->QueryPathInfo.use().apply(printStorePath(path))); + auto use(state.stmts->QueryValidPathId.use().apply(printStorePath(path))); if (!use.next()) throw InvalidPath("path '%s' is not valid", printStorePath(path)); return use.getInt(0); @@ -865,7 +867,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) bool LocalStore::isValidPath_(State & state, const StorePath & path) { - return state.stmts->QueryPathInfo.use().apply(printStorePath(path)).next(); + return state.stmts->QueryValidPathId.use().apply(printStorePath(path)).next(); } bool LocalStore::isValidPathUncached(const StorePath & path) From b4b2960461f9856bf9e9c2194d9a70ef2ed3d958 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jul 2026 12:35:51 +0200 Subject: [PATCH 573/927] LocalStore::queryValidPathId(): Use pathInfoCache While adding references, we do a huge number of valid path id lookups that most of the time are for paths already in pathInfoCache. However, ValidPathInfo doesn't contain the SQLite id. So local-store.cc now uses LocalStorePathInfo, a subclass of ValidPathInfo that adds an id field (similar to how BinaryCacheStore uses NarInfo), for the path infos it creates and caches. queryValidPathId() then returns the id from the cache if available, and only falls back to a database query on a cache miss. This further reduces the number of SQLite calls on my NixOS config from 352955 to 225397. Assisted-by: Claude Fable 5 --- src/libstore/local-store.cc | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5124e62f51..64426a29be 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -103,6 +103,24 @@ bool LocalStoreConfig::getDefaultRequireSigs() return settings.requireSigs; } +/** + * A `ValidPathInfo` that also records the id of the corresponding row + * in the `ValidPaths` table of the SQLite database. This allows + * `queryValidPathId()` to return the id from `pathInfoCache` without + * consulting the database. + */ +struct LocalStorePathInfo : ValidPathInfo +{ + uint64_t id; + + LocalStorePathInfo(ValidPathInfo info, uint64_t id) + : UnkeyedValidPathInfo(info) + , ValidPathInfo(std::move(info)) + , id(id) + { + } +}; + struct LocalStore::State::Stmts { /* Some precompiled SQLite statements. */ @@ -773,7 +791,8 @@ uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info) } } - pathInfoCache->lock()->upsert(info.path, PathInfoCacheValue{.value = std::make_shared(info)}); + pathInfoCache->lock()->upsert( + info.path, PathInfoCacheValue{.value = std::make_shared(info, id)}); return id; } @@ -808,7 +827,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); } - auto info = std::make_shared(path, UnkeyedValidPathInfo(*this, narHash)); + auto info = std::make_shared(ValidPathInfo(path, UnkeyedValidPathInfo(*this, narHash)), id); info->registrationTime = useQueryPathInfo.getInt(2); @@ -859,6 +878,14 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) { + /* Avoid a database query if the id is in `pathInfoCache`. */ + { + auto cache(pathInfoCache->lock()); + if (auto res = cache->getOrNullptr(path)) + if (auto info = dynamic_cast(res->value.get())) + return info->id; + } + auto use(state.stmts->QueryValidPathId.use().apply(printStorePath(path))); if (!use.next()) throw InvalidPath("path '%s' is not valid", printStorePath(path)); @@ -1016,6 +1043,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) auto referrer = ids.at(i.path); for (auto & j : i.references) { auto k = ids.find(j); + /* Note that queryValidPathId() uses pathInfoCache, so it's possible that path `j` has been + * garbage-collected. But that's fine: the foreign key constraint on Refs will detect that the ID is no + * longer valid. Path IDs are never reused so there is no possibility that ID now refers to another + * path. */ state->stmts->AddReference.use() .apply(referrer) .apply(k != ids.end() ? k->second : queryValidPathId(*state, j)) From b800e6515729b7bd92adabb636274774244d6eb2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jul 2026 18:53:00 +0200 Subject: [PATCH 574/927] Fix build --- src/libstore/local-store.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 64426a29be..9fc56975c1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -119,8 +119,12 @@ struct LocalStorePathInfo : ValidPathInfo , id(id) { } + + void anchor() override; }; +void LocalStorePathInfo::anchor() {} + struct LocalStore::State::Stmts { /* Some precompiled SQLite statements. */ From 8051880e0a6842c10afd4710688af49d38028e9a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jul 2026 14:18:10 +0200 Subject: [PATCH 575/927] Remove redundant unity settings --- src/libexpr/meson.build | 2 -- src/libfetchers/meson.build | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 484ee8993f..995202dbcb 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -7,8 +7,6 @@ project( # TODO(Qyriad): increase the warning level 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail - 'unity=on', - 'unity_size=1024', ], meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 1a38b1905a..134fd496a7 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -7,8 +7,6 @@ project( # TODO(Qyriad): increase the warning level 'warning_level=1', 'errorlogs=true', # Please print logs for tests that fail - 'unity=on', - 'unity_size=1024', ], meson_version : '>= 1.8', license : 'LGPL-2.1-or-later', From fbd4c3f7eb66693c290e9e79c5240c8a0e49eddd Mon Sep 17 00:00:00 2001 From: Rohit1 Singh Date: Mon, 27 Jul 2026 11:09:24 +0000 Subject: [PATCH 576/927] daemon: unlink worker's temproots file on exit A forked nix-daemon worker ends with exit(0), which does not unwind the stack, so the LocalStore it owns is never destroyed and ~LocalStore() (which unlinks /nix/var/nix/temproots/) never runs. Each connection therefore leaks a stale temproots file until a later GC reaps it. Before 2.34.0 the store was a temporary passed straight into processConnection(), so it was destroyed at the end of that statement, before exit(0). c4e408459 hoisted it into a named local to call init(), extending its lifetime past exit(0). processConnection() takes the store by value, so std::move it in: the store is then destroyed when that call returns, restoring the cleanup. (cherry picked from commit f7f5c5dee7a268442e268e325f53d711c87ef24f) --- src/nix/unix/daemon.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 956adc43c9..954a2eea38 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -375,7 +375,12 @@ static void daemonLoop( // Handle the connection. auto store = storeConfig->openStore(); store->init(); - processConnection(store, FdSource(remote.get()), FdSink(remote.get()), trusted, NotRecursive); + processConnection( + std::move(store), + FdSource(remote.get()), + FdSink(remote.get()), + trusted, + RecursiveFlag::NotRecursive); exit(0); }, From fba3f6e49bea5f6b7b5b75ab794e32afaabd5d26 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 27 Jul 2026 03:35:26 +0300 Subject: [PATCH 577/927] libstore: Fix lost waiting for slot goal wakeups In 6fae3a25f1a07472b45c56862c0ffe2050159e0f I neglected the fact that a jobCategory == Build (and less so Substitution) doesn't necessarily use up and free up a build slot. This apparently led to https://github.com/NixOS/nix/issues/16005. From attached --debug logs in the issue: > skipping build of derivation '/nix/store/6gc23jl32lnaqv2ycch1m1igqm6hys35-unit-systemd-journald-.service.drv', someone beat us to it Which directly precedes the crash, seemingly caused by us failing to wake more goals waiting for build slots when a Build goal terminates while someone else has successfully built the same path. It's now the responsibility of the Worker::run loop to pick up just enough goals waiting for slots - overestimating a bit there is ok since the goal implementations avoid overconsuming slots by waiting more. This way we keep the bounded complexity on each iteration, while hopefully also fixing the root cause of the bug. (cherry picked from commit 332976977c61f92085ef9a7c0738af845b86241d) --- src/libstore/build/worker.cc | 30 +++++++++++-------- .../include/nix/store/build/worker.hh | 3 +- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 208bf5ca3e..38b12e7107 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -249,19 +249,6 @@ void Worker::childTerminated(Goal * goal, JobCategory jobCategory) } children.erase(i); - auto & waiting = jobCategory == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild; - - /* Wake up goals waiting for a build slot. Wake at most one waiter to avoid - starting unnecessary work (that is accompanied by coroutine frame allocation). */ - auto it = waiting.begin(); - while (it != waiting.end()) { - if (auto goal = it->lock()) { - waiting.erase(it); - wakeUp(goal); - break; - } - it = waiting.erase(it); - } } void Worker::waitForBuildSlot(GoalPtr goal) @@ -337,6 +324,22 @@ void Worker::run(const Goals & _topGoals) if (topGoals.empty()) break; // stuff may have been cancelled } + + auto wakeSlotWaiters = [this](WeakGoals & waiting, size_t running, size_t limit) { + auto it = waiting.begin(); + while (it != waiting.end() && running < limit) { + auto goal = it->lock(); + it = waiting.erase(it); + if (!goal) + continue; + wakeUp(goal); + ++running; + } + }; + + wakeSlotWaiters( + wantingToSubstitute, getNrSubstitutions(), std::max(1, settings.maxSubstitutionJobs)); + wakeSlotWaiters(wantingToBuild, getNrLocalBuilds(), settings.maxBuildJobs); } if (topGoals.empty()) @@ -365,6 +368,7 @@ void Worker::run(const Goals & _topGoals) --keep-going *is* set, then they must all be finished now. */ assert(!settings.keepGoing || awake.empty()); assert(!settings.keepGoing || wantingToBuild.empty()); + assert(!settings.keepGoing || wantingToSubstitute.empty()); assert(!settings.keepGoing || children.empty()); } diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index 12e6c0122e..861a86b485 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -326,8 +326,7 @@ public: bool respectTimeouts); /** - * Unregisters a running child process. Wakes at most a single goal that is - * awaiting on the corresponding build slot type (building or substitution). + * Unregisters a running child process. * * This overload requires `goal` to point to a fully constructed, * valid goal object, as it calls `goal->jobCategory()`. From 95cfc5da57b54dfeaade8e9f4cdafd68f7f86f1e Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Jul 2026 23:10:11 +0300 Subject: [PATCH 578/927] libutil-tests: Add test for sourceToSink boost.context bug This surfaces the bug fixed by https://github.com/boostorg/context/pull/337 in our test suite. (cherry picked from commit 88579a87f43a88b8ea5c0a0bbaa737620ad1241e) --- src/libutil-tests/serialise.cc | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/libutil-tests/serialise.cc b/src/libutil-tests/serialise.cc index 01e5579117..24ef6d2c60 100644 --- a/src/libutil-tests/serialise.cc +++ b/src/libutil-tests/serialise.cc @@ -1,5 +1,6 @@ #include "nix/util/serialise.hh" +#include #include namespace nix { @@ -37,4 +38,34 @@ TEST(readPadding, works) } } +TEST(sourceToSink, forcedUnwindUcaughtExceptions) +{ + int uncaughtExceptions = 42; + bool caught = false; + + auto sink = sourceToSink([&](Source & source) { + auto recordUncaughtExceptions = Finally([&]() { uncaughtExceptions = std::uncaught_exceptions(); }); + try { + StringSink s; + source.drainInto(s, 8); + source.drainInto(s, 8); + } catch (const boost::context::detail::forced_unwind &) { + caught = true; + throw; + } + }); + + *sink << 42; + + // Abandon the coroutine. This will trigger it to unwind with boost::context::detail::forced_unwind. + sink.reset(); + + ASSERT_TRUE(caught); + // This is a test for boost.context regression fixed by https://github.com/boostorg/context/pull/337. + // Without the fix std::uncaught_exceptions() *misreports* 0 while there's stack unwinding in progress. + // The issue only surfaces with libstdc++ and fcontext when boost.context + // uses fiber-specific exception states and messes with __cxa_get_globals(). + ASSERT_EQ(uncaughtExceptions, 0); +} + } // namespace nix From f8d28f3b841aff58740c52260fb65fbcd78a4dea Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Jul 2026 23:01:17 +0300 Subject: [PATCH 579/927] Add boost.context patch for std::uncaught_exceptions() misreporting 0 while abandoning the coroutine Applies a patch to our boost.context dependency to fix https://github.com/NixOS/nix/issues/16174. An alternative would be to apply basically the same workaround to our suspension points, but that would be more fragile. Other packagers might want to apply the boost patch too (maybe once that's merged upstream), since the issue is likely to affect more stuff than just nix. The bug is subtle enough that it went unnoticed for quite some time. (cherry picked from commit f8b102f902ee98bf051232a001e1e55b13ff70a4) --- packaging/dependencies.nix | 3 + ...eptions-not-accounting-for-forced_un.patch | 102 ++++++++++++++++++ src/libutil-tests/serialise.cc | 2 +- 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 5777dd5649..7edb7413b9 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -98,6 +98,9 @@ scope: { "--with-iostreams" "--with-url" ]; + patches = [ + ./patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch + ]; enableIcu = false; }).overrideAttrs (old: { diff --git a/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch b/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch new file mode 100644 index 0000000000..7ec13724c4 --- /dev/null +++ b/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch @@ -0,0 +1,102 @@ +From 5883212311535a0046031d74d1568ae173c1e35b Mon Sep 17 00:00:00 2001 +From: Sergei Zimmerman +Date: Tue, 21 Jul 2026 21:15:51 +0000 +Subject: [PATCH] Fix uncaught_exceptions() not accounting for forced_unwind + +Unwound fibers would see std::uncaught_exceptions() == 0, while a +forced_unwind exception is in "flight". This goes against the contract +of std::uncaught_exceptions() that scope guards rely upon. Failing +to report the correct number of uncaught exceptions (especially +misreporting zero) will lead to scope guards to misbehave badly and skip +running cleanup code which branches on whether the destructor is called +during stack unwinding or not. + +This is because the "throw" would happen before the destructor is run on +the fiber stack being switched to, but the increment would be clobbered +by the destructor of manage_exception_state. + +I'm not sure what the contract of run ontop_fcontext is wrt to whether +the the caller provided function can throw or not, but in my best +understanding the forced_unwind mechanism is mostly internal and so is +throwing from ontop_fcontext in the switched-to fiber. Thus, I've kept +the catch block scoped to detail::forced_unwind. +--- + include/boost/context/fiber_fcontext.hpp | 37 +++++++++++++++++------- + test/test_fiber.cpp | 24 +++++++++++++++ + 2 files changed, 51 insertions(+), 10 deletions(-) + +diff --git a/include/boost/context/fiber_fcontext.hpp b/include/boost/context/fiber_fcontext.hpp +index 543ba6c..38476c9 100644 +--- a/boost/context/fiber_fcontext.hpp ++++ b/boost/context/fiber_fcontext.hpp +@@ -70,7 +70,9 @@ namespace context { + namespace detail { + + // manage_exception_state is a dummy struct unless we have specific support +-struct manage_exception_state {}; ++struct manage_exception_state { ++ void from_forced_unwind() noexcept {} ++}; + + } // namespace detail + } // namespace context +@@ -90,6 +92,11 @@ public: + manage_exception_state() { + exception_state_ = *__cxa_get_globals(); + } ++ // Hack to account for the forced_unwind exception thrown in fiber_unwind ++ // that's run ontop before the destructor. ++ void from_forced_unwind() noexcept { ++ exception_state_.uncaughtExceptions += 1; ++ } + ~manage_exception_state() { + *__cxa_get_globals() = exception_state_; + } +@@ -376,13 +383,18 @@ public: + BOOST_ASSERT( nullptr != fctx_); + detail::manage_exception_state exstate; + boost::ignore_unused(exstate); +- return { detail::jump_fcontext( ++ try { ++ return { detail::jump_fcontext( + #if defined(BOOST_NO_CXX14_STD_EXCHANGE) +- detail::exchange( fctx_, nullptr), ++ detail::exchange( fctx_, nullptr), + #else +- std::exchange( fctx_, nullptr), ++ std::exchange( fctx_, nullptr), + #endif +- nullptr).fctx }; ++ nullptr).fctx }; ++ } catch ( detail::forced_unwind const& ) { ++ exstate.from_forced_unwind(); ++ throw; ++ } + } + + template< typename Fn > +@@ -391,14 +403,19 @@ public: + detail::manage_exception_state exstate; + boost::ignore_unused(exstate); + auto p = std::forward< Fn >( fn); +- return { detail::ontop_fcontext( ++ try { ++ return { detail::ontop_fcontext( + #if defined(BOOST_NO_CXX14_STD_EXCHANGE) +- detail::exchange( fctx_, nullptr), ++ detail::exchange( fctx_, nullptr), + #else +- std::exchange( fctx_, nullptr), ++ std::exchange( fctx_, nullptr), + #endif +- & p, +- detail::fiber_ontop< fiber, decltype(p) >).fctx }; ++ & p, ++ detail::fiber_ontop< fiber, decltype(p) >).fctx }; ++ } catch ( detail::forced_unwind const& ) { ++ exstate.from_forced_unwind(); ++ throw; ++ } + } + + explicit operator bool() const noexcept { diff --git a/src/libutil-tests/serialise.cc b/src/libutil-tests/serialise.cc index 24ef6d2c60..8af591a5e6 100644 --- a/src/libutil-tests/serialise.cc +++ b/src/libutil-tests/serialise.cc @@ -65,7 +65,7 @@ TEST(sourceToSink, forcedUnwindUcaughtExceptions) // Without the fix std::uncaught_exceptions() *misreports* 0 while there's stack unwinding in progress. // The issue only surfaces with libstdc++ and fcontext when boost.context // uses fiber-specific exception states and messes with __cxa_get_globals(). - ASSERT_EQ(uncaughtExceptions, 0); + ASSERT_EQ(uncaughtExceptions, 1); } } // namespace nix From b15a3d0034157c50cb8b5f3fe189fb218fa5f78f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 23 Jul 2026 00:04:26 +0300 Subject: [PATCH 580/927] libutil-tests: Disable sourceToSink.forcedUnwindUcaughtExceptions under ASan (cherry picked from commit 54aa8880be35e7d169e8d84d3dc135893cc4ff12) --- src/libutil-tests/serialise.cc | 10 ++++++++++ src/libutil/meson.build | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/src/libutil-tests/serialise.cc b/src/libutil-tests/serialise.cc index 8af591a5e6..40424c6853 100644 --- a/src/libutil-tests/serialise.cc +++ b/src/libutil-tests/serialise.cc @@ -1,4 +1,5 @@ #include "nix/util/serialise.hh" +#include "nix/util/config.hh" #include #include @@ -38,6 +39,13 @@ TEST(readPadding, works) } } +// The following test catches the bug only under fcontext backend, +// but Boost.Coroutine2 stack switching when abandoning confuses the +// hell out of ASan. The workaround to getting ASan working isn't +// immediately useful because it works only with ucontext implementation. +// https://www.boost.org/doc/libs/1_89_0/libs/coroutine2/doc/html/coroutine2/stack/sanitizers.html +#if !NIX_ASAN_ENABLED + TEST(sourceToSink, forcedUnwindUcaughtExceptions) { int uncaughtExceptions = 42; @@ -68,4 +76,6 @@ TEST(sourceToSink, forcedUnwindUcaughtExceptions) ASSERT_EQ(uncaughtExceptions, 1); } +#endif + } // namespace nix diff --git a/src/libutil/meson.build b/src/libutil/meson.build index fad7167617..b61403b51c 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -44,6 +44,12 @@ configdata_pub.set( description : 'Whether nix has been built with UBSan enabled', ) +configdata_pub.set( + 'NIX_ASAN_ENABLED', + ('address' in get_option('b_sanitize')).to_int(), + description : 'Whether nix has been built with ASan enabled', +) + subdir('nix-meson-build-support/libatomic') if host_machine.system() == 'windows' From c1f381b93c937f899fb971baa47c00084de2c76f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 31 Jul 2026 18:22:27 +0200 Subject: [PATCH 581/927] packaging/secure-packages/flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'path:../..' → 'path:../..' • Updated input 'nix/nixpkgs': 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-25.11/0.1.915354%2Brev-dca15ad98e9ae434bc8338133e9528120f5d35fc/019fae10-e6ee-7cc0-8f8a-fc79a3778326/source.tar.gz' (2026-07-29) → 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1006668%2Brev-e87516da91e53b694a46eda0de22b782eb008d82/019fb45e-7a22-75f0-a9f9-a895378372a0/source.tar.gz' (2026-07-30) --- packaging/secure-packages/flake.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index 76d082ff3b..651372b846 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -77,12 +77,12 @@ }, "nixpkgs": { "locked": { - "lastModified": 1785330376, - "narHash": "sha256-/rQoD1bBp3Fdeu+SsHYLF+ciIOOPqH5DgVypmmvEIvs=", - "rev": "dca15ad98e9ae434bc8338133e9528120f5d35fc", - "revCount": 915354, + "lastModified": 1785435372, + "narHash": "sha256-veAo2hsRG1Off47F+1HHgwarJVo/FjXUFv3A40ZUViY=", + "rev": "e87516da91e53b694a46eda0de22b782eb008d82", + "revCount": 1006668, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-25.11/0.1.915354%2Brev-dca15ad98e9ae434bc8338133e9528120f5d35fc/019fae10-e6ee-7cc0-8f8a-fc79a3778326/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1006668%2Brev-e87516da91e53b694a46eda0de22b782eb008d82/019fb45e-7a22-75f0-a9f9-a895378372a0/source.tar.gz" }, "original": { "type": "tarball", From bfe0fe7c7d69ff88cf9344bbc0694b4b935bf5ae Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 17 Jul 2026 23:27:45 +0300 Subject: [PATCH 582/927] libutil: Use O_PATH when opening parent dirFds Also extend the symlinked-home functional NixOS VM test to catch such issues. (cherry picked from commit a58925d4e41150e2ebde6e27579c556d9514e099) --- src/libutil/posix-source-accessor.cc | 16 ++++++++++++++-- tests/nixos/functional/symlinked-home.nix | 14 ++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 5103d62640..a067653d1b 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -383,8 +383,20 @@ std::pair> PosixDirectorySourceAccessor } try { - AutoCloseFD parentFdOwning = - openFileEnsureBeneathNoSymlinks(startFd, relPath, O_DIRECTORY | O_RDONLY | O_CLOEXEC, 0, std::move(cb)); + AutoCloseFD parentFdOwning = openFileEnsureBeneathNoSymlinks( + startFd, + relPath, +# ifdef O_PATH + /* As to not require read permissions on the directory. */ + O_PATH | +# else + /* Sadly this will require read permissison for path resolution, + but without O_PATH that's unavoidable. */ + O_RDONLY | +# endif + O_DIRECTORY | O_CLOEXEC, + 0, + std::move(cb)); return {parentFdOwning.get(), make_ref(std::move(parentFdOwning))}; } catch (SymlinkNotAllowed & e) { /* Need to fixup the error message to include the actual path relative to the (possibly) cached fd. */ diff --git a/tests/nixos/functional/symlinked-home.nix b/tests/nixos/functional/symlinked-home.nix index 900543d0cf..09780339da 100644 --- a/tests/nixos/functional/symlinked-home.nix +++ b/tests/nixos/functional/symlinked-home.nix @@ -1,9 +1,11 @@ /** This test runs the functional tests on a NixOS system where the home directory - is symlinked to another location. + is symlinked to another location that also happens to reside in parent directory + that we don't have read permissions for (only execute). The purpose of this test is to find cases where Nix uses low-level operations - that don't support symlinks on paths that include them. + that don't support symlinks on paths that include them or requires excessive + permissions for path resolution. It is not a substitute for more intricate, use case-specific tests, but helps catch common issues. @@ -27,8 +29,12 @@ machine.succeed(""" ( set -x - mv /home/alice /home/alice.real - ln -s alice.real /home/alice + mkdir -p /home/alice.parent + chown alice:users /home/alice.parent + # Make the parent unreadable for good measure + chmod 0110 /home/alice.parent + mv /home/alice /home/alice.parent/alice.real + ln -s alice.parent/alice.real /home/alice ) 1>&2 """) machine.succeed(""" From e987624177c657cc7129619bdfe60ecfcdfe0d7f Mon Sep 17 00:00:00 2001 From: Rohit1 Singh Date: Mon, 27 Jul 2026 11:09:24 +0000 Subject: [PATCH 583/927] daemon: unlink worker's temproots file on exit A forked nix-daemon worker ends with exit(0), which does not unwind the stack, so the LocalStore it owns is never destroyed and ~LocalStore() (which unlinks /nix/var/nix/temproots/) never runs. Each connection therefore leaks a stale temproots file until a later GC reaps it. Before 2.34.0 the store was a temporary passed straight into processConnection(), so it was destroyed at the end of that statement, before exit(0). c4e408459 hoisted it into a named local to call init(), extending its lifetime past exit(0). processConnection() takes the store by value, so std::move it in: the store is then destroyed when that call returns, restoring the cleanup. (cherry picked from commit f7f5c5dee7a268442e268e325f53d711c87ef24f) --- src/nix/unix/daemon.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 43ad2f046c..d7018b9c32 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -384,7 +384,12 @@ static void daemonLoop( // Handle the connection. auto store = storeConfig->openStore(); store->init(); - processConnection(store, FdSource(remote.get()), FdSink(remote.get()), trusted, NotRecursive); + processConnection( + std::move(store), + FdSource(remote.get()), + FdSink(remote.get()), + trusted, + RecursiveFlag::NotRecursive); exit(0); }, From 586a309e25e7e3d16bda8851d950e0bc905cc1a8 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Mon, 27 Jul 2026 03:35:26 +0300 Subject: [PATCH 584/927] libstore: Fix lost waiting for slot goal wakeups In 6fae3a25f1a07472b45c56862c0ffe2050159e0f I neglected the fact that a jobCategory == Build (and less so Substitution) doesn't necessarily use up and free up a build slot. This apparently led to https://github.com/NixOS/nix/issues/16005. From attached --debug logs in the issue: > skipping build of derivation '/nix/store/6gc23jl32lnaqv2ycch1m1igqm6hys35-unit-systemd-journald-.service.drv', someone beat us to it Which directly precedes the crash, seemingly caused by us failing to wake more goals waiting for build slots when a Build goal terminates while someone else has successfully built the same path. It's now the responsibility of the Worker::run loop to pick up just enough goals waiting for slots - overestimating a bit there is ok since the goal implementations avoid overconsuming slots by waiting more. This way we keep the bounded complexity on each iteration, while hopefully also fixing the root cause of the bug. (cherry picked from commit 332976977c61f92085ef9a7c0738af845b86241d) --- src/libstore/build/worker.cc | 30 +++++++++++-------- .../include/nix/store/build/worker.hh | 3 +- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 2229df9f2f..fbf9b8e62f 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -249,19 +249,6 @@ void Worker::childTerminated(Goal * goal, JobCategory jobCategory) } children.erase(i); - auto & waiting = jobCategory == JobCategory::Substitution ? wantingToSubstitute : wantingToBuild; - - /* Wake up goals waiting for a build slot. Wake at most one waiter to avoid - starting unnecessary work (that is accompanied by coroutine frame allocation). */ - auto it = waiting.begin(); - while (it != waiting.end()) { - if (auto goal = it->lock()) { - waiting.erase(it); - wakeUp(goal); - break; - } - it = waiting.erase(it); - } } void Worker::waitForBuildSlot(GoalPtr goal) @@ -337,6 +324,22 @@ void Worker::run(const Goals & _topGoals) if (topGoals.empty()) break; // stuff may have been cancelled } + + auto wakeSlotWaiters = [this](WeakGoals & waiting, size_t running, size_t limit) { + auto it = waiting.begin(); + while (it != waiting.end() && running < limit) { + auto goal = it->lock(); + it = waiting.erase(it); + if (!goal) + continue; + wakeUp(goal); + ++running; + } + }; + + wakeSlotWaiters( + wantingToSubstitute, getNrSubstitutions(), std::max(1, settings.maxSubstitutionJobs)); + wakeSlotWaiters(wantingToBuild, getNrLocalBuilds(), settings.maxBuildJobs); } if (topGoals.empty()) @@ -365,6 +368,7 @@ void Worker::run(const Goals & _topGoals) --keep-going *is* set, then they must all be finished now. */ assert(!settings.keepGoing || awake.empty()); assert(!settings.keepGoing || wantingToBuild.empty()); + assert(!settings.keepGoing || wantingToSubstitute.empty()); assert(!settings.keepGoing || children.empty()); } diff --git a/src/libstore/include/nix/store/build/worker.hh b/src/libstore/include/nix/store/build/worker.hh index cd84551d00..8bb35221e7 100644 --- a/src/libstore/include/nix/store/build/worker.hh +++ b/src/libstore/include/nix/store/build/worker.hh @@ -329,8 +329,7 @@ public: bool respectTimeouts); /** - * Unregisters a running child process. Wakes at most a single goal that is - * awaiting on the corresponding build slot type (building or substitution). + * Unregisters a running child process. * * This overload requires `goal` to point to a fully constructed, * valid goal object, as it calls `goal->jobCategory()`. From f31c9093e5d8bf5a15f04b966c09f349ba593359 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Jul 2026 23:10:11 +0300 Subject: [PATCH 585/927] libutil-tests: Add test for sourceToSink boost.context bug This surfaces the bug fixed by https://github.com/boostorg/context/pull/337 in our test suite. (cherry picked from commit 88579a87f43a88b8ea5c0a0bbaa737620ad1241e) --- src/libutil-tests/serialise.cc | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/libutil-tests/serialise.cc b/src/libutil-tests/serialise.cc index 01e5579117..24ef6d2c60 100644 --- a/src/libutil-tests/serialise.cc +++ b/src/libutil-tests/serialise.cc @@ -1,5 +1,6 @@ #include "nix/util/serialise.hh" +#include #include namespace nix { @@ -37,4 +38,34 @@ TEST(readPadding, works) } } +TEST(sourceToSink, forcedUnwindUcaughtExceptions) +{ + int uncaughtExceptions = 42; + bool caught = false; + + auto sink = sourceToSink([&](Source & source) { + auto recordUncaughtExceptions = Finally([&]() { uncaughtExceptions = std::uncaught_exceptions(); }); + try { + StringSink s; + source.drainInto(s, 8); + source.drainInto(s, 8); + } catch (const boost::context::detail::forced_unwind &) { + caught = true; + throw; + } + }); + + *sink << 42; + + // Abandon the coroutine. This will trigger it to unwind with boost::context::detail::forced_unwind. + sink.reset(); + + ASSERT_TRUE(caught); + // This is a test for boost.context regression fixed by https://github.com/boostorg/context/pull/337. + // Without the fix std::uncaught_exceptions() *misreports* 0 while there's stack unwinding in progress. + // The issue only surfaces with libstdc++ and fcontext when boost.context + // uses fiber-specific exception states and messes with __cxa_get_globals(). + ASSERT_EQ(uncaughtExceptions, 0); +} + } // namespace nix From 8b06e2ab080eb482e84acfec3dba66fc6ab7e4d2 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 22 Jul 2026 23:01:17 +0300 Subject: [PATCH 586/927] Add boost.context patch for std::uncaught_exceptions() misreporting 0 while abandoning the coroutine Applies a patch to our boost.context dependency to fix https://github.com/NixOS/nix/issues/16174. An alternative would be to apply basically the same workaround to our suspension points, but that would be more fragile. Other packagers might want to apply the boost patch too (maybe once that's merged upstream), since the issue is likely to affect more stuff than just nix. The bug is subtle enough that it went unnoticed for quite some time. (cherry picked from commit f8b102f902ee98bf051232a001e1e55b13ff70a4) --- packaging/dependencies.nix | 3 + ...eptions-not-accounting-for-forced_un.patch | 102 ++++++++++++++++++ src/libutil-tests/serialise.cc | 2 +- 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 7f6a94346d..480c4079e9 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -198,6 +198,9 @@ scope: { "--with-url" "--with-thread" ]; + patches = [ + ./patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch + ]; enableIcu = false; inherit stdenv; }).overrideAttrs diff --git a/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch b/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch new file mode 100644 index 0000000000..7ec13724c4 --- /dev/null +++ b/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch @@ -0,0 +1,102 @@ +From 5883212311535a0046031d74d1568ae173c1e35b Mon Sep 17 00:00:00 2001 +From: Sergei Zimmerman +Date: Tue, 21 Jul 2026 21:15:51 +0000 +Subject: [PATCH] Fix uncaught_exceptions() not accounting for forced_unwind + +Unwound fibers would see std::uncaught_exceptions() == 0, while a +forced_unwind exception is in "flight". This goes against the contract +of std::uncaught_exceptions() that scope guards rely upon. Failing +to report the correct number of uncaught exceptions (especially +misreporting zero) will lead to scope guards to misbehave badly and skip +running cleanup code which branches on whether the destructor is called +during stack unwinding or not. + +This is because the "throw" would happen before the destructor is run on +the fiber stack being switched to, but the increment would be clobbered +by the destructor of manage_exception_state. + +I'm not sure what the contract of run ontop_fcontext is wrt to whether +the the caller provided function can throw or not, but in my best +understanding the forced_unwind mechanism is mostly internal and so is +throwing from ontop_fcontext in the switched-to fiber. Thus, I've kept +the catch block scoped to detail::forced_unwind. +--- + include/boost/context/fiber_fcontext.hpp | 37 +++++++++++++++++------- + test/test_fiber.cpp | 24 +++++++++++++++ + 2 files changed, 51 insertions(+), 10 deletions(-) + +diff --git a/include/boost/context/fiber_fcontext.hpp b/include/boost/context/fiber_fcontext.hpp +index 543ba6c..38476c9 100644 +--- a/boost/context/fiber_fcontext.hpp ++++ b/boost/context/fiber_fcontext.hpp +@@ -70,7 +70,9 @@ namespace context { + namespace detail { + + // manage_exception_state is a dummy struct unless we have specific support +-struct manage_exception_state {}; ++struct manage_exception_state { ++ void from_forced_unwind() noexcept {} ++}; + + } // namespace detail + } // namespace context +@@ -90,6 +92,11 @@ public: + manage_exception_state() { + exception_state_ = *__cxa_get_globals(); + } ++ // Hack to account for the forced_unwind exception thrown in fiber_unwind ++ // that's run ontop before the destructor. ++ void from_forced_unwind() noexcept { ++ exception_state_.uncaughtExceptions += 1; ++ } + ~manage_exception_state() { + *__cxa_get_globals() = exception_state_; + } +@@ -376,13 +383,18 @@ public: + BOOST_ASSERT( nullptr != fctx_); + detail::manage_exception_state exstate; + boost::ignore_unused(exstate); +- return { detail::jump_fcontext( ++ try { ++ return { detail::jump_fcontext( + #if defined(BOOST_NO_CXX14_STD_EXCHANGE) +- detail::exchange( fctx_, nullptr), ++ detail::exchange( fctx_, nullptr), + #else +- std::exchange( fctx_, nullptr), ++ std::exchange( fctx_, nullptr), + #endif +- nullptr).fctx }; ++ nullptr).fctx }; ++ } catch ( detail::forced_unwind const& ) { ++ exstate.from_forced_unwind(); ++ throw; ++ } + } + + template< typename Fn > +@@ -391,14 +403,19 @@ public: + detail::manage_exception_state exstate; + boost::ignore_unused(exstate); + auto p = std::forward< Fn >( fn); +- return { detail::ontop_fcontext( ++ try { ++ return { detail::ontop_fcontext( + #if defined(BOOST_NO_CXX14_STD_EXCHANGE) +- detail::exchange( fctx_, nullptr), ++ detail::exchange( fctx_, nullptr), + #else +- std::exchange( fctx_, nullptr), ++ std::exchange( fctx_, nullptr), + #endif +- & p, +- detail::fiber_ontop< fiber, decltype(p) >).fctx }; ++ & p, ++ detail::fiber_ontop< fiber, decltype(p) >).fctx }; ++ } catch ( detail::forced_unwind const& ) { ++ exstate.from_forced_unwind(); ++ throw; ++ } + } + + explicit operator bool() const noexcept { diff --git a/src/libutil-tests/serialise.cc b/src/libutil-tests/serialise.cc index 24ef6d2c60..8af591a5e6 100644 --- a/src/libutil-tests/serialise.cc +++ b/src/libutil-tests/serialise.cc @@ -65,7 +65,7 @@ TEST(sourceToSink, forcedUnwindUcaughtExceptions) // Without the fix std::uncaught_exceptions() *misreports* 0 while there's stack unwinding in progress. // The issue only surfaces with libstdc++ and fcontext when boost.context // uses fiber-specific exception states and messes with __cxa_get_globals(). - ASSERT_EQ(uncaughtExceptions, 0); + ASSERT_EQ(uncaughtExceptions, 1); } } // namespace nix From 8030508e60f94329bf6af4a594da00de94b49c7f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 23 Jul 2026 00:04:26 +0300 Subject: [PATCH 587/927] libutil-tests: Disable sourceToSink.forcedUnwindUcaughtExceptions under ASan (cherry picked from commit 54aa8880be35e7d169e8d84d3dc135893cc4ff12) --- src/libutil-tests/serialise.cc | 10 ++++++++++ src/libutil/meson.build | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/src/libutil-tests/serialise.cc b/src/libutil-tests/serialise.cc index 8af591a5e6..40424c6853 100644 --- a/src/libutil-tests/serialise.cc +++ b/src/libutil-tests/serialise.cc @@ -1,4 +1,5 @@ #include "nix/util/serialise.hh" +#include "nix/util/config.hh" #include #include @@ -38,6 +39,13 @@ TEST(readPadding, works) } } +// The following test catches the bug only under fcontext backend, +// but Boost.Coroutine2 stack switching when abandoning confuses the +// hell out of ASan. The workaround to getting ASan working isn't +// immediately useful because it works only with ucontext implementation. +// https://www.boost.org/doc/libs/1_89_0/libs/coroutine2/doc/html/coroutine2/stack/sanitizers.html +#if !NIX_ASAN_ENABLED + TEST(sourceToSink, forcedUnwindUcaughtExceptions) { int uncaughtExceptions = 42; @@ -68,4 +76,6 @@ TEST(sourceToSink, forcedUnwindUcaughtExceptions) ASSERT_EQ(uncaughtExceptions, 1); } +#endif + } // namespace nix diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 37cc9ff94d..bf3a1f3b08 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -44,6 +44,12 @@ configdata_pub.set( description : 'Whether nix has been built with UBSan enabled', ) +configdata_pub.set( + 'NIX_ASAN_ENABLED', + ('address' in get_option('b_sanitize')).to_int(), + description : 'Whether nix has been built with ASan enabled', +) + subdir('nix-meson-build-support/libatomic') if host_machine.system() == 'windows' From a80eb82de1e25ef5aee1a55505727bb73a696822 Mon Sep 17 00:00:00 2001 From: John Bargman Date: Thu, 16 Jul 2026 10:47:38 +0000 Subject: [PATCH 588/927] fix: replace LocalCommand in-place and fix stale master detection When useMaster=true, command SSHs through a live master socket do not run LocalCommand. But when the master dies (ControlPersist=no) and the command SSH falls back to a direct connection, LocalCommand fires and 'echo started' leaks into the nix protocol stream, causing: error: protocol mismatch, got 'started' This fix: 1. Replaces all -oLocalCommand=* args with -oLocalCommand=true on command SSHs when useMaster=true (OpenSSH uses first-match-wins, so appending would be silently ignored; prefix match handles NIX_SSHOPTS and extraSshArgs injection). 2. Skips the readLine('started') readiness check when useMaster=true, since LocalCommand is now a no-op. The command SSH either connects through the live master or fails loudly via stderr. 3. Fixes startMaster() stale state: checks master process liveness with waitpid(WNOHANG) before returning cached socket path. Uses Pid::release() (not operator=) to avoid kill()/wait() on the already-reaped child. Handles EINTR/ECHILD properly. Refs: DeterminateSystems/nix-src#441, NixOS/nix#14132 --- src/libstore/ssh.cc | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index c4dff18bba..b0ea165814 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -6,6 +6,11 @@ #include "nix/util/exec.hh" #include "nix/util/base-n.hh" +#ifndef _WIN32 +# include +# include +#endif + namespace nix { static std::string parsePublicHostKey(std::string_view host, std::string_view sshPublicHostKey) @@ -193,6 +198,16 @@ std::unique_ptr SSHMaster::startCommand(OsStrings && comm if (verbosity >= lvlChatty) args.push_back("-v"); args.splice(args.end(), std::move(extraSshArgs)); + // Override LocalCommand to no-op on command SSHs; master + // already consumed "started". On fallback, "echo started" + // would leak into the nix protocol stream. #441 + if (useMaster) { + for (auto & arg : args) { + if (arg.starts_with(OS_STR("-oLocalCommand="))) { + arg = OS_STR("-oLocalCommand=true"); + } + } + } args.push_back("--"); } @@ -208,9 +223,9 @@ std::unique_ptr SSHMaster::startCommand(OsStrings && comm in.readSide = INVALID_DESCRIPTOR; out.writeSide = INVALID_DESCRIPTOR; - // Wait for the SSH connection to be established, - // So that we don't overwrite the password prompt with our progress bar. - if (!fakeSSH && !(socketPath && isMasterRunning(*socketPath))) { + // Skip readLine when useMaster: SSH connects via master socket, or + // falls back to direct connection where LocalCommand is now a no-op. + if (!fakeSSH && !useMaster && !(socketPath && isMasterRunning(*socketPath))) { std::string reply; try { reply = readLine(out.readSide.get()); @@ -239,8 +254,21 @@ std::optional SSHMaster::startMaster() auto state(state_.lock()); - if (state->sshMaster != INVALID_DESCRIPTOR) - return state->socketPath; + if (state->sshMaster != INVALID_DESCRIPTOR) { + // Check if master is still alive before returning cached socket. + pid_t result = waitpid(state->sshMaster, nullptr, WNOHANG); + if (result == 0) + return state->socketPath; + if (result == -1) { + if (errno == EINTR) + return state->socketPath; // assume alive; worst case SSH falls back + if (errno != ECHILD) + warn("waitpid failed for SSH master %d: %s", + (pid_t) state->sshMaster, strerror(errno)); + } + // Master gone — release Pid to avoid kill()/wait() on already-reaped child. + state->sshMaster.release(); + } state->socketPath = tmpDir->path() / "ssh.sock"; From 40c62739648f9204c26c13579c455d68154318b3 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Tue, 4 Aug 2026 00:05:20 +0300 Subject: [PATCH 589/927] libstore: Actually check that hash part is followed by a dash in StorePath This was missing validation and meant that potentially any byte could be included after the dash (even NUL one - though it probably would have segfaulted earlier due to std::filesystem::path). The latter is also very much necessary to fix, but let's start here. (cherry picked from commit 089ac360af56c8812546de6b93af687ab723d8a6) --- src/libstore-tests/path.cc | 9 +++++++++ src/libstore/path.cc | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/libstore-tests/path.cc b/src/libstore-tests/path.cc index 65755b7f96..4e4f9befe4 100644 --- a/src/libstore-tests/path.cc +++ b/src/libstore-tests/path.cc @@ -139,6 +139,15 @@ RC_GTEST_FIXTURE_PROP(StorePathTest, prop_check_regex_eq_parse, ()) RC_ASSERT(parsed == std::regex_match(std::string{name}, nameRegex)); } +TEST_F(StorePathTest, mustBeDashAfterHashPart) +{ + std::string hashPart = "575s52sh487i0ylmbs9pvi606ljdszr0"; + for (char c : {'_', '\0'}) { + EXPECT_THROW(StorePath(hashPart + c + "name"), BadStorePath); + } + EXPECT_NO_THROW(StorePath(hashPart + "-" + "name")); +} + /* ---------------------------------------------------------------------------- * JSON * --------------------------------------------------------------------------*/ diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 1052043110..e2e53551b9 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -51,6 +51,8 @@ StorePath::StorePath(std::string_view _baseName) { if (baseName.size() < HashLen + 1) throw BadStorePath("'%s' is too short to be a valid store path", baseName); + if (baseName[HashLen] != '-') + throw BadStorePath("'%s' can't name a store path because the hash part is not followed by a '-'", baseName); for (auto c : hashPart()) if (c == 'e' || c == 'o' || c == 'u' || c == 't' || !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'))) throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c); From d9ef66984618b06c418730cff59210e2abcbaafa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Aug 2026 14:29:54 +0200 Subject: [PATCH 590/927] Executor: Use std::move_only_function for work items std::function requires its callable to be copy-constructible, so work items that need to root a Value had to wrap the move-only RootValue in a std::shared_ptr, costing a heap-allocated control block and atomic reference counting per work item. Since we build with C++23, we can use std::move_only_function instead, allowing work items to capture a RootValue directly. Note that FutureVector::spawn() can no longer brace-initialize the WorkItems vector, since std::initializer_list requires copyable elements. Assisted-by: Claude Fable 5 --- src/libexpr/include/nix/expr/parallel-eval.hh | 7 ++++--- src/libexpr/parallel-eval.cc | 4 +--- src/libexpr/value-to-json.cc | 5 ++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/libexpr/include/nix/expr/parallel-eval.hh b/src/libexpr/include/nix/expr/parallel-eval.hh index 6e771dc72d..12b336d33b 100644 --- a/src/libexpr/include/nix/expr/parallel-eval.hh +++ b/src/libexpr/include/nix/expr/parallel-eval.hh @@ -21,8 +21,7 @@ namespace nix { struct Executor { - // FIXME: support std::moveable_function. - using work_t = std::function; + using work_t = std::move_only_function; struct Item { @@ -84,7 +83,9 @@ struct FutureVector void spawn(uint8_t prioPrefix, Executor::work_t && work) { - spawn({{std::move(work), prioPrefix}}); + Executor::WorkItems items; + items.emplace_back(std::move(work), prioPrefix); + spawn(std::move(items)); } void finishAll(); diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index e0cdab93a7..1c89b9c4db 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -294,9 +294,7 @@ static void prim_parallel(EvalState & state, const PosIdx pos, Value ** args, Va Executor::WorkItems work; for (auto value : args[0]->listView()) if (!value->isFinished()) - state.addWork(work, 0, [value(std::make_shared(value)), &state, pos]() { - state.forceValue(***value, pos); - }); + state.addWork(work, 0, [value(RootValue(value)), &state, pos]() { state.forceValue(**value, pos); }); state.executor->spawn(std::move(work)); } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 33477e1bc8..134417e165 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -28,9 +28,8 @@ static void parallelForceDeep(EvalState & state, Value & v, PosIdx pos) if (v.attrs()->get(state.s.outPath)) return; for (auto & a : *v.attrs()) - state.addWork(work, 0, [value(std::make_shared(a.value)), pos(a.pos), &state]() { - parallelForceDeep(state, ***value, pos); - }); + state.addWork( + work, 0, [value(RootValue(a.value)), pos(a.pos), &state]() { parallelForceDeep(state, **value, pos); }); break; } From 539d5f97ead33b1111a0490c7bab24043bd1bf64 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Aug 2026 17:30:35 +0200 Subject: [PATCH 591/927] Add MoveOnlyFunction fallback for standard libraries without std::move_only_function libc++ (as of version 21) doesn't implement std::move_only_function yet, breaking the libcxxStdenv build. Add nix::MoveOnlyFunction, which aliases std::move_only_function when available (per the __cpp_lib_move_only_function feature macro) and otherwise provides a minimal type-erased wrapper that only requires the callable to be move-constructible. Assisted-by: Claude Fable 5 --- src/libexpr/include/nix/expr/parallel-eval.hh | 3 +- src/libutil/include/nix/util/meson.build | 1 + .../include/nix/util/move-only-function.hh | 83 +++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 src/libutil/include/nix/util/move-only-function.hh diff --git a/src/libexpr/include/nix/expr/parallel-eval.hh b/src/libexpr/include/nix/expr/parallel-eval.hh index 12b336d33b..4f611328f4 100644 --- a/src/libexpr/include/nix/expr/parallel-eval.hh +++ b/src/libexpr/include/nix/expr/parallel-eval.hh @@ -7,6 +7,7 @@ #include +#include "nix/util/move-only-function.hh" #include "nix/util/sync.hh" #include "nix/util/logging.hh" #include "nix/util/environment-variables.hh" @@ -21,7 +22,7 @@ namespace nix { struct Executor { - using work_t = std::move_only_function; + using work_t = MoveOnlyFunction; struct Item { diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 56f8298f10..3af780eb29 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -64,6 +64,7 @@ headers = [ config_pub_h ] + files( 'memo.hh', 'memory-source-accessor.hh', 'mounted-source-accessor.hh', + 'move-only-function.hh', 'muxable-pipe.hh', 'nar-accessor.hh', 'nar-cache.hh', diff --git a/src/libutil/include/nix/util/move-only-function.hh b/src/libutil/include/nix/util/move-only-function.hh new file mode 100644 index 0000000000..4798122078 --- /dev/null +++ b/src/libutil/include/nix/util/move-only-function.hh @@ -0,0 +1,83 @@ +#pragma once +///@file + +#include +#include +#include +#include + +namespace nix { + +#ifdef __cpp_lib_move_only_function + +template +using MoveOnlyFunction = std::move_only_function; + +#else + +/** + * Fallback implementation of `std::move_only_function` for standard + * libraries that don't provide it yet (e.g. libc++ as of version 21): + * a type-erased callable wrapper that, unlike `std::function`, only + * requires the callable to be move-constructible, so it can hold + * lambdas that capture move-only types. + */ +template +class MoveOnlyFunction; + +template +class MoveOnlyFunction +{ + struct Base + { + virtual Ret call(Args &&... args) = 0; + virtual ~Base() = default; + }; + + template + struct Impl final : Base + { + F f; + + template + requires std::is_same_v, F> + Impl(G && g) + : f(std::forward(g)) + { + } + + Ret call(Args &&... args) override + { + return f(std::forward(args)...); + } + }; + + std::unique_ptr impl; + +public: + MoveOnlyFunction() = default; + + template + requires(!std::is_same_v, MoveOnlyFunction> && std::is_invocable_r_v) + MoveOnlyFunction(F && f) + : impl(std::make_unique>>(std::forward(f))) + { + } + + MoveOnlyFunction(MoveOnlyFunction &&) noexcept = default; + MoveOnlyFunction & operator=(MoveOnlyFunction &&) noexcept = default; + + Ret operator()(Args... args) + { + return impl->call(std::forward(args)...); + } + + explicit operator bool() const + { + return (bool) impl; + } +}; + +#endif + +} // namespace nix From 25e8a171c8a64a15d3056cdacbbbad07d088f288 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Aug 2026 11:05:05 +0200 Subject: [PATCH 592/927] Redraw the progress bar on Ctrl-Z / resume When the user suspends Nix (Ctrl-Z) and resumes it, the progress bar was not redrawn until the next logging event, and the cursor was left hidden while stopped. Generalize the InterruptCallbacks mechanism into a per-signal-type callback registry (createSignalCallback()), keyed by a portable SignalType enum. The signal handler thread now also handles: * SIGTSTP: run Stop callbacks (ProgressBar unhides the cursor), then stop the process explicitly via SIGSTOP, since sigwait() consumed the signal and suppressed its default action. Not on FreeBSD, where SIGTSTP doubles as NIX_SIG_MULTI_INT. * SIGCONT: run Cont callbacks (ProgressBar forces a redraw and re-hides the cursor). Also trigger Winch callbacks on SIGWINCH, so the progress bar reflows immediately on terminal resize instead of waiting for the next event. Assisted-by: Claude Fable 5 --- src/libmain/progress-bar.cc | 26 +++- src/libmain/shared.cc | 7 ++ src/libutil/include/nix/util/signals.hh | 31 +++++ src/libutil/unix/signals.cc | 111 ++++++++++++------ .../windows/include/nix/util/signals-impl.hh | 5 + 5 files changed, 141 insertions(+), 39 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index baf29adcdd..48d44a3a4b 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -106,7 +106,7 @@ class ProgressBar final : public Logger bool printBuildLogs = false; bool isTTY; - std::unique_ptr interruptCallback; + std::unique_ptr interruptCallback, stopCallback, contCallback, winchCallback; void hideCursorIfNeeded() const { @@ -131,6 +131,30 @@ class ProgressBar final : public Logger { hideCursorIfNeeded(); state_.lock()->active = isTTY; + + /* On Ctrl-Z, unhide the cursor before the process is + stopped. */ + stopCallback = createSignalCallback(SignalType::Stop, [&]() { + auto state(state_.lock()); + if (state->active && !state->isPaused()) + unhideCursorIfNeeded(); + }); + + /* Redraw the progress bar when the process is resumed or the + terminal size changes. */ + auto redrawCallback = [&]() { + auto state(state_.lock()); + if (state->active && !state->isPaused()) { + /* Force a redraw, since the new output may be + identical to the one cached by redraw(). */ + invalidateRedrawCache(); + hideCursorIfNeeded(); + update(*state); + } + }; + contCallback = createSignalCallback(SignalType::Cont, redrawCallback); + winchCallback = createSignalCallback(SignalType::Winch, redrawCallback); + updateThread = std::thread([&]() { auto state(state_.lock()); auto nextWakeup = std::chrono::milliseconds::max(); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 3fae87761e..abf6e49e11 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -205,6 +205,13 @@ void initNix(bool loadConfig) if (sigaction(SIGWINCH, &act, 0)) throw SysError("handling SIGWINCH"); + /* Same for SIGCONT and SIGTSTP, which are also handled by + * signalHandlerThread. */ + if (sigaction(SIGCONT, &act, 0)) + throw SysError("handling SIGCONT"); + if (sigaction(SIGTSTP, &act, 0)) + throw SysError("handling SIGTSTP"); + /* Disable SA_RESTART for interrupts, so that system calls on this thread * error with EINTR like they do on Linux. * Most signals on BSD systems default to SA_RESTART on, but Nix diff --git a/src/libutil/include/nix/util/signals.hh b/src/libutil/include/nix/util/signals.hh index 5124ebacca..0cfa7f3754 100644 --- a/src/libutil/include/nix/util/signals.hh +++ b/src/libutil/include/nix/util/signals.hh @@ -50,6 +50,37 @@ struct InterruptCallback virtual ~InterruptCallback(); }; +/** + * Portable identifiers for the signal events that callbacks can be + * registered for via `createSignalCallback()`. + */ +enum class SignalType { + /** + * SIGINT/SIGTERM/SIGHUP, or a programmatic `triggerInterrupt()`. + */ + Int, + /** + * SIGTSTP, called just before the process stops itself. + */ + Stop, + /** + * SIGCONT, i.e. the process was resumed after being stopped. + */ + Cont, + /** + * SIGWINCH, i.e. the terminal size changed. + */ + Winch, +}; + +/** + * Register a function that gets called from the signal handler thread + * (in a non-signal context) when the given signal type is received. + * + * @note Does nothing on Windows + */ +std::unique_ptr createSignalCallback(SignalType type, fun callback); + /** * Register a function that gets called on SIGINT (in a non-signal * context). diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index 8605647f1e..810ad5517c 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -29,11 +29,11 @@ void unix::_interrupted() ////////////////////////////////////////////////////////////////////// -/* We keep track of interrupt callbacks using integer tokens, so we can iterate +/* We keep track of signal callbacks using integer tokens, so we can iterate safely without having to lock the data structure while executing arbitrary functions. */ -struct InterruptCallbacks +struct SignalCallbacks { typedef int64_t Token; @@ -41,21 +41,47 @@ struct InterruptCallbacks handler because of an erroneous double delete. */ Token nextToken = 0; - /* Used as a list, see InterruptCallbacks comment. */ - std::map> callbacks; + /* Each per-signal map is used as a list, see SignalCallbacks comment. */ + std::map>> callbacks; }; InterruptCallback::~InterruptCallback() {} /* Required to avoid static initialization order fiasco. This allows global objects to safely register callbacks. */ -static Sync & getInterruptCallbacks() +static Sync & getSignalCallbacks() { /* Intentionally leak, according to the Construct On First Use Idiom. An alternative is to use the Nifty Counter Idiom, but - InterruptCallbacks' destructor is not very important. */ - static Sync * _interruptCallbacks = new Sync(); - return *_interruptCallbacks; + SignalCallbacks' destructor is not very important. */ + static Sync * _signalCallbacks = new Sync(); + return *_signalCallbacks; +} + +static void triggerSignalCallbacks(SignalType type) +{ + SignalCallbacks::Token i = 0; + while (true) { + std::function callback; + { + auto signalCallbacks(getSignalCallbacks().lock()); + auto it = signalCallbacks->callbacks.find(type); + if (it == signalCallbacks->callbacks.end()) + break; + auto lb = it->second.lower_bound(i); + if (lb == it->second.end()) + break; + + callback = lb->second; + i = lb->first + 1; + } + + try { + callback(); + } catch (...) { + ignoreExceptionInDestructor(); + } + } } static void signalHandlerThread(sigset_t set) @@ -70,7 +96,23 @@ static void signalHandlerThread(sigset_t set) else if (signal == SIGWINCH) { updateWindowSize(); + triggerSignalCallbacks(SignalType::Winch); } + + else if (signal == SIGCONT) + triggerSignalCallbacks(SignalType::Cont); + +#ifndef __FreeBSD__ + else if (signal == SIGTSTP) { + triggerSignalCallbacks(SignalType::Stop); + /* Since we consumed SIGTSTP via sigwait(), its default + action (stopping the process) no longer happens, so stop + ourselves explicitly. SIGSTOP cannot be blocked. On + resumption, SIGCONT is delivered via sigwait() as + usual. */ + kill(getpid(), SIGSTOP); + } +#endif } } @@ -78,27 +120,7 @@ void unix::triggerInterrupt() { _isInterrupted = true; - { - InterruptCallbacks::Token i = 0; - while (true) { - std::function callback; - { - auto interruptCallbacks(getInterruptCallbacks().lock()); - auto lb = interruptCallbacks->callbacks.lower_bound(i); - if (lb == interruptCallbacks->callbacks.end()) - break; - - callback = lb->second; - i = lb->first + 1; - } - - try { - callback(); - } catch (...) { - ignoreExceptionInDestructor(); - } - } - } + triggerSignalCallbacks(SignalType::Int); } static sigset_t savedSignalMask; @@ -125,6 +147,12 @@ void unix::startSignalHandlerThread() sigaddset(&set, SIGHUP); sigaddset(&set, SIGPIPE); sigaddset(&set, SIGWINCH); + sigaddset(&set, SIGCONT); +#ifndef __FreeBSD__ + /* Not on FreeBSD, where SIGTSTP doubles as NIX_SIG_MULTI_INT and is + delivered to specific threads using pthread_kill(). */ + sigaddset(&set, SIGTSTP); +#endif if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) throw SysError("blocking signals"); @@ -156,10 +184,12 @@ namespace { /* RAII helper to automatically deregister a callback. */ struct InterruptCallbackImpl : InterruptCallback { - InterruptCallbacks::Token token; + SignalType type; + SignalCallbacks::Token token; - InterruptCallbackImpl(InterruptCallbacks::Token token) - : token(token) + InterruptCallbackImpl(SignalType type, SignalCallbacks::Token token) + : type(type) + , token(token) { } @@ -170,19 +200,24 @@ struct InterruptCallbackImpl : InterruptCallback ~InterruptCallbackImpl() override { - auto interruptCallbacks(getInterruptCallbacks().lock()); - interruptCallbacks->callbacks.erase(token); + auto signalCallbacks(getSignalCallbacks().lock()); + signalCallbacks->callbacks[type].erase(token); } }; } // namespace +std::unique_ptr createSignalCallback(SignalType type, fun callback) +{ + auto signalCallbacks(getSignalCallbacks().lock()); + auto token = signalCallbacks->nextToken++; + signalCallbacks->callbacks[type].emplace(token, callback); + return std::make_unique(type, token); +} + std::unique_ptr createInterruptCallback(fun callback) { - auto interruptCallbacks(getInterruptCallbacks().lock()); - auto token = interruptCallbacks->nextToken++; - interruptCallbacks->callbacks.emplace(token, callback); - return std::make_unique(token); + return createSignalCallback(SignalType::Int, std::move(callback)); } } // namespace nix diff --git a/src/libutil/windows/include/nix/util/signals-impl.hh b/src/libutil/windows/include/nix/util/signals-impl.hh index 379397c72c..db899b2921 100644 --- a/src/libutil/windows/include/nix/util/signals-impl.hh +++ b/src/libutil/windows/include/nix/util/signals-impl.hh @@ -44,6 +44,11 @@ struct ReceiveInterrupts ~ReceiveInterrupts() {} }; +inline std::unique_ptr createSignalCallback(SignalType type, fun callback) +{ + return {}; +} + inline std::unique_ptr createInterruptCallback(fun callback) { return {}; From e196a673d0d3d435da8fe45b35e2dddf714f9b88 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Aug 2026 12:25:38 +0200 Subject: [PATCH 593/927] Diagnose randomly failing build.sh test These fail randomly in GHA with messages like builder-fast-fail.sh: line 3: /cancelled-builds-fifo/fifo: No such file or directory that suggest that the sandbox was not actually set up properly. This could happen if the clone() call in mountAndPidNamespacesSupported() fails under load. So let's ensure an explicit error message in that case. --- tests/functional/build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/functional/build.sh b/tests/functional/build.sh index 64aa47487b..e470e6d12c 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -218,7 +218,8 @@ if isDaemonNewer "2.34pre" && canUseSandbox; then if ! isTestOnNixOS; then sandboxPathsArg=(--option sandbox-paths "/nix/store") fi - out="$(nix flake check ./cancelled-builds --impure -L -j2 \ + cat /proc/sys/user/max_user_namespaces /proc/sys/user/max_pid_namespaces >&2 || true + out="$(nix flake check ./cancelled-builds --no-sandbox-fallback --impure -L -j2 \ --option sandbox true \ "${sandboxPathsArg[@]}" \ --option sandbox-build-dir /build-tmp \ @@ -244,7 +245,7 @@ if isDaemonNewer "2.34pre" && canUseSandbox; then sandboxPathsArg=(--option sandbox-paths "/nix/store") fi system=$(nix eval --raw --impure --expr builtins.currentSystem) - out="$(nix build --impure -L -j2 \ + out="$(nix build --no-sandbox-fallback --impure -L -j2 \ --option sandbox true \ "${sandboxPathsArg[@]}" \ --option sandbox-build-dir /build-tmp \ From 08b40b50b605048497d8db5a4334fb9cffccaabd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Aug 2026 14:24:58 +0200 Subject: [PATCH 594/927] build.sh: Require unprivileged user namespace support --- tests/functional/build.sh | 3 +-- tests/functional/common/functions.sh | 8 +++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/functional/build.sh b/tests/functional/build.sh index e470e6d12c..0dd3d289cd 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -205,7 +205,7 @@ fi # Only fast-fail should be reported as a failure. # Uses fifo for synchronization to ensure deterministic behavior. # Requires -j2 so slow and fast-fail run concurrently (fifo deadlocks if serialized). -if isDaemonNewer "2.34pre" && canUseSandbox; then +if isDaemonNewer "2.34pre" && canUseSandbox && unprivilegedUserNamespacesSupported; then fifoDir="$TEST_ROOT/cancelled-builds-fifo" mkdir -p "$fifoDir" mkfifo "$fifoDir/fifo" @@ -218,7 +218,6 @@ if isDaemonNewer "2.34pre" && canUseSandbox; then if ! isTestOnNixOS; then sandboxPathsArg=(--option sandbox-paths "/nix/store") fi - cat /proc/sys/user/max_user_namespaces /proc/sys/user/max_pid_namespaces >&2 || true out="$(nix flake check ./cancelled-builds --no-sandbox-fallback --impure -L -j2 \ --option sandbox true \ "${sandboxPathsArg[@]}" \ diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index e13c753ad0..a180730d39 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -353,8 +353,14 @@ count() { trap onError ERR +# Note: with newer AppArmor stacks, this restriction doesn't prevent *creating* +# a user namespace, but denies mounting inside it, which Nix's sandbox needs. +unprivilegedUserNamespacesSupported() { + ! { [[ -f /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]] && [[ $(< /proc/sys/kernel/apparmor_restrict_unprivileged_userns) -eq 1 ]]; } +} + requiresUnprivilegedUserNamespaces() { - if [[ -f /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]] && [[ $(< /proc/sys/kernel/apparmor_restrict_unprivileged_userns) -eq 1 ]]; then + if ! unprivilegedUserNamespacesSupported; then skipTest "Unprivileged user namespaces are disabled. Run 'sudo sysctl -w /proc/sys/kernel/apparmor_restrict_unprivileged_userns=0' to allow, and run these tests." fi } From 0a283e2242ae20bf22e3cde0d935d6b149bec8ea Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Aug 2026 15:57:06 +0200 Subject: [PATCH 595/927] Fix formatting --- src/libstore/ssh.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index b0ea165814..b77c40558f 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -263,8 +263,7 @@ std::optional SSHMaster::startMaster() if (errno == EINTR) return state->socketPath; // assume alive; worst case SSH falls back if (errno != ECHILD) - warn("waitpid failed for SSH master %d: %s", - (pid_t) state->sshMaster, strerror(errno)); + warn("waitpid failed for SSH master %d: %s", (pid_t) state->sshMaster, strerror(errno)); } // Master gone — release Pid to avoid kill()/wait() on already-reaped child. state->sshMaster.release(); From 6181935644b04ef3d376903b2c31041e3772e00a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Aug 2026 16:11:03 +0200 Subject: [PATCH 596/927] Update the window size on SIGCONT The kernel only sends SIGWINCH to the terminal's foreground process group, so if the terminal is resized while we're stopped, we never hear about it. Re-query the window size on resume, like less(1) does. Assisted-by: Claude Fable 5 --- src/libutil/unix/signals.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index 810ad5517c..9611bdcf36 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -99,8 +99,14 @@ static void signalHandlerThread(sigset_t set) triggerSignalCallbacks(SignalType::Winch); } - else if (signal == SIGCONT) + else if (signal == SIGCONT) { + /* The terminal may have been resized while we were + stopped, and we won't have received SIGWINCH for it, + since the kernel only sends it to the foreground process + group. */ + updateWindowSize(); triggerSignalCallbacks(SignalType::Cont); + } #ifndef __FreeBSD__ else if (signal == SIGTSTP) { From 0d2935e32715db81d881378388bc38bf8939a5d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Aug 2026 17:22:28 +0200 Subject: [PATCH 597/927] Move non-blocking child liveness check into Pid::isAlive() Refactor the inline waitpid(WNOHANG) logic in SSHMaster::startMaster() into a reusable Pid::isAlive() method (similar to Pid::wait()). If the child has exited, it is reaped and the Pid is reset so the destructor won't kill()/wait() an already-dead process. Assisted-by: Claude Fable 5 --- src/libstore/ssh.cc | 22 +++------------------- src/libutil/include/nix/util/processes.hh | 7 +++++++ src/libutil/unix/processes.cc | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index b77c40558f..fd6f3b3243 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -6,11 +6,6 @@ #include "nix/util/exec.hh" #include "nix/util/base-n.hh" -#ifndef _WIN32 -# include -# include -#endif - namespace nix { static std::string parsePublicHostKey(std::string_view host, std::string_view sshPublicHostKey) @@ -254,20 +249,9 @@ std::optional SSHMaster::startMaster() auto state(state_.lock()); - if (state->sshMaster != INVALID_DESCRIPTOR) { - // Check if master is still alive before returning cached socket. - pid_t result = waitpid(state->sshMaster, nullptr, WNOHANG); - if (result == 0) - return state->socketPath; - if (result == -1) { - if (errno == EINTR) - return state->socketPath; // assume alive; worst case SSH falls back - if (errno != ECHILD) - warn("waitpid failed for SSH master %d: %s", (pid_t) state->sshMaster, strerror(errno)); - } - // Master gone — release Pid to avoid kill()/wait() on already-reaped child. - state->sshMaster.release(); - } + // Check if the master is still alive before returning the cached socket. + if (state->sshMaster != INVALID_DESCRIPTOR && state->sshMaster.isAlive()) + return state->socketPath; state->socketPath = tmpDir->path() / "ssh.sock"; diff --git a/src/libutil/include/nix/util/processes.hh b/src/libutil/include/nix/util/processes.hh index 64053fab49..3927807b44 100644 --- a/src/libutil/include/nix/util/processes.hh +++ b/src/libutil/include/nix/util/processes.hh @@ -61,6 +61,13 @@ public: // TODO: Implement for Windows #ifndef _WIN32 + /** + * Check whether the child process is still running, without + * blocking. If it has exited (or was reaped elsewhere), the child + * is reaped if necessary and this object is reset so that the + * destructor won't kill()/wait() an already-dead process. + */ + bool isAlive(); void setSeparatePG(bool separatePG); void setKillSignal(int signal); void setKillTimeout(std::chrono::milliseconds duration); diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 32d5953588..40f4785da7 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -130,6 +130,25 @@ int Pid::wait(bool allowInterrupts) } } +bool Pid::isAlive() +{ + assert(pid != -1); + pid_t res = waitpid(pid, nullptr, WNOHANG); + if (res == 0) + return true; + if (res == -1) { + if (errno == EINTR) + /* Assume it's still alive; the caller can check again later. */ + return true; + if (errno != ECHILD) + warn("waitpid failed for PID %d: %s", pid, strerror(errno)); + } + /* The process has exited and been reaped (or was already reaped + elsewhere). Reset so the destructor doesn't kill()/wait() it. */ + release(); + return false; +} + void Pid::setSeparatePG(bool separatePG) { this->separatePG = separatePG; From 5e294925aa66baa952d46035bac34ae4b181a101 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:26:03 +0000 Subject: [PATCH 598/927] Prepare release v3.22.0 From 3bc9027ac7f96cf16487266235b1b13196b13a9a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:26:06 +0000 Subject: [PATCH 599/927] packaging/secure-packages/flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'path:../..' → 'path:../..' • Updated input 'nix/nixpkgs': 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1006668%2Brev-e87516da91e53b694a46eda0de22b782eb008d82/019fb45e-7a22-75f0-a9f9-a895378372a0/source.tar.gz' (2026-07-30) → 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1011423%2Brev-11bcae4108b62d9423d6eebc4b2bde45286d9fd2/019fd38f-01a2-7781-872d-a64af7252926/source.tar.gz' (2026-08-05) --- packaging/secure-packages/flake.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index 651372b846..77288f9b41 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -77,12 +77,12 @@ }, "nixpkgs": { "locked": { - "lastModified": 1785435372, - "narHash": "sha256-veAo2hsRG1Off47F+1HHgwarJVo/FjXUFv3A40ZUViY=", - "rev": "e87516da91e53b694a46eda0de22b782eb008d82", - "revCount": 1006668, + "lastModified": 1785959040, + "narHash": "sha256-XqYoFGaP8eGRiRRzAtfHFRUMDrN4sPdXXsdmdLKEDvc=", + "rev": "11bcae4108b62d9423d6eebc4b2bde45286d9fd2", + "revCount": 1011423, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1006668%2Brev-e87516da91e53b694a46eda0de22b782eb008d82/019fb45e-7a22-75f0-a9f9-a895378372a0/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1011423%2Brev-11bcae4108b62d9423d6eebc4b2bde45286d9fd2/019fd38f-01a2-7781-872d-a64af7252926/source.tar.gz" }, "original": { "type": "tarball", From c21985d928aa4dc594a98268c58068f52dae37c5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:26:09 +0000 Subject: [PATCH 600/927] Set .version-determinate to 3.22.0 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index ac3a4bd51e..a7e7070f80 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.21.9 +3.22.0 From 1547b0b277f7cd25e6b329b914c37d752e1fbc19 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:26:14 +0000 Subject: [PATCH 601/927] Generate release notes for 3.22.0 --- doc/manual/source/SUMMARY.md.in | 1 + .../release-notes-determinate/changes.md | 21 ++++++++++++++++++- .../release-notes-determinate/v3.22.0.md | 17 +++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/v3.22.0.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 5b16561a04..783dcb12b5 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -154,6 +154,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.22.0 (2026-08-06)](release-notes-determinate/v3.22.0.md) - [Release 3.21.9 (2026-07-30)](release-notes-determinate/v3.21.9.md) - [Release 3.21.8 (2026-07-20)](release-notes-determinate/v3.21.8.md) - [Release 3.21.7 (2026-07-15)](release-notes-determinate/v3.21.7.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index f557e8f7fc..97eea8a024 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.34 and Determinate Nix 3.21.9. +This section lists the differences between upstream Nix 2.35 and Determinate Nix 3.22.0. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -224,3 +224,22 @@ This section lists the differences between upstream Nix 2.34 and Determinate Nix + + + +* Sync with upstream 2.35.1 by @edolstra in [DeterminateSystems/nix-src#530](https://github.com/DeterminateSystems/nix-src/pull/530) + +* LocalStore: Reduce number of SQLite calls by @edolstra in [DeterminateSystems/nix-src#574](https://github.com/DeterminateSystems/nix-src/pull/574) + +* Fixes from upstream 2.35-maintenance by @edolstra in [DeterminateSystems/nix-src#584](https://github.com/DeterminateSystems/nix-src/pull/584) + +* Use std::move_only_function for parallel evel work items by @edolstra in [DeterminateSystems/nix-src#585](https://github.com/DeterminateSystems/nix-src/pull/585) + +* Fix randomly failing build.sh test by @edolstra in [DeterminateSystems/nix-src#587](https://github.com/DeterminateSystems/nix-src/pull/587) + +* Progress bar signal handling improvements by @edolstra in [DeterminateSystems/nix-src#586](https://github.com/DeterminateSystems/nix-src/pull/586) + +* Prevent LocalCommand 'started' leak on stale SSH master socket by @DarthPJB in [DeterminateSystems/nix-src#569](https://github.com/DeterminateSystems/nix-src/pull/569) + + +* @DarthPJB made their first contribution in [DeterminateSystems/nix-src#569](https://github.com/DeterminateSystems/nix-src/pull/569) diff --git a/doc/manual/source/release-notes-determinate/v3.22.0.md b/doc/manual/source/release-notes-determinate/v3.22.0.md new file mode 100644 index 0000000000..ec814286cf --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.22.0.md @@ -0,0 +1,17 @@ +# Release 3.22.0 (2026-08-06) + +* Based on [upstream Nix 2.35.1](../release-notes/rl-2.35.md). + +## What's Changed +* Sync with upstream 2.35.1 by @edolstra in [DeterminateSystems/nix-src#530](https://github.com/DeterminateSystems/nix-src/pull/530) +* LocalStore: Reduce number of SQLite calls by @edolstra in [DeterminateSystems/nix-src#574](https://github.com/DeterminateSystems/nix-src/pull/574) +* Fixes from upstream 2.35-maintenance by @edolstra in [DeterminateSystems/nix-src#584](https://github.com/DeterminateSystems/nix-src/pull/584) +* Use std::move_only_function for parallel evel work items by @edolstra in [DeterminateSystems/nix-src#585](https://github.com/DeterminateSystems/nix-src/pull/585) +* Fix randomly failing build.sh test by @edolstra in [DeterminateSystems/nix-src#587](https://github.com/DeterminateSystems/nix-src/pull/587) +* Progress bar signal handling improvements by @edolstra in [DeterminateSystems/nix-src#586](https://github.com/DeterminateSystems/nix-src/pull/586) +* Prevent LocalCommand 'started' leak on stale SSH master socket by @DarthPJB in [DeterminateSystems/nix-src#569](https://github.com/DeterminateSystems/nix-src/pull/569) + +## New Contributors +* @DarthPJB made their first contribution in [DeterminateSystems/nix-src#569](https://github.com/DeterminateSystems/nix-src/pull/569) + +**Full Changelog**: [v3.21.9...v3.22.0](https://github.com/DeterminateSystems/nix-src/compare/v3.21.9...v3.22.0) From 09cfbb56f727e06a1596f54cab4fa6473ffc966c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 6 Aug 2026 21:33:41 +0200 Subject: [PATCH 602/927] Tweak release notes --- .../release-notes-determinate/changes.md | 17 ----------------- .../release-notes-determinate/v3.22.0.md | 18 +++++++++++++----- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 97eea8a024..b157d9c603 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -226,20 +226,3 @@ This section lists the differences between upstream Nix 2.35 and Determinate Nix - -* Sync with upstream 2.35.1 by @edolstra in [DeterminateSystems/nix-src#530](https://github.com/DeterminateSystems/nix-src/pull/530) - -* LocalStore: Reduce number of SQLite calls by @edolstra in [DeterminateSystems/nix-src#574](https://github.com/DeterminateSystems/nix-src/pull/574) - -* Fixes from upstream 2.35-maintenance by @edolstra in [DeterminateSystems/nix-src#584](https://github.com/DeterminateSystems/nix-src/pull/584) - -* Use std::move_only_function for parallel evel work items by @edolstra in [DeterminateSystems/nix-src#585](https://github.com/DeterminateSystems/nix-src/pull/585) - -* Fix randomly failing build.sh test by @edolstra in [DeterminateSystems/nix-src#587](https://github.com/DeterminateSystems/nix-src/pull/587) - -* Progress bar signal handling improvements by @edolstra in [DeterminateSystems/nix-src#586](https://github.com/DeterminateSystems/nix-src/pull/586) - -* Prevent LocalCommand 'started' leak on stale SSH master socket by @DarthPJB in [DeterminateSystems/nix-src#569](https://github.com/DeterminateSystems/nix-src/pull/569) - - -* @DarthPJB made their first contribution in [DeterminateSystems/nix-src#569](https://github.com/DeterminateSystems/nix-src/pull/569) diff --git a/doc/manual/source/release-notes-determinate/v3.22.0.md b/doc/manual/source/release-notes-determinate/v3.22.0.md index ec814286cf..c7a0580bdd 100644 --- a/doc/manual/source/release-notes-determinate/v3.22.0.md +++ b/doc/manual/source/release-notes-determinate/v3.22.0.md @@ -2,14 +2,22 @@ * Based on [upstream Nix 2.35.1](../release-notes/rl-2.35.md). -## What's Changed -* Sync with upstream 2.35.1 by @edolstra in [DeterminateSystems/nix-src#530](https://github.com/DeterminateSystems/nix-src/pull/530) +## Upstream Nix 2.35 + +This release is based on upstream Nix 2.35.1, which includes numerous new features and performance improvements. See the [upstream release notes](../release-notes/rl-2.35.md) for more details. + +PRs: [DeterminateSystems/nix-src#530](https://github.com/DeterminateSystems/nix-src/pull/530), [DeterminateSystems/nix-src#584](https://github.com/DeterminateSystems/nix-src/pull/584) + +## Performance improvements + * LocalStore: Reduce number of SQLite calls by @edolstra in [DeterminateSystems/nix-src#574](https://github.com/DeterminateSystems/nix-src/pull/574) -* Fixes from upstream 2.35-maintenance by @edolstra in [DeterminateSystems/nix-src#584](https://github.com/DeterminateSystems/nix-src/pull/584) -* Use std::move_only_function for parallel evel work items by @edolstra in [DeterminateSystems/nix-src#585](https://github.com/DeterminateSystems/nix-src/pull/585) -* Fix randomly failing build.sh test by @edolstra in [DeterminateSystems/nix-src#587](https://github.com/DeterminateSystems/nix-src/pull/587) +* Use std::move_only_function for parallel eval work items by @edolstra in [DeterminateSystems/nix-src#585](https://github.com/DeterminateSystems/nix-src/pull/585) + +## Bug fixes + * Progress bar signal handling improvements by @edolstra in [DeterminateSystems/nix-src#586](https://github.com/DeterminateSystems/nix-src/pull/586) * Prevent LocalCommand 'started' leak on stale SSH master socket by @DarthPJB in [DeterminateSystems/nix-src#569](https://github.com/DeterminateSystems/nix-src/pull/569) +* Fix randomly failing build.sh test by @edolstra in [DeterminateSystems/nix-src#587](https://github.com/DeterminateSystems/nix-src/pull/587) ## New Contributors * @DarthPJB made their first contribution in [DeterminateSystems/nix-src#569](https://github.com/DeterminateSystems/nix-src/pull/569) From ae51952255b3f65ac716ea58a211242f05038d5f Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 12 Aug 2026 19:45:01 +0300 Subject: [PATCH 603/927] Revert "Drop old hacks for unachhored vtables and exception handling" Seems like LTO is still wreaking havoc on x86_64-darwin builds: https://hydra.nixos.org/build/341537794/step/1/log This reverts commit 3146ac72b0ebf57bb773752f177fe15282f9079b. (cherry picked from commit 36d5881905cab1853bcfcfbec71ee20548cca2d6) --- packaging/components.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/components.nix b/packaging/components.nix index 57bb08c4ac..54d49bc60b 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -139,6 +139,8 @@ let !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isCygwin) # build failure && !stdenv.hostPlatform.isStatic + # LTO breaks exception handling on x86-64-darwin. + && stdenv.system != "x86_64-darwin" ) '' case "$mesonBuildType" in From 131a0b5d97bc492b53e7f00ff1e6ad38a16d5ce7 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 13 Aug 2026 01:02:21 +0300 Subject: [PATCH 604/927] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index c0ac27fe01..c819688d6b 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.35.2 +2.35.3 From 53f642b3b0661158c1c6f2cfb50b8710b0881ec8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 13 Aug 2026 19:45:31 +0200 Subject: [PATCH 605/927] Don't let missing store paths abort 'nix store optimise' Previously, a valid path that was missing from disk caused the whole optimisation pass to fail: error: getting status of "/nix/store/0mqak3v82iwqz1v3d7v1r6nwrbw4pwij-gn-fd3d768.drv": No such file or directory Missing valid paths are a problem, but they should not prevent store optimisation from finishing, so warn and skip them instead. Assisted-by: Claude Fable 5 --- src/libstore/optimise-store.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index cf6a34addf..05d35c935e 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -106,7 +106,14 @@ void LocalStore::optimisePath_( { checkInterrupt(); - auto st = lstat(path); + /* Missing valid paths are a problem, but they should not prevent + store optimisation from finishing, so warn and skip them. */ + auto optSt = maybeLstat(path); + if (!optSt) { + warn("skipping missing path %s", PathFmt(path)); + return; + } + auto & st = *optSt; #ifdef __APPLE__ /* HFS/macOS has some undocumented security feature disabling hardlinking for From 1e4f3876570c7e5d5527597c2279b66ae42fe1f2 Mon Sep 17 00:00:00 2001 From: Jens Petersen Date: Sat, 15 Aug 2026 20:25:12 +0800 Subject: [PATCH 606/927] rl-2.35.md: correct meson option to disable mimalloc `-Dmimalloc=disabled` is not accepted but `nix:mimalloc` seems to be. (cherry picked from commit ea5d36edec1e73cc0a99ccdc94f07d4f3a3a4c5b) --- doc/manual/source/release-notes/rl-2.35.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/release-notes/rl-2.35.md b/doc/manual/source/release-notes/rl-2.35.md index a712b63856..6c2de4b3b9 100644 --- a/doc/manual/source/release-notes/rl-2.35.md +++ b/doc/manual/source/release-notes/rl-2.35.md @@ -61,7 +61,7 @@ The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) by default, replacing glibc's malloc for all non-GC allocations. This yields a **5–12% wall-clock improvement** on evaluation workloads, ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS configurations. - The allocator can be disabled at build time with `-Dmimalloc=disabled`. + The allocator can be disabled at build time with `-Dnix:mimalloc=disabled`. - The `revCount` attribute of the Git fetchers is now lazily computed and passed-through as-is when explicitly specified [#15772](https://github.com/NixOS/nix/pull/15772) [#14596](https://github.com/NixOS/nix/pull/14596) From 6e3fa7a53bc37e301bb4de994809727554fe2fc4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Aug 2026 14:06:35 +0200 Subject: [PATCH 607/927] Make ExprLet::eval a tail call This reduces stack usage for evaluating a NixOS configuration by ~5%. Also add gnu::musttail to a couple of sites that should be tail calls. --- src/libexpr/eval.cc | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0023c52ca7..29ea9908db 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1109,7 +1109,7 @@ Value * ExprVar::maybeThunk(EvalState & state, Env & env) state.nrAvoided++; return v; } - return Expr::maybeThunk(state, env); + [[gnu::musttail]] return Expr::maybeThunk(state, env); } Value * ExprString::maybeThunk(EvalState & state, Env & env) @@ -1422,11 +1422,12 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) env2.values[displ++] = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv)); } - auto dts = state.debugRepl - ? makeDebugTraceStacker(state, *this, env2, getPos(), "while evaluating a '%1%' expression", "let") - : nullptr; - - body->eval(state, env2, v); + if (state.debugRepl) { + auto dts = makeDebugTraceStacker(state, *this, env2, getPos(), "while evaluating a '%1%' expression", "let"); + return body->eval(state, env2, v); + } else { + [[gnu::musttail]] return body->eval(state, env2, v); + } } void ExprList::eval(EvalState & state, Env & env, Value & v) @@ -1871,7 +1872,7 @@ void EvalState::autoCallFunction(const Bindings & args, Value & fun, Value & res Value * v = allocValue(); callFunction(*found->value, fun, *v, pos); forceValue(*v, pos); - return autoCallFunction(args, *v, res); + [[gnu::musttail]] return autoCallFunction(args, *v, res); } } @@ -1910,7 +1911,7 @@ values, or passed explicitly with '--arg' or '--argstr'. See } } - callFunction(fun, allocValue()->mkAttrs(attrs), res, pos); + [[gnu::musttail]] return callFunction(fun, allocValue()->mkAttrs(attrs), res, pos); } void ExprWith::eval(EvalState & state, Env & env, Value & v) @@ -1919,13 +1920,14 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) env2.up = &env; env2.values[0] = attrs->maybeThunk(state, env); - body->eval(state, env2, v); + [[gnu::musttail]] return body->eval(state, env2, v); } void ExprIf::eval(EvalState & state, Env & env, Value & v) { // We cheat in the parser, and pass the position of the condition as the position of the if itself. - (state.evalBool(env, cond, pos, "while evaluating a branch condition") ? then : else_)->eval(state, env, v); + [[gnu::musttail]] return (state.evalBool(env, cond, pos, "while evaluating a branch condition") ? then : else_) + ->eval(state, env, v); } void ExprAssert::eval(EvalState & state, Env & env, Value & v) @@ -1950,7 +1952,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) state.error("assertion '%1%' failed", exprStr).atPos(pos).withFrame(env, *this).debugThrow(); } - body->eval(state, env, v); + [[gnu::musttail]] return body->eval(state, env, v); } void ExprOpNot::eval(EvalState & state, Env & env, Value & v) From 3589ffe67b4a09af17bed7f765a9cda40794a750 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Aug 2026 21:06:14 +0200 Subject: [PATCH 608/927] Make 'waiting for exclusive access' show up in the interactive CLI --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9fc56975c1..24f42e1162 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -295,7 +295,7 @@ LocalStore::LocalStore(ref config) auto acquireWriteLock = [&]() { if (!lockFile(globalLock.get(), ltWrite, false)) { - printInfo("waiting for exclusive access to the Nix store..."); + Activity act(*logger, lvlInfo, actUnknown, "waiting for exclusive access to the Nix store"); // We have acquired a shared lock; release it to prevent deadlocks. // This can happen if someone else is trying to promote their read // lock into a write lock. From 317810ab841993982e3868cfdd31348f898c862d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Aug 2026 21:15:25 +0200 Subject: [PATCH 609/927] Increase verbosity of schema migration messages --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 24f42e1162..e1734b6520 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -655,7 +655,7 @@ bool LocalStore::upgradeDBSchema(State & state, bool dryRun) if (dryRun) return; - debug("executing Nix database schema migration '%s'...", migrationName); + notice("executing Nix database schema migration '%s'...", migrationName); SQLiteTxn txn(state.db); state.db.exec(stmt + fmt(";\ninsert or ignore into SchemaMigrations values('%s')", migrationName)); From bb54cabbf4a451dc98bed994de6134fb6233a2fb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Aug 2026 21:25:00 +0200 Subject: [PATCH 610/927] Revert "libstore: only acquire big write lock when needed, hold read lock" This reverts commits 41e2fabc224d128d2cb5621b2f662c94c4a99093, 9934b5f44ae3d4c3d824da381aadee68e8d6f6b4. If multiple processes start the migration at the same time, and one of them holds the global lock open indefinitely (e.g. magic-nix-cache or a `nix repl`), then the other processes will hang indefinitly waiting for an exclusive lock that isn't actually needed anymore. --- src/libstore/include/nix/store/local-store.hh | 7 +- src/libstore/local-store.cc | 67 ++++--------------- 2 files changed, 15 insertions(+), 59 deletions(-) diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 67a0d57162..5b64c4e117 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -507,12 +507,7 @@ private: void openDB(State & state, bool create); - /** - * Perform or check if a database schema upgrade is needed. - * @param dryRun only check if an upgrade is needed. - * @return true if an upgrade is needed or was performed, false otherwise. - */ - bool upgradeDBSchema(State & state, bool dryRun); + void upgradeDBSchema(State & state); void makeStoreWritable(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e1734b6520..237df855d0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -293,17 +293,6 @@ LocalStore::LocalStore(ref config) : "database schema needs migrating, but this cannot be done in read-only mode"); } - auto acquireWriteLock = [&]() { - if (!lockFile(globalLock.get(), ltWrite, false)) { - Activity act(*logger, lvlInfo, actUnknown, "waiting for exclusive access to the Nix store"); - // We have acquired a shared lock; release it to prevent deadlocks. - // This can happen if someone else is trying to promote their read - // lock into a write lock. - lockFile(globalLock.get(), ltNone, false); - lockFile(globalLock.get(), ltWrite, true); - } - }; - if (curSchema > nixSchemaVersion) throw Error("current Nix store schema is version %1%, but I only support %2%", curSchema, nixSchemaVersion); @@ -326,7 +315,12 @@ LocalStore::LocalStore(ref config) "which is no longer supported. To convert to the new format,\n" "please upgrade Nix to version 1.11 first."); - acquireWriteLock(); + if (!lockFile(globalLock.get(), ltWrite, false)) { + Activity act(*logger, lvlInfo, actUnknown, "waiting for exclusive access to the Nix store"); + lockFile( + globalLock.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks + lockFile(globalLock.get(), ltWrite, true); + } /* Get the schema version again, because another process may have performed the upgrade already. */ @@ -359,21 +353,13 @@ LocalStore::LocalStore(ref config) writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, FsSync::Yes); - // Downgrade to a read lock and hold to prevent other processes from - // upgrading the schema while we're using the store lockFile(globalLock.get(), ltRead, true); } else openDB(*state, false); - if (!config->readOnly && upgradeDBSchema(*state, true)) { - acquireWriteLock(); - upgradeDBSchema(*state, false); - // Downgrade to a read lock and hold to prevent other processes from - // upgrading the schema while we're using the store - lockFile(globalLock.get(), ltRead, true); - } + upgradeDBSchema(*state); /* Prepare SQL statements. */ state->stmts->RegisterValidPath.create( @@ -614,24 +600,9 @@ void LocalStore::openDB(State & state, bool create) } } -bool LocalStore::upgradeDBSchema(State & state, bool dryRun) +void LocalStore::upgradeDBSchema(State & state) { - bool ret = false; - - { - SQLiteStmt queryHasSchemaMigrations; - queryHasSchemaMigrations.create( - state.db, "SELECT 1 FROM sqlite_master WHERE type='table' AND name='SchemaMigrations';"); - auto useQueryHasSchemaMigrations(queryHasSchemaMigrations.use()); - if (!useQueryHasSchemaMigrations.next()) { - if (dryRun) - return true; - else { - state.db.exec("create table SchemaMigrations (migration text primary key not null);"); - ret = true; - } - } - } + state.db.exec("create table if not exists SchemaMigrations (migration text primary key not null);"); StringSet schemaMigrations; @@ -643,16 +614,8 @@ bool LocalStore::upgradeDBSchema(State & state, bool dryRun) schemaMigrations.insert(useQuerySchemaMigrations.getStr(0)); } - auto needsMigration = [&](const std::string & migrationName) -> bool { - return !schemaMigrations.contains(migrationName); - }; - - auto maybeUpgrade = [&](const std::string & migrationName, const std::string & stmt) { - if (!needsMigration(migrationName)) - return; - - ret = true; - if (dryRun) + auto doUpgrade = [&](const std::string & migrationName, const std::string & stmt) { + if (schemaMigrations.contains(migrationName)) return; notice("executing Nix database schema migration '%s'...", migrationName); @@ -665,17 +628,15 @@ bool LocalStore::upgradeDBSchema(State & state, bool dryRun) }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) - maybeUpgrade( + doUpgrade( "20251017-ca-derivations", #include "ca-specific-schema.sql.gen.hh" ); - maybeUpgrade("20260309-drop-redundant-indexreferrer", "drop index if exists IndexReferrer"); + doUpgrade("20260309-drop-redundant-indexreferrer", "drop index if exists IndexReferrer"); if (experimentalFeatureSettings.isEnabled(Xp::Provenance)) - maybeUpgrade("20241024-provenance", "alter table ValidPaths add column provenance text"); - - return ret; + doUpgrade("20241024-provenance", "alter table ValidPaths add column provenance text"); } /* To improve purity, users may want to make the Nix store a read-only From 95db28ca01e403691e395d815f86a8d3441f28e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Aug 2026 22:07:26 +0200 Subject: [PATCH 611/927] Correctly handle SQLITE_BUSY errors when multiple processes do a schema migration We should never use SQLite::exec() inside a transaction, because it catches SQLiteBusy exceptions and retries the statement, which will just fail again. It's the entire transaction that must be retried. In addition, we now check inside the transaction whether the migration has already been done. Thus, on a retry, we will notice that another process has already done the migration. --- src/libstore/include/nix/store/sqlite.hh | 9 ++++++++ src/libstore/local-store.cc | 26 ++++++++++++++++-------- src/libstore/sqlite.cc | 11 ++++++---- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/libstore/include/nix/store/sqlite.hh b/src/libstore/include/nix/store/sqlite.hh index 294bb372d6..e416c44bd0 100644 --- a/src/libstore/include/nix/store/sqlite.hh +++ b/src/libstore/include/nix/store/sqlite.hh @@ -74,8 +74,17 @@ struct SQLite */ void isCache(); + /** + * Execute a statement. Retry if the database is busy. Do not call this inside a transaction; use execNoRetry() + * instead. + */ void exec(const std::string & stmt); + /** + * Execute a statement. + */ + void execNoRetry(const std::string & stmt); + uint64_t getLastInsertedRowId(); /** diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 237df855d0..b1f2d5fed8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -144,6 +144,7 @@ struct LocalStore::State::Stmts SQLiteStmt QueryRealisedOutput; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; + SQLiteStmt QuerySchemaMigration; }; LocalStore::LocalStore(ref config) @@ -334,21 +335,17 @@ LocalStore::LocalStore(ref config) if (curSchema < 8) { SQLiteTxn txn(state->db); - state->db.exec("alter table ValidPaths add column ultimate integer"); - state->db.exec("alter table ValidPaths add column sigs text"); + state->db.execNoRetry("alter table ValidPaths add column ultimate integer"); + state->db.execNoRetry("alter table ValidPaths add column sigs text"); txn.commit(); } if (curSchema < 9) { - SQLiteTxn txn(state->db); state->db.exec("drop table FailedPaths"); - txn.commit(); } if (curSchema < 10) { - SQLiteTxn txn(state->db); state->db.exec("alter table ValidPaths add column ca text"); - txn.commit(); } writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, FsSync::Yes); @@ -604,6 +601,8 @@ void LocalStore::upgradeDBSchema(State & state) { state.db.exec("create table if not exists SchemaMigrations (migration text primary key not null);"); + state.stmts->QuerySchemaMigration.create(state.db, "select 1 from SchemaMigrations where migration = ?;"); + StringSet schemaMigrations; { @@ -620,9 +619,18 @@ void LocalStore::upgradeDBSchema(State & state) notice("executing Nix database schema migration '%s'...", migrationName); - SQLiteTxn txn(state.db); - state.db.exec(stmt + fmt(";\ninsert or ignore into SchemaMigrations values('%s')", migrationName)); - txn.commit(); + retrySQLite([&]() { + SQLiteTxn txn(state.db); + + /* Another process may have executed this migration in the + meantime, so check for it again within the + transaction. */ + if (state.stmts->QuerySchemaMigration.use().apply(migrationName).next()) + return; + + state.db.execNoRetry(stmt + fmt(";\ninsert or ignore into SchemaMigrations values('%s')", migrationName)); + txn.commit(); + }); schemaMigrations.insert(migrationName); }; diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 5e62d96ebb..3a7cef27a6 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -133,10 +133,13 @@ void SQLite::isCache() void SQLite::exec(const std::string & stmt) { - retrySQLite([&]() { - if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK) - SQLiteError::throw_(db, "executing SQLite statement '%s'", stmt); - }); + retrySQLite([&]() { execNoRetry(stmt); }); +} + +void SQLite::execNoRetry(const std::string & stmt) +{ + if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK) + SQLiteError::throw_(db, "executing SQLite statement '%s'", stmt); } uint64_t SQLite::getLastInsertedRowId() From 76c15a2739fe2c5e5d9b7ba56b50f0305e66777a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Aug 2026 15:58:34 -0400 Subject: [PATCH 612/927] libstore: Merge, don't clobber, colliding `inputDrvs` hashes modulo Fix #16307 `hashDerivationModulo` built the intermediate `inputs2` map with `insert_or_assign`, so on such a collision the second entry silently dropped the first entry's output names. Merge the output-name sets instead. This changes computed store path of any derivation that hits the collision. But that's good, we're undoing the bug. Two *distinct* input derivations can share a hash modulo --- that is the whole point of the modulo, e.g. they differ only in the provenance of a fixed-output input. Usually this comes up *temporally*, e.g. between two different versions of Nixpkgs that have different curls used in `fetchurl` derivations, but it can also arise *spatially*. Assisted-by: Claude Code (Claude Opus 5) (cherry picked from commit b26204ba1ddae673985a3b05aa4b23d091cd0e21) --- src/libstore/derivations.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 0cf79e82f7..1dfb507ada 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -945,7 +945,8 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m [&](const DrvHashModulo::DeferredDrv &) { return true; }, // Regular non-CA derivation, replace derivation [&](const DrvHashModulo::DrvHash & drvHash) { - inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node); + inputs2[drvHash.to_string(HashFormat::Base16, false)].value.insert( + node.value.begin(), node.value.end()); return false; }, // CA derivation's output hashes @@ -955,11 +956,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m const auto h = get(outputHashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2.insert_or_assign( - h->to_string(HashFormat::Base16, false), - DerivedPathMap::ChildNode{ - .value = {"out"}, - }); + inputs2[h->to_string(HashFormat::Base16, false)].value.insert("out"); } return false; }, From bc3c50149a638f44842de430a19a07e58d556df5 Mon Sep 17 00:00:00 2001 From: Jens Petersen Date: Sat, 15 Aug 2026 20:25:12 +0800 Subject: [PATCH 613/927] rl-2.35.md: correct meson option to disable mimalloc `-Dmimalloc=disabled` is not accepted but `nix:mimalloc` seems to be. (cherry picked from commit ea5d36edec1e73cc0a99ccdc94f07d4f3a3a4c5b) (cherry picked from commit 1e4f3876570c7e5d5527597c2279b66ae42fe1f2) --- doc/manual/source/release-notes/rl-2.35.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/release-notes/rl-2.35.md b/doc/manual/source/release-notes/rl-2.35.md index a712b63856..6c2de4b3b9 100644 --- a/doc/manual/source/release-notes/rl-2.35.md +++ b/doc/manual/source/release-notes/rl-2.35.md @@ -61,7 +61,7 @@ The `nix` binary now links [mimalloc](https://github.com/microsoft/mimalloc) by default, replacing glibc's malloc for all non-GC allocations. This yields a **5–12% wall-clock improvement** on evaluation workloads, ranging from `nix-instantiate hello` to `nix-env -qa` and full NixOS configurations. - The allocator can be disabled at build time with `-Dmimalloc=disabled`. + The allocator can be disabled at build time with `-Dnix:mimalloc=disabled`. - The `revCount` attribute of the Git fetchers is now lazily computed and passed-through as-is when explicitly specified [#15772](https://github.com/NixOS/nix/pull/15772) [#14596](https://github.com/NixOS/nix/pull/14596) From c7ba515596dab8287be49a752b9da3c3fee2d14d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Aug 2026 15:58:34 -0400 Subject: [PATCH 614/927] libstore: Merge, don't clobber, colliding `inputDrvs` hashes modulo Fix #16307 `hashDerivationModulo` built the intermediate `inputs2` map with `insert_or_assign`, so on such a collision the second entry silently dropped the first entry's output names. Merge the output-name sets instead. This changes computed store path of any derivation that hits the collision. But that's good, we're undoing the bug. Two *distinct* input derivations can share a hash modulo --- that is the whole point of the modulo, e.g. they differ only in the provenance of a fixed-output input. Usually this comes up *temporally*, e.g. between two different versions of Nixpkgs that have different curls used in `fetchurl` derivations, but it can also arise *spatially*. Assisted-by: Claude Code (Claude Opus 5) (cherry picked from commit b26204ba1ddae673985a3b05aa4b23d091cd0e21) (cherry picked from commit 76c15a2739fe2c5e5d9b7ba56b50f0305e66777a) --- src/libstore/derivations.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b4f1f5a27f..0f469ab812 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -961,7 +961,8 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m [&](const DrvHashModulo::DeferredDrv &) { return true; }, // Regular non-CA derivation, replace derivation [&](const DrvHashModulo::DrvHash & drvHash) { - inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node); + inputs2[drvHash.to_string(HashFormat::Base16, false)].value.insert( + node.value.begin(), node.value.end()); return false; }, // CA derivation's output hashes @@ -971,11 +972,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m const auto h = get(outputHashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2.insert_or_assign( - h->to_string(HashFormat::Base16, false), - DerivedPathMap::ChildNode{ - .value = {"out"}, - }); + inputs2[h->to_string(HashFormat::Base16, false)].value.insert("out"); } return false; }, From acac0ed2d0f700cf88cf56af548e57a6601009b2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:53:15 +0000 Subject: [PATCH 615/927] Prepare release v3.22.1 From 03c4281aaba103bd8f48ecdebdcfb799b13b67ac Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:53:18 +0000 Subject: [PATCH 616/927] packaging/secure-packages/flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'path:../..' → 'path:../..' • Updated input 'nix/nixpkgs': 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1011423%2Brev-11bcae4108b62d9423d6eebc4b2bde45286d9fd2/019fd38f-01a2-7781-872d-a64af7252926/source.tar.gz' (2026-08-05) → 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1012116%2Brev-b5d2789f4c19edaa14cc30a9aed6b15605ced159/01a002f6-c976-7971-ae43-841e5c7df492/source.tar.gz' (2026-08-15) --- packaging/secure-packages/flake.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index 77288f9b41..296f47b686 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -77,12 +77,12 @@ }, "nixpkgs": { "locked": { - "lastModified": 1785959040, - "narHash": "sha256-XqYoFGaP8eGRiRRzAtfHFRUMDrN4sPdXXsdmdLKEDvc=", - "rev": "11bcae4108b62d9423d6eebc4b2bde45286d9fd2", - "revCount": 1011423, + "lastModified": 1786753867, + "narHash": "sha256-OJ+0LGs1qTMY0dURtdvA7t53yN8PNYqOgznXfMLh3/s=", + "rev": "b5d2789f4c19edaa14cc30a9aed6b15605ced159", + "revCount": 1012116, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1011423%2Brev-11bcae4108b62d9423d6eebc4b2bde45286d9fd2/019fd38f-01a2-7781-872d-a64af7252926/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1012116%2Brev-b5d2789f4c19edaa14cc30a9aed6b15605ced159/01a002f6-c976-7971-ae43-841e5c7df492/source.tar.gz" }, "original": { "type": "tarball", From 8603a817bd14da7ea2e759037f6a7cd741bb182d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:53:21 +0000 Subject: [PATCH 617/927] Set .version-determinate to 3.22.1 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index a7e7070f80..732c06105d 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.22.0 +3.22.1 From 91892737e68cb2d2ba9b0430958f7dfac61f447f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:53:26 +0000 Subject: [PATCH 618/927] Generate release notes for 3.22.1 --- doc/manual/source/SUMMARY.md.in | 1 + .../source/release-notes-determinate/changes.md | 14 +++++++++++++- .../source/release-notes-determinate/v3.22.1.md | 13 +++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/v3.22.1.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 783dcb12b5..37ba61d026 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -154,6 +154,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.22.1 (2026-08-18)](release-notes-determinate/v3.22.1.md) - [Release 3.22.0 (2026-08-06)](release-notes-determinate/v3.22.0.md) - [Release 3.21.9 (2026-07-30)](release-notes-determinate/v3.21.9.md) - [Release 3.21.8 (2026-07-20)](release-notes-determinate/v3.21.8.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index b157d9c603..4dfd18824f 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.35 and Determinate Nix 3.22.0. +This section lists the differences between upstream Nix 2.35 and Determinate Nix 3.22.1. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -226,3 +226,15 @@ This section lists the differences between upstream Nix 2.35 and Determinate Nix + + + +* Don't let missing store paths abort 'nix store optimise' by @edolstra in [DeterminateSystems/nix-src#590](https://github.com/DeterminateSystems/nix-src/pull/590) + +* Sync with upstream 2.35.2 by @edolstra in [DeterminateSystems/nix-src#592](https://github.com/DeterminateSystems/nix-src/pull/592) + +* Make ExprLet::eval a tail call by @edolstra in [DeterminateSystems/nix-src#593](https://github.com/DeterminateSystems/nix-src/pull/593) + +* Fix "waiting for exclusive access" hangs during schema migrations by @edolstra in [DeterminateSystems/nix-src#594](https://github.com/DeterminateSystems/nix-src/pull/594) + +* Upstream fixes by @edolstra in [DeterminateSystems/nix-src#595](https://github.com/DeterminateSystems/nix-src/pull/595) diff --git a/doc/manual/source/release-notes-determinate/v3.22.1.md b/doc/manual/source/release-notes-determinate/v3.22.1.md new file mode 100644 index 0000000000..effe5fd80c --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.22.1.md @@ -0,0 +1,13 @@ +# Release 3.22.1 (2026-08-18) + +* Based on [upstream Nix 2.35.2](../release-notes/rl-2.35.md). + +## What's Changed +* Don't let missing store paths abort 'nix store optimise' by @edolstra in [DeterminateSystems/nix-src#590](https://github.com/DeterminateSystems/nix-src/pull/590) +* Sync with upstream 2.35.2 by @edolstra in [DeterminateSystems/nix-src#592](https://github.com/DeterminateSystems/nix-src/pull/592) +* Make ExprLet::eval a tail call by @edolstra in [DeterminateSystems/nix-src#593](https://github.com/DeterminateSystems/nix-src/pull/593) +* Fix "waiting for exclusive access" hangs during schema migrations by @edolstra in [DeterminateSystems/nix-src#594](https://github.com/DeterminateSystems/nix-src/pull/594) +* Upstream fixes by @edolstra in [DeterminateSystems/nix-src#595](https://github.com/DeterminateSystems/nix-src/pull/595) + + +**Full Changelog**: [v3.22.0...v3.22.1](https://github.com/DeterminateSystems/nix-src/compare/v3.22.0...v3.22.1) From c0861e89441b0fe991720e87ebc455cde5d41d87 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Wed, 19 Aug 2026 09:12:37 -0400 Subject: [PATCH 619/927] Update release notes --- .../release-notes-determinate/changes.md | 10 +------- .../release-notes-determinate/v3.22.0.md | 25 ------------------- .../release-notes-determinate/v3.22.1.md | 24 ++++++++++++++---- 3 files changed, 20 insertions(+), 39 deletions(-) delete mode 100644 doc/manual/source/release-notes-determinate/v3.22.0.md diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 4dfd18824f..c77a995040 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -229,12 +229,4 @@ This section lists the differences between upstream Nix 2.35 and Determinate Nix -* Don't let missing store paths abort 'nix store optimise' by @edolstra in [DeterminateSystems/nix-src#590](https://github.com/DeterminateSystems/nix-src/pull/590) - -* Sync with upstream 2.35.2 by @edolstra in [DeterminateSystems/nix-src#592](https://github.com/DeterminateSystems/nix-src/pull/592) - -* Make ExprLet::eval a tail call by @edolstra in [DeterminateSystems/nix-src#593](https://github.com/DeterminateSystems/nix-src/pull/593) - -* Fix "waiting for exclusive access" hangs during schema migrations by @edolstra in [DeterminateSystems/nix-src#594](https://github.com/DeterminateSystems/nix-src/pull/594) - -* Upstream fixes by @edolstra in [DeterminateSystems/nix-src#595](https://github.com/DeterminateSystems/nix-src/pull/595) +* Fix "waiting for exclusive access" hangs during schema migrations [DeterminateSystems/nix-src#594](https://github.com/DeterminateSystems/nix-src/pull/594) diff --git a/doc/manual/source/release-notes-determinate/v3.22.0.md b/doc/manual/source/release-notes-determinate/v3.22.0.md deleted file mode 100644 index c7a0580bdd..0000000000 --- a/doc/manual/source/release-notes-determinate/v3.22.0.md +++ /dev/null @@ -1,25 +0,0 @@ -# Release 3.22.0 (2026-08-06) - -* Based on [upstream Nix 2.35.1](../release-notes/rl-2.35.md). - -## Upstream Nix 2.35 - -This release is based on upstream Nix 2.35.1, which includes numerous new features and performance improvements. See the [upstream release notes](../release-notes/rl-2.35.md) for more details. - -PRs: [DeterminateSystems/nix-src#530](https://github.com/DeterminateSystems/nix-src/pull/530), [DeterminateSystems/nix-src#584](https://github.com/DeterminateSystems/nix-src/pull/584) - -## Performance improvements - -* LocalStore: Reduce number of SQLite calls by @edolstra in [DeterminateSystems/nix-src#574](https://github.com/DeterminateSystems/nix-src/pull/574) -* Use std::move_only_function for parallel eval work items by @edolstra in [DeterminateSystems/nix-src#585](https://github.com/DeterminateSystems/nix-src/pull/585) - -## Bug fixes - -* Progress bar signal handling improvements by @edolstra in [DeterminateSystems/nix-src#586](https://github.com/DeterminateSystems/nix-src/pull/586) -* Prevent LocalCommand 'started' leak on stale SSH master socket by @DarthPJB in [DeterminateSystems/nix-src#569](https://github.com/DeterminateSystems/nix-src/pull/569) -* Fix randomly failing build.sh test by @edolstra in [DeterminateSystems/nix-src#587](https://github.com/DeterminateSystems/nix-src/pull/587) - -## New Contributors -* @DarthPJB made their first contribution in [DeterminateSystems/nix-src#569](https://github.com/DeterminateSystems/nix-src/pull/569) - -**Full Changelog**: [v3.21.9...v3.22.0](https://github.com/DeterminateSystems/nix-src/compare/v3.21.9...v3.22.0) diff --git a/doc/manual/source/release-notes-determinate/v3.22.1.md b/doc/manual/source/release-notes-determinate/v3.22.1.md index effe5fd80c..98c19440d6 100644 --- a/doc/manual/source/release-notes-determinate/v3.22.1.md +++ b/doc/manual/source/release-notes-determinate/v3.22.1.md @@ -2,12 +2,26 @@ * Based on [upstream Nix 2.35.2](../release-notes/rl-2.35.md). -## What's Changed +## Upstream Nix 2.35 + +This release is based on upstream Nix 2.35.2, which includes numerous new features and performance improvements. See the [upstream release notes](../release-notes/rl-2.35.md) for more details. + +PRs: [DeterminateSystems/nix-src#530](https://github.com/DeterminateSystems/nix-src/pull/530), [DeterminateSystems/nix-src#584](https://github.com/DeterminateSystems/nix-src/pull/584) + +## Performance improvements + +* LocalStore: Reduce number of SQLite calls by @edolstra in [DeterminateSystems/nix-src#574](https://github.com/DeterminateSystems/nix-src/pull/574) +* Use std::move_only_function for parallel eval work items by @edolstra in [DeterminateSystems/nix-src#585](https://github.com/DeterminateSystems/nix-src/pull/585) + +## Bug fixes + +* Progress bar signal handling improvements by @edolstra in [DeterminateSystems/nix-src#586](https://github.com/DeterminateSystems/nix-src/pull/586) +* Prevent LocalCommand 'started' leak on stale SSH master socket by @DarthPJB in [DeterminateSystems/nix-src#569](https://github.com/DeterminateSystems/nix-src/pull/569) +* Fix randomly failing build.sh test by @edolstra in [DeterminateSystems/nix-src#587](https://github.com/DeterminateSystems/nix-src/pull/587) * Don't let missing store paths abort 'nix store optimise' by @edolstra in [DeterminateSystems/nix-src#590](https://github.com/DeterminateSystems/nix-src/pull/590) -* Sync with upstream 2.35.2 by @edolstra in [DeterminateSystems/nix-src#592](https://github.com/DeterminateSystems/nix-src/pull/592) -* Make ExprLet::eval a tail call by @edolstra in [DeterminateSystems/nix-src#593](https://github.com/DeterminateSystems/nix-src/pull/593) * Fix "waiting for exclusive access" hangs during schema migrations by @edolstra in [DeterminateSystems/nix-src#594](https://github.com/DeterminateSystems/nix-src/pull/594) -* Upstream fixes by @edolstra in [DeterminateSystems/nix-src#595](https://github.com/DeterminateSystems/nix-src/pull/595) +## New Contributors +* @DarthPJB made their first contribution in [DeterminateSystems/nix-src#569](https://github.com/DeterminateSystems/nix-src/pull/569) -**Full Changelog**: [v3.22.0...v3.22.1](https://github.com/DeterminateSystems/nix-src/compare/v3.22.0...v3.22.1) +**Full Changelog**: [v3.21.9...v3.22.1](https://github.com/DeterminateSystems/nix-src/compare/v3.21.9...v3.22.1) From 4284b5768080bcaa884befbd55d519ed72d1a5c6 Mon Sep 17 00:00:00 2001 From: Jamie Magee Date: Sun, 16 Aug 2026 21:16:10 -0700 Subject: [PATCH 620/927] Validate padding after skipped NAR contents The skipped-content path treats file contents and padding as one block. It accepts non-zero padding that the draining path rejects. Skip only the declared contents, then validate padding on both paths. Assisted-by: GitHub Copilot CLI (GPT-5.6 Sol) --- src/libutil/archive.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 56172bf32b..9f4ef43913 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -1,10 +1,11 @@ +#include #include +#include #include #include // for strcasecmp #include "nix/util/archive.hh" -#include "nix/util/alignment.hh" #include "nix/util/config-global.hh" #include "nix/util/source-accessor.hh" #include "nix/util/source-path.hh" @@ -151,7 +152,7 @@ static void parseContents(CreateRegularFileSink & sink, Source & source) sink.preallocateContents(size); if (sink.skipContents) { - uint64_t left = alignUp(size, 8); + uint64_t left = size; /* Source::skip takes a size_t, which might be narrower on 32 bit systems, so be careful around truncations. */ while (left) { @@ -159,11 +160,10 @@ static void parseContents(CreateRegularFileSink & sink, Source & source) source.skip(toSkip); left -= toSkip; } - return; + } else { + source.drainInto(sink, size); } - source.drainInto(sink, size); - readPadding(size, source); } From deb3de4616ea6911d56ef738033a40d1c0f3723d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 20 Aug 2026 15:58:46 +0200 Subject: [PATCH 621/927] Remove fallback behaviour for RemoteStore::addTempRoots() The fallback makes `nix copy` very slow when copying to a older daemon that doesn't support the AddTempRoots worker protocol operation. Of course, this is less correct, but it's the behaviour we've had for years prior to 7beda55f66a4cef7780a46a93ca975313fcbbc74. This reverts commit 334f160af34bdae2fa9439a3c2ddfd44a93ab9f6. --- src/libstore/build/derivation-goal.cc | 5 +---- src/libstore/remote-store.cc | 8 +------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 55bf392390..72733516aa 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -62,12 +62,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) if (!drv->type().hasKnownOutputPaths()) experimentalFeatureSettings.require(Xp::CaDerivations); - StorePathSet outputPaths; for (auto & i : drv->outputsAndOptPaths(worker.store)) if (i.second.second) - outputPaths.insert(*i.second.second); - - worker.store.addTempRoots(outputPaths); + worker.store.addTempRoot(*i.second.second); /* We don't yet have any safe way to cache an impure derivation at this step. */ diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6620636c7b..e285a06f1d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -759,14 +759,8 @@ void RemoteStore::addTempRoots(const StorePathSet & paths) WorkerProto::write(*this, *conn, paths); conn.processStderr(); readInt(conn->from); - } else { - /* Fallback for daemons that don't support the batched - operation. Note that this is very slow for large sets of - paths on high-latency links, due to a network round-trip per - path. */ - for (auto & path : paths) - conn->addTempRoot(*this, &conn.daemonException, path); } + /* Note: there is no fallback for old daemons to prevent performance regressions. */ } Roots RemoteStore::findRoots(bool censor) From 1081c2b21631ebf32a6af74a4cd6e526d6f2cabe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 20 Aug 2026 17:14:49 +0200 Subject: [PATCH 622/927] nix eval: Add --drv-link flag to create symlinks to top-level derivations This creates GC-rooted symlinks for the store paths in the string context of the result of `nix eval` (with --raw or --json). Unlike `nix build --out-link`, derivations are linked to their .drv files rather than their outputs, since nothing was built. Assisted-by: Claude Fable 5 --- src/nix/eval.cc | 39 +++++++++++++++++++++++++++++++ tests/functional/flakes/flakes.sh | 10 ++++++++ 2 files changed, 49 insertions(+) diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 0802ca2dfb..d5db8e4b1d 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -2,6 +2,7 @@ #include "nix/main/common-args.hh" #include "nix/main/shared.hh" #include "nix/store/store-api.hh" +#include "nix/store/local-fs-store.hh" #include "nix/expr/eval.hh" #include "nix/expr/eval-inline.hh" #include "nix/expr/value-to-json.hh" @@ -15,6 +16,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption bool raw = false; std::optional apply; std::optional writeTo; + std::optional drvLink; CmdEval() : InstallableValueCommand() @@ -38,6 +40,15 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption .labels = {"path"}, .handler = {&writeTo}, }); + + addFlag({ + .longName = "drv-link", + .description = + "Use *path* as prefix for symlinks to derivations produced by the evaluation at top-level. By default, no symlinks are created.", + .labels = {"path"}, + .handler = {&drvLink}, + .completer = completePath, + }); } std::string description() override @@ -62,6 +73,9 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption if (raw && json) throw UsageError("--raw and --json are mutually exclusive"); + if (drvLink && !raw && !json) + throw UsageError("--drv-link requires --raw or --json"); + auto state = getEvalState(); auto [v, pos] = installable->toValue(*state); @@ -125,6 +139,31 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption else { logger->cout("%s", ValuePrinter(*state, *v, PrintOptions{.force = true, .derivationPaths = true})); } + + if (drvLink) { + // Create out links for the store paths in `context`. Similar to createOutLinks(), except that for + // derivations, we create symlinks to the .drv files rather than the outputs (since we didn't build + // anything). + if (auto store2 = store.dynamic_pointer_cast()) { + std::set roots; + for (auto & c : context) + std::visit( + overloaded{ + [&](const NixStringContextElem::Opaque & o) { roots.insert(state->devirtualize(o.path)); }, + [&](const NixStringContextElem::DrvDeep & d) { roots.insert(d.drvPath); }, + [&](const NixStringContextElem::Built & b) { roots.insert(b.drvPath->getBaseStorePath()); }, + [&](const NixStringContextElem::Path &) {}, + }, + c.raw); + for (const auto & [i, path] : enumerate(roots)) { + auto symlink = *drvLink; + if (i) + symlink += fmt("-%d", i); + state->waitForPath(path); + store2->addPermRoot(path, absPath(symlink).string()); + } + } + } } }; diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 50e06bca91..b9963e47f5 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -102,6 +102,16 @@ nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir" (cd "$flake1Dir" && nix build -o "$TEST_ROOT/result" "path:.") (cd "$flake1Dir" && nix build -o "$TEST_ROOT/result" "git+file:.") +# Test 'nix eval --drv-link'. +drvPath=$(nix eval --raw --drv-link "$TEST_ROOT/drv" flake1#foo.drvPath) +[[ $(readlink "$TEST_ROOT/drv") = "$drvPath" ]] +[[ $(nix path-info "$TEST_ROOT/drv") = "$drvPath" ]] +rm "$TEST_ROOT/drv" + +# '--drv-link' requires '--raw' or '--json'. +expectStderr 1 nix eval --drv-link "$TEST_ROOT/drv" flake1#foo.drvPath | grepQuiet -- "--drv-link requires --raw or --json" +[[ ! -e "$TEST_ROOT/drv" ]] + # Test explicit packages.default. nix build -o "$TEST_ROOT/result" "$flake1Dir#default" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default" From 5deb4384620b11806a5188303a78d616e432d16f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 20 Aug 2026 19:01:50 +0200 Subject: [PATCH 623/927] upgradeDBSchema(): Don't show "executing migration" messages on a new store --- src/libstore/include/nix/store/local-store.hh | 2 +- src/libstore/local-store.cc | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 5b64c4e117..440dc67188 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -507,7 +507,7 @@ private: void openDB(State & state, bool create); - void upgradeDBSchema(State & state); + void upgradeDBSchema(State & state, bool isNew); void makeStoreWritable(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b1f2d5fed8..d840f5e323 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -294,6 +294,8 @@ LocalStore::LocalStore(ref config) : "database schema needs migrating, but this cannot be done in read-only mode"); } + bool isNew = false; + if (curSchema > nixSchemaVersion) throw Error("current Nix store schema is version %1%, but I only support %2%", curSchema, nixSchemaVersion); @@ -301,6 +303,7 @@ LocalStore::LocalStore(ref config) curSchema = nixSchemaVersion; openDB(*state, true); writeFile(schemaPath, fmt("%1%", curSchema), 0666, FsSync::Yes); + isNew = true; } else if (curSchema < nixSchemaVersion) { @@ -356,7 +359,7 @@ LocalStore::LocalStore(ref config) else openDB(*state, false); - upgradeDBSchema(*state); + upgradeDBSchema(*state, isNew); /* Prepare SQL statements. */ state->stmts->RegisterValidPath.create( @@ -597,7 +600,7 @@ void LocalStore::openDB(State & state, bool create) } } -void LocalStore::upgradeDBSchema(State & state) +void LocalStore::upgradeDBSchema(State & state, bool isNew) { state.db.exec("create table if not exists SchemaMigrations (migration text primary key not null);"); @@ -617,7 +620,8 @@ void LocalStore::upgradeDBSchema(State & state) if (schemaMigrations.contains(migrationName)) return; - notice("executing Nix database schema migration '%s'...", migrationName); + if (!isNew) + notice("executing Nix database schema migration '%s'...", migrationName); retrySQLite([&]() { SQLiteTxn txn(state.db); From 7089df49cd0ed4a9c0fd8d62e6c21753d8d1edf9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Aug 2026 13:29:25 +0200 Subject: [PATCH 624/927] CachingFilteringSourceAccessor: Make the isAllowed() cache thread-safe The cache in CachingFilteringSourceAccessor was a plain std::map mutated on every cache miss. With parallel evaluation, multiple eval threads can call isAllowed() on the same accessor concurrently (e.g. via builtins.readDir on a Git input fetched with exportIgnore), racing find() against emplace() and corrupting the tree. This caused segfaults like DETERMINATE-NIX-8W (a SEGV_MAPERR at offset 0x10, i.e. reading _M_left of a null red-black tree node). Use boost::concurrent_flat_map instead, following the same pattern as CachingSourceAccessor. Assisted-by: Claude Fable 5 --- src/libfetchers/filtering-source-accessor.cc | 16 ++++++++++++---- .../nix/fetchers/filtering-source-accessor.hh | 6 ++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index c881718fe5..0b809e7824 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -1,6 +1,8 @@ #include "nix/fetchers/filtering-source-accessor.hh" #include "nix/util/sync.hh" +#include "nix/util/util.hh" +#include #include namespace nix { @@ -115,13 +117,19 @@ ref AllowListSourceAccessor::create( return make_ref(next, allowedPrefixes, allowedPaths, std::move(makeNotAllowedError)); } +CachingFilteringSourceAccessor::CachingFilteringSourceAccessor( + const SourcePath & src, MakeNotAllowedError && makeNotAllowedError) + : FilteringSourceAccessor(src, std::move(makeNotAllowedError)) + , cache(make_ref>()) +{ +} + bool CachingFilteringSourceAccessor::isAllowed(const CanonPath & path) { - auto i = cache.find(path); - if (i != cache.end()) - return i->second; + if (auto allowed = getConcurrent(*cache, path)) + return *allowed; auto res = isAllowedUncached(path); - cache.emplace(path, res); + cache->emplace(path, res); return res; } diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 6b7a9d83ad..e77ae0dfed 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -5,6 +5,8 @@ #include #include +#include + namespace nix { /** @@ -97,9 +99,9 @@ struct AllowListSourceAccessor : public FilteringSourceAccessor */ struct CachingFilteringSourceAccessor : FilteringSourceAccessor { - std::map cache; + const ref> cache; - using FilteringSourceAccessor::FilteringSourceAccessor; + CachingFilteringSourceAccessor(const SourcePath & src, MakeNotAllowedError && makeNotAllowedError); bool isAllowed(const CanonPath & path) override; From a23a9c0416c8cf4e822462d0739c102d17df2f93 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:56:10 +0000 Subject: [PATCH 625/927] Prepare release v3.22.2 From 211c98ad7848cddd2229a0d463f51b567fb014bc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:56:12 +0000 Subject: [PATCH 626/927] packaging/secure-packages/flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'path:../..' → 'path:../..' • Updated input 'nix/nixpkgs': 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1012116%2Brev-b5d2789f4c19edaa14cc30a9aed6b15605ced159/01a002f6-c976-7971-ae43-841e5c7df492/source.tar.gz' (2026-08-15) → 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1012125%2Brev-86c2aa5aa3e08175d183edb2581a7fd32c358115/01a01b96-88bb-7c70-84ca-1cbd5a037ecf/source.tar.gz' (2026-08-19) --- packaging/secure-packages/flake.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index 296f47b686..4a2f30abe4 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -77,12 +77,12 @@ }, "nixpkgs": { "locked": { - "lastModified": 1786753867, - "narHash": "sha256-OJ+0LGs1qTMY0dURtdvA7t53yN8PNYqOgznXfMLh3/s=", - "rev": "b5d2789f4c19edaa14cc30a9aed6b15605ced159", - "revCount": 1012116, + "lastModified": 1787167317, + "narHash": "sha256-Ec5+Im+BNJ4tVuD5Em6bsvrSjIDX6lBDnGd64QEorAg=", + "rev": "86c2aa5aa3e08175d183edb2581a7fd32c358115", + "revCount": 1012125, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1012116%2Brev-b5d2789f4c19edaa14cc30a9aed6b15605ced159/01a002f6-c976-7971-ae43-841e5c7df492/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1012125%2Brev-86c2aa5aa3e08175d183edb2581a7fd32c358115/01a01b96-88bb-7c70-84ca-1cbd5a037ecf/source.tar.gz" }, "original": { "type": "tarball", From 114e4c9110d41484a5a9f3f65e8fb77b5b714b88 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:56:16 +0000 Subject: [PATCH 627/927] Set .version-determinate to 3.22.2 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index 732c06105d..2fec58b681 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.22.1 +3.22.2 From 84e60a8fb8fb38d021bee50b0870080449e0ccd4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:56:21 +0000 Subject: [PATCH 628/927] Generate release notes for 3.22.2 --- doc/manual/source/SUMMARY.md.in | 1 + .../source/release-notes-determinate/changes.md | 12 +++++++++++- .../source/release-notes-determinate/v3.22.2.md | 12 ++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/v3.22.2.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 37ba61d026..224d901c2e 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -154,6 +154,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.22.2 (2026-08-21)](release-notes-determinate/v3.22.2.md) - [Release 3.22.1 (2026-08-18)](release-notes-determinate/v3.22.1.md) - [Release 3.22.0 (2026-08-06)](release-notes-determinate/v3.22.0.md) - [Release 3.21.9 (2026-07-30)](release-notes-determinate/v3.21.9.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index c77a995040..e60e757279 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.35 and Determinate Nix 3.22.1. +This section lists the differences between upstream Nix 2.35 and Determinate Nix 3.22.2. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -230,3 +230,13 @@ This section lists the differences between upstream Nix 2.35 and Determinate Nix * Fix "waiting for exclusive access" hangs during schema migrations [DeterminateSystems/nix-src#594](https://github.com/DeterminateSystems/nix-src/pull/594) + + + +* upgradeDBSchema(): Don't show "executing migration" messages on a new store by @edolstra in [DeterminateSystems/nix-src#600](https://github.com/DeterminateSystems/nix-src/pull/600) + +* Remove fallback behaviour for RemoteStore::addTempRoots() by @edolstra in [DeterminateSystems/nix-src#598](https://github.com/DeterminateSystems/nix-src/pull/598) + +* CachingFilteringSourceAccessor: Make the isAllowed() cache thread-safe by @edolstra in [DeterminateSystems/nix-src#601](https://github.com/DeterminateSystems/nix-src/pull/601) + +* nix eval: Add --drv-link flag to create symlinks to top-level derivations by @edolstra in [DeterminateSystems/nix-src#599](https://github.com/DeterminateSystems/nix-src/pull/599) diff --git a/doc/manual/source/release-notes-determinate/v3.22.2.md b/doc/manual/source/release-notes-determinate/v3.22.2.md new file mode 100644 index 0000000000..49a2a28196 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.22.2.md @@ -0,0 +1,12 @@ +# Release 3.22.2 (2026-08-21) + +* Based on [upstream Nix 2.35.2](../release-notes/rl-2.35.md). + +## What's Changed +* upgradeDBSchema(): Don't show "executing migration" messages on a new store by @edolstra in [DeterminateSystems/nix-src#600](https://github.com/DeterminateSystems/nix-src/pull/600) +* Remove fallback behaviour for RemoteStore::addTempRoots() by @edolstra in [DeterminateSystems/nix-src#598](https://github.com/DeterminateSystems/nix-src/pull/598) +* CachingFilteringSourceAccessor: Make the isAllowed() cache thread-safe by @edolstra in [DeterminateSystems/nix-src#601](https://github.com/DeterminateSystems/nix-src/pull/601) +* nix eval: Add --drv-link flag to create symlinks to top-level derivations by @edolstra in [DeterminateSystems/nix-src#599](https://github.com/DeterminateSystems/nix-src/pull/599) + + +**Full Changelog**: [v3.22.1...v3.22.2](https://github.com/DeterminateSystems/nix-src/compare/v3.22.1...v3.22.2) From 8c1b5cf2aa374f325c3a0f5a4513074978d064a4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Aug 2026 19:06:50 +0200 Subject: [PATCH 629/927] Tweak release notes --- .../release-notes-determinate/changes.md | 10 +--------- .../release-notes-determinate/v3.22.2.md | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index e60e757279..4de2651e51 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -229,14 +229,6 @@ This section lists the differences between upstream Nix 2.35 and Determinate Nix -* Fix "waiting for exclusive access" hangs during schema migrations [DeterminateSystems/nix-src#594](https://github.com/DeterminateSystems/nix-src/pull/594) - -* upgradeDBSchema(): Don't show "executing migration" messages on a new store by @edolstra in [DeterminateSystems/nix-src#600](https://github.com/DeterminateSystems/nix-src/pull/600) - -* Remove fallback behaviour for RemoteStore::addTempRoots() by @edolstra in [DeterminateSystems/nix-src#598](https://github.com/DeterminateSystems/nix-src/pull/598) - -* CachingFilteringSourceAccessor: Make the isAllowed() cache thread-safe by @edolstra in [DeterminateSystems/nix-src#601](https://github.com/DeterminateSystems/nix-src/pull/601) - -* nix eval: Add --drv-link flag to create symlinks to top-level derivations by @edolstra in [DeterminateSystems/nix-src#599](https://github.com/DeterminateSystems/nix-src/pull/599) +* `nix eval --json` has a flag `--drv-link` to create symlinks to top-level derivations. This prevents them from being garbage-collected. [DeterminateSystems/nix-src#599](https://github.com/DeterminateSystems/nix-src/pull/599) diff --git a/doc/manual/source/release-notes-determinate/v3.22.2.md b/doc/manual/source/release-notes-determinate/v3.22.2.md index 49a2a28196..2a8e5b030b 100644 --- a/doc/manual/source/release-notes-determinate/v3.22.2.md +++ b/doc/manual/source/release-notes-determinate/v3.22.2.md @@ -2,11 +2,21 @@ * Based on [upstream Nix 2.35.2](../release-notes/rl-2.35.md). -## What's Changed +## `nix eval --json` has a new `--drv-link` flag + +When you're using `nix eval --json` to instantiate derivations, there is a possibility that a concurrent garbage collection run deletes the store derivations that were just created. Nix now has a new `--drv-link` flag that creates symlinks to the top-level derivations returned by the evaluations, avoiding this race condition. + +PR: [DeterminateSystems/nix-src#599](https://github.com/DeterminateSystems/nix-src/pull/599) + +## `nix copy` does not register remote store GC roots when talking to older daemons + +Older versions of Determinate Nix as well as upstream Nix lack the ability to create multiple GC roots in a single RPC call. This made `nix copy` over high-latency connections very slow. Therefore, we no longer create GC roots when talking to such daemons. Note that this means that a concurrent garbage collection run on the remote store may delete the store paths that were just copied. + +PR: [DeterminateSystems/nix-src#598](https://github.com/DeterminateSystems/nix-src/pull/598) + +## Bug fixes + * upgradeDBSchema(): Don't show "executing migration" messages on a new store by @edolstra in [DeterminateSystems/nix-src#600](https://github.com/DeterminateSystems/nix-src/pull/600) -* Remove fallback behaviour for RemoteStore::addTempRoots() by @edolstra in [DeterminateSystems/nix-src#598](https://github.com/DeterminateSystems/nix-src/pull/598) * CachingFilteringSourceAccessor: Make the isAllowed() cache thread-safe by @edolstra in [DeterminateSystems/nix-src#601](https://github.com/DeterminateSystems/nix-src/pull/601) -* nix eval: Add --drv-link flag to create symlinks to top-level derivations by @edolstra in [DeterminateSystems/nix-src#599](https://github.com/DeterminateSystems/nix-src/pull/599) - **Full Changelog**: [v3.22.1...v3.22.2](https://github.com/DeterminateSystems/nix-src/compare/v3.22.1...v3.22.2) From de3a27708c4f5ac3568a7068981f4dd173a35806 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 12 Aug 2026 23:15:45 +0300 Subject: [PATCH 630/927] Improve 'sandbox-fallback' diagnostics, disallow fallback with relocated store (cherry picked from commit 1d5a8c8b27e4c0e3fe5c51a22022382fc86734de) --- src/libstore/unix/build/derivation-builder.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 3b02dfc4f1..386d3f863c 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1777,7 +1777,9 @@ std::unique_ptr makeDerivationBuild useSandbox = params.drv.type().isSandboxed() && !params.drvOptions.noChroot; } - if (store.storeDir != store.config->realStoreDir.get()) { + const bool isRelocatedStore = store.storeDir != store.config->realStoreDir.get(); + + if (isRelocatedStore) { #if defined(__linux__) || defined(__FreeBSD__) useSandbox = true; #else @@ -1790,10 +1792,19 @@ std::unique_ptr makeDerivationBuild if (!localSettings.sandboxFallback) throw Error( "this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); - debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); + + if (isRelocatedStore) + throw Error( + "this system does not support the kernel namespaces that are required for sandboxing, which is required for building in a diverted store"); + + static std::atomic warned = false; + warnOnce( + warned, + "auto-disabling sandboxing because the prerequisite namespaces are not available and '%1%' is enabled; use '--no-sandbox' or specify 'sandbox = false' setting to silence this warning", + localSettings.sandboxFallback.name); + useSandbox = false; } - #endif if (!useSandbox && params.drvOptions.useUidRange(params.drv)) From b8abc0832096af8ec32edec7a05031732778ea80 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jul 2026 18:33:52 +0200 Subject: [PATCH 631/927] Populate the srcToStore cache when we have a fetcher cache hit This saves tens of thousands of calls to SQLite and to makeFixedOutputPathFromCA(). (E.g. instantiating nixpkgs#firefox was doing 4744 fetcher cache checks for /pkgs/stdenv/generic/source-stdenv.sh.) (cherry picked from commit 30820a54b112f4842bdb7df28b61b2a607e54033) --- src/libfetchers/fetch-to-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index acedca6dbe..085c97da7d 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -82,6 +82,7 @@ std::pair fetchToStore2( path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); + settings.srcToStore->cache.insert_or_assign(srcToStoreKey, std::make_tuple(storePath, hash, mode)); return {storePath, hash}; } debug("source path '%s' not in store", path); From 31c6f6e67e70ec5c38c072ad352d04a80e65a085 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Aug 2026 12:23:52 +0200 Subject: [PATCH 632/927] Print a proper error message when memory allocation fails Previously, running out of memory resulted in an opaque SIGABRT (e.g. Sentry issue DETERMINATE-NIX-8P): mimalloc's `operator new` override is compiled as C, so it cannot throw `std::bad_alloc` and instead calls abort() when an allocation fails and no new handler is installed. So install a `std::new_handler` that panics with a "ran out of memory" message. However, that alone doesn't help under mimalloc: libmimalloc is linked with `-Bsymbolic`, so its weak null stub of `std::get_new_handler()` shadows the real one from libstdc++ and the installed handler is never seen. Therefore also register a mimalloc error callback via `mi_register_error()` that panics on ENOMEM/EOVERFLOW. This also sets the `panic_msg` Sentry tag, so OOM crash reports now self-identify instead of showing up as unexplained aborts in allocation paths. Assisted-by: Claude Fable 5 --- src/libutil/error.cc | 5 +++++ src/libutil/include/nix/util/error.hh | 10 ++++++++++ src/libutil/util.cc | 6 ++++++ src/nix/main.cc | 19 +++++++++++++++++++ src/nix/meson.build | 2 ++ 5 files changed, 42 insertions(+) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 59b82ad3af..4f9c9c96f4 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -470,6 +470,11 @@ void panic(std::string_view msg) std::terminate(); } +void outOfMemory() +{ + panic("ran out of memory"); +} + void unreachable(std::source_location loc) { char buf[512]; diff --git a/src/libutil/include/nix/util/error.hh b/src/libutil/include/nix/util/error.hh index 3f399e0598..f493f5b52b 100644 --- a/src/libutil/include/nix/util/error.hh +++ b/src/libutil/include/nix/util/error.hh @@ -470,6 +470,16 @@ void throwExceptionSelfCheck(); [[noreturn]] void panic(std::string_view msg); +/** + * Print an error message and std::terminate(). Installed as the + * `std::new_handler` so that failing memory allocations produce a + * clear "out of memory" error instead of an opaque abort (in + * particular when `operator new` is overridden by mimalloc, which + * cannot throw `std::bad_alloc`). + */ +[[noreturn]] +void outOfMemory(); + /** * Log the current exception (if any) and call abort(). */ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 839fe79da7..6c885f3ec0 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -63,6 +64,11 @@ void initLibUtil() effect. */ if (OPENSSL_init_crypto(OPENSSL_INIT_NO_ATEXIT, nullptr) != 1) throw Error("could not initialise OpenSSL"); + + /* Make sure that failing memory allocations don't result in an + opaque abort() (e.g. from mimalloc's `operator new` override, + which cannot throw `std::bad_alloc`). */ + std::set_new_handler(outOfMemory); } ////////////////////////////////////////////////////////////////////// diff --git a/src/nix/main.cc b/src/nix/main.cc index b9e60f53ab..63e30b1324 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -35,6 +35,10 @@ # include #endif +#if HAVE_MIMALLOC +# include +#endif + #ifndef _WIN32 # include # include @@ -426,6 +430,21 @@ void mainWrapped(int argc, char ** argv) /* This must be called before Sentry since both initialize OpenSSL. */ initLibUtil(); +#if HAVE_MIMALLOC + /* Make allocation failures print a proper error message instead + of aborting silently. Note that the `std::new_handler` installed + by `initLibUtil()` does not suffice, since mimalloc's `operator + new` override never sees it: libmimalloc is linked with + `-Bsymbolic`, so its weak null stub of `std::get_new_handler()` + shadows the real one from libstdc++. */ + mi_register_error( + [](int err, void *) { + if (err == ENOMEM || err == EOVERFLOW) + outOfMemory(); + }, + nullptr); +#endif + bool sentryEnabled = false; #if HAVE_SENTRY diff --git a/src/nix/meson.build b/src/nix/meson.build index 9515101546..fc232a8154 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -69,6 +69,8 @@ mandir = get_option('mandir') mandir = fs.is_absolute(mandir) ? mandir : prefix / mandir configdata.set_quoted('NIX_MAN_DIR', mandir) +configdata.set('HAVE_MIMALLOC', mimalloc.found().to_int()) + sentry_required = get_option('sentry') configdata.set('HAVE_SENTRY', sentry_required.enabled().to_int()) if sentry_required.enabled() From b247ae7e53ed19d355fb0a02f5d7074d9337e670 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Aug 2026 12:32:13 +0200 Subject: [PATCH 633/927] Replace throws of std::bad_alloc by outOfMemory() We have no ability to meaningfully handle bad_alloc, and it can lead to terminate() calls when it happens in a destructor or some other context where throwing is not allowed. So just panic with a clear "ran out of memory" message. Assisted-by: Claude Fable 5 --- src/libexpr/eval-gc.cc | 3 +-- src/libexpr/eval.cc | 4 ++-- src/libexpr/include/nix/expr/eval-inline.hh | 6 +++--- src/libexpr/root-value.cc | 2 +- src/libstore/filetransfer.cc | 2 +- src/libstore/local-store.cc | 2 +- 6 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 1aef8b844d..4c6d6457c1 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -40,8 +40,7 @@ static_assert(sizeof(void *) * 2 == GC_GRANULE_BYTES, "Boehm GC must use GC_GRAN /* Called when the Boehm GC runs out of memory. */ static void * oomHandler(size_t requested) { - /* Convert this to a proper C++ exception. */ - throw std::bad_alloc(); + outOfMemory(); } static size_t getFreeMem() diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 29ea9908db..f0effb0add 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -70,7 +70,7 @@ static char * allocString(size_t size) char * t; t = (char *) GC_MALLOC_ATOMIC(size); if (!t) - throw std::bad_alloc(); + outOfMemory(); return t; } @@ -96,7 +96,7 @@ StringData & StringData::alloc(EvalMemory & mem, size_t size) { void * t = mem.allocBytes(sizeof(StringData) + size + 1); if (!t) - throw std::bad_alloc(); + outOfMemory(); auto res = new (t) StringData(size); return *res; } diff --git a/src/libexpr/include/nix/expr/eval-inline.hh b/src/libexpr/include/nix/expr/eval-inline.hh index b724762a80..6d439111cf 100644 --- a/src/libexpr/include/nix/expr/eval-inline.hh +++ b/src/libexpr/include/nix/expr/eval-inline.hh @@ -22,7 +22,7 @@ inline void * EvalMemory::allocBytes(size_t n) p = calloc(n, 1); #endif if (!p) - throw std::bad_alloc(); + outOfMemory(); return p; } @@ -42,7 +42,7 @@ Value * EvalMemory::allocValue() if (!*valueAllocCache) { *valueAllocCache = GC_malloc_many(sizeof(Value)); if (!*valueAllocCache) - throw std::bad_alloc(); + outOfMemory(); } /* GC_NEXT is a convenience macro for accessing the first word of an object. @@ -76,7 +76,7 @@ Env & EvalMemory::allocEnv(size_t size) if (!*env1AllocCache) { *env1AllocCache = GC_malloc_many(sizeof(Env) + sizeof(Value *)); if (!*env1AllocCache) - throw std::bad_alloc(); + outOfMemory(); } void * p = *env1AllocCache; diff --git a/src/libexpr/root-value.cc b/src/libexpr/root-value.cc index 7ad2c645db..d1ba0e3d5f 100644 --- a/src/libexpr/root-value.cc +++ b/src/libexpr/root-value.cc @@ -46,7 +46,7 @@ RootValue::RootValue(Value * v) constexpr size_t slabSize = 4096; auto slab = (Slot *) GC_MALLOC_UNCOLLECTABLE(slabSize * sizeof(Slot)); if (!slab) - throw std::bad_alloc(); + outOfMemory(); for (size_t i = 0; i + 1 < slabSize; ++i) slab[i].nextFree = &slab[i + 1]; slab[slabSize - 1].nextFree = nullptr; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 80f6a6be07..d2b4359e20 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -246,7 +246,7 @@ struct curlFileTransfer : public FileTransfer { curlSList tmpSList = curlSList(::curl_slist_append(requestHeaders.get(), requireCString(header))); if (!tmpSList) - throw std::bad_alloc(); + outOfMemory(); requestHeaders.release(); requestHeaders = std::move(tmpSList); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d840f5e323..6ecdafb26d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1417,7 +1417,7 @@ StorePath LocalStore::addToStoreFromDump( dumpBuffer.release(); dumpBuffer.reset((char *) tmp); } else { - throw std::bad_alloc(); + outOfMemory(); } auto got = 0; Finally cleanup([&]() { dump = {dumpBuffer.get(), dump.size() + got}; }); From 02cf0ce64cf6cbc1119b8aff8aa0cc7f062c3c68 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Aug 2026 12:34:58 +0200 Subject: [PATCH 634/927] Remove redundant bad_alloc catch in the daemon Allocation failures now call outOfMemory(), which panics instead of throwing bad_alloc, so this catch is unreachable. Assisted-by: Claude Fable 5 --- src/libstore/daemon.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 24cb920711..534b54b3a4 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1178,10 +1178,6 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted tunnelLogger->stopWork(&e); if (!errorAllowed) throw; - } catch (std::bad_alloc & e) { - auto ex = Error("Nix daemon out of memory"); - tunnelLogger->stopWork(&ex); - throw; } conn.to.flush(); From 67cf40d9fa6a7350bec6e1925c58209e62eb0e3f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Aug 2026 17:27:33 +0200 Subject: [PATCH 635/927] Don't mask flake ref URL interpretation errors Errors that occur while interpreting a syntactically valid flake ref URL (such as an unsupported query parameter) were swallowed by parseURLFlakeRef(), causing the flake ref to be reinterpreted as a path and leading to a confusing error, e.g. $ nix --option nix-219-compat true eval --expr 'builtins.parseFlakeRef "github:NixOS/nix?dir=perl"' error: flake reference 'github:NixOS/nix?dir=perl' is not an absolute path instead of the underlying error error: URL 'github:NixOS/nix?dir=perl' contains unknown parameter 'dir' Now we only fall back to path parsing if the string doesn't parse as a URL at all. Assisted-by: Claude Fable 5 --- src/libflake-tests/flakeref.cc | 19 +++++++++++++++++++ src/libflake/flakeref.cc | 25 +++++++++++++++++-------- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/libflake-tests/flakeref.cc b/src/libflake-tests/flakeref.cc index 332ffe18b7..5cf03db13f 100644 --- a/src/libflake-tests/flakeref.cc +++ b/src/libflake-tests/flakeref.cc @@ -10,6 +10,7 @@ #include "nix/util/configuration.hh" #include "nix/util/error.hh" #include "nix/util/experimental-features.hh" +#include "nix/util/terminal.hh" namespace nix { @@ -278,6 +279,24 @@ TEST(to_string, doesntReencodeUrl) ASSERT_EQ(unparsed, expected); } +TEST(parseFlakeRef, urlInterpretationErrorsAreNotMasked) +{ + fetchers::Settings fetchSettings; + + // Errors that occur while interpreting a syntactically valid URL + // (such as an unsupported query parameter) should be shown to the + // user, rather than causing the flake ref to be reinterpreted as a + // path, leading to a confusing "not an absolute path" error. + try { + parseFlakeRef(fetchSettings, "github:foo/bar?xyzzy=1"); + FAIL() << "expected parseFlakeRef to throw"; + } catch (BadURL & e) { + auto msg = filterANSIEscapes(e.msg()); + EXPECT_NE(msg.find("xyzzy"), std::string::npos) << msg; + EXPECT_EQ(msg.find("not an absolute path"), std::string::npos) << msg; + } +} + TEST(parseFlakeRef, malformedGithubUrlDoesNotCrash) { fetchers::Settings fetchSettings; diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index 3af499ae6b..eb72db8440 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -250,18 +250,27 @@ std::optional> parseURLFlakeRef( const std::optional & baseDir, bool isFlake) { + std::optional parsed; try { - auto parsed = parseURL(url, /*lenient=*/true); - if (baseDir && (parsed.scheme == "path" || parsed.scheme == "git+file")) { - /* Here we know that the path must not contain encoded '/' or NUL bytes. */ - auto path = urlPathToPath(parsed.path); - if (!path.is_absolute()) - parsed.path = pathToUrlPath(absPath(path, get(baseDir))); - } - return fromParsedURL(fetchSettings, std::move(parsed), isFlake); + parsed = parseURL(url, /*lenient=*/true); } catch (BadURL &) { + /* The string is not a URL, so try to interpret it as a path + instead (in the caller). Note that errors that occur while + *interpreting* a syntactically valid URL (such as an + unsupported query parameter) are not caught here; they + should be shown to the user rather than masked by a + confusing error about the flake ref not being a valid + path. */ return std::nullopt; } + + if (baseDir && (parsed->scheme == "path" || parsed->scheme == "git+file")) { + /* Here we know that the path must not contain encoded '/' or NUL bytes. */ + auto path = urlPathToPath(parsed->path); + if (!path.is_absolute()) + parsed->path = pathToUrlPath(absPath(path, get(baseDir))); + } + return fromParsedURL(fetchSettings, std::move(*parsed), isFlake); } std::pair parseFlakeRefWithFragment( From 9e5e156b4603dde141b6e7fc35c2951b2b60f7a2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Aug 2026 17:39:53 +0200 Subject: [PATCH 636/927] Fix Nix < 2.20 compat mode for flake refs like 'github:foo/bar?dir=sub' Since f3aeeca2f, nix-219-compat mode kept the 'dir' query parameter in the URL passed to Input::fromURL(), in order to reproduce the 'url' attribute that Nix < 2.20 wrote to lock files (e.g. "url": "https://foo/bar?dir=subdir"). However, since 6cae299bd, the GitHub input scheme rejects unknown query parameters, so flake refs like 'github:NixOS/nix?dir=perl' failed to parse in compat mode. Nix 2.19.0 only retained 'dir' in the 'url' attribute of input types that have one (git, hg, tarball, file); for input types like 'github', 'dir' was never part of the input attributes. So instead of keeping 'dir' in the query (which input schemes may reject), we now always erase it before calling Input::fromURL(), and in compat mode re-add it to the resulting 'url' attribute if the input has one. Verified against Nix 2.19.0 that 'builtins.parseFlakeRef' now produces identical attributes in compat mode for git, hg, tarball, file, github, gitlab and sourcehut flake refs with a 'dir' parameter, and that lock files generated in compat mode are identical to those generated by 2.19.0 for a 'git+file://...?dir=sub' input. Assisted-by: Claude Fable 5 --- src/libflake-tests/flakeref.cc | 36 ++++++++++++++++++++++++++++++++++ src/libflake/flakeref.cc | 31 ++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/libflake-tests/flakeref.cc b/src/libflake-tests/flakeref.cc index 5cf03db13f..8ef0b996ec 100644 --- a/src/libflake-tests/flakeref.cc +++ b/src/libflake-tests/flakeref.cc @@ -268,6 +268,42 @@ INSTANTIATE_TEST_SUITE_P( }), [](const ::testing::TestParamInfo & info) { return info.param.description; }); +TEST(parseFlakeRef, nix219CompatRetainsDirInUrlAttr) +{ + fetchers::Settings fetchSettings; + fetchSettings.nix219Compat = true; + + /* Nix < 2.20 retained the `dir` query parameter in the `url` + attribute of input types that have one. */ + { + auto flakeref = parseFlakeRef(fetchSettings, "git+https://example.org/repo?dir=sub"); + ASSERT_EQ(flakeref.subdir, "sub"); + ASSERT_EQ(flakeref.toAttrs().at("url"), Attr("https://example.org/repo?dir=sub")); + } + + { + auto flakeref = parseFlakeRef(fetchSettings, "https://example.org/foo.tar.gz?dir=sub"); + ASSERT_EQ(flakeref.subdir, "sub"); + ASSERT_EQ(flakeref.toAttrs().at("url"), Attr("https://example.org/foo.tar.gz?dir=sub")); + } + + /* But for input types that don't have a `url` attribute (such as + `github`), the `dir` parameter was never part of the input + attributes, so it should not cause a parse failure. */ + { + auto flakeref = parseFlakeRef(fetchSettings, "github:NixOS/nix?dir=perl"); + ASSERT_EQ(flakeref.subdir, "perl"); + ASSERT_EQ( + flakeref.toAttrs(), + (fetchers::Attrs{ + {"dir", Attr("perl")}, + {"owner", Attr("NixOS")}, + {"repo", Attr("nix")}, + {"type", Attr("github")}, + })); + } +} + TEST(to_string, doesntReencodeUrl) { fetchers::Settings fetchSettings; diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index eb72db8440..86ca67b557 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -91,13 +91,38 @@ static std::pair fromParsedURL(const fetchers::Settings & fetchSettings, ParsedURL && parsedURL, bool isFlake) { auto dir = getOr(parsedURL.query, "dir", ""); - if (!fetchSettings.nix219Compat) - parsedURL.query.erase("dir"); + parsedURL.query.erase("dir"); std::string fragment; std::swap(fragment, parsedURL.fragment); - return {FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment}; + auto input = fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake); + + /* Backward compatibility hack: Nix < 2.20 retained the `dir` + query parameter in the `url` attribute of input types that have + one (i.e. `git`, `hg`, `tarball` and `file`), resulting in lock + file entries like + + "original": { + "dir": "subdir", + "type": "git", + "url": "https://foo/bar?dir=subdir" + } + + In compat mode, reproduce that behavior so that we write lock + files that Nix < 2.20 considers up to date. Input types that + don't have a `url` attribute (such as `github`) never included + the `dir` parameter in their attributes, so they're left + alone. */ + if (fetchSettings.nix219Compat && dir != "") { + if (auto url = fetchers::maybeGetStrAttr(input.attrs, "url")) { + auto parsedUrlAttr = parseURL(*url, /*lenient=*/true); + parsedUrlAttr.query.insert_or_assign("dir", dir); + input.attrs.insert_or_assign("url", parsedUrlAttr.to_string()); + } + } + + return {FlakeRef(std::move(input), dir), fragment}; } std::pair parsePathFlakeRefWithFragment( From 91ba67976eb4fa61a0137cf5c0f456f23267bb63 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Aug 2026 12:08:52 +0200 Subject: [PATCH 637/927] Write the schema file atomically Write the schema version to a temporary file, fsync it, and atomically rename it into place. This ensures the schema file is never observed empty or partially written. Assisted-by: Claude Fable 5 --- src/libstore/local-store.cc | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d840f5e323..f170ee2940 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -147,6 +147,22 @@ struct LocalStore::State::Stmts SQLiteStmt QuerySchemaMigration; }; +/** + * Write the schema version to `schemaPath` atomically: write it to a + * temporary file, fsync it, and atomically rename it into place. This + * ensures the schema file is never empty or partially written. + */ +static void writeSchemaFile(const std::filesystem::path & schemaPath, int version) +{ + auto tmp = schemaPath; + tmp += fmt(".tmp.%d", getpid()); + AutoDelete del(tmp, false); + writeFile(tmp, std::to_string(version), 0666, FsSync::Yes); + std::filesystem::rename(tmp, schemaPath); + del.cancel(); + syncParent(schemaPath); +} + LocalStore::LocalStore(ref config) : Store{*config} , LocalFSStore{*config} @@ -302,7 +318,7 @@ LocalStore::LocalStore(ref config) else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; openDB(*state, true); - writeFile(schemaPath, fmt("%1%", curSchema), 0666, FsSync::Yes); + writeSchemaFile(schemaPath, curSchema); isNew = true; } @@ -351,7 +367,7 @@ LocalStore::LocalStore(ref config) state->db.exec("alter table ValidPaths add column ca text"); } - writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, FsSync::Yes); + writeSchemaFile(schemaPath, nixSchemaVersion); lockFile(globalLock.get(), ltRead, true); } From 0a28caa84d8e40354ae81d291d321bbb74abe97e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Aug 2026 14:39:37 +0200 Subject: [PATCH 638/927] Serve GC allocations up to 1520 bytes from the thread-local freelists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Profiling 'nix search nixpkgs --no-eval-cache' showed that ~65% of all futex waits came from the Boehm GC, the largest share being the global allocation lock (GC_allocate_ml): with the default GC_TINY_FREELISTS of 25, only allocations up to 384 bytes are served from the per-thread freelists, and everything larger takes the global lock. Nixpkgs evaluation does ~755k such allocations per search — e.g. a typical derivation attrset (~46 attrs) is a 752-byte Bindings, of which one is allocated per package — all serialized on the lock across the worker threads. Building bdwgc with GC_TINY_FREELISTS=96 (thread-local up to 1520 bytes) eliminates 93% of the global-lock allocations (755k -> 56k; what remains is essentially the >16 KiB attrset giants), halves the context-switch count at 8-16 workers, and makes the search ~10% faster at 24 workers (~3.7s -> ~3.3s). The per-thread memory cost is at most a partial heap block per size class and object kind. Also add a patch (from the bdwgc "nix-patches" branch) that makes gctest's fixed "unexpected heap growth" limit scale with GC_TINY_FREELISTS, since the larger freelists exceed it by design. Assisted-by: Claude Fable 5 --- packaging/dependencies.nix | 19 ++++++---- ...gc-gctest-tiny-freelists-heap-growth.patch | 38 +++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 packaging/patches/boehmgc-gctest-tiny-freelists-heap-growth.patch diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 480c4079e9..180e194339 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -36,13 +36,10 @@ scope: { inherit stdenv; }).overrideAttrs (attrs: { - # Reduce contention on the GC allocation lock during parallel - # evaluation by handing out multiple heap blocks worth of - # objects per lock acquisition in GC_generic_malloc_many(). - # The default batch size is set via GC_MANY_BLOCKS_DEFAULT - # below and can be overridden at runtime through the - # GC_MALLOC_MANY_BLOCKS environment variable. - patches = (attrs.patches or [ ]) ++ [ ./patches/boehmgc-batch-malloc-many.patch ]; + patches = (attrs.patches or [ ]) ++ [ + ./patches/boehmgc-batch-malloc-many.patch + ./patches/boehmgc-gctest-tiny-freelists-heap-growth.patch + ]; env = (attrs.env or { }) // { # Increase the initial mark stack size to avoid stack @@ -55,6 +52,14 @@ scope: { [ "-DINITIAL_MARK_STACK_SIZE=1048576" "-DGC_MANY_BLOCKS_DEFAULT=64" + # Serve allocations up to 1520 bytes (95 granules) from + # the per-thread freelists instead of taking the global + # allocation lock. The default (25, i.e. <= 384 bytes) is + # too small for parallel evaluation: e.g. a typical + # derivation attrset (~46 attrs) is a 752-byte Bindings, + # of which nixpkgs evaluation does hundreds of thousands, + # all serialized on GC_allocate_ml. + "-DGC_TINY_FREELISTS=96" ] # For some reason that is not clear, it is wanting to use libgcc_eh which is not available. # Force this to be built with compiler-rt & libunwind over libgcc_eh works. diff --git a/packaging/patches/boehmgc-gctest-tiny-freelists-heap-growth.patch b/packaging/patches/boehmgc-gctest-tiny-freelists-heap-growth.patch new file mode 100644 index 0000000000..a7cd51d694 --- /dev/null +++ b/packaging/patches/boehmgc-gctest-tiny-freelists-heap-growth.patch @@ -0,0 +1,38 @@ +From 7cd4f090acc0e801502f918060106bba926c7153 Mon Sep 17 00:00:00 2001 +From: Eelco Dolstra +Date: Tue, 25 Aug 2026 14:37:18 +0200 +Subject: [PATCH] Scale gctest's heap growth limit with GC_TINY_FREELISTS + +The 'unexpected heap growth' check in check_heap_stats() uses a fixed +limit that is exceeded when the collector is built with a larger +GC_TINY_FREELISTS: bigger thread-local free lists cause each thread to +retain more nearly-empty heap blocks (up to one per size class and +object kind) and increase block-level fragmentation. Scale the limit +proportionally so that gctest passes with such configurations. + +Assisted-by: Claude Fable 5 +--- + tests/test.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/tests/test.c b/tests/test.c +index 0ae3f95b..b62699fa 100644 +--- a/tests/test.c ++++ b/tests/test.c +@@ -1753,6 +1753,13 @@ void check_heap_stats(void) + # endif + # ifdef MEMORY_SANITIZER + max_heap_sz += max_heap_sz / 4; ++# endif ++# if defined(GC_TINY_FREELISTS) && GC_TINY_FREELISTS > 25 ++ /* Larger thread-local free lists cause each thread to retain */ ++ /* more nearly-empty heap blocks (up to one per size class and */ ++ /* object kind) and increase block-level fragmentation, so */ ++ /* allow the heap to grow proportionally. */ ++ max_heap_sz = max_heap_sz / 25 * GC_TINY_FREELISTS; + # endif + max_heap_sz *= n_tests; + # if defined(USE_MMAP) || defined(MSWIN32) +-- +2.54.0 + From d26495d920c121d340e490e876dbbaa6e886749a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Aug 2026 15:21:25 +0200 Subject: [PATCH 639/927] Use a single GC root for all values created during a JSON parse The JSON parser used GC-visible (uncollectable) containers in every object/array parser state (ValueMap/ValueVector) to keep the values under construction alive. Since GC_MALLOC_UNCOLLECTABLE() and GC_FREE() take the global GC allocation lock, a big JSON document did one or more global-lock round trips per object/array node. Profiling 'nix search nixpkgs --no-eval-cache' showed ~183k such allocations (nixpkgs evaluation parses a lot of JSON), making this the largest remaining source of allocation-lock contention after GC_TINY_FREELISTS was increased. Instead, root every value allocated during the parse in a single GC-visible vector for the duration of the parse (they all end up in the result anyway), and use ordinary (malloc) containers in the per-node parser states. This reduces the parser's uncollectable allocations from ~183k to zero, cutting the context-switch count of the search benchmark by another ~15% at 24 eval cores. Assisted-by: Claude Fable 5 --- src/libexpr/json-to-value.cc | 64 +++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 378a046687..04a6dc5f4b 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -19,7 +19,18 @@ class JSONSax : nlohmann::json_sax { protected: std::unique_ptr parent; - RootValue v; + + /** + * The value being built by this state, or null. Kept alive by + * `JSONSax::rootValues`. + */ + Value * v = nullptr; + + /** + * Reference to `JSONSax::rootValues`. + */ + ValueVector & rootValues; + public: virtual std::unique_ptr resolve(EvalState &) { @@ -28,11 +39,13 @@ class JSONSax : nlohmann::json_sax explicit JSONState(std::unique_ptr && p) : parent(std::move(p)) + , rootValues(parent->rootValues) { } - explicit JSONState(Value * v) - : v(RootValue(v)) + JSONState(Value * v, ValueVector & rootValues) + : v(v) + , rootValues(rootValues) { } @@ -40,9 +53,13 @@ class JSONSax : nlohmann::json_sax Value & value(EvalState & state) { - if (!v) - v = RootValue(state.allocValue()); - return **v; + if (!v) { + v = state.allocValue(); + /* Root the value for the duration of the parse (see + `JSONSax::rootValues`). */ + rootValues.push_back(v); + } + return *v; } virtual ~JSONState() {} @@ -53,7 +70,14 @@ class JSONSax : nlohmann::json_sax class JSONObjectState : public JSONState { using JSONState::JSONState; - ValueMap attrs; + + /** + * Note: deliberately not `ValueMap`: the values are rooted via + * `JSONSax::rootValues`, so there is no need for a GC-visible + * (uncollectable) container, whose allocations would take the + * global GC allocation lock. + */ + boost::unordered_flat_map> attrs; std::unique_ptr resolve(EvalState & state) override { @@ -66,7 +90,7 @@ class JSONSax : nlohmann::json_sax void add() override { - v.reset(); + v = nullptr; } public: void key(string_t & name, EvalState & state) @@ -78,7 +102,11 @@ class JSONSax : nlohmann::json_sax class JSONListState : public JSONState { - ValueVector values; + /** + * Note: deliberately not `ValueVector` (see + * `JSONObjectState::attrs`). + */ + std::vector values; std::unique_ptr resolve(EvalState & state) override { @@ -91,8 +119,8 @@ class JSONSax : nlohmann::json_sax void add() override { - values.push_back(*v); - v.reset(); + values.push_back(v); + v = nullptr; } public: JSONListState(std::unique_ptr && p, std::size_t reserve) @@ -103,12 +131,24 @@ class JSONSax : nlohmann::json_sax }; EvalState & state; + + /** + * Keeps all values allocated during the parse alive. This is the + * only GC-visible container of the parser: the per-node containers + * in `JSONObjectState`/`JSONListState` use ordinary allocators, + * since GC-visible (uncollectable) allocations take the global GC + * allocation lock, and a big JSON document would otherwise do one + * or more of them per object/array node — a significant source of + * lock contention during parallel evaluation. + */ + ValueVector rootValues; + std::unique_ptr rs; public: JSONSax(EvalState & state, Value & v) : state(state) - , rs(new JSONState(&v)) {}; + , rs(new JSONState(&v, rootValues)) {}; bool null() override { From e0a0f821336e790a56c8b2c178f916db1575183a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Aug 2026 15:35:18 +0200 Subject: [PATCH 640/927] fromJSON: Replace the per-object unordered_flat_map with a plain vector BindingsBuilder sorts the attributes anyway, so hashing every key on insertion is unnecessary. Keys are now appended to a vector; on object close, a stable sort plus a keep-last-of-run scan preserves JSON's last-occurrence-wins semantics for duplicate keys, and the pre-sorted result lets us use BindingsBuilder::alreadySorted() to skip the builder's own sort. Assisted-by: Claude Fable 5 --- src/libexpr/json-to-value.cc | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 04a6dc5f4b..8c84ffe3b2 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -77,14 +77,21 @@ class JSONSax : nlohmann::json_sax * (uncollectable) container, whose allocations would take the * global GC allocation lock. */ - boost::unordered_flat_map> attrs; + std::vector> attrs; std::unique_ptr resolve(EvalState & state) override { + /* JSON allows duplicate keys, and the last occurrence + wins. `Bindings` requires unique keys, so drop all but + the last entry of every run of equal keys. The stable + sort keeps equal keys in insertion order. */ + std::stable_sort( + attrs.begin(), attrs.end(), [](const auto & a, const auto & b) { return a.first < b.first; }); auto attrs2 = state.buildBindings(attrs.size()); - for (auto & i : attrs) - attrs2.insert(i.first, i.second); - parent->value(state).mkAttrs(attrs2); + for (auto i = attrs.begin(); i != attrs.end(); ++i) + if (std::next(i) == attrs.end() || std::next(i)->first != i->first) + attrs2.insert(i->first, i->second); + parent->value(state).mkAttrs(attrs2.alreadySorted()); return std::move(parent); } @@ -96,7 +103,7 @@ class JSONSax : nlohmann::json_sax void key(string_t & name, EvalState & state) { forceNoNullByte(name); - attrs.insert_or_assign(state.symbols.create(name), &value(state)); + attrs.emplace_back(state.symbols.create(name), &value(state)); } }; From 6ab349d7f63e02c1fadc3d4b8dc6a49d9187c356 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Aug 2026 15:43:05 +0200 Subject: [PATCH 641/927] fromJSON: Use small_vector for the per-node parse states Small JSON objects and lists (up to 16 entries) now build up in the state's inline storage instead of a separately heap-allocated vector. Also drop the speculative reserve(128) for arrays: nlohmann's text parser never knows the length in advance (only binary formats like CBOR do), so it only served to force a heap allocation. Assisted-by: Claude Fable 5 --- src/libexpr/json-to-value.cc | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 8c84ffe3b2..cee95813fb 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -3,6 +3,7 @@ #include "nix/expr/eval.hh" #include +#include #include using json = nlohmann::json; @@ -75,9 +76,10 @@ class JSONSax : nlohmann::json_sax * Note: deliberately not `ValueMap`: the values are rooted via * `JSONSax::rootValues`, so there is no need for a GC-visible * (uncollectable) container, whose allocations would take the - * global GC allocation lock. + * global GC allocation lock. The inline capacity avoids a heap + * allocation for small objects. */ - std::vector> attrs; + boost::container::small_vector, 16> attrs; std::unique_ptr resolve(EvalState & state) override { @@ -109,11 +111,13 @@ class JSONSax : nlohmann::json_sax class JSONListState : public JSONState { + using JSONState::JSONState; + /** * Note: deliberately not `ValueVector` (see * `JSONObjectState::attrs`). */ - std::vector values; + boost::container::small_vector values; std::unique_ptr resolve(EvalState & state) override { @@ -129,12 +133,6 @@ class JSONSax : nlohmann::json_sax values.push_back(v); v = nullptr; } - public: - JSONListState(std::unique_ptr && p, std::size_t reserve) - : JSONState(std::move(p)) - { - values.reserve(reserve); - } }; EvalState & state; @@ -239,7 +237,7 @@ class JSONSax : nlohmann::json_sax bool start_array(size_t len) override { - rs = std::make_unique(std::move(rs), len != std::numeric_limits::max() ? len : 128); + rs = std::make_unique(std::move(rs)); return true; } From 7d886df353de7c0085601ec90df31ff0a8fbedef Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 26 Aug 2026 11:47:12 +0200 Subject: [PATCH 642/927] Remove unused ValueMap type --- src/libexpr/include/nix/expr/value.hh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index 60c29eeddf..aadb05f620 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -1534,13 +1534,6 @@ public: }; typedef std::vector> ValueVector; -typedef boost::unordered_flat_map< - Symbol, - Value *, - std::hash, - std::equal_to, - traceable_allocator>> - ValueMap; void forceNoNullByte(std::string_view s, std::function = nullptr); } // namespace nix From d6f5cef6c0e5c5fd02e4506391ecab0c98b1bcfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 27 Aug 2026 03:21:30 +0200 Subject: [PATCH 643/927] nix flake prefetch-inputs: fix inputs being randomly skipped Nodes were deduplicated by address, but std::bind copies its argument, so &node pointed at a temporary inside the bind object. A later work item could reuse that address and be treated as already visited, so inputs were silently not fetched while the command still exited 0. Pass the lock graph node by reference so the address is stable. (cherry picked from commit f0008d095dad1554de5f54233f8b5b3541865a89) --- src/nix/flake-prefetch-inputs.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/flake-prefetch-inputs.cc b/src/nix/flake-prefetch-inputs.cc index eb73c6c915..32f8387714 100644 --- a/src/nix/flake-prefetch-inputs.cc +++ b/src/nix/flake-prefetch-inputs.cc @@ -56,11 +56,11 @@ struct CmdFlakePrefetchInputs : FlakeCommand for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) - pool.enqueue(std::bind(visit, **inputNode)); + pool.enqueue([&visit, inputNode(*inputNode)] { visit(*inputNode); }); } }; - pool.enqueue(std::bind(visit, *flake.lockFile.root)); + pool.enqueue([&] { visit(*flake.lockFile.root); }); pool.process(); From f5014fca235dbc5ccfd2762e9d353b940f5abbba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 27 Aug 2026 16:37:16 +0200 Subject: [PATCH 644/927] Revert "Remove fallback behaviour for RemoteStore::addTempRoots()" This reverts commit deb3de4616ea6911d56ef738033a40d1c0f3723d. This was a bad idea because it broke RemoteStore::addTempRoot() entirely. --- src/libstore/build/derivation-goal.cc | 5 ++++- src/libstore/remote-store.cc | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 72733516aa..55bf392390 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -62,9 +62,12 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation) if (!drv->type().hasKnownOutputPaths()) experimentalFeatureSettings.require(Xp::CaDerivations); + StorePathSet outputPaths; for (auto & i : drv->outputsAndOptPaths(worker.store)) if (i.second.second) - worker.store.addTempRoot(*i.second.second); + outputPaths.insert(*i.second.second); + + worker.store.addTempRoots(outputPaths); /* We don't yet have any safe way to cache an impure derivation at this step. */ diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e285a06f1d..6620636c7b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -759,8 +759,14 @@ void RemoteStore::addTempRoots(const StorePathSet & paths) WorkerProto::write(*this, *conn, paths); conn.processStderr(); readInt(conn->from); + } else { + /* Fallback for daemons that don't support the batched + operation. Note that this is very slow for large sets of + paths on high-latency links, due to a network round-trip per + path. */ + for (auto & path : paths) + conn->addTempRoot(*this, &conn.daemonException, path); } - /* Note: there is no fallback for old daemons to prevent performance regressions. */ } Roots RemoteStore::findRoots(bool censor) From 78fc6a4c15c0d78283aa5ab2e07c7bc6a8df08cd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 27 Aug 2026 17:11:07 +0200 Subject: [PATCH 645/927] nix copy: Don't register temp roots if the remote doesn't have AddTempRoots --- src/libstore/gc.cc | 2 +- src/libstore/include/nix/store/local-store.hh | 2 +- src/libstore/include/nix/store/remote-store.hh | 2 +- src/libstore/include/nix/store/store-api.hh | 2 +- src/libstore/remote-store.cc | 4 +++- src/libstore/restricted-store.cc | 2 +- src/libstore/store-api.cc | 4 +++- 7 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 73adbac343..a7d9077004 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -81,7 +81,7 @@ void LocalStore::createTempRootsFile() } } -void LocalStore::addTempRoots(const StorePathSet & paths) +void LocalStore::addTempRoots(const StorePathSet & paths, bool _skipIfSlow) { if (config->readOnly) { debug( diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 440dc67188..872add7148 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -313,7 +313,7 @@ public: RepairFlag repair, std::shared_ptr provenance) override; - void addTempRoots(const StorePathSet & paths) override; + void addTempRoots(const StorePathSet & paths, bool skipIfSlow) override; private: diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index c4ae96cbd6..0769afd6b3 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -136,7 +136,7 @@ public: void ensurePath(const StorePath & path) override; - void addTempRoots(const StorePathSet & paths) override; + void addTempRoots(const StorePathSet & paths, bool skipIfSlow) override; Roots findRoots(bool censor) override; diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index d09a7ae04e..78a819c215 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -895,7 +895,7 @@ public: * Add multiple store paths as temporary roots of the garbage collector. * The roots disappears as soon as we exit. */ - virtual void addTempRoots(const StorePathSet & paths) + virtual void addTempRoots(const StorePathSet & paths, bool skipIfSlow = false) { debug("not creating temporary roots, store doesn't support GC"); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6620636c7b..956a15eb57 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -747,7 +747,7 @@ void RemoteStore::ensurePath(const StorePath & path) readInt(conn->from); } -void RemoteStore::addTempRoots(const StorePathSet & paths) +void RemoteStore::addTempRoots(const StorePathSet & paths, bool skipIfSlow) { if (paths.empty()) return; @@ -760,6 +760,8 @@ void RemoteStore::addTempRoots(const StorePathSet & paths) conn.processStderr(); readInt(conn->from); } else { + if (skipIfSlow) + return; /* Fallback for daemons that don't support the batched operation. Note that this is very slow for large sets of paths on high-latency links, due to a network round-trip per diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index 926a414efd..6ea7752db4 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -130,7 +130,7 @@ struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStor unsupported("buildDerivation"); } - void addTempRoots(const StorePathSet & paths) override {} + void addTempRoots(const StorePathSet & paths, bool skipIfSlow) override {} void addIndirectRoot(const std::filesystem::path & path) override {} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d23fb930d5..b4e5fd11b9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1066,7 +1066,9 @@ std::map copyPaths( CheckSigsFlag checkSigs, SubstituteFlag substitute) { - dstStore.addTempRoots(storePaths); + /* This is optional to avoid a performance regression when talking to old daemons that don't have the AddTempRoots + * operation. */ + dstStore.addTempRoots(storePaths, true); auto valid = dstStore.queryValidPaths(storePaths, substitute); From 5f430f56a341317f5c90329a0b44c0c900233863 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Aug 2026 15:58:58 +0200 Subject: [PATCH 646/927] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2605.1009383%2Brev-4382ed2b7a6839d4280a9b386db49cbc5907414d/019f6c11-ad78-7500-a194-d18a5bad0fbe/source.tar.gz' (2026-07-16) → 'https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2605.1012930%2Brev-c5c4a43b0e8056328ec4529f735cabdb8f1942bb/01a053c0-7061-7382-b001-102e9148a5b5/source.tar.gz' (2026-08-29) --- flake.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 99e0a8ec1c..f11f48a2aa 100644 --- a/flake.lock +++ b/flake.lock @@ -57,12 +57,12 @@ }, "nixpkgs": { "locked": { - "lastModified": 1784160687, - "narHash": "sha256-iYL/bixrb6FlHFu/gIuBYzq6c6lM5AAXsXNSWXtIgQc=", - "rev": "4382ed2b7a6839d4280a9b386db49cbc5907414d", - "revCount": 1009383, + "lastModified": 1787962033, + "narHash": "sha256-u6z9VTZA4Kf3RkHQo9sQI7NI4Ei/uiU9vrMqOiwWP1Y=", + "rev": "c5c4a43b0e8056328ec4529f735cabdb8f1942bb", + "revCount": 1012930, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2605.1009383%2Brev-4382ed2b7a6839d4280a9b386db49cbc5907414d/019f6c11-ad78-7500-a194-d18a5bad0fbe/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2605.1012930%2Brev-c5c4a43b0e8056328ec4529f735cabdb8f1942bb/01a053c0-7061-7382-b001-102e9148a5b5/source.tar.gz" }, "original": { "type": "tarball", From 95af23df251cd95e49531372c250bc6b73046227 Mon Sep 17 00:00:00 2001 From: Emre Sahin Date: Tue, 25 Aug 2026 11:18:52 +0200 Subject: [PATCH 647/927] Remove backported boost patch See https://github.com/NixOS/nixpkgs/commit/b5e044308f120cbcaaeff4b5c9326c48739c3ff6 --- packaging/dependencies.nix | 3 - ...eptions-not-accounting-for-forced_un.patch | 102 ------------------ 2 files changed, 105 deletions(-) delete mode 100644 packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 180e194339..183ec3fb6a 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -203,9 +203,6 @@ scope: { "--with-url" "--with-thread" ]; - patches = [ - ./patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch - ]; enableIcu = false; inherit stdenv; }).overrideAttrs diff --git a/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch b/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch deleted file mode 100644 index 7ec13724c4..0000000000 --- a/packaging/patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch +++ /dev/null @@ -1,102 +0,0 @@ -From 5883212311535a0046031d74d1568ae173c1e35b Mon Sep 17 00:00:00 2001 -From: Sergei Zimmerman -Date: Tue, 21 Jul 2026 21:15:51 +0000 -Subject: [PATCH] Fix uncaught_exceptions() not accounting for forced_unwind - -Unwound fibers would see std::uncaught_exceptions() == 0, while a -forced_unwind exception is in "flight". This goes against the contract -of std::uncaught_exceptions() that scope guards rely upon. Failing -to report the correct number of uncaught exceptions (especially -misreporting zero) will lead to scope guards to misbehave badly and skip -running cleanup code which branches on whether the destructor is called -during stack unwinding or not. - -This is because the "throw" would happen before the destructor is run on -the fiber stack being switched to, but the increment would be clobbered -by the destructor of manage_exception_state. - -I'm not sure what the contract of run ontop_fcontext is wrt to whether -the the caller provided function can throw or not, but in my best -understanding the forced_unwind mechanism is mostly internal and so is -throwing from ontop_fcontext in the switched-to fiber. Thus, I've kept -the catch block scoped to detail::forced_unwind. ---- - include/boost/context/fiber_fcontext.hpp | 37 +++++++++++++++++------- - test/test_fiber.cpp | 24 +++++++++++++++ - 2 files changed, 51 insertions(+), 10 deletions(-) - -diff --git a/include/boost/context/fiber_fcontext.hpp b/include/boost/context/fiber_fcontext.hpp -index 543ba6c..38476c9 100644 ---- a/boost/context/fiber_fcontext.hpp -+++ b/boost/context/fiber_fcontext.hpp -@@ -70,7 +70,9 @@ namespace context { - namespace detail { - - // manage_exception_state is a dummy struct unless we have specific support --struct manage_exception_state {}; -+struct manage_exception_state { -+ void from_forced_unwind() noexcept {} -+}; - - } // namespace detail - } // namespace context -@@ -90,6 +92,11 @@ public: - manage_exception_state() { - exception_state_ = *__cxa_get_globals(); - } -+ // Hack to account for the forced_unwind exception thrown in fiber_unwind -+ // that's run ontop before the destructor. -+ void from_forced_unwind() noexcept { -+ exception_state_.uncaughtExceptions += 1; -+ } - ~manage_exception_state() { - *__cxa_get_globals() = exception_state_; - } -@@ -376,13 +383,18 @@ public: - BOOST_ASSERT( nullptr != fctx_); - detail::manage_exception_state exstate; - boost::ignore_unused(exstate); -- return { detail::jump_fcontext( -+ try { -+ return { detail::jump_fcontext( - #if defined(BOOST_NO_CXX14_STD_EXCHANGE) -- detail::exchange( fctx_, nullptr), -+ detail::exchange( fctx_, nullptr), - #else -- std::exchange( fctx_, nullptr), -+ std::exchange( fctx_, nullptr), - #endif -- nullptr).fctx }; -+ nullptr).fctx }; -+ } catch ( detail::forced_unwind const& ) { -+ exstate.from_forced_unwind(); -+ throw; -+ } - } - - template< typename Fn > -@@ -391,14 +403,19 @@ public: - detail::manage_exception_state exstate; - boost::ignore_unused(exstate); - auto p = std::forward< Fn >( fn); -- return { detail::ontop_fcontext( -+ try { -+ return { detail::ontop_fcontext( - #if defined(BOOST_NO_CXX14_STD_EXCHANGE) -- detail::exchange( fctx_, nullptr), -+ detail::exchange( fctx_, nullptr), - #else -- std::exchange( fctx_, nullptr), -+ std::exchange( fctx_, nullptr), - #endif -- & p, -- detail::fiber_ontop< fiber, decltype(p) >).fctx }; -+ & p, -+ detail::fiber_ontop< fiber, decltype(p) >).fctx }; -+ } catch ( detail::forced_unwind const& ) { -+ exstate.from_forced_unwind(); -+ throw; -+ } - } - - explicit operator bool() const noexcept { From a76fcaac6867cd26ebd973649b889bd318eb8a41 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Aug 2026 16:11:37 +0200 Subject: [PATCH 648/927] packaging/secure-packages/flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix/nixpkgs': 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1012125%2Brev-86c2aa5aa3e08175d183edb2581a7fd32c358115/01a01b96-88bb-7c70-84ca-1cbd5a037ecf/source.tar.gz' (2026-08-19) → 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013487%2Brev-01649628bfcaf28024737daec63d559fe0463e6c/01a04eb4-9c90-76b3-a3fb-a92559e04618/source.tar.gz' (2026-08-29) --- packaging/secure-packages/flake.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index 4a2f30abe4..ed100b9a76 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -77,12 +77,12 @@ }, "nixpkgs": { "locked": { - "lastModified": 1787167317, - "narHash": "sha256-Ec5+Im+BNJ4tVuD5Em6bsvrSjIDX6lBDnGd64QEorAg=", - "rev": "86c2aa5aa3e08175d183edb2581a7fd32c358115", - "revCount": 1012125, + "lastModified": 1788025223, + "narHash": "sha256-TWYxqK6AnH3/AXPSowct3pxzafZ501CjVuJGu1kBUJw=", + "rev": "01649628bfcaf28024737daec63d559fe0463e6c", + "revCount": 1013487, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1012125%2Brev-86c2aa5aa3e08175d183edb2581a7fd32c358115/01a01b96-88bb-7c70-84ca-1cbd5a037ecf/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013487%2Brev-01649628bfcaf28024737daec63d559fe0463e6c/01a04eb4-9c90-76b3-a3fb-a92559e04618/source.tar.gz" }, "original": { "type": "tarball", From d4d023e94c07b056af69a58641905f549cbe3f5d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 31 Aug 2026 12:17:03 +0200 Subject: [PATCH 649/927] Fix crash when GC is interrupted The GC roots server's per-client thread only caught Error, but Interrupted derives from BaseError, not Error. So when the daemon was interrupted during garbage collection while a client thread was blocked in readLine(), the Interrupted exception escaped the thread, calling std::terminate() and crashing the daemon. Fixes Sentry report DETERMINATE-NIX-31. Assisted-by: Claude Fable 5 --- src/libstore/gc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index a7d9077004..30242a5db3 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -509,7 +509,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } else printError("received garbage instead of a root from client"); writeFull(fdClient.get(), "1", false); - } catch (Error & e) { + } catch (BaseError & e) { debug("reading GC root from client: %s", e.msg()); break; } From 765608ceda4de0309933e911a3d2791674484b51 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 31 Aug 2026 12:21:10 +0200 Subject: [PATCH 650/927] PR creation hint --- CLAUDE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 27268bf11f..abd454bed0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,3 +3,5 @@ Observe the following rules when contributing to this repository: * Before committing, run ./maintainers/format.sh to detect/fix any formatting issues. * Use "Assisted-by:" instead of "Co-Authored-By:" for the Claude trailer in commits. + +* Do not create PRs unless prompted. Create PRs using `gh pr create --repo DeterminateSystems/nix-src --base main`, observing `.github/PULL_REQUEST_TEMPLATE.md`. From 02002da656041237503b74aae73481418327449e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 31 Aug 2026 12:55:28 +0200 Subject: [PATCH 651/927] upload-debug-info-to-sentry.py: Support zstd-compressed NARs cache.nixos.org compresses newer NARs with zstd instead of xz, but the script unconditionally piped downloaded debuginfo NARs through `xz -d`. This broke after the Nixpkgs update pulled in glibc 2.42, whose debuginfo NAR is served as .nar.zst. Pick the decompressor based on the archive extension, and add xz/zstd to the nix shell shebang so both are guaranteed to be on PATH. Assisted-by: Claude Fable 5 --- maintainers/upload-debug-info-to-sentry.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/maintainers/upload-debug-info-to-sentry.py b/maintainers/upload-debug-info-to-sentry.py index 87804d32a5..d183311ea9 100755 --- a/maintainers/upload-debug-info-to-sentry.py +++ b/maintainers/upload-debug-info-to-sentry.py @@ -1,5 +1,5 @@ #!/usr/bin/env nix -#!nix shell --inputs-from . nixpkgs#sentry-cli nixpkgs#python3 nixpkgs#binutils --command python3 +#!nix shell --inputs-from . nixpkgs#sentry-cli nixpkgs#python3 nixpkgs#binutils nixpkgs#xz nixpkgs#zstd --command python3 import argparse import json @@ -65,21 +65,27 @@ def download_nar(build_id: str, archive: str) -> str: def extract_debug_symbols(nar_path: str, member: str, build_id: str) -> str: - """Extract a member from a .nar.xz into /tmp/debug-info/.debug. Returns the output path.""" + """Extract a member from a compressed NAR into /tmp/debug-info/.debug. Returns the output path.""" out_path = os.path.join(DEBUG_INFO_DIR, f"{build_id}.debug") if os.path.exists(out_path): print(f" already extracted {out_path}", file=sys.stderr) return out_path os.makedirs(DEBUG_INFO_DIR, exist_ok=True) print(f" extracting {member} -> {out_path} ...", file=sys.stderr) - xz = subprocess.Popen(["xz", "-d"], stdin=open(nar_path, "rb"), stdout=subprocess.PIPE) + if nar_path.endswith(".zst"): + decompress_cmd = ["zstd", "-d", "-c"] + elif nar_path.endswith(".xz"): + decompress_cmd = ["xz", "-d"] + else: + raise Exception(f"unknown NAR compression format: {nar_path}") + decompress = subprocess.Popen(decompress_cmd, stdin=open(nar_path, "rb"), stdout=subprocess.PIPE) nar_cat = subprocess.run( ["nix", "nar", "cat", "/dev/stdin", member], - stdin=xz.stdout, + stdin=decompress.stdout, capture_output=True, check=True, ) - xz.wait() + decompress.wait() with open(out_path, "wb") as f: f.write(nar_cat.stdout) return out_path From 7734b0989259335d4aafecc7a5b0a6c7ca2d7729 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 31 Aug 2026 16:24:53 +0200 Subject: [PATCH 652/927] Register the mimalloc error handler earlier and panic on heap corruption Move the mi_register_error() call before initLibUtil() so that allocation failures during initialization also invoke outOfMemory() instead of mimalloc's silent abort. Also handle EFAULT (heap corruption detected in secure mode) by calling panic(), preserving fatal behavior but with a proper error message. Assisted-by: Claude Fable 5 --- src/nix/main.cc | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 63e30b1324..0151ec45e5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -427,24 +427,32 @@ void mainWrapped(int argc, char ** argv) } #endif - /* This must be called before Sentry since both initialize OpenSSL. */ - initLibUtil(); - #if HAVE_MIMALLOC /* Make allocation failures print a proper error message instead of aborting silently. Note that the `std::new_handler` installed by `initLibUtil()` does not suffice, since mimalloc's `operator new` override never sees it: libmimalloc is linked with `-Bsymbolic`, so its weak null stub of `std::get_new_handler()` - shadows the real one from libstdc++. */ + shadows the real one from libstdc++. Register this before + `initLibUtil()` so allocation failures during initialization + are handled as well. */ mi_register_error( [](int err, void *) { if (err == ENOMEM || err == EOVERFLOW) outOfMemory(); + /* EFAULT means mimalloc detected heap corruption (e.g. a + double free) in secure mode; don't continue with a + corrupted heap. EINVAL (bad pointer passed to + `mi_free()`) is non-fatal, so just return. */ + if (err == EFAULT) + panic("mimalloc detected heap corruption"); }, nullptr); #endif + /* This must be called before Sentry since both initialize OpenSSL. */ + initLibUtil(); + bool sentryEnabled = false; #if HAVE_SENTRY From 9c5d36a0c6addc42dcb5e297faa87ff88f320c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 30 Aug 2026 09:34:59 +0200 Subject: [PATCH 653/927] libfetchers/git: use HEAD as default ref for detached local repos getDefaultRef() guessed 'master' (with a warning) when a local repo had a detached HEAD, e.g. a CI worktree fetched as git+file:///wt?rev=.... That branch may not even exist. libgit2 resolves 'HEAD' just fine, so use it instead of guessing. Co-authored-by: Gaetan Lepage (cherry picked from commit c34d30cfb277ec8ac684765bde2f49f2666e47fc) --- src/libfetchers/git.cc | 5 ++++- tests/functional/fetchGit.sh | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 29fcb5dfc3..70e159eca5 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -767,7 +767,10 @@ struct GitInputScheme : InputScheme { auto head = std::visit( overloaded{ - [&](const std::filesystem::path & path) { return GitRepo::openRepo(path, {})->getWorkdirRef(); }, + [&](const std::filesystem::path & path) -> std::optional { + /* A detached HEAD is still resolvable by libgit2. */ + return GitRepo::openRepo(path, {})->getWorkdirRef().value_or("HEAD"); + }, [&](const ParsedURL & url) { return readHeadCached(settings, url.to_string(), shallow); }}, repoInfo.location); if (!head) { diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index 7255b5d753..b3aab573b1 100755 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -211,6 +211,9 @@ createGitRepo "$TEST_ROOT"/minimal git -C "$TEST_ROOT"/minimal fetch "$repo" "$rev2" git -C "$TEST_ROOT"/minimal checkout "$rev2" [[ $(nix eval --impure --raw --expr "(builtins.fetchGit { url = $TEST_ROOT/minimal; }).rev") = "$rev2" ]] +# Detached HEAD must not fall back to a guessed 'master' ref. +expectStderr 0 nix eval --raw --expr "(builtins.fetchGit { url = $TEST_ROOT/minimal; rev = \"$rev2\"; }).outPath" \ + | grepQuietInverse "could not read HEAD ref" # Explicit ref = "HEAD" should work, and produce the same outPath as without ref path7=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).outPath") From 81dca68c74198e3636b66ef84f57c2d5f7833cf0 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Sun, 30 Aug 2026 09:28:31 +0200 Subject: [PATCH 654/927] libfetchers/git: render submodules as empty dirs in workdir accessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With submodules=false, getAccessorFromCommit() renders a gitlink as an empty directory (like `git archive`), but getAccessorFromWorkdir() omitted the path entirely. Fetching git+file:///repo and git+file:///repo?rev=HEAD thus gave different narHashes for the same rev, and since both share a fingerprint, one poisoned the fetchToStore cache for the other. This breaks nix-eval-jobs on repos with submodules: the flake is locked from the workdir, workers re-fetch the __final input by rev and fail with a narHash mismatch. Enumerate submodules in the workdir from gitlinks in the index instead of from .gitmodules (which only supplies URLs), and mount an empty directory over each one that is not fetched, so the workdir accessor matches the commit accessor in all cases, including gitlinks without a .gitmodules entry. Fixes #13698. Addresses #15423 for the workdir case. Supersedes #14779. Co-authored-by: Jörg Thalheim Co-authored-by: Julian Kuners Assisted-by: OpenCode (GPT-5.6 Sol) (cherry picked from commit 52546bafddab82a41246706619f527c5134fa3d6) --- src/libfetchers/git-utils.cc | 23 +++++++++++++-- src/libfetchers/git.cc | 20 ++++++++----- tests/functional/fetchGitSubmodules.sh | 39 ++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index cd04bfee77..bf5f1b24bc 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -117,6 +118,7 @@ typedef std::unique_ptr> typedef std::unique_ptr> ObjectDb; typedef std::unique_ptr> PackBuilder; typedef std::unique_ptr> Indexer; +typedef std::unique_ptr> Index; static Hash toHash(const git_oid & oid) { @@ -572,10 +574,27 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback)) throw GitError("getting working directory status"); - /* Get submodule info. */ + /* Gitlinks in the index are the submodules; .gitmodules only + supplies their URLs. */ + std::map configured; auto modulesFile = path / ".gitmodules"; if (pathExists(modulesFile)) - info.submodules = parseSubmodules(modulesFile); + for (auto & submodule : parseSubmodules(modulesFile)) + configured.emplace(submodule.path, submodule); + + Index index; + if (git_repository_index(Setter(index), *this)) + throw GitError("opening index of Git repository %s", PathFmt(path)); + for (size_t n = 0, count = git_index_entrycount(index.get()); n < count; ++n) { + auto entry = git_index_get_byindex(index.get(), n); + if (entry->mode != GIT_FILEMODE_COMMIT || git_index_entry_stage(entry) != 0) + continue; + CanonPath entryPath(entry->path); + if (auto it = configured.find(entryPath); it != configured.end()) + info.submodules.push_back(std::move(it->second)); + else + info.submodules.push_back(Submodule{.path = entryPath}); + } return info; } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 29fcb5dfc3..1716e0f0d3 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -989,10 +989,9 @@ struct GitInputScheme : InputScheme { auto repoPath = repoInfo.getPath().value(); - if (getSubmodulesAttr(input)) - /* Create mountpoints for the submodules. */ - for (auto & submodule : repoInfo.workdirInfo.submodules) - repoInfo.workdirInfo.files.insert(submodule.path); + /* Create mountpoints for the submodules. */ + for (auto & submodule : repoInfo.workdirInfo.submodules) + repoInfo.workdirInfo.files.insert(submodule.path); auto repo = GitRepo::openRepo(repoPath, {}); @@ -1002,12 +1001,19 @@ struct GitInputScheme : InputScheme repo->getAccessor(repoInfo.workdirInfo, {.exportIgnore = exportIgnore}, makeNotAllowedError(repoPath)); /* If the repo has submodules, return a mounted input accessor - consisting of the accessor for the top-level repo and the - accessors for the submodule workdirs. */ - if (getSubmodulesAttr(input) && !repoInfo.workdirInfo.submodules.empty()) { + consisting of the accessor for the top-level repo and, per + submodule, either its workdir accessor or an empty directory + where getAccessorFromCommit() would produce one (submodules + disabled, or a gitlink without a .gitmodules entry). */ + if (!repoInfo.workdirInfo.submodules.empty()) { std::map> mounts; for (auto & submodule : repoInfo.workdirInfo.submodules) { + if (!getSubmodulesAttr(input) || submodule.url.empty()) { + mounts.insert_or_assign(submodule.path, makeEmptySourceAccessor()); + continue; + } + auto submodulePath = repoPath / submodule.path.rel(); fetchers::Attrs attrs; attrs.insert_or_assign("type", "git"); diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh index ac77020d84..d3f9e4da57 100755 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -36,7 +36,21 @@ git -C "$rootRepo" commit -m "Add submodule" rev=$(git -C "$rootRepo" rev-parse HEAD) +# A clean workdir must yield the same tree as fetching its HEAD rev: the +# submodule is an empty directory, whether or not it is checked out. +r0=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; }).outPath") +[[ -d $r0/sub ]] +[[ -z "$(ls -A "$r0/sub")" ]] +# Both paths share a fingerprint, so compare narHashes with separate caches. +hashWorkdir=$(XDG_CACHE_HOME=$TEST_ROOT/cache-workdir nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; }).narHash") +hashRev=$(XDG_CACHE_HOME=$TEST_ROOT/cache-rev nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev\"; }).narHash") +[[ $hashWorkdir == "$hashRev" ]] +# The non-local (clone-to-cache) code path must agree as well. +hashHttp=$(_NIX_FORCE_HTTP=1 XDG_CACHE_HOME=$TEST_ROOT/cache-http nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev\"; }).narHash") +[[ $hashWorkdir == "$hashHttp" ]] + r1=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev\"; }).outPath") +[[ $r0 == "$r1" ]] r2=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev\"; submodules = false; }).outPath") r3=$(nix eval --raw --expr "(builtins.fetchGit { url = \"file://$rootRepo\"; rev = \"$rev\"; submodules = true; }).outPath") @@ -226,3 +240,28 @@ test_submodule_nested() { } test_submodule_nested + + +# A gitlink without a .gitmodules entry is still a submodule (#15423): it must +# render the same (empty directory) from the workdir and by rev. +test_gitlink_without_gitmodules() { + local repo=$TEST_ROOT/nogitmodules + createGitRepo "$repo" + git -C "$repo" submodule add "$subRepo" sub + git -C "$repo" commit -m "Add submodule" + git -C "$repo" rm .gitmodules + git -C "$repo" commit -m "Remove .gitmodules" + local rev + rev=$(git -C "$repo" rev-parse HEAD) + + for submodules in true false; do + local hashWorkdir hashRev outWorkdir + hashWorkdir=$(XDG_CACHE_HOME=$TEST_ROOT/ngm-workdir-$submodules nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; submodules = $submodules; }).narHash") + hashRev=$(XDG_CACHE_HOME=$TEST_ROOT/ngm-rev-$submodules nix eval --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; rev = \"$rev\"; submodules = $submodules; }).narHash") + [[ $hashWorkdir == "$hashRev" ]] + outWorkdir=$(XDG_CACHE_HOME=$TEST_ROOT/ngm-workdir-$submodules nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; submodules = $submodules; }).outPath") + [[ -d $outWorkdir/sub ]] + [[ -z "$(ls -A "$outWorkdir/sub")" ]] + done +} +test_gitlink_without_gitmodules From c39e300dbb3530312734847d93d6167eca78d9b5 Mon Sep 17 00:00:00 2001 From: Sandro Date: Mon, 31 Aug 2026 17:08:56 +0200 Subject: [PATCH 655/927] Convert stdenv.isLinux to stdenv.hostPlatform.isLinux (cherry picked from commit fa4f25547cda44714cd06afafbf4a6806220d7cc) --- packaging/everything.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/everything.nix b/packaging/everything.nix index 74629d6840..152e633a11 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -161,7 +161,7 @@ stdenv.mkDerivation (finalAttrs: { ln -sT ${nix-manual} $doc ln -sT ${nix-manual.man} $man '' - + lib.optionalString stdenv.isLinux '' + + lib.optionalString stdenv.hostPlatform.isLinux '' lndir ${nix-nswrapper} $out ''; From ea74b87f7087ac5d2cb2f49ca020dff3c5ca3b38 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Sep 2026 11:48:40 +0200 Subject: [PATCH 656/927] Fix forwarding netrc-file to the daemon This was introduced by a bad merge. Fixes #591. --- src/libstore/daemon.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 534b54b3a4..56e563d64c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -291,9 +291,10 @@ struct ClientSettings trusted || name == settings.getWorkerSettings().buildTimeout.name || name == settings.getWorkerSettings().maxSilentTime.name || name == settings.getWorkerSettings().pollInterval.name || name == "connect-timeout" - || name == loggerSettings.sessionId.name || (name == "builders" && value == "")) + || name == loggerSettings.sessionId.name || (name == "builders" && value == "")) { settings.set(name, value); - else if (setSubstituters(settings.getWorkerSettings().substituters)) + fileTransferSettings.set(name, value); + } else if (setSubstituters(settings.getWorkerSettings().substituters)) ; else warn( From afa04f7c64efd5826dddf37c1717d3a84570070c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Sep 2026 12:29:50 +0200 Subject: [PATCH 657/927] Initialize Sentry after initNix() to fix Ctrl-C handling sentry_init() spawns worker threads. When it ran before initNix() (and thus before startSignalHandlerThread() blocks SIGINT etc. in the calling thread), those threads inherited a signal mask with SIGINT unblocked, so the kernel could deliver a Ctrl-C's SIGINT to one of them. Its default disposition then killed the process immediately, without printing "error: interrupted by the user" or restoring the terminal cursor hidden by the progress bar. Initializing Sentry after initNix() makes its threads inherit the blocked mask, so SIGINT is only received by the signal handler thread via sigwait(), as intended. The SIGQUIT reset that undoes Sentry's crash handler for it moves along, since it must run after sentry_init(). Assisted-by: Claude Fable 5 --- src/libmain/shared.cc | 9 --------- src/nix/main.cc | 46 +++++++++++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index abf6e49e11..8cf1489003 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -186,15 +186,6 @@ void initNix(bool loadConfig) act.sa_handler = sigHandler; if (sigaction(NIX_SIG_MULTI_INT, &act, 0)) throw SysError("handling multiplexed interrupt"); - - /* Reset SIGQUIT to its default disposition. In particular, this - unregisters any crash handler installed by `sentry_init()` - (which runs before us): SIGQUIT is a user-initiated "quit with - core dump" action (e.g. Ctrl-\ at a terminal), not a crash, so - it should not be reported. */ - act.sa_handler = SIG_DFL; - if (sigaction(SIGQUIT, &act, 0)) - throw SysError("handling SIGQUIT"); #endif #ifdef __APPLE__ diff --git a/src/nix/main.cc b/src/nix/main.cc index 0151ec45e5..5acf75257d 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -44,6 +44,7 @@ # include # include # include +# include #endif #ifdef __linux__ @@ -453,6 +454,27 @@ void mainWrapped(int argc, char ** argv) /* This must be called before Sentry since both initialize OpenSSL. */ initLibUtil(); + /* Set the build hook location + + For builds we perform a self-invocation, so Nix has to be + self-aware. That is, it has to know where it is installed. We + don't think it's sentient. + */ + settings.getWorkerSettings().buildHook.setDefault( + Strings{ + getNixBin({}).string(), + "__build-remote", + }); + + initNix(); + + /* Initialize Sentry only after initNix(), i.e. after + startSignalHandlerThread() has blocked SIGINT etc. in the + calling thread. The worker threads spawned by `sentry_init()` + inherit that signal mask; if they were started earlier, the + kernel could deliver a Ctrl-C's SIGINT to one of them, where its + default disposition would kill the process immediately (without + printing an error or restoring the terminal cursor). */ bool sentryEnabled = false; #if HAVE_SENTRY @@ -481,6 +503,17 @@ void mainWrapped(int argc, char ** argv) setSentryTag("nix_command", argc > 0 ? std::string(baseNameOf(argv[0])).c_str() : ""); std::set_terminate(terminateHandler); sentryEnabled = true; + + /* Reset SIGQUIT, for which `sentry_init()` installed a crash + handler, to its default disposition: SIGQUIT is a + user-initiated "quit with core dump" action (e.g. Ctrl-\ at + a terminal), not a crash, so it should not be reported. */ + struct sigaction act; + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + act.sa_handler = SIG_DFL; + if (sigaction(SIGQUIT, &act, 0)) + throw SysError("handling SIGQUIT"); } Finally cleanupSentry([&]() { @@ -492,19 +525,6 @@ void mainWrapped(int argc, char ** argv) if (!sentryEnabled) registerCrashHandler(); - /* Set the build hook location - - For builds we perform a self-invocation, so Nix has to be - self-aware. That is, it has to know where it is installed. We - don't think it's sentient. - */ - settings.getWorkerSettings().buildHook.setDefault( - Strings{ - getNixBin({}).string(), - "__build-remote", - }); - - initNix(); initGC(); flakeSettings.configureEvalSettings(evalSettings); From a82c6e5a4df19748fb1ac3b93d700eed917c4ac4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 18 Feb 2026 04:26:35 +0100 Subject: [PATCH 658/927] fix(eval-cache): preserve string context when caching The in-memory cached value was created with an empty context `{}` instead of copying the actual context from the value. This caused string contexts to be lost even though they were correctly written to the database. --- src/libexpr/eval-cache.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index d2f041a06b..db509f09da 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -477,9 +477,13 @@ Value & AttrCursor::forceValue() } if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { - if (v.type() == nString) - cachedValue = {root->db->setString(getKey(), v.string_view(), v.context()), string_t{v.string_view(), {}}}; - else if (v.type() == nPath) { + if (v.type() == nString) { + NixStringContext context; + copyContext(v, context); + cachedValue = { + root->db->setString(getKey(), v.string_view(), v.context()), + string_t{v.string_view(), std::move(context)}}; + } else if (v.type() == nPath) { auto path = v.path().path; cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; } else if (v.type() == nBool) From b3649bb3dfae15b1a0018d6cb1cadd881a5f3cdf Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 18 Feb 2026 04:31:31 +0100 Subject: [PATCH 659/927] fix(eval-cache): preserve rowids when sealing attribute names When setAttrs "seals" an attrset (transitions from Placeholder to FullAttrs), the previous INSERT OR REPLACE would delete and recreate the row with a new rowid. This orphaned any children cached under the old rowid, causing cache misses on subsequent builds. This explains the "third time's a charm" behavior: on the 2nd build, orphaned children are re-cached under the new parent rowid. With deeper nesting, even more evaluations could be required as each level of sealing orphans the next level's children. Fix by using INSERT ... ON CONFLICT DO UPDATE (upsert) to preserve the rowid, and INSERT OR IGNORE for children to preserve existing cached values. Also delete redundant Missing entries since we now know the complete set. This requires SQLite >= 3.35 for the RETURNING clause, now declared as an explicit dependency of libexpr. --- src/libexpr/eval-cache.cc | 34 +++++++++++++++++++++++++--------- src/libexpr/meson.build | 4 ++++ src/libexpr/package.nix | 2 ++ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index db509f09da..30aedd6467 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -58,6 +58,9 @@ struct AttrDb SQLiteStmt insertAttributeWithContext; SQLiteStmt queryAttribute; SQLiteStmt queryAttributes; + SQLiteStmt upsertAttribute; + SQLiteStmt insertAttributeIfNotExists; + SQLiteStmt deleteMissingChildren; std::unique_ptr txn; }; @@ -92,6 +95,17 @@ struct AttrDb state->queryAttributes.create(state->db, "select name from Attributes where parent = ?"); + state->upsertAttribute.create( + state->db, + "insert into Attributes(parent, name, type, value) values (?, ?, ?, ?) " + "on conflict(parent, name) do update set type = excluded.type, value = excluded.value " + "returning rowid"); + + state->insertAttributeIfNotExists.create( + state->db, "insert or ignore into Attributes(parent, name, type, value) values (?, ?, ?, ?)"); + + state->deleteMissingChildren.create(state->db, "delete from Attributes where parent = ? and type = 3"); + state->txn = std::make_unique(state->db); } @@ -126,18 +140,20 @@ struct AttrDb return doSQLite([&]() { auto state(_state->lock()); - state->insertAttribute.use() - .apply(key.first) - .apply(symbols[key.second]) - .apply(AttrType::FullAttrs) - .apply(0, false) - .exec(); - - AttrId rowId = state->db.getLastInsertedRowId(); + // Seal the attribute names: we now know the complete set. + auto upsertAttribute(state->upsertAttribute.use() + .apply(key.first) + .apply(symbols[key.second]) + .apply(AttrType::FullAttrs) + .apply(0, false)); + upsertAttribute.next(); + auto rowId = (AttrId) upsertAttribute.getInt(0); assert(rowId); + state->deleteMissingChildren.use().apply(rowId).exec(); + // Insert children as placeholders, but don't replace existing entries for (auto & attr : attrs) - state->insertAttribute.use() + state->insertAttributeIfNotExists.use() .apply(rowId) .apply(symbols[attr]) .apply(AttrType::Placeholder) diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 995202dbcb..129f32e155 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -54,6 +54,10 @@ deps_other += boost nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json +# eval-cache uses `INSERT ... RETURNING` which requires >= 3.35 +sqlite = dependency('sqlite3', 'sqlite', version : '>= 3.35') +deps_private += sqlite + bdw_gc_required = get_option('gc').disable_if( 'address' in get_option('b_sanitize'), error_message : 'Building with Boehm GC and ASAN is not supported', diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index 6eccc06451..b5bccc9079 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -13,6 +13,7 @@ boost, boehmgc, nlohmann_json, + sqlite, toml11, wasmtime, @@ -69,6 +70,7 @@ mkMesonLibrary (finalAttrs: { ]; buildInputs = [ + sqlite toml11 ] ++ lib.optional enableWasm wasmtime; From 3b7d4c97294f695d19e48b3e163e8e2238b72d17 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 18 Feb 2026 04:32:44 +0100 Subject: [PATCH 660/927] test(eval-cache): add basic caching test Verify that evaluation caching works on the second build by checking that builtins.trace output only appears on the first evaluation. --- tests/functional/flakes/eval-cache.sh | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/functional/flakes/eval-cache.sh b/tests/functional/flakes/eval-cache.sh index f4e48d8bb7..450e59e95a 100755 --- a/tests/functional/flakes/eval-cache.sh +++ b/tests/functional/flakes/eval-cache.sh @@ -4,6 +4,42 @@ source ./common.sh requireGit +# Test basic caching: trace should only appear on first evaluation +basicCacheDir="$TEST_ROOT/basic-cache-flake" +createGitRepo "$basicCacheDir" "" +cp "${config_nix}" "$basicCacheDir/" +git -C "$basicCacheDir" add config.nix +git -C "$basicCacheDir" commit -m "config.nix" + +cat >"$basicCacheDir/flake.nix" < throw "pure-eval not enabled"; + builtins.trace "basic-test-evaluating" (mkDerivation { + name = "cached-build"; + buildCommand = "echo hello > \\\$out"; + }); + }; +} +EOF + +git -C "$basicCacheDir" add flake.nix +git -C "$basicCacheDir" commit -m "Init" + +clearBinaryCache + +# First build should show trace +nix build --no-link "$basicCacheDir" 2>&1 | grepQuiet 'trace: basic-test-evaluating' + +# Second build should use cache, no trace output +nix build --no-link "$basicCacheDir" 2>&1 | grepQuietInverse 'trace: basic-test-evaluating' + +# Third build should also use cache, no trace output +nix build --no-link "$basicCacheDir" 2>&1 | grepQuietInverse 'trace: basic-test-evaluating' + +# Test edge cases with separate flake flake1Dir="$TEST_ROOT/eval-cache-flake" createGitRepo "$flake1Dir" "" From 4131627604a0963a11d2a7f504ecd83a1f99ea57 Mon Sep 17 00:00:00 2001 From: Riccardo Mazzarini Date: Mon, 31 Aug 2026 19:50:41 +0200 Subject: [PATCH 661/927] libexpr-c: allow null primop docs Problem: `nix_alloc_primop` crashes when the doc pointer is null, despite the C API documenting it as optional. Solution: treat null as `std::nullopt` (and add a regression test). (cherry picked from commit 53440e1bfae871f8c0cc5ee57af77a9e5568b57e) --- src/libexpr-c/nix_api_value.cc | 2 +- src/libexpr-tests/nix_api_expr.cc | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 3cb5705898..b52dab7024 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -156,7 +156,7 @@ PrimOp * nix_alloc_primop( .name = name, .args = {}, .arity = (size_t) arity, - .doc = doc, + .doc = doc ? std::optional{doc} : std::nullopt, .impl = std::bind(nix_c_primop_wrapper, fun, user_data, arity, _1, _2, _3, _4)}; if (args) for (size_t i = 0; args[i]; i++) diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index 8362e68508..3ceb0a24c0 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -244,6 +244,16 @@ primop_square(void * user_data, nix_c_context * context, EvalState * state, nix_ nix_init_int(context, ret, i * i); } +TEST_F(nix_api_expr_test, nix_alloc_primop_without_doc) +{ + PrimOp * primop = nix_alloc_primop(ctx, primop_square, 1, "undocumented", nullptr, nullptr, nullptr); + assert_ctx_ok(); + ASSERT_NE(nullptr, primop); + + nix_gc_decref(ctx, primop); + assert_ctx_ok(); +} + TEST_F(nix_api_expr_test, nix_expr_primop) { PrimOp * primop = nix_alloc_primop( From 97dce1f68aa8905386be7381696d29f323b695ce Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Sep 2026 17:07:13 +0200 Subject: [PATCH 662/927] doc: mention that flake metadata's Path field is absent with lazy-trees Fixes https://github.com/DeterminateSystems/nix-src/issues/589. Assisted-by: Claude Fable 5 --- src/nix/flake-metadata.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/flake-metadata.md b/src/nix/flake-metadata.md index b997700499..6625a9356f 100644 --- a/src/nix/flake-metadata.md +++ b/src/nix/flake-metadata.md @@ -69,7 +69,7 @@ data. This includes: * `Description`: A one-line description of the flake, taken from the `description` field in `flake.nix`. -* `Path`: The store path containing the source code of the flake. +* `Path`: The store path containing the source code of the flake. This field is absent if the [`lazy-trees`](@docroot@/command-ref/conf-file.md#conf-lazy-trees) setting is enabled, since then the flake is not required to reside in the Nix store. * `Revision`: The Git or Mercurial commit hash of the locked flake. @@ -97,7 +97,7 @@ With `--json`, the output is a JSON object with the following fields: * `description`: See `Description` above. -* `path`: See `Path` above. +* `path`: See `Path` above. This field is absent if the [`lazy-trees`](@docroot@/command-ref/conf-file.md#conf-lazy-trees) setting is enabled. * `revision`: See `Revision` above. From 34105bd1bade54b06fe7254d0e0def56c8e9ba29 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Sep 2026 15:56:25 +0200 Subject: [PATCH 663/927] fix(cli): don't return duplicate installables results quadratically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With duplicate installables (e.g. 'nix build nixpkgs#hello nixpkgs#hello nixpkgs#hello'), Installable::build2() pushed the same DerivedPath into pathsToBuild once per installable, while also recording each installable in backmap. Since buildPathsWithResults() returns one result per request, fanning each of the n results out to all n backmap entries produced n² results — printing every output path n² times with --print-out-paths, and likewise duplicating --json entries and --out-link symlinks. Instead, record all requests in a vector in command-line order and deduplicate pathsToBuild separately, then produce results by looking up each request's build result. This yields exactly one result per installable while preserving command-line order even when duplicates are interleaved (e.g. 'nix build A B A' yields results for A, B, A rather than grouping the duplicates). Assisted-by: Claude Fable 5 --- src/libcmd/installables.cc | 91 ++++++++++++++++++++------------------ tests/functional/build.sh | 16 +++++++ 2 files changed, 64 insertions(+), 43 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 2285e0c3cf..cdb11914cc 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -535,19 +535,24 @@ std::vector Installable::build2( if (mode == Realise::Nothing) settings.readOnlyMode = true; - struct Aux + struct Request { + DerivedPath path; ref info; ref installable; }; + /* All requests in command-line order, which the results must + preserve; pathsToBuild holds each distinct derived path once. */ + std::vector requests; std::vector pathsToBuild; - std::map> backmap; + std::set seen; for (auto & i : installables) { for (auto b : i->toDerivedPaths()) { - pathsToBuild.push_back(b.path); - backmap[b.path].push_back({.info = b.info, .installable = i}); + if (seen.insert(b.path).second) + pathsToBuild.push_back(b.path); + requests.push_back({.path = b.path, .info = b.info, .installable = i}); } } @@ -559,32 +564,30 @@ std::vector Installable::build2( case Realise::Derivation: printMissing(store, pathsToBuild, lvlError); - for (auto & path : pathsToBuild) { - for (auto & aux : backmap[path]) { - std::visit( - overloaded{ - [&](const DerivedPath::Built & bfd) { - auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); - res.push_back( - {.installable = aux.installable, - .result = InstallableWithBuildResult::Success{ - .path = - BuiltPath::Built{ - .drvPath = make_ref( - getBuiltPath(evalStore, store, *bfd.drvPath)), - .outputs = outputs, - }, - .info = aux.info}}); - }, - [&](const DerivedPath::Opaque & bo) { - res.push_back( - {.installable = aux.installable, - .result = InstallableWithBuildResult::Success{ - .path = BuiltPath::Opaque{bo.path}, .info = aux.info}}); - }, + for (auto & req : requests) { + std::visit( + overloaded{ + [&](const DerivedPath::Built & bfd) { + auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); + res.push_back( + {.installable = req.installable, + .result = InstallableWithBuildResult::Success{ + .path = + BuiltPath::Built{ + .drvPath = + make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, + .info = req.info}}); }, - path.raw()); - } + [&](const DerivedPath::Opaque & bo) { + res.push_back( + {.installable = req.installable, + .result = InstallableWithBuildResult::Success{ + .path = BuiltPath::Opaque{bo.path}, .info = req.info}}); + }, + }, + req.path.raw()); } break; @@ -594,23 +597,25 @@ std::vector Installable::build2( printMissing(store, pathsToBuild, lvlInfo); auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); - for (auto & buildResult : buildResults) { + + std::map resultsByPath; + for (auto & buildResult : buildResults) + resultsByPath.emplace(buildResult.path, &buildResult); + + for (auto & req : requests) { + auto & buildResult = *resultsByPath.at(req.path); if (buildResult.tryGetFailure()) { - for (auto & aux : backmap[buildResult.path]) { - res.push_back({.installable = aux.installable, .result = buildResult}); - } + res.push_back({.installable = req.installable, .result = buildResult}); continue; } - for (auto & aux : backmap[buildResult.path]) { - res.push_back({ - aux.installable, - BuiltPathWithResult{ - .path = toBuiltPath(buildResult, evalStore, store), - .info = aux.info, - .result = buildResult, - }, - }); - } + res.push_back({ + req.installable, + BuiltPathWithResult{ + .path = toBuiltPath(buildResult, evalStore, store), + .info = req.info, + .result = buildResult, + }, + }); } break; diff --git a/tests/functional/build.sh b/tests/functional/build.sh index 0dd3d289cd..705759d641 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -17,6 +17,22 @@ nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' (.out | match(".*multiple-outputs-b")))) ' +# Duplicate installables should yield one result per command-line +# argument (not one per argument squared), in command-line order even +# when duplicates are interleaved with other installables. +[[ $(nix build -f multiple-outputs.nix --no-link --print-out-paths b b b | wc -l) = 3 ]] +out=$(nix build -f multiple-outputs.nix --no-link --print-out-paths b nothing-to-install b) +[[ $(echo "$out" | wc -l) = 3 ]] +[[ $(echo "$out" | sed -n 1p) = $(echo "$out" | sed -n 3p) ]] +echo "$out" | sed -n 1p | grepQuiet "multiple-outputs-b" +echo "$out" | sed -n 2p | grepQuiet "nothing-to-install" +nix build -f multiple-outputs.nix --no-link --json b nothing-to-install b | jq --exit-status ' + length == 3 + and (.[0].drvPath | match(".*multiple-outputs-b.drv")) + and (.[1].drvPath | match(".*nothing-to-install.drv")) + and (.[2].drvPath | match(".*multiple-outputs-b.drv")) +' + # Test output selection using the '^' syntax. nix build -f multiple-outputs.nix --json a^first --no-link | jq --exit-status ' (.[0] | From 4cbb12c5cafdc89b61e12cb67b049516a5d7b72b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:06:13 +0000 Subject: [PATCH 664/927] Prepare release v3.22.3 From 98b3f0ec6bb6677fbaf703f9d928155453aa6505 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:06:16 +0000 Subject: [PATCH 665/927] packaging/secure-packages/flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'path:../..' → 'path:../..' • Updated input 'nix/nixpkgs': 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013487%2Brev-01649628bfcaf28024737daec63d559fe0463e6c/01a04eb4-9c90-76b3-a3fb-a92559e04618/source.tar.gz' (2026-08-29) → 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013502%2Brev-88369aab48f8d2b0f69126d306c5af06e997849b/01a05deb-f862-7c32-a065-599b037b2e53/source.tar.gz' (2026-09-01) --- packaging/secure-packages/flake.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index ed100b9a76..ff01da9861 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -77,12 +77,12 @@ }, "nixpkgs": { "locked": { - "lastModified": 1788025223, - "narHash": "sha256-TWYxqK6AnH3/AXPSowct3pxzafZ501CjVuJGu1kBUJw=", - "rev": "01649628bfcaf28024737daec63d559fe0463e6c", - "revCount": 1013487, + "lastModified": 1788280191, + "narHash": "sha256-mB3qK5ql+QFtajxSI/qwCPRPaJMuWyqIKnTZvADKwU8=", + "rev": "88369aab48f8d2b0f69126d306c5af06e997849b", + "revCount": 1013502, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013487%2Brev-01649628bfcaf28024737daec63d559fe0463e6c/01a04eb4-9c90-76b3-a3fb-a92559e04618/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013502%2Brev-88369aab48f8d2b0f69126d306c5af06e997849b/01a05deb-f862-7c32-a065-599b037b2e53/source.tar.gz" }, "original": { "type": "tarball", From 96e755aa8b6602e236b6891663b83c527684e326 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:06:19 +0000 Subject: [PATCH 666/927] Set .version-determinate to 3.22.3 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index 2fec58b681..ce070a60eb 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.22.2 +3.22.3 From 648a0ea3275d5c09b522ad7247354a5c9ca15e2f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:06:24 +0000 Subject: [PATCH 667/927] Generate release notes for 3.22.3 --- doc/manual/source/SUMMARY.md.in | 1 + .../release-notes-determinate/changes.md | 32 ++++++++++++++++++- .../release-notes-determinate/v3.22.3.md | 22 +++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/v3.22.3.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 224d901c2e..308569fb90 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -154,6 +154,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.22.3 (2026-09-02)](release-notes-determinate/v3.22.3.md) - [Release 3.22.2 (2026-08-21)](release-notes-determinate/v3.22.2.md) - [Release 3.22.1 (2026-08-18)](release-notes-determinate/v3.22.1.md) - [Release 3.22.0 (2026-08-06)](release-notes-determinate/v3.22.0.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 4de2651e51..53560f33de 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.35 and Determinate Nix 3.22.2. +This section lists the differences between upstream Nix 2.35 and Determinate Nix 3.22.3. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -232,3 +232,33 @@ This section lists the differences between upstream Nix 2.35 and Determinate Nix * `nix eval --json` has a flag `--drv-link` to create symlinks to top-level derivations. This prevents them from being garbage-collected. [DeterminateSystems/nix-src#599](https://github.com/DeterminateSystems/nix-src/pull/599) + + + +* Fix `dir` attribute handling in nix-219-compat mode by @edolstra in [DeterminateSystems/nix-src#604](https://github.com/DeterminateSystems/nix-src/pull/604) + +* Update the schema file atomically by @edolstra in [DeterminateSystems/nix-src#606](https://github.com/DeterminateSystems/nix-src/pull/606) + +* Reduce contention on the bdwgc global lock by @edolstra in [DeterminateSystems/nix-src#608](https://github.com/DeterminateSystems/nix-src/pull/608) + +* fromJSON: Eliminate use of uncollectable containers by @edolstra in [DeterminateSystems/nix-src#609](https://github.com/DeterminateSystems/nix-src/pull/609) + +* nix copy: Don't register temp roots if the remote doesn't have AddTempRoots by @edolstra in [DeterminateSystems/nix-src#610](https://github.com/DeterminateSystems/nix-src/pull/610) + +* Update Nixpkgs by @edolstra in [DeterminateSystems/nix-src#607](https://github.com/DeterminateSystems/nix-src/pull/607) + +* Fix crash when GC is interrupted by @edolstra in [DeterminateSystems/nix-src#612](https://github.com/DeterminateSystems/nix-src/pull/612) + +* Handle out-of-memory crashes better by @edolstra in [DeterminateSystems/nix-src#603](https://github.com/DeterminateSystems/nix-src/pull/603) + +* Fix forwarding netrc-file to the daemon by @edolstra in [DeterminateSystems/nix-src#616](https://github.com/DeterminateSystems/nix-src/pull/616) + +* Sync with upstream 2.35-maintenance by @edolstra in [DeterminateSystems/nix-src#617](https://github.com/DeterminateSystems/nix-src/pull/617) + +* Fix Ctrl-C handling when Sentry is enabled by @edolstra in [DeterminateSystems/nix-src#618](https://github.com/DeterminateSystems/nix-src/pull/618) + +* Eval cache: make the second evaluation hit the cache by @edolstra in [DeterminateSystems/nix-src#619](https://github.com/DeterminateSystems/nix-src/pull/619) + +* doc: mention that flake metadata's `Path` field is absent with lazy-trees by @edolstra in [DeterminateSystems/nix-src#620](https://github.com/DeterminateSystems/nix-src/pull/620) + +* Deduplicate `nix build --print-out-paths` output by @edolstra in [DeterminateSystems/nix-src#622](https://github.com/DeterminateSystems/nix-src/pull/622) diff --git a/doc/manual/source/release-notes-determinate/v3.22.3.md b/doc/manual/source/release-notes-determinate/v3.22.3.md new file mode 100644 index 0000000000..1203ed34c1 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.22.3.md @@ -0,0 +1,22 @@ +# Release 3.22.3 (2026-09-02) + +* Based on [upstream Nix 2.35.3](../release-notes/rl-2.35.md). + +## What's Changed +* Fix `dir` attribute handling in nix-219-compat mode by @edolstra in [DeterminateSystems/nix-src#604](https://github.com/DeterminateSystems/nix-src/pull/604) +* Update the schema file atomically by @edolstra in [DeterminateSystems/nix-src#606](https://github.com/DeterminateSystems/nix-src/pull/606) +* Reduce contention on the bdwgc global lock by @edolstra in [DeterminateSystems/nix-src#608](https://github.com/DeterminateSystems/nix-src/pull/608) +* fromJSON: Eliminate use of uncollectable containers by @edolstra in [DeterminateSystems/nix-src#609](https://github.com/DeterminateSystems/nix-src/pull/609) +* nix copy: Don't register temp roots if the remote doesn't have AddTempRoots by @edolstra in [DeterminateSystems/nix-src#610](https://github.com/DeterminateSystems/nix-src/pull/610) +* Update Nixpkgs by @edolstra in [DeterminateSystems/nix-src#607](https://github.com/DeterminateSystems/nix-src/pull/607) +* Fix crash when GC is interrupted by @edolstra in [DeterminateSystems/nix-src#612](https://github.com/DeterminateSystems/nix-src/pull/612) +* Handle out-of-memory crashes better by @edolstra in [DeterminateSystems/nix-src#603](https://github.com/DeterminateSystems/nix-src/pull/603) +* Fix forwarding netrc-file to the daemon by @edolstra in [DeterminateSystems/nix-src#616](https://github.com/DeterminateSystems/nix-src/pull/616) +* Sync with upstream 2.35-maintenance by @edolstra in [DeterminateSystems/nix-src#617](https://github.com/DeterminateSystems/nix-src/pull/617) +* Fix Ctrl-C handling when Sentry is enabled by @edolstra in [DeterminateSystems/nix-src#618](https://github.com/DeterminateSystems/nix-src/pull/618) +* Eval cache: make the second evaluation hit the cache by @edolstra in [DeterminateSystems/nix-src#619](https://github.com/DeterminateSystems/nix-src/pull/619) +* doc: mention that flake metadata's `Path` field is absent with lazy-trees by @edolstra in [DeterminateSystems/nix-src#620](https://github.com/DeterminateSystems/nix-src/pull/620) +* Deduplicate `nix build --print-out-paths` output by @edolstra in [DeterminateSystems/nix-src#622](https://github.com/DeterminateSystems/nix-src/pull/622) + + +**Full Changelog**: [v3.22.2...v3.22.3](https://github.com/DeterminateSystems/nix-src/compare/v3.22.2...v3.22.3) From 4ede2f57d5c8b44a34ba54221893d74e54066c32 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Sep 2026 22:00:34 +0200 Subject: [PATCH 668/927] Tweak release notes Also drop the Nix version number down to 2.35.2 since there is no 2.35.3 yet. --- .version | 2 +- .../release-notes-determinate/changes.md | 28 ------------------- .../release-notes-determinate/v3.22.3.md | 23 +++++++++------ 3 files changed, 16 insertions(+), 37 deletions(-) diff --git a/.version b/.version index c819688d6b..c0ac27fe01 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.35.3 +2.35.2 diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 53560f33de..27aa8517a5 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -234,31 +234,3 @@ This section lists the differences between upstream Nix 2.35 and Determinate Nix * `nix eval --json` has a flag `--drv-link` to create symlinks to top-level derivations. This prevents them from being garbage-collected. [DeterminateSystems/nix-src#599](https://github.com/DeterminateSystems/nix-src/pull/599) - -* Fix `dir` attribute handling in nix-219-compat mode by @edolstra in [DeterminateSystems/nix-src#604](https://github.com/DeterminateSystems/nix-src/pull/604) - -* Update the schema file atomically by @edolstra in [DeterminateSystems/nix-src#606](https://github.com/DeterminateSystems/nix-src/pull/606) - -* Reduce contention on the bdwgc global lock by @edolstra in [DeterminateSystems/nix-src#608](https://github.com/DeterminateSystems/nix-src/pull/608) - -* fromJSON: Eliminate use of uncollectable containers by @edolstra in [DeterminateSystems/nix-src#609](https://github.com/DeterminateSystems/nix-src/pull/609) - -* nix copy: Don't register temp roots if the remote doesn't have AddTempRoots by @edolstra in [DeterminateSystems/nix-src#610](https://github.com/DeterminateSystems/nix-src/pull/610) - -* Update Nixpkgs by @edolstra in [DeterminateSystems/nix-src#607](https://github.com/DeterminateSystems/nix-src/pull/607) - -* Fix crash when GC is interrupted by @edolstra in [DeterminateSystems/nix-src#612](https://github.com/DeterminateSystems/nix-src/pull/612) - -* Handle out-of-memory crashes better by @edolstra in [DeterminateSystems/nix-src#603](https://github.com/DeterminateSystems/nix-src/pull/603) - -* Fix forwarding netrc-file to the daemon by @edolstra in [DeterminateSystems/nix-src#616](https://github.com/DeterminateSystems/nix-src/pull/616) - -* Sync with upstream 2.35-maintenance by @edolstra in [DeterminateSystems/nix-src#617](https://github.com/DeterminateSystems/nix-src/pull/617) - -* Fix Ctrl-C handling when Sentry is enabled by @edolstra in [DeterminateSystems/nix-src#618](https://github.com/DeterminateSystems/nix-src/pull/618) - -* Eval cache: make the second evaluation hit the cache by @edolstra in [DeterminateSystems/nix-src#619](https://github.com/DeterminateSystems/nix-src/pull/619) - -* doc: mention that flake metadata's `Path` field is absent with lazy-trees by @edolstra in [DeterminateSystems/nix-src#620](https://github.com/DeterminateSystems/nix-src/pull/620) - -* Deduplicate `nix build --print-out-paths` output by @edolstra in [DeterminateSystems/nix-src#622](https://github.com/DeterminateSystems/nix-src/pull/622) diff --git a/doc/manual/source/release-notes-determinate/v3.22.3.md b/doc/manual/source/release-notes-determinate/v3.22.3.md index 1203ed34c1..db6cab11ff 100644 --- a/doc/manual/source/release-notes-determinate/v3.22.3.md +++ b/doc/manual/source/release-notes-determinate/v3.22.3.md @@ -1,22 +1,29 @@ # Release 3.22.3 (2026-09-02) -* Based on [upstream Nix 2.35.3](../release-notes/rl-2.35.md). +* Based on [upstream Nix 2.35.2](../release-notes/rl-2.35.md). + +## Parallel evaluation performance improvements + +We're continuing to make performance improvements to parallel evaluation, in particular reducing contention on the Boehm-Demers-Weiser garbage collector's global lock by making more use of per-thread freelists and by removing the use of uncollectable allocations in `builtins.fromJSON`. + +PRs: [DeterminateSystems/nix-src#608](https://github.com/DeterminateSystems/nix-src/pull/608), [DeterminateSystems/nix-src#609](https://github.com/DeterminateSystems/nix-src/pull/609) + +## Miscellaneous changes + +* Update Nixpkgs by @edolstra in [DeterminateSystems/nix-src#607](https://github.com/DeterminateSystems/nix-src/pull/607) +* doc: mention that flake metadata's `Path` field is absent with lazy-trees by @edolstra in [DeterminateSystems/nix-src#620](https://github.com/DeterminateSystems/nix-src/pull/620) + +## Bug fixes -## What's Changed * Fix `dir` attribute handling in nix-219-compat mode by @edolstra in [DeterminateSystems/nix-src#604](https://github.com/DeterminateSystems/nix-src/pull/604) * Update the schema file atomically by @edolstra in [DeterminateSystems/nix-src#606](https://github.com/DeterminateSystems/nix-src/pull/606) -* Reduce contention on the bdwgc global lock by @edolstra in [DeterminateSystems/nix-src#608](https://github.com/DeterminateSystems/nix-src/pull/608) -* fromJSON: Eliminate use of uncollectable containers by @edolstra in [DeterminateSystems/nix-src#609](https://github.com/DeterminateSystems/nix-src/pull/609) * nix copy: Don't register temp roots if the remote doesn't have AddTempRoots by @edolstra in [DeterminateSystems/nix-src#610](https://github.com/DeterminateSystems/nix-src/pull/610) -* Update Nixpkgs by @edolstra in [DeterminateSystems/nix-src#607](https://github.com/DeterminateSystems/nix-src/pull/607) * Fix crash when GC is interrupted by @edolstra in [DeterminateSystems/nix-src#612](https://github.com/DeterminateSystems/nix-src/pull/612) * Handle out-of-memory crashes better by @edolstra in [DeterminateSystems/nix-src#603](https://github.com/DeterminateSystems/nix-src/pull/603) * Fix forwarding netrc-file to the daemon by @edolstra in [DeterminateSystems/nix-src#616](https://github.com/DeterminateSystems/nix-src/pull/616) * Sync with upstream 2.35-maintenance by @edolstra in [DeterminateSystems/nix-src#617](https://github.com/DeterminateSystems/nix-src/pull/617) * Fix Ctrl-C handling when Sentry is enabled by @edolstra in [DeterminateSystems/nix-src#618](https://github.com/DeterminateSystems/nix-src/pull/618) -* Eval cache: make the second evaluation hit the cache by @edolstra in [DeterminateSystems/nix-src#619](https://github.com/DeterminateSystems/nix-src/pull/619) -* doc: mention that flake metadata's `Path` field is absent with lazy-trees by @edolstra in [DeterminateSystems/nix-src#620](https://github.com/DeterminateSystems/nix-src/pull/620) +* Eval cache: make the second evaluation hit the cache by @roberth in [DeterminateSystems/nix-src#619](https://github.com/DeterminateSystems/nix-src/pull/619) * Deduplicate `nix build --print-out-paths` output by @edolstra in [DeterminateSystems/nix-src#622](https://github.com/DeterminateSystems/nix-src/pull/622) - **Full Changelog**: [v3.22.2...v3.22.3](https://github.com/DeterminateSystems/nix-src/compare/v3.22.2...v3.22.3) From beed3dd4a2cf43a044521a8b53a0b8600e8660b4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 2 Sep 2026 22:11:28 +0200 Subject: [PATCH 669/927] fix(packaging): do not add boost fiber patch if duplicate Makes the flake with newer nixpkgs, e.g. with input overrides. Also takes into account the applicable version range. (cherry picked from commit 67b94a7abdf040a7c43ec2fbdc4a0903b74d5434) --- packaging/dependencies.nix | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 7edb7413b9..69c25b2b81 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -98,9 +98,14 @@ scope: { "--with-iostreams" "--with-url" ]; - patches = [ - ./patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch - ]; + patches = lib.optional ( + lib.versionAtLeast pkgs.boost.version "1.88" + && lib.versionOlder pkgs.boost.version "1.92" + # may already be done in nixpkgs: https://github.com/NixOS/nixpkgs/pull/546405 + && !lib.any (patch: lib.hasInfix "5883212311535a0046031d74d1568ae173c1e35b" (baseNameOf patch)) ( + pkgs.boost.patches or [ ] + ) + ) ./patches/0001-Fix-uncaught_exceptions-not-accounting-for-forced_un.patch; enableIcu = false; }).overrideAttrs (old: { From 22af94b73c3d263d9199dfe841db6c5c367c043a Mon Sep 17 00:00:00 2001 From: Josh Spence Date: Thu, 3 Sep 2026 17:59:43 +1000 Subject: [PATCH 670/927] Fix the Nix < 2.20 NAR hash fallback for inputs with submodules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `GitInputScheme::getAccessorFromCommit()` decided whether a lock was produced with Nix < 2.20 or Nix >= 2.20 export semantics *before* mounting submodules. For an input with `submodules = true` the expected NAR hash covers the mounted tree, while both candidate hashes covered only the bare top-level repo, so neither could ever match. Instead of falling back to `git archive`/`git checkout` and warning, Nix hard-failed with a NAR hash mismatch: error: NAR hash mismatch in input 'git+ssh://git@example.org/repo.git?rev=...&shallow=1&submodules=1', expected 'sha256-CnkK...' but got 'sha256-XNEq...' `nix flake update ` recomputes rather than verifies the hash, so the failure only showed up on the read path — anyone evaluating a lock file written by Nix < 2.20 (or by Nix >= 2.20 with `nix-219-compat` enabled) for a repo with submodules could not evaluate it at all. Move the submodule mounting into a `getTree()` lambda so both candidate trees are fully mounted before their NAR hashes are compared. Since the Git filters that Nix < 2.20 applied also affect submodule contents, reproducing such a hash requires exporting the submodules with the same semantics; propagate this through the synthesized submodule inputs via a new internal `__legacyExport` attribute (not part of `allowedAttrs()`, never serialized into a lock file, following the `__final` convention). When it's absent we're a top-level input and `nix-219-compat` decides; when it's present we follow the top-level repo regardless of the setting. `GitAccessorOptions` gains a matching `legacy` field so that `makeFingerprint()` accounts for it. It is appended last, keeping the legacy cache keys byte-identical to the previous hand-concatenated `makeFingerprint(rev) + ";legacy"`. Assisted-by: Claude Opus 5 --- src/libfetchers/fetchers.cc | 4 +- src/libfetchers/git-utils.cc | 2 +- src/libfetchers/git.cc | 183 ++++++++++-------- .../include/nix/fetchers/git-utils.hh | 7 + tests/functional/fetchGitSubmodules.sh | 59 ++++++ 5 files changed, 176 insertions(+), 79 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index b1bc134ac3..dde72d79e2 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -105,7 +105,9 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs) auto allowedAttrs = inputScheme->allowedAttrs(); for (auto & [name, _] : attrs) - if (name != "type" && name != "__final" && allowedAttrs.count(name) == 0) + /* `__final` and `__legacyExport` are purely internal attributes, so they're not listed in + `allowedAttrs()`. */ + if (name != "type" && name != "__final" && name != "__legacyExport" && allowedAttrs.count(name) == 0) throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName); auto res = inputScheme->inputFromAttrs(settings, attrs); diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 52986a6b45..7dc4d7c5e7 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -817,7 +817,7 @@ ref GitRepo::openRepo(const std::filesystem::path & path, GitRepo::Opti std::string GitAccessorOptions::makeFingerprint(const Hash & rev) const { - return "git:" + rev.gitRev() + (exportIgnore ? ";e" : "") + (smudgeLfs ? ";l" : ""); + return "git:" + rev.gitRev() + (exportIgnore ? ";e" : "") + (smudgeLfs ? ";l" : "") + (legacy ? ";legacy" : ""); } /** diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7f560e0d2b..5b74b104cb 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -618,6 +618,19 @@ struct GitInputScheme : InputScheme return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); } + /** + * Whether to export this input using Nix < 2.20 semantics. This is a purely internal + * attribute: it is set only on the submodule inputs synthesized by + * `getAccessorFromCommit()`, so that the export of a repository composes the export of the + * top-level repo with exports of its submodules using the *same* semantics. If it's absent, + * we're a top-level input and the `nix-219-compat` setting decides. It is deliberately not + * part of `allowedAttrs()`. + */ + std::optional getLegacyExportAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "__legacyExport"); + } + RepoInfo getRepoInfo(const Input & input) const { auto checkHashAlgorithm = [&](const std::optional & hash) { @@ -832,6 +845,7 @@ struct GitInputScheme : InputScheme .exportIgnore = getExportIgnoreAttr(input), .smudgeLfs = getLfsAttr(input), .submodules = getSubmodulesAttr(input), + .legacy = getLegacyExportAttr(input).value_or(false), }; } @@ -850,7 +864,9 @@ struct GitInputScheme : InputScheme if (!options.submodules) options.exportIgnore = true; - auto fingerprint = options.makeFingerprint(rev) + ";legacy"; + options.legacy = true; + + auto fingerprint = options.makeFingerprint(rev); auto cacheKey = makeSourcePathToHashCacheKey(fingerprint, ContentAddressMethod::Raw::NixArchive, CanonPath::root); @@ -1073,95 +1089,108 @@ struct GitInputScheme : InputScheme auto expectedNarHash = input.getNarHash(); - auto accessor = repo->getAccessor(rev, options, "«" + input.to_string(true) + "»"); + /* Return an accessor for the complete tree denoted by `rev`, that is, the top-level repo + with any submodules mounted into it. If `legacy` is set, Nix < 2.20 semantics are used + (i.e. `git archive` / `git checkout`, which apply Git filters, `export-ignore` and + `export-subst`); the submodules are then exported using those semantics as well. - if (settings.nix219Compat && !options.smudgeLfs) { - /* Use Nix 2.19 semantics to generate locks, but if a NAR hash is specified, support Nix >= 2.20 semantics - * as well. */ - warn("Using Nix 2.19 semantics to export Git repository '%s'.", input.to_string()); - auto accessorModern = accessor; - accessor = getLegacyGitAccessor(settings, store, repoInfo, repoDir, rev, options); - if (expectedNarHash) { - auto narHashLegacy = - fetchToStore2(settings, store, {accessor}, FetchMode::DryRun, input.getName()).second; - if (expectedNarHash != narHashLegacy) { - auto narHashModern = - fetchToStore2(settings, store, {accessorModern}, FetchMode::DryRun, input.getName()).second; - if (expectedNarHash == narHashModern) - accessor = accessorModern; + Note that submodules must be mounted before the NAR hash of the tree can be compared + against `expectedNarHash`, since that hash covers the submodule contents as well. */ + auto getTree = [&](bool legacy) -> nix::ref { + auto options2 = options; + options2.legacy = legacy; + + auto accessor = legacy ? getLegacyGitAccessor(settings, store, repoInfo, repoDir, rev, options2) + : repo->getAccessor(rev, options2, "«" + input.to_string(true) + "»"); + + /* If the repo has submodules, fetch them and return a mounted + input accessor consisting of the accessor for the top-level + repo and the accessors for the submodules. */ + if (options2.submodules) { + std::map> mounts; + + for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, options2.exportIgnore)) { + auto resolved = repo->resolveSubmoduleUrl(submodule.url); + debug( + "Git submodule %s: %s %s %s -> %s", + submodule.path, + submodule.url, + submodule.branch, + submoduleRev.gitRev(), + resolved); + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", resolved); + if (submodule.branch != "") { + // A special value of . is used to indicate that the name of the branch in the submodule + // should be the same name as the current branch in the current repository. + // https://git-scm.com/docs/gitmodules + if (submodule.branch == ".") { + attrs.insert_or_assign("ref", ref); + } else { + attrs.insert_or_assign("ref", submodule.branch); + } + } + attrs.insert_or_assign("rev", submoduleRev.gitRev()); + attrs.insert_or_assign("exportIgnore", Explicit{options2.exportIgnore}); + attrs.insert_or_assign("submodules", Explicit{true}); + attrs.insert_or_assign("lfs", Explicit{options2.smudgeLfs}); + attrs.insert_or_assign("allRefs", Explicit{true}); + /* Export the submodule using the same semantics as the top-level repo, + regardless of the `nix-219-compat` setting. */ + attrs.insert_or_assign("__legacyExport", Explicit{options2.legacy}); + auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); + submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string(true) + "»"); + mounts.insert_or_assign(submodule.path, submoduleAccessor); + } + + if (!mounts.empty()) { + auto newFingerprint = accessor->getFingerprint(CanonPath::root).second->append(";s"); + mounts.insert_or_assign(CanonPath::root, accessor); + auto mounted = makeMountedSourceAccessor(std::move(mounts)); + mounted->fingerprint = newFingerprint; + return mounted; } } - } else { - /* Backward compatibility hack for locks produced by Nix < 2.20 that depend on Nix applying Git filters, - * `export-ignore` or `export-subst`. Nix >= 2.20 doesn't do those, so we may get a NAR hash mismatch. If - * that happens, try again using `git archive`. */ - if (expectedNarHash) { - auto narHashNew = fetchToStore2(settings, store, {accessor}, FetchMode::DryRun, input.getName()).second; - if (expectedNarHash != narHashNew) { - auto accessorLegacy = getLegacyGitAccessor(settings, store, repoInfo, repoDir, rev, options); - auto narHashLegacy = - fetchToStore2(settings, store, {accessorLegacy}, FetchMode::DryRun, input.getName()).second; - if (expectedNarHash == narHashLegacy) { + + return accessor; + }; + + /* Use Nix 2.19 semantics if requested. If we're a submodule, follow whatever the top-level + repo is being exported with instead of the setting. */ + auto legacyExport = getLegacyExportAttr(input); + bool useLegacy = legacyExport.value_or(settings.nix219Compat) && !options.smudgeLfs; + + if (useLegacy && !legacyExport) + warn("Using Nix 2.19 semantics to export Git repository '%s'.", input.to_string()); + + auto accessor = getTree(useLegacy); + + /* Backward compatibility hack: a lock may have been produced by a Nix version that used the + other export semantics. In particular, locks produced by Nix < 2.20 depend on Nix applying + Git filters, `export-ignore` or `export-subst`, while Nix >= 2.20 doesn't do those. So if + we get a NAR hash mismatch, try again using the other semantics. */ + if (expectedNarHash) { + auto narHash = fetchToStore2(settings, store, {accessor}, FetchMode::DryRun, input.getName()).second; + if (*expectedNarHash != narHash) { + auto accessorOther = getTree(!useLegacy); + auto narHashOther = + fetchToStore2(settings, store, {accessorOther}, FetchMode::DryRun, input.getName()).second; + if (*expectedNarHash == narHashOther) { + if (!useLegacy) warn( "Git input '%s' specifies a NAR hash '%s' that was created by Nix < 2.20.\n" "Nix >= 2.20 does not apply Git filters, `export-ignore` and `export-subst` by default, which changes the NAR hash.\n" "Please update the NAR hash to '%s'.", input.to_string(), expectedNarHash->to_string(HashFormat::SRI, true), - narHashNew.to_string(HashFormat::SRI, true)); - accessor = accessorLegacy; - } + narHash.to_string(HashFormat::SRI, true)); + accessor = accessorOther; } } } - /* If the repo has submodules, fetch them and return a mounted - input accessor consisting of the accessor for the top-level - repo and the accessors for the submodules. */ - if (options.submodules) { - std::map> mounts; - - for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, options.exportIgnore)) { - auto resolved = repo->resolveSubmoduleUrl(submodule.url); - debug( - "Git submodule %s: %s %s %s -> %s", - submodule.path, - submodule.url, - submodule.branch, - submoduleRev.gitRev(), - resolved); - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "git"); - attrs.insert_or_assign("url", resolved); - if (submodule.branch != "") { - // A special value of . is used to indicate that the name of the branch in the submodule - // should be the same name as the current branch in the current repository. - // https://git-scm.com/docs/gitmodules - if (submodule.branch == ".") { - attrs.insert_or_assign("ref", ref); - } else { - attrs.insert_or_assign("ref", submodule.branch); - } - } - attrs.insert_or_assign("rev", submoduleRev.gitRev()); - attrs.insert_or_assign("exportIgnore", Explicit{options.exportIgnore}); - attrs.insert_or_assign("submodules", Explicit{true}); - attrs.insert_or_assign("lfs", Explicit{options.smudgeLfs}); - attrs.insert_or_assign("allRefs", Explicit{true}); - auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs)); - auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); - submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string(true) + "»"); - mounts.insert_or_assign(submodule.path, submoduleAccessor); - } - - if (!mounts.empty()) { - auto newFingerprint = accessor->getFingerprint(CanonPath::root).second->append(";s"); - mounts.insert_or_assign(CanonPath::root, accessor); - accessor = makeMountedSourceAccessor(std::move(mounts)); - accessor->fingerprint = newFingerprint; - } - } - assert(!origRev || origRev == rev); return {{accessor, std::move(input)}}; diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 6ffc7372df..f2926e6766 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -32,6 +32,13 @@ struct GitAccessorOptions bool smudgeLfs = false; bool submodules = false; // Currently implemented in GitInputScheme rather than GitAccessor + /** + * Whether to export the repository using Nix < 2.20 semantics, i.e. using `git archive` or + * `git checkout` (which apply Git filters, `export-ignore` and `export-subst`) rather than + * libgit2. Currently implemented in GitInputScheme rather than GitAccessor. + */ + bool legacy = false; + std::string makeFingerprint(const Hash & rev) const; }; diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh index d3f9e4da57..db34dedfc0 100755 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -265,3 +265,62 @@ test_gitlink_without_gitmodules() { done } test_gitlink_without_gitmodules + +# The backward compatibility hack for Nix < 2.20 locks must also work for repos +# fetched with `submodules = true`. Since the NAR hash of such a repo covers the +# submodule contents, deciding whether a lock was produced with Nix < 2.20 +# semantics requires hashing the *mounted* tree, and the submodules have to be +# exported using those same semantics. +test_legacy_export_with_submodules() { + local root=$TEST_ROOT/legacySubmodulesRoot + local sub=$TEST_ROOT/legacySubmodulesSub + + rm -rf "$TEST_HOME"/.cache/nix + + createGitRepo "$sub" + # `text eol=crlf` is a Git filter, so it's applied by `git checkout` (which is + # what Nix < 2.20 used for repos with submodules) but not by libgit2. + printf "crlf text eol=crlf\n" > "$sub"/.gitattributes + printf "Hello\nWorld\n" > "$sub"/crlf + git -C "$sub" add .gitattributes crlf + git -C "$sub" commit -m "Add crlf" + + createGitRepo "$root" + git -C "$root" submodule add "$sub" sub + git -C "$root" commit -m "Add submodule" + + local rev + rev=$(git -C "$root" rev-parse HEAD) + + local input="{ type = \"git\"; url = \"file://$root\"; rev = \"$rev\"; submodules = true; }" + + # Determine the two NAR hashes in a throwaway store: `builtins.fetchTree` marks + # inputs with a `narHash` as final, so if a tree with the expected hash is + # already in the store, it is returned without running the fetcher at all, + # which would mask the behaviour we want to test below. + local scratch=$TEST_ROOT/legacySubmodulesStore + local legacyHash modernHash + legacyHash=$(nix eval --store "$scratch" --nix-219-compat --raw --expr "(builtins.fetchTree $input).narHash") + modernHash=$(nix eval --store "$scratch" --raw --expr "(builtins.fetchTree $input).narHash") + + # If the Git filter in the submodule didn't make a difference, this test + # wouldn't be testing anything. + [[ $legacyHash != "$modernHash" ]] + + # A NAR hash produced by Nix < 2.20 must still be accepted by default (with a + # warning), and must yield the filtered submodule contents. + expectStderr 0 nix eval --expr \ + "let tree = builtins.fetchTree ($input // { narHash = \"$legacyHash\"; }); in assert builtins.readFile \"\${tree}/sub/crlf\" == \"Hello\r\nWorld\r\n\"; true" \ + | grepQuiet "Please update the NAR hash to '$modernHash'" + + # Conversely, a NAR hash produced by Nix >= 2.20 must be accepted even when + # `nix-219-compat` is enabled. + nix eval --nix-219-compat --expr \ + "let tree = builtins.fetchTree ($input // { narHash = \"$modernHash\"; }); in assert builtins.readFile \"\${tree}/sub/crlf\" == \"Hello\nWorld\n\"; true" + + # A NAR hash that matches neither must still be an error. + expectStderr 102 nix eval --expr \ + "builtins.fetchTree ($input // { narHash = \"sha256-DLDvcwdcwCxnuPTxSQ6gLAyopB20lD0bOQoQB3i2hsA=\"; })" \ + | grepQuiet "NAR hash mismatch" +} +test_legacy_export_with_submodules From 94e26ee348038dfd779cad28dec739d203366585 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Aug 2026 15:54:50 +0200 Subject: [PATCH 671/927] libutil: Add optional OpenTelemetry tracing facade Add a vendor-neutral distributed tracing facade (nix::otel) as the foundation for tracing binary cache requests from the Nix client through the daemon to FlakeHub Cache. The facade (nix/util/otel.hh) is flag-independent: no conditionals or OpenTelemetry includes in the installed header. The implementation is gated behind a new 'otel' meson feature option; when disabled (or when enabled but OTEL_EXPORTER_OTLP_ENDPOINT is not set), every operation is a cheap no-op. Design notes: * Spans are copyable shared-handle values, safe to move across threads and to end out of stack order, so they can later back FileTransfer requests (which complete on the curl worker thread) and Activities. * Span::injectContext() returns W3C traceparent/tracestate headers for propagation to HTTP servers and the daemon; startSpanFromRemoteParent() continues a trace on the receiving side. * The tracer provider is owned by a deliberately leaked global instead of opentelemetry's Provider singleton so that no exporter teardown can run from static destructors at exit() time (cf. the OPENSSL_INIT_NO_ATEXIT note in util.cc); flushing happens only via an explicit forceFlushAndShutdown() with bounded timeouts. * Only the OTLP/HTTP exporter is used (not gRPC, which is heavy and fork-unsafe). nixpkgs' opentelemetry-cpp ships no OTLP exporter by default, so packaging/dependencies.nix overrides it with enableHttp = true. Nothing calls init() yet; wiring into the CLI, the daemon, and FileTransfer comes separately. Assisted-by: Claude Fable 5 --- packaging/dependencies.nix | 6 + src/libutil/include/nix/util/meson.build | 1 + src/libutil/include/nix/util/otel.hh | 134 ++++++++++ src/libutil/meson.build | 7 + src/libutil/meson.options | 6 + src/libutil/otel.cc | 309 +++++++++++++++++++++++ src/libutil/package.nix | 7 +- 7 files changed, 469 insertions(+), 1 deletion(-) create mode 100644 src/libutil/include/nix/util/otel.hh create mode 100644 src/libutil/otel.cc diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 183ec3fb6a..0eb3c5e346 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -219,6 +219,12 @@ scope: { inherit (scope) curl; }; + opentelemetry-cpp = pkgs.opentelemetry-cpp.override { + # The default build has no OTLP exporters at all; we need the + # OTLP/HTTP one (and specifically not the gRPC one). + enableHttp = true; + }; + libmicrohttpd = pkgs.libmicrohttpd.overrideDerivation (old: { # Don't pull in gnutls since it's pretty big and we don't need it. configureFlags = old.configureFlags or [ ] ++ [ "--without-gnutls" ]; diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 3af780eb29..994bd479f5 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -70,6 +70,7 @@ headers = [ config_pub_h ] + files( 'nar-cache.hh', 'nar-listing.hh', 'os-string.hh', + 'otel.hh', 'override-provenance-source-accessor.hh', 'pool.hh', 'pos-idx.hh', diff --git a/src/libutil/include/nix/util/otel.hh b/src/libutil/include/nix/util/otel.hh new file mode 100644 index 0000000000..10f1559188 --- /dev/null +++ b/src/libutil/include/nix/util/otel.hh @@ -0,0 +1,134 @@ +#pragma once +///@file +/** + * Vendor-neutral distributed tracing facade. Backed by OpenTelemetry + * (exporting via OTLP/HTTP) when built with `-Dotel=enabled`; + * otherwise all operations are cheap no-ops. Even when compiled in, + * tracing stays disabled unless `init()` is called and + * `OTEL_EXPORTER_OTLP_ENDPOINT` (or + * `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`) is set in the environment. + */ + +#include "nix/util/types.hh" + +#include +#include +#include + +namespace nix::otel { + +enum class SpanKind { + Internal, + Client, + Server, +}; + +struct SpanImpl; + +/** + * A handle to an in-flight trace span. + * + * Value type with shared-handle semantics: copies refer to the same + * underlying span. The span ends when `end()` is first called, or, as + * a fallback, when the last handle is destroyed. Handles may be moved + * or copied across threads; the underlying span is thread-safe. + * + * A default-constructed handle (or any handle obtained while tracing + * is disabled) is inert: all operations are cheap no-ops. + */ +class Span +{ + friend Span startSpan(std::string_view name, SpanKind kind); + friend Span startSpan(std::string_view name, const Span & parent, SpanKind kind); + friend Span startSpanFromRemoteParent( + std::string_view name, std::string_view traceparent, std::string_view tracestate, SpanKind kind); + + std::shared_ptr impl; + + explicit Span(std::shared_ptr impl); + +public: + Span() noexcept; + Span(const Span &); + Span(Span &&) noexcept; + Span & operator=(const Span &); + Span & operator=(Span &&) noexcept; + ~Span(); + + /** + * Whether this span actually records data (i.e. tracing is + * compiled in, initialized, and this span is sampled). + */ + bool isRecording() const noexcept; + + void setAttribute(std::string_view key, std::string_view value); + void setAttribute(std::string_view key, int64_t value); + void setAttribute(std::string_view key, bool value); + + /** + * Mark this span as failed, with a description of the error. + */ + void setError(std::string_view description); + + /** + * End the span. Idempotent; only the first call records the end + * timestamp. + */ + void end(); + + /** + * Return W3C Trace Context headers (`traceparent`, `tracestate`) + * that propagate this span's context to another process, e.g. as + * HTTP request headers. Returns an empty list if this span is + * disabled. + */ + Headers injectContext() const; +}; + +/** + * Initialize tracing for this process. Does nothing (and is cheap) if + * `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` + * is not set in the environment, if tracing support is not compiled + * in, or if tracing is already initialized. Not called from library + * code; the program's main entry point decides. + */ +void init(std::string_view serviceName); + +/** + * Whether tracing is compiled in, initialized and not yet shut down. + */ +bool isEnabled() noexcept; + +/** + * Start a new root span (no parent). + */ +Span startSpan(std::string_view name, SpanKind kind = SpanKind::Internal); + +/** + * Start a span as a child of `parent`. If `parent` is disabled, this + * behaves like the parentless overload. + */ +Span startSpan(std::string_view name, const Span & parent, SpanKind kind = SpanKind::Internal); + +/** + * Start a span whose parent is a span in another process, identified + * by W3C `traceparent` / `tracestate` header values (e.g. received by + * the daemon from a client). Invalid values yield a root span. + */ +Span startSpanFromRemoteParent( + std::string_view name, + std::string_view traceparent, + std::string_view tracestate = {}, + SpanKind kind = SpanKind::Internal); + +/** + * Flush all pending spans and shut down the exporter. Safe to call if + * tracing was never initialized, and safe to call more than once. + * Must be called explicitly before process exit: nothing is flushed + * from static destructors (cf. the OPENSSL_INIT_NO_ATEXIT note in + * util.cc). Must not be called in a fork()ed child of the process + * that called `init()`. + */ +void forceFlushAndShutdown(std::chrono::milliseconds timeout = std::chrono::milliseconds(5000)); + +} // namespace nix::otel diff --git a/src/libutil/meson.build b/src/libutil/meson.build index bf3a1f3b08..cc314742be 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -124,6 +124,12 @@ cpuid = dependency( configdata_priv.set('HAVE_LIBCPUID', cpuid.found().to_int()) deps_private += cpuid +otel_required = get_option('otel') +otel = dependency('opentelemetry_exporter_otlp_http', required : otel_required) +otel_trace = dependency('opentelemetry_trace', required : otel_required) +configdata_priv.set('HAVE_OTEL', (otel.found() and otel_trace.found()).to_int()) +deps_private += [ otel, otel_trace ] + nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -178,6 +184,7 @@ sources = [ config_priv_h ] + files( 'nar-accessor.cc', 'nar-cache.cc', 'nar-listing.cc', + 'otel.cc', 'pos-table.cc', 'position.cc', 'posix-source-accessor.cc', diff --git a/src/libutil/meson.options b/src/libutil/meson.options index a828061629..9c40227c16 100644 --- a/src/libutil/meson.options +++ b/src/libutil/meson.options @@ -5,3 +5,9 @@ option( type : 'feature', description : 'determine microarchitecture levels with libcpuid (only relevant on x86_64)', ) + +option( + 'otel', + type : 'feature', + description : 'distributed tracing via OpenTelemetry (OTLP/HTTP exporter)', +) diff --git a/src/libutil/otel.cc b/src/libutil/otel.cc new file mode 100644 index 0000000000..5a5008c754 --- /dev/null +++ b/src/libutil/otel.cc @@ -0,0 +1,309 @@ +#include "nix/util/otel.hh" + +#include "util-config-private.hh" + +#if HAVE_OTEL +# include "nix/util/environment-variables.hh" + +# include + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + +namespace nix::otel { + +#if HAVE_OTEL + +struct SpanImpl +{ + opentelemetry::nostd::shared_ptr span; +}; + +namespace { + +struct State +{ + std::unique_ptr provider; + opentelemetry::nostd::shared_ptr tracer; +}; + +/* Owned here rather than via opentelemetry's global Provider + singleton, and deliberately leaked: the provider's destructor joins + the batch exporter's worker thread, which must never happen from + static destructors at exit() time (cf. the OPENSSL_INIT_NO_ATEXIT + note in util.cc; in the daemon's forked children that thread does + not even exist). Teardown happens only via an explicit + forceFlushAndShutdown(). */ +std::atomic theState{nullptr}; + +opentelemetry::trace::SpanKind mapKind(SpanKind kind) +{ + switch (kind) { + case SpanKind::Client: + return opentelemetry::trace::SpanKind::kClient; + case SpanKind::Server: + return opentelemetry::trace::SpanKind::kServer; + case SpanKind::Internal: + default: + return opentelemetry::trace::SpanKind::kInternal; + } +} + +inline opentelemetry::nostd::string_view toNostd(std::string_view sv) noexcept +{ + return {sv.data(), sv.size()}; +} + +struct ExtractCarrier : opentelemetry::context::propagation::TextMapCarrier +{ + opentelemetry::nostd::string_view traceparent, tracestate; + + opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view key) const noexcept override + { + if (key == "traceparent") + return traceparent; + if (key == "tracestate") + return tracestate; + return {}; + } + + void Set(opentelemetry::nostd::string_view, opentelemetry::nostd::string_view) noexcept override {} +}; + +struct InjectCarrier : opentelemetry::context::propagation::TextMapCarrier +{ + Headers headers; + + opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view) const noexcept override + { + return {}; + } + + void Set(opentelemetry::nostd::string_view key, opentelemetry::nostd::string_view value) noexcept override + { + /* Copy immediately: `value` may point into a stack buffer of + the propagator. */ + headers.emplace_back(std::string(key.data(), key.size()), std::string(value.data(), value.size())); + } +}; + +} // namespace + +void init(std::string_view serviceName) +{ + if (theState.load(std::memory_order_acquire)) + return; + + /* Without an explicitly configured endpoint, stay off: the SDK's + default would silently export to http://localhost:4318. */ + if (!getEnv("OTEL_EXPORTER_OTLP_ENDPOINT") && !getEnv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")) + return; + + namespace sdktrace = opentelemetry::sdk::trace; + + /* Default-constructed options read the OTEL_EXPORTER_OTLP_* + environment variables (endpoint, headers, TLS, compression). */ + auto exporter = opentelemetry::exporter::otlp::OtlpHttpExporterFactory::Create(); + auto processor = + sdktrace::BatchSpanProcessorFactory::Create(std::move(exporter), sdktrace::BatchSpanProcessorOptions{}); + auto resource = opentelemetry::sdk::resource::Resource::Create({ + {opentelemetry::semconv::service::kServiceName, std::string(serviceName)}, + }); + + auto state = std::make_unique(); + state->provider = sdktrace::TracerProviderFactory::Create(std::move(processor), resource); + state->tracer = state->provider->GetTracer("nix"); + + State * expected = nullptr; + if (theState.compare_exchange_strong(expected, state.get())) + state.release(); +} + +bool isEnabled() noexcept +{ + return theState.load(std::memory_order_acquire) != nullptr; +} + +void forceFlushAndShutdown(std::chrono::milliseconds timeout) +{ + auto * state = theState.exchange(nullptr); + if (!state) + return; + /* Bound the timeouts: the SDK defaults are microseconds::max(), + and a hung collector must not hang process exit. */ + auto us = std::chrono::duration_cast(timeout); + state->provider->ForceFlush(us); + state->provider->Shutdown(us); + /* Deliberately leak `state`: outstanding Span handles may still + reference the tracer; after Shutdown() their spans are simply + dropped. */ +} + +Span startSpan(std::string_view name, SpanKind kind) +{ + auto * state = theState.load(std::memory_order_acquire); + if (!state) + return Span(); + opentelemetry::trace::StartSpanOptions options; + options.kind = mapKind(kind); + return Span(std::make_shared(SpanImpl{state->tracer->StartSpan(toNostd(name), options)})); +} + +Span startSpan(std::string_view name, const Span & parent, SpanKind kind) +{ + auto * state = theState.load(std::memory_order_acquire); + if (!state) + return Span(); + opentelemetry::trace::StartSpanOptions options; + options.kind = mapKind(kind); + if (parent.impl) + options.parent = parent.impl->span->GetContext(); + return Span(std::make_shared(SpanImpl{state->tracer->StartSpan(toNostd(name), options)})); +} + +Span startSpanFromRemoteParent( + std::string_view name, std::string_view traceparent, std::string_view tracestate, SpanKind kind) +{ + auto * state = theState.load(std::memory_order_acquire); + if (!state) + return Span(); + opentelemetry::trace::StartSpanOptions options; + options.kind = mapKind(kind); + ExtractCarrier carrier; + carrier.traceparent = toNostd(traceparent); + carrier.tracestate = toNostd(tracestate); + /* Extract() returns the input context unchanged on a parse + failure, leaving an invalid SpanContext, i.e. a root span. */ + opentelemetry::context::Context emptyCtx; + auto ctx = opentelemetry::trace::propagation::HttpTraceContext{}.Extract(carrier, emptyCtx); + auto spanContext = opentelemetry::trace::GetSpan(ctx)->GetContext(); + if (spanContext.IsValid()) + options.parent = spanContext; + return Span(std::make_shared(SpanImpl{state->tracer->StartSpan(toNostd(name), options)})); +} + +bool Span::isRecording() const noexcept +{ + return impl && impl->span->IsRecording(); +} + +void Span::setAttribute(std::string_view key, std::string_view value) +{ + if (impl) + impl->span->SetAttribute(toNostd(key), toNostd(value)); +} + +void Span::setAttribute(std::string_view key, int64_t value) +{ + if (impl) + impl->span->SetAttribute(toNostd(key), value); +} + +void Span::setAttribute(std::string_view key, bool value) +{ + if (impl) + impl->span->SetAttribute(toNostd(key), value); +} + +void Span::setError(std::string_view description) +{ + if (impl) + impl->span->SetStatus(opentelemetry::trace::StatusCode::kError, toNostd(description)); +} + +void Span::end() +{ + if (impl) + impl->span->End(); +} + +Headers Span::injectContext() const +{ + if (!impl || !impl->span->GetContext().IsValid()) + return {}; + InjectCarrier carrier; + opentelemetry::context::Context ctx; + auto withSpan = opentelemetry::trace::SetSpan(ctx, impl->span); + opentelemetry::trace::propagation::HttpTraceContext{}.Inject(carrier, withSpan); + return std::move(carrier.headers); +} + +#else + +struct SpanImpl +{}; + +void init(std::string_view) {} + +bool isEnabled() noexcept +{ + return false; +} + +void forceFlushAndShutdown(std::chrono::milliseconds) {} + +Span startSpan(std::string_view, SpanKind) +{ + return Span(); +} + +Span startSpan(std::string_view, const Span &, SpanKind) +{ + return Span(); +} + +Span startSpanFromRemoteParent(std::string_view, std::string_view, std::string_view, SpanKind) +{ + return Span(); +} + +bool Span::isRecording() const noexcept +{ + return false; +} + +void Span::setAttribute(std::string_view, std::string_view) {} + +void Span::setAttribute(std::string_view, int64_t) {} + +void Span::setAttribute(std::string_view, bool) {} + +void Span::setError(std::string_view) {} + +void Span::end() {} + +Headers Span::injectContext() const +{ + return {}; +} + +#endif // HAVE_OTEL + +Span::Span() noexcept = default; +Span::Span(const Span &) = default; +Span::Span(Span &&) noexcept = default; +Span & Span::operator=(const Span &) = default; +Span & Span::operator=(Span &&) noexcept = default; +Span::~Span() = default; + +Span::Span(std::shared_ptr impl) + : impl(std::move(impl)) +{ +} + +} // namespace nix::otel diff --git a/src/libutil/package.nix b/src/libutil/package.nix index fe6fcb75f4..fc6a95eb2c 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -11,11 +11,14 @@ libsodium, nlohmann_json, openssl, + opentelemetry-cpp, zstd, # Configuration Options version, + + withOtel ? !stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform, }: let @@ -55,7 +58,8 @@ mkMesonLibrary (finalAttrs: { openssl zstd ] - ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + ++ lib.optional withOtel opentelemetry-cpp; propagatedBuildInputs = [ boost @@ -65,6 +69,7 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ (lib.mesonEnable "cpuid" stdenv.hostPlatform.isx86_64) + (lib.mesonEnable "otel" withOtel) ]; meta = { From 0b30dd932846c7876e417566e79975d2ffdc943b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Aug 2026 16:28:48 +0200 Subject: [PATCH 672/927] nix: Initialize OpenTelemetry tracing and trace CLI invocations Wire the nix::otel facade into the CLI entry point: * otel::init() runs after programName is resolved and before the legacy command dispatch, so nix, nix-daemon, build-remote etc. all get a provider with the appropriate service.name. It remains a no-op unless OTEL_EXPORTER_OTLP_ENDPOINT (or OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) is set in the environment. * A Finally flushes and shuts down the exporter on every exit path, including exceptions and legacy commands' early returns. * Each command invocation gets a root span ("nix "), ended by RAII. On failure the span records an error status with the ANSI-stripped error message; Exit is rethrown unmarked since it can represent a clean exit. Assisted-by: Claude Fable 5 --- src/nix/main.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 5acf75257d..afa6837209 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -23,6 +23,8 @@ #include "nix/expr/eval-cache.hh" #include "nix/flake/flake.hh" #include "nix/flake/settings.hh" +#include "nix/util/exit.hh" +#include "nix/util/otel.hh" #include "nix/util/sentry.hh" #include "self-exe.hh" @@ -547,6 +549,12 @@ void mainWrapped(int argc, char ** argv) argc--; } + /* No-op unless OTEL_EXPORTER_OTLP_ENDPOINT is set in the + environment. */ + otel::init(programName); + + Finally flushOtel([] { otel::forceFlushAndShutdown(); }); + { if (auto legacy = get(RegisterLegacyCommand::commands(), programName)) return (*legacy)(argc, argv); @@ -706,6 +714,9 @@ void mainWrapped(int argc, char ** argv) setSentryTag("nix_subcommand", concatStringsSep(" ", subcommand).c_str()); + /* Ended by RAII, including on exceptions. */ + auto rootSpan = otel::startSpan("nix " + concatStringsSep(" ", subcommand)); + try { args.command->second->run(); } catch (eval_cache::CachedEvalError & e) { @@ -713,6 +724,12 @@ void mainWrapped(int argc, char ** argv) cached error so that we can show the original error to the user. */ e.force(); + } catch (Exit &) { + throw; + } catch (std::exception & e) { + // FIXME: privacy + rootSpan.setError(filterANSIEscapes(e.what(), true)); + throw; } } From 6f1e05363de93bd59724634ee20793aed4a731c1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Aug 2026 16:44:59 +0200 Subject: [PATCH 673/927] libstore: Trace HTTP transfers and propagate W3C trace context Give every FileTransfer a span, parented under the process root span and exported as a CLIENT span covering the whole transfer including retries. On success it records http.response.status_code (and http.request.resend_count when retries happened); every failure path funnels through failEx(), which records an error status with the ANSI-stripped message. The span's W3C context (traceparent/tracestate) is injected into the request headers, so any otel-instrumented server (e.g. FlakeHub Cache) can join its server-side spans to the client's trace. To parent transfer spans correctly, the facade gains otel::setRootSpan() / otel::rootSpan(): a process-wide default parent registered by main.cc. Only a weak reference is kept so the root span's RAII lifetime is unaffected. Assisted-by: Claude Fable 5 --- src/libstore/filetransfer.cc | 22 ++++++++++++++++++++++ src/libutil/include/nix/util/otel.hh | 16 ++++++++++++++++ src/libutil/otel.cc | 17 +++++++++++++++++ src/nix/main.cc | 1 + 4 files changed, 56 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index d2b4359e20..851a92c011 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -4,7 +4,9 @@ #include "nix/util/config-global.hh" #include "nix/util/finally.hh" #include "nix/util/callback.hh" +#include "nix/util/otel.hh" #include "nix/util/signals.hh" +#include "nix/util/terminal.hh" #include "nix/util/util.hh" #include "nix/store/s3-url.hh" @@ -161,6 +163,7 @@ struct curlFileTransfer : public FileTransfer FileTransferRequest request; FileTransferResult result; std::unique_ptr _act; + otel::Span span; Callback callback; CURL * req = 0; // buffer to accompany the `req` above @@ -303,6 +306,11 @@ struct curlFileTransfer : public FileTransfer for (auto it = request.headers.begin(); it != request.headers.end(); ++it) { appendHeaders(fmt("%s: %s", it->first, it->second)); } + + span = otel::startSpan(request.verb(), otel::rootSpan(), otel::SpanKind::Client); + span.setAttribute("url.full", request.displayUri()); + for (auto & [name, value] : span.injectContext()) + appendHeaders(fmt("%s: %s", name, value)); } ~TransferItem() @@ -330,6 +338,15 @@ struct curlFileTransfer : public FileTransfer } catch (...) { /* Can't add more context to the error. */ } + try { + std::rethrow_exception(ex); + } catch (std::exception & e) { + // FIXME: privacy + span.setError(filterANSIEscapes(e.what(), true)); + } catch (...) { + span.setError("unknown error"); + } + span.end(); callback.rethrow(ex); } @@ -800,6 +817,11 @@ struct curlFileTransfer : public FileTransfer curl_easy_getinfo(req, CURLINFO_SIZE_DOWNLOAD_T, &dlSize); act().progress(dlSize, dlSize); done = true; + if (httpStatus) + span.setAttribute("http.response.status_code", (int64_t) httpStatus); + if (attempt > 0) + span.setAttribute("http.request.resend_count", (int64_t) attempt); + span.end(); callback(std::move(result)); } diff --git a/src/libutil/include/nix/util/otel.hh b/src/libutil/include/nix/util/otel.hh index 10f1559188..35daeeb566 100644 --- a/src/libutil/include/nix/util/otel.hh +++ b/src/libutil/include/nix/util/otel.hh @@ -42,6 +42,8 @@ class Span friend Span startSpan(std::string_view name, const Span & parent, SpanKind kind); friend Span startSpanFromRemoteParent( std::string_view name, std::string_view traceparent, std::string_view tracestate, SpanKind kind); + friend void setRootSpan(const Span & span); + friend Span rootSpan(); std::shared_ptr impl; @@ -121,6 +123,20 @@ Span startSpanFromRemoteParent( std::string_view tracestate = {}, SpanKind kind = SpanKind::Internal); +/** + * Register the process-wide root span, used as the default parent by + * call sites that have no more specific parent (e.g. HTTP transfers). + * Only a weak reference is kept, so this does not extend the span's + * lifetime. + */ +void setRootSpan(const Span & span); + +/** + * The span registered with setRootSpan(), or an inert span if none + * was registered or it no longer exists. + */ +Span rootSpan(); + /** * Flush all pending spans and shut down the exporter. Safe to call if * tracing was never initialized, and safe to call more than once. diff --git a/src/libutil/otel.cc b/src/libutil/otel.cc index 5a5008c754..697d11ee69 100644 --- a/src/libutil/otel.cc +++ b/src/libutil/otel.cc @@ -2,6 +2,8 @@ #include "util-config-private.hh" +#include + #if HAVE_OTEL # include "nix/util/environment-variables.hh" @@ -306,4 +308,19 @@ Span::Span(std::shared_ptr impl) { } +static std::mutex rootSpanMutex; +static std::weak_ptr rootSpanImpl; + +void setRootSpan(const Span & span) +{ + std::lock_guard lock(rootSpanMutex); + rootSpanImpl = span.impl; +} + +Span rootSpan() +{ + std::lock_guard lock(rootSpanMutex); + return Span(rootSpanImpl.lock()); +} + } // namespace nix::otel diff --git a/src/nix/main.cc b/src/nix/main.cc index afa6837209..812036bcaa 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -716,6 +716,7 @@ void mainWrapped(int argc, char ** argv) /* Ended by RAII, including on exceptions. */ auto rootSpan = otel::startSpan("nix " + concatStringsSep(" ", subcommand)); + otel::setRootSpan(rootSpan); try { args.command->second->run(); From c07d977eb35f2eebcee57987be3a9dcbfd34f3b7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Aug 2026 16:55:58 +0200 Subject: [PATCH 674/927] libstore: Use separate spans for transfers and HTTP attempts Split the FileTransfer span into two levels: a span covering the lifetime of the TransferItem (including retries and backoff), and a child CLIENT span per HTTP request, recreated on every attempt. Each attempt span records its own http.response.status_code, its resend count, and its own error status when that attempt fails -- even if the transfer later succeeds on a retry. Since the traceparent now differs per attempt, the request headers are rebuilt in init() rather than once in the constructor, so every HTTP request propagates the context of exactly the attempt that produced it. Assisted-by: Claude Fable 5 --- src/libstore/filetransfer.cc | 68 +++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 851a92c011..06d255e843 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -163,7 +163,18 @@ struct curlFileTransfer : public FileTransfer FileTransferRequest request; FileTransferResult result; std::unique_ptr _act; - otel::Span span; + + /** + * Span covering the lifetime of this transfer, including + * retries. + */ + otel::Span transferSpan; + + /** + * Span covering the current HTTP request. Recreated on every + * attempt. + */ + otel::Span requestSpan; Callback callback; CURL * req = 0; // buffer to accompany the `req` above @@ -299,18 +310,8 @@ struct curlFileTransfer : public FileTransfer { result.urls.push_back(request.uri.to_string()); - if (!request.expectedETag.empty()) - appendHeaders("If-None-Match: " + request.expectedETag); - if (!request.mimeType.empty()) - appendHeaders("Content-Type: " + request.mimeType); - for (auto it = request.headers.begin(); it != request.headers.end(); ++it) { - appendHeaders(fmt("%s: %s", it->first, it->second)); - } - - span = otel::startSpan(request.verb(), otel::rootSpan(), otel::SpanKind::Client); - span.setAttribute("url.full", request.displayUri()); - for (auto & [name, value] : span.injectContext()) - appendHeaders(fmt("%s: %s", name, value)); + transferSpan = otel::startSpan(request.verb(), otel::rootSpan()); + transferSpan.setAttribute("url.full", request.displayUri()); } ~TransferItem() @@ -342,11 +343,15 @@ struct curlFileTransfer : public FileTransfer std::rethrow_exception(ex); } catch (std::exception & e) { // FIXME: privacy - span.setError(filterANSIEscapes(e.what(), true)); + auto description = filterANSIEscapes(e.what(), true); + requestSpan.setError(description); + transferSpan.setError(description); } catch (...) { - span.setError("unknown error"); + requestSpan.setError("unknown error"); + transferSpan.setError("unknown error"); } - span.end(); + requestSpan.end(); + transferSpan.end(); callback.rethrow(ex); } @@ -624,6 +629,23 @@ struct curlFileTransfer : public FileTransfer bytesReceived = 0; + requestSpan = otel::startSpan(request.verb(), transferSpan, otel::SpanKind::Client); + if (attempt > 0) + requestSpan.setAttribute("http.request.resend_count", (int64_t) attempt); + + /* (Re)build the request headers, since the traceparent + header differs per attempt. */ + requestHeaders = curlSList{}; + if (!request.expectedETag.empty()) + appendHeaders("If-None-Match: " + request.expectedETag); + if (!request.mimeType.empty()) + appendHeaders("Content-Type: " + request.mimeType); + for (auto it = request.headers.begin(); it != request.headers.end(); ++it) { + appendHeaders(fmt("%s: %s", it->first, it->second)); + } + for (auto & [name, value] : requestSpan.injectContext()) + appendHeaders(fmt("%s: %s", name, value)); + if (verbosity >= lvlVomit) { curl_easy_setopt(req, CURLOPT_VERBOSE, 1); curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback); @@ -801,6 +823,9 @@ struct curlFileTransfer : public FileTransfer httpStatus = std::to_underlying(HttpStatus::NotModified); } + if (httpStatus) + requestSpan.setAttribute("http.response.status_code", (int64_t) httpStatus); + if (callbackException) failEx(callbackException); @@ -817,11 +842,8 @@ struct curlFileTransfer : public FileTransfer curl_easy_getinfo(req, CURLINFO_SIZE_DOWNLOAD_T, &dlSize); act().progress(dlSize, dlSize); done = true; - if (httpStatus) - span.setAttribute("http.response.status_code", (int64_t) httpStatus); - if (attempt > 0) - span.setAttribute("http.request.resend_count", (int64_t) attempt); - span.end(); + requestSpan.end(); + transferSpan.end(); callback(std::move(result)); } @@ -946,6 +968,10 @@ struct curlFileTransfer : public FileTransfer code, errbuf); + // FIXME: privacy + requestSpan.setError(filterANSIEscapes(exc.what(), true)); + requestSpan.end(); + maybeRetry(err, httpStatus, std::move(exc)); } } From 538247f39ec5a423216cea65941e2617e873d2d9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Aug 2026 16:59:23 +0200 Subject: [PATCH 675/927] libutil: Strip ANSI escapes inside otel::Span::setError() Every caller wants this (error messages are formatted for the terminal), so do it in the facade instead of at each call site. Assisted-by: Claude Fable 5 --- src/libstore/filetransfer.cc | 8 +++----- src/libutil/include/nix/util/otel.hh | 3 ++- src/libutil/otel.cc | 7 +++++-- src/nix/main.cc | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 06d255e843..5abdde268b 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -6,7 +6,6 @@ #include "nix/util/callback.hh" #include "nix/util/otel.hh" #include "nix/util/signals.hh" -#include "nix/util/terminal.hh" #include "nix/util/util.hh" #include "nix/store/s3-url.hh" @@ -343,9 +342,8 @@ struct curlFileTransfer : public FileTransfer std::rethrow_exception(ex); } catch (std::exception & e) { // FIXME: privacy - auto description = filterANSIEscapes(e.what(), true); - requestSpan.setError(description); - transferSpan.setError(description); + requestSpan.setError(e.what()); + transferSpan.setError(e.what()); } catch (...) { requestSpan.setError("unknown error"); transferSpan.setError("unknown error"); @@ -969,7 +967,7 @@ struct curlFileTransfer : public FileTransfer errbuf); // FIXME: privacy - requestSpan.setError(filterANSIEscapes(exc.what(), true)); + requestSpan.setError(exc.what()); requestSpan.end(); maybeRetry(err, httpStatus, std::move(exc)); diff --git a/src/libutil/include/nix/util/otel.hh b/src/libutil/include/nix/util/otel.hh index 35daeeb566..30f7fa7af7 100644 --- a/src/libutil/include/nix/util/otel.hh +++ b/src/libutil/include/nix/util/otel.hh @@ -68,7 +68,8 @@ public: void setAttribute(std::string_view key, bool value); /** - * Mark this span as failed, with a description of the error. + * Mark this span as failed, with a description of the error. ANSI + * escape sequences are stripped from the description. */ void setError(std::string_view description); diff --git a/src/libutil/otel.cc b/src/libutil/otel.cc index 697d11ee69..5e57c91393 100644 --- a/src/libutil/otel.cc +++ b/src/libutil/otel.cc @@ -6,6 +6,7 @@ #if HAVE_OTEL # include "nix/util/environment-variables.hh" +# include "nix/util/terminal.hh" # include @@ -224,8 +225,10 @@ void Span::setAttribute(std::string_view key, bool value) void Span::setError(std::string_view description) { - if (impl) - impl->span->SetStatus(opentelemetry::trace::StatusCode::kError, toNostd(description)); + if (impl) { + auto filtered = filterANSIEscapes(description, true); + impl->span->SetStatus(opentelemetry::trace::StatusCode::kError, toNostd(filtered)); + } } void Span::end() diff --git a/src/nix/main.cc b/src/nix/main.cc index 812036bcaa..e530831dfb 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -729,7 +729,7 @@ void mainWrapped(int argc, char ** argv) throw; } catch (std::exception & e) { // FIXME: privacy - rootSpan.setError(filterANSIEscapes(e.what(), true)); + rootSpan.setError(e.what()); throw; } } From c90f043716b18e67fbfe5cfe11639939829f5b0d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Aug 2026 17:19:58 +0200 Subject: [PATCH 676/927] libstore: Propagate OpenTelemetry trace context to the daemon Add an "open-telemetry" worker protocol feature. When both sides support it, the client sends the W3C traceparent of its root span (or an empty string if it is not tracing) right after the handshake in initConnection(). The daemon then opens a "daemon connection" SERVER span parented under the client's trace and registers it as its root span, so everything it does on behalf of the connection -- including binary cache requests once they happen daemon-side -- joins the client's trace. Since the daemon forks a child per connection and the OpenTelemetry batch exporter's worker thread does not survive fork(), the facade gains otel::resetAfterFork(), which discards the inherited provider without touching it. The connection child calls it, initializes tracing afresh as "nix-daemon", and flushes before exit. Feature negotiation keeps old clients and daemons compatible in both directions: without agreement, no traceparent is exchanged. Assisted-by: Claude Fable 5 --- src/libstore/daemon.cc | 10 ++++++++++ src/libstore/include/nix/store/worker-protocol.hh | 7 +++++++ src/libstore/remote-store.cc | 12 ++++++++++++ src/libstore/worker-protocol.cc | 1 + src/libutil/include/nix/util/otel.hh | 9 +++++++++ src/libutil/otel.cc | 11 +++++++++++ src/nix/unix/daemon.cc | 13 +++++++++++++ 7 files changed, 63 insertions(+) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 56e563d64c..3974e62d19 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -12,6 +12,7 @@ #include "nix/store/remote-store.hh" #include "nix/store/path-with-outputs.hh" #include "nix/util/finally.hh" +#include "nix/util/otel.hh" #include "nix/util/archive.hh" #include "nix/store/derivations.hh" #include "nix/util/args.hh" @@ -1115,6 +1116,15 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted if (conn.protoVersion.number < WorkerProto::minimum.number) throw Error("the Nix client version is too old"); + std::string traceparent; + if (conn.protoVersion.features.contains(WorkerProto::featureOpenTelemetry)) + traceparent = readString(from); + + /* Covers the lifetime of this connection; parented under the + client's root span if it sent a trace context. */ + auto connectionSpan = otel::startSpanFromRemoteParent("daemon connection", traceparent, {}, otel::SpanKind::Server); + otel::setRootSpan(connectionSpan); + conn.to = std::move(to); conn.from = std::move(from); diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index 88ed08ed23..8c87b3f10e 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -142,6 +142,13 @@ struct WorkerProto */ static constexpr std::string_view featureDisableSetOptions = "disable-set-options"; + /** + * Feature for propagating OpenTelemetry trace context from the + * client to the daemon: after the handshake, the client sends a + * W3C `traceparent` string (empty if the client is not tracing). + */ + static constexpr std::string_view featureOpenTelemetry = "open-telemetry"; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 956a15eb57..36d0c2d4dd 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -13,6 +13,7 @@ #include "nix/util/archive.hh" #include "nix/store/globals.hh" #include "nix/store/derivations.hh" +#include "nix/util/otel.hh" #include "nix/util/pool.hh" #include "nix/util/finally.hh" #include "nix/util/logging.hh" @@ -112,6 +113,17 @@ void RemoteStore::initConnection(Connection & conn) throw Error("protocol mismatch, got '%s'", chomp(saved.s)); } + if (conn.protoVersion.features.contains(WorkerProto::featureOpenTelemetry)) { + /* Send our trace context so that the daemon can parent + its spans under ours. */ + std::string traceparent; + for (auto & [name, value] : otel::rootSpan().injectContext()) + if (name == "traceparent") + traceparent = value; + conn.to << traceparent; + conn.to.flush(); + } + static_cast(conn) = conn.postHandshake(*this); for (auto & feature : conn.protoVersion.features) diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 4949062e83..05b941a3e9 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -35,6 +35,7 @@ const WorkerProto::Version WorkerProto::latest = { std::string{WorkerProto::featureVersionedAddToStoreMultiple}, std::string{WorkerProto::featureAddTempRoots}, std::string{WorkerProto::featureQueryPathInfos}, + std::string{WorkerProto::featureOpenTelemetry}, }, }; diff --git a/src/libutil/include/nix/util/otel.hh b/src/libutil/include/nix/util/otel.hh index 30f7fa7af7..2890b886e8 100644 --- a/src/libutil/include/nix/util/otel.hh +++ b/src/libutil/include/nix/util/otel.hh @@ -138,6 +138,15 @@ void setRootSpan(const Span & span); */ Span rootSpan(); +/** + * Discard all tracing state inherited from the parent process after a + * fork(): the exporter's worker thread does not exist in the child, + * so the inherited provider can neither export nor be shut down. + * Also unregisters the root span. Afterwards `init()` can be called + * again to start fresh tracing in the child. + */ +void resetAfterFork(); + /** * Flush all pending spans and shut down the exporter. Safe to call if * tracing was never initialized, and safe to call more than once. diff --git a/src/libutil/otel.cc b/src/libutil/otel.cc index 5e57c91393..5911e5094f 100644 --- a/src/libutil/otel.cc +++ b/src/libutil/otel.cc @@ -142,6 +142,15 @@ bool isEnabled() noexcept return theState.load(std::memory_order_acquire) != nullptr; } +void resetAfterFork() +{ + /* Deliberately leak the old state: it references a worker thread + that does not exist in this process, so it can neither flush + nor be destroyed safely. */ + theState.exchange(nullptr); + setRootSpan(Span()); +} + void forceFlushAndShutdown(std::chrono::milliseconds timeout) { auto * state = theState.exchange(nullptr); @@ -260,6 +269,8 @@ bool isEnabled() noexcept return false; } +void resetAfterFork() {} + void forceFlushAndShutdown(std::chrono::milliseconds) {} Span startSpan(std::string_view, SpanKind) diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index d7018b9c32..2ecd7f139d 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -8,6 +8,7 @@ #include "nix/store/remote-store.hh" #include "nix/store/remote-store-connection.hh" #include "nix/store/store-open.hh" +#include "nix/util/otel.hh" #include "nix/util/serialise.hh" #include "nix/store/globals.hh" #include "nix/util/config-global.hh" @@ -366,6 +367,13 @@ static void daemonLoop( [&, storeConfig, closeListeners = std::move(closeListeners)]() { setInterrupted(false); + /* The OpenTelemetry exporter's worker thread + does not survive the fork, so set up + tracing afresh. */ + otel::resetAfterFork(); + otel::init("nix-daemon"); + Finally flushOtel([] { otel::forceFlushAndShutdown(); }); + closeListeners(); // Background the daemon. @@ -391,6 +399,11 @@ static void daemonLoop( trusted, RecursiveFlag::NotRecursive); + /* Flush explicitly, since exit() does not + unwind the stack and so won't run + `flushOtel`. */ + otel::forceFlushAndShutdown(); + exit(0); }, options); From b40895ceb73c166223f78c6b1648d609f8260934 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 31 Aug 2026 12:10:51 +0200 Subject: [PATCH 677/927] packaging: Enable gzip compression in the OTLP/HTTP exporter Build opentelemetry-cpp with WITH_OTLP_HTTP_COMPRESSION=ON (and zlib), so that OTEL_EXPORTER_OTLP_COMPRESSION=gzip works instead of warning and falling back to uncompressed exports. The flag also enables two gzip HTTP round-trip tests that time out flakily in the build sandbox (their sibling BasicCurlHttpTests are already shipped disabled), so exclude them from the check phase. Assisted-by: Claude Fable 5 --- packaging/dependencies.nix | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 0eb3c5e346..3449443bc0 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -219,11 +219,25 @@ scope: { inherit (scope) curl; }; - opentelemetry-cpp = pkgs.opentelemetry-cpp.override { - # The default build has no OTLP exporters at all; we need the - # OTLP/HTTP one (and specifically not the gRPC one). - enableHttp = true; - }; + opentelemetry-cpp = + (pkgs.opentelemetry-cpp.override { + # The default build has no OTLP exporters at all; we need the + # OTLP/HTTP one (and specifically not the gRPC one). + enableHttp = true; + }).overrideAttrs + (old: { + # Support OTEL_EXPORTER_OTLP_COMPRESSION=gzip. + cmakeFlags = old.cmakeFlags or [ ] ++ [ (pkgs.lib.cmakeBool "WITH_OTLP_HTTP_COMPRESSION" true) ]; + buildInputs = old.buildInputs or [ ] ++ [ pkgs.zlib ]; + # The gzip tests enabled by WITH_OTLP_HTTP_COMPRESSION do an + # HTTP round trip against a local test server and time out + # flakily in the build sandbox. + checkPhase = '' + runHook preCheck + ctest --output-on-failure -E 'BasicCurlHttpTests.Gzip' + runHook postCheck + ''; + }); libmicrohttpd = pkgs.libmicrohttpd.overrideDerivation (old: { # Don't pull in gnutls since it's pretty big and we don't need it. From cf6566561b87ef2f5d8b550ccd21e02647fe9189 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 13:00:07 +0200 Subject: [PATCH 678/927] Reimplement OpenTelemetry tracing as a Logger subclass Instead of creating spans explicitly at instrumented call sites, derive them from Nix's existing Activity system: an OpenTelemetryLogger (analogous to JSONLogger) maps activities onto spans and is added to the global logger via the new applyExtraLogger(). All OpenTelemetry code moves from libutil to src/nix (otel-logger.{cc,hh}), along with the build-time dependency; libutil and libstore are now free of it. Anything that emits an Activity gets traced for free, which is also how future instrumentation (e.g. the evaluator) should hook in. Vendor-neutral additions to libutil's logging interface: * Logger::getTraceContext(): returns W3C trace context headers for an activity, used by FileTransfer to add a traceparent header to HTTP requests and by RemoteStore to propagate the trace context to the daemon during the handshake (the "open-telemetry" protocol feature is unchanged on the wire). * RemoteLogSource: marks Logger calls that replay messages from another process. The client-side handling of activities forwarded from the daemon uses it, so that the client does not export telemetry for them. * actFileTransferAttempt / resHttpStatus: FileTransfer now starts a child activity per HTTP request (there can be several per transfer due to retries) and reports the response status on it. The transfer activity is started eagerly in init() so that it exists before the request headers are built. Export ownership: every process exports exactly the activities it originates. The daemon keeps its own exporter lifecycle; its connection span (created via processConnection()'s new setupTelemetry callback) is parented under the client's trace context, and its activities are exported by the daemon itself while still being forwarded to the client for display. The client skips forwarded activities, so nothing is exported twice. Known limitations: span error status is mostly lossy (stopActivity carries no status; the root span still records command failures), and top-level daemon activities parent under the connection span rather than the client-side operation that triggered them. Assisted-by: Claude Fable 5 --- src/libstore/daemon.cc | 17 +- src/libstore/filetransfer.cc | 59 ++- src/libstore/include/nix/store/daemon.hh | 19 +- src/libstore/remote-store.cc | 3 +- src/libstore/worker-protocol-connection.cc | 3 + src/libutil/include/nix/util/logging.hh | 52 +++ src/libutil/include/nix/util/meson.build | 1 - src/libutil/include/nix/util/otel.hh | 160 -------- src/libutil/logging.cc | 26 +- src/libutil/meson.build | 7 - src/libutil/meson.options | 6 - src/libutil/otel.cc | 340 ----------------- src/libutil/package.nix | 7 +- src/libutil/tee-logger.cc | 31 ++ src/nix/main.cc | 26 +- src/nix/meson.build | 7 + src/nix/meson.options | 6 + src/nix/otel-logger.cc | 409 +++++++++++++++++++++ src/nix/otel-logger.hh | 74 ++++ src/nix/package.nix | 6 +- src/nix/unix/daemon.cc | 39 +- 21 files changed, 703 insertions(+), 595 deletions(-) delete mode 100644 src/libutil/include/nix/util/otel.hh delete mode 100644 src/libutil/otel.cc create mode 100644 src/nix/otel-logger.cc create mode 100644 src/nix/otel-logger.hh diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 3974e62d19..e3e268db2e 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -12,7 +12,6 @@ #include "nix/store/remote-store.hh" #include "nix/store/path-with-outputs.hh" #include "nix/util/finally.hh" -#include "nix/util/otel.hh" #include "nix/util/archive.hh" #include "nix/store/derivations.hh" #include "nix/util/args.hh" @@ -1085,7 +1084,13 @@ static void performOp( } } -void processConnection(ref store, FdSource && from, FdSink && to, TrustedFlag trusted, RecursiveFlag recursive) +void processConnection( + ref store, + FdSource && from, + FdSink && to, + TrustedFlag trusted, + RecursiveFlag recursive, + std::function setupTelemetry) { #ifndef _WIN32 // TODO need graceful async exit support on Windows? auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; @@ -1120,11 +1125,6 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted if (conn.protoVersion.features.contains(WorkerProto::featureOpenTelemetry)) traceparent = readString(from); - /* Covers the lifetime of this connection; parented under the - client's root span if it sent a trace context. */ - auto connectionSpan = otel::startSpanFromRemoteParent("daemon connection", traceparent, {}, otel::SpanKind::Server); - otel::setRootSpan(connectionSpan); - conn.to = std::move(to); conn.from = std::move(from); @@ -1136,6 +1136,9 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted applyJSONLogger(); } + if (setupTelemetry) + setupTelemetry(traceparent); + unsigned int opCount = 0; Finally finally([&]() { diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 5abdde268b..f3aa5e2976 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -4,7 +4,6 @@ #include "nix/util/config-global.hh" #include "nix/util/finally.hh" #include "nix/util/callback.hh" -#include "nix/util/otel.hh" #include "nix/util/signals.hh" #include "nix/util/util.hh" @@ -23,6 +22,8 @@ #include +#include + #include #include #include @@ -164,16 +165,10 @@ struct curlFileTransfer : public FileTransfer std::unique_ptr _act; /** - * Span covering the lifetime of this transfer, including - * retries. - */ - otel::Span transferSpan; - - /** - * Span covering the current HTTP request. Recreated on every - * attempt. + * Activity covering the current HTTP request. Recreated on + * every attempt, as a child of the transfer activity. */ - otel::Span requestSpan; + std::unique_ptr attemptAct; Callback callback; CURL * req = 0; // buffer to accompany the `req` above @@ -308,9 +303,6 @@ struct curlFileTransfer : public FileTransfer }) { result.urls.push_back(request.uri.to_string()); - - transferSpan = otel::startSpan(request.verb(), otel::rootSpan()); - transferSpan.setAttribute("url.full", request.displayUri()); } ~TransferItem() @@ -338,18 +330,6 @@ struct curlFileTransfer : public FileTransfer } catch (...) { /* Can't add more context to the error. */ } - try { - std::rethrow_exception(ex); - } catch (std::exception & e) { - // FIXME: privacy - requestSpan.setError(e.what()); - transferSpan.setError(e.what()); - } catch (...) { - requestSpan.setError("unknown error"); - transferSpan.setError("unknown error"); - } - requestSpan.end(); - transferSpan.end(); callback.rethrow(ex); } @@ -627,9 +607,17 @@ struct curlFileTransfer : public FileTransfer bytesReceived = 0; - requestSpan = otel::startSpan(request.verb(), transferSpan, otel::SpanKind::Client); - if (attempt > 0) - requestSpan.setAttribute("http.request.resend_count", (int64_t) attempt); + /* Start an activity for this HTTP request. Note that + act() is called here (rather than lazily from a libcurl + callback) so that the activity exists before the + request headers are built. */ + attemptAct = std::make_unique( + *logger, + lvlDebug, + actFileTransferAttempt, + "", + Logger::Fields{request.displayUri(), (uint64_t) attempt}, + act().id); /* (Re)build the request headers, since the traceparent header differs per attempt. */ @@ -641,7 +629,7 @@ struct curlFileTransfer : public FileTransfer for (auto it = request.headers.begin(); it != request.headers.end(); ++it) { appendHeaders(fmt("%s: %s", it->first, it->second)); } - for (auto & [name, value] : requestSpan.injectContext()) + for (auto & [name, value] : logger->getTraceContext(attemptAct->id)) appendHeaders(fmt("%s: %s", name, value)); if (verbosity >= lvlVomit) { @@ -821,8 +809,11 @@ struct curlFileTransfer : public FileTransfer httpStatus = std::to_underlying(HttpStatus::NotModified); } - if (httpStatus) - requestSpan.setAttribute("http.response.status_code", (int64_t) httpStatus); + if (attemptAct) { + if (httpStatus) + logger->result(attemptAct->id, resHttpStatus, nlohmann::json{{"httpStatus", httpStatus}}); + attemptAct.reset(); + } if (callbackException) failEx(callbackException); @@ -840,8 +831,6 @@ struct curlFileTransfer : public FileTransfer curl_easy_getinfo(req, CURLINFO_SIZE_DOWNLOAD_T, &dlSize); act().progress(dlSize, dlSize); done = true; - requestSpan.end(); - transferSpan.end(); callback(std::move(result)); } @@ -966,10 +955,6 @@ struct curlFileTransfer : public FileTransfer code, errbuf); - // FIXME: privacy - requestSpan.setError(exc.what()); - requestSpan.end(); - maybeRetry(err, httpStatus, std::move(exc)); } } diff --git a/src/libstore/include/nix/store/daemon.hh b/src/libstore/include/nix/store/daemon.hh index 4d550696e8..4534695003 100644 --- a/src/libstore/include/nix/store/daemon.hh +++ b/src/libstore/include/nix/store/daemon.hh @@ -4,10 +4,27 @@ #include "nix/util/serialise.hh" #include "nix/store/store-api.hh" +#include + namespace nix::daemon { enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; -void processConnection(ref store, FdSource && from, FdSink && to, TrustedFlag trusted, RecursiveFlag recursive); +/** + * Serve a client on the given file descriptors. + * + * @param setupTelemetry Called once after the handshake and logger + * setup, with the W3C `traceparent` string received from the client + * (empty if the client sent none or the protocol feature was not + * negotiated). Allows the caller to set up distributed tracing for + * the connection, e.g. by adding a tracing logger. + */ +void processConnection( + ref store, + FdSource && from, + FdSink && to, + TrustedFlag trusted, + RecursiveFlag recursive, + std::function setupTelemetry = {}); } // namespace nix::daemon diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 36d0c2d4dd..f02d1c6e16 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -13,7 +13,6 @@ #include "nix/util/archive.hh" #include "nix/store/globals.hh" #include "nix/store/derivations.hh" -#include "nix/util/otel.hh" #include "nix/util/pool.hh" #include "nix/util/finally.hh" #include "nix/util/logging.hh" @@ -117,7 +116,7 @@ void RemoteStore::initConnection(Connection & conn) /* Send our trace context so that the daemon can parent its spans under ours. */ std::string traceparent; - for (auto & [name, value] : otel::rootSpan().injectContext()) + for (auto & [name, value] : logger->getTraceContext(0)) if (name == "traceparent") traceparent = value; conn.to << traceparent; diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index cb07f397de..c7fdf9736f 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -82,11 +82,13 @@ WorkerProto::BasicClientConnection::processStderrReturn(Sink * sink, Source * so auto s = readString(from); auto fields = readFields(from); auto parent = readNum(from); + RemoteLogSource remoteLogSource; logger->startActivity(act, lvl, type, s, fields, parent); } else if (msg == STDERR_STOP_ACTIVITY) { auto act = readNum(from); + RemoteLogSource remoteLogSource; logger->stopActivity(act); } @@ -94,6 +96,7 @@ WorkerProto::BasicClientConnection::processStderrReturn(Sink * sink, Source * so auto act = readNum(from); auto type = (ResultType) readInt(from); auto fields = readFields(from); + RemoteLogSource remoteLogSource; logger->result(act, type, fields); } diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index 9dc5195e2a..79cbb11c26 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -3,6 +3,7 @@ #include "nix/util/error.hh" #include "nix/util/configuration.hh" +#include "nix/util/types.hh" #include "nix/util/file-descriptor.hh" #include "nix/util/finally.hh" #include "nix/util/fun.hh" @@ -28,6 +29,12 @@ typedef enum { actPostBuildHook = 110, actBuildWaiting = 111, actFetchTree = 112, + /** + * A single HTTP request within an `actFileTransfer` activity + * (there can be several due to retries). Fields: [0] the URI + * (string), [1] the number of preceding attempts (int). + */ + actFileTransferAttempt = 113, } ActivityType; typedef enum { @@ -42,6 +49,12 @@ typedef enum { resFetchStatus = 108, resHashMismatch = 109, resBuildResult = 110, + /** + * Emitted via the JSON `result()` overload: an object with an + * `httpStatus` field containing the status code of an HTTP + * response. More fields may be added in the future. + */ + resHttpStatus = 111, } ResultType; typedef uint64_t ActivityId; @@ -180,6 +193,19 @@ public: virtual void result(ActivityId act, ResultType type, const nlohmann::json & json) noexcept {}; + /** + * Return distributed tracing context for the given activity as + * W3C Trace Context headers (`traceparent`, `tracestate`), for + * propagation to another process (e.g. as HTTP request headers). + * `act == 0` denotes the root context of this process. Returns an + * empty list if this logger does not do tracing or has no context + * for `act`. + */ + virtual Headers getTraceContext(ActivityId act) + { + return {}; + } + virtual void writeToStdout(std::string_view s); template @@ -297,8 +323,34 @@ std::unique_ptr makeJSONLogger(Descriptor fd, bool includeNixPrefix = tr std::unique_ptr makeJSONLogger(const std::filesystem::path & path, bool includeNixPrefix = true); +/** + * Add an additional logger to the global `logger` by combining them + * into a `TeeLogger`. The current logger keeps responsibility for + * stdout and user interaction. + */ +void applyExtraLogger(std::unique_ptr extraLogger); + void applyJSONLogger(); +/** + * Marks, for the duration of its existence, Logger calls made on this + * thread as replaying messages that originated in another process + * (e.g. activities forwarded from the daemon to its client). Loggers + * that export telemetry should ignore such messages, since the + * originating process is responsible for exporting them. + */ +struct RemoteLogSource +{ + RemoteLogSource(); + ~RemoteLogSource(); +}; + +/** + * Whether Logger calls on this thread are currently replaying + * messages from another process, cf. `RemoteLogSource`. + */ +bool isRemoteLogSource(); + /** * @param source A noun phrase describing the source of the message, e.g. "the builder". */ diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 994bd479f5..3af780eb29 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -70,7 +70,6 @@ headers = [ config_pub_h ] + files( 'nar-cache.hh', 'nar-listing.hh', 'os-string.hh', - 'otel.hh', 'override-provenance-source-accessor.hh', 'pool.hh', 'pos-idx.hh', diff --git a/src/libutil/include/nix/util/otel.hh b/src/libutil/include/nix/util/otel.hh deleted file mode 100644 index 2890b886e8..0000000000 --- a/src/libutil/include/nix/util/otel.hh +++ /dev/null @@ -1,160 +0,0 @@ -#pragma once -///@file -/** - * Vendor-neutral distributed tracing facade. Backed by OpenTelemetry - * (exporting via OTLP/HTTP) when built with `-Dotel=enabled`; - * otherwise all operations are cheap no-ops. Even when compiled in, - * tracing stays disabled unless `init()` is called and - * `OTEL_EXPORTER_OTLP_ENDPOINT` (or - * `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`) is set in the environment. - */ - -#include "nix/util/types.hh" - -#include -#include -#include - -namespace nix::otel { - -enum class SpanKind { - Internal, - Client, - Server, -}; - -struct SpanImpl; - -/** - * A handle to an in-flight trace span. - * - * Value type with shared-handle semantics: copies refer to the same - * underlying span. The span ends when `end()` is first called, or, as - * a fallback, when the last handle is destroyed. Handles may be moved - * or copied across threads; the underlying span is thread-safe. - * - * A default-constructed handle (or any handle obtained while tracing - * is disabled) is inert: all operations are cheap no-ops. - */ -class Span -{ - friend Span startSpan(std::string_view name, SpanKind kind); - friend Span startSpan(std::string_view name, const Span & parent, SpanKind kind); - friend Span startSpanFromRemoteParent( - std::string_view name, std::string_view traceparent, std::string_view tracestate, SpanKind kind); - friend void setRootSpan(const Span & span); - friend Span rootSpan(); - - std::shared_ptr impl; - - explicit Span(std::shared_ptr impl); - -public: - Span() noexcept; - Span(const Span &); - Span(Span &&) noexcept; - Span & operator=(const Span &); - Span & operator=(Span &&) noexcept; - ~Span(); - - /** - * Whether this span actually records data (i.e. tracing is - * compiled in, initialized, and this span is sampled). - */ - bool isRecording() const noexcept; - - void setAttribute(std::string_view key, std::string_view value); - void setAttribute(std::string_view key, int64_t value); - void setAttribute(std::string_view key, bool value); - - /** - * Mark this span as failed, with a description of the error. ANSI - * escape sequences are stripped from the description. - */ - void setError(std::string_view description); - - /** - * End the span. Idempotent; only the first call records the end - * timestamp. - */ - void end(); - - /** - * Return W3C Trace Context headers (`traceparent`, `tracestate`) - * that propagate this span's context to another process, e.g. as - * HTTP request headers. Returns an empty list if this span is - * disabled. - */ - Headers injectContext() const; -}; - -/** - * Initialize tracing for this process. Does nothing (and is cheap) if - * `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` - * is not set in the environment, if tracing support is not compiled - * in, or if tracing is already initialized. Not called from library - * code; the program's main entry point decides. - */ -void init(std::string_view serviceName); - -/** - * Whether tracing is compiled in, initialized and not yet shut down. - */ -bool isEnabled() noexcept; - -/** - * Start a new root span (no parent). - */ -Span startSpan(std::string_view name, SpanKind kind = SpanKind::Internal); - -/** - * Start a span as a child of `parent`. If `parent` is disabled, this - * behaves like the parentless overload. - */ -Span startSpan(std::string_view name, const Span & parent, SpanKind kind = SpanKind::Internal); - -/** - * Start a span whose parent is a span in another process, identified - * by W3C `traceparent` / `tracestate` header values (e.g. received by - * the daemon from a client). Invalid values yield a root span. - */ -Span startSpanFromRemoteParent( - std::string_view name, - std::string_view traceparent, - std::string_view tracestate = {}, - SpanKind kind = SpanKind::Internal); - -/** - * Register the process-wide root span, used as the default parent by - * call sites that have no more specific parent (e.g. HTTP transfers). - * Only a weak reference is kept, so this does not extend the span's - * lifetime. - */ -void setRootSpan(const Span & span); - -/** - * The span registered with setRootSpan(), or an inert span if none - * was registered or it no longer exists. - */ -Span rootSpan(); - -/** - * Discard all tracing state inherited from the parent process after a - * fork(): the exporter's worker thread does not exist in the child, - * so the inherited provider can neither export nor be shut down. - * Also unregisters the root span. Afterwards `init()` can be called - * again to start fresh tracing in the child. - */ -void resetAfterFork(); - -/** - * Flush all pending spans and shut down the exporter. Safe to call if - * tracing was never initialized, and safe to call more than once. - * Must be called explicitly before process exit: nothing is flushed - * from static destructors (cf. the OPENSSL_INIT_NO_ATEXIT note in - * util.cc). Must not be called in a fork()ed child of the process - * that called `init()`. - */ -void forceFlushAndShutdown(std::chrono::milliseconds timeout = std::chrono::milliseconds(5000)); - -} // namespace nix::otel diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index dffa4a90bb..b231b573e8 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -36,6 +36,23 @@ void setCurActivity(const ActivityId activityId) curActivity = activityId; } +[[gnu::tls_model("initial-exec")]] static thread_local unsigned int remoteLogSourceDepth = 0; + +RemoteLogSource::RemoteLogSource() +{ + remoteLogSourceDepth++; +} + +RemoteLogSource::~RemoteLogSource() +{ + remoteLogSourceDepth--; +} + +bool isRemoteLogSource() +{ + return remoteLogSourceDepth > 0; +} + /** * This is a raw pointer to allow it to leak. * Avoids races in activity teardown. @@ -420,14 +437,7 @@ void applyJSONLogger() { if (auto & opt = loggerSettings.jsonLogPath.get()) { try { - std::vector> loggers; - loggers.push_back(makeJSONLogger(*opt, false)); - try { - logger = makeTeeLogger(std::unique_ptr(logger), std::move(loggers)).release(); - } catch (...) { - // `logger` is now gone so give up. - abort(); - } + applyExtraLogger(makeJSONLogger(*opt, false)); } catch (...) { ignoreExceptionExceptInterrupt(); } diff --git a/src/libutil/meson.build b/src/libutil/meson.build index cc314742be..bf3a1f3b08 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -124,12 +124,6 @@ cpuid = dependency( configdata_priv.set('HAVE_LIBCPUID', cpuid.found().to_int()) deps_private += cpuid -otel_required = get_option('otel') -otel = dependency('opentelemetry_exporter_otlp_http', required : otel_required) -otel_trace = dependency('opentelemetry_trace', required : otel_required) -configdata_priv.set('HAVE_OTEL', (otel.found() and otel_trace.found()).to_int()) -deps_private += [ otel, otel_trace ] - nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -184,7 +178,6 @@ sources = [ config_priv_h ] + files( 'nar-accessor.cc', 'nar-cache.cc', 'nar-listing.cc', - 'otel.cc', 'pos-table.cc', 'position.cc', 'posix-source-accessor.cc', diff --git a/src/libutil/meson.options b/src/libutil/meson.options index 9c40227c16..a828061629 100644 --- a/src/libutil/meson.options +++ b/src/libutil/meson.options @@ -5,9 +5,3 @@ option( type : 'feature', description : 'determine microarchitecture levels with libcpuid (only relevant on x86_64)', ) - -option( - 'otel', - type : 'feature', - description : 'distributed tracing via OpenTelemetry (OTLP/HTTP exporter)', -) diff --git a/src/libutil/otel.cc b/src/libutil/otel.cc deleted file mode 100644 index 5911e5094f..0000000000 --- a/src/libutil/otel.cc +++ /dev/null @@ -1,340 +0,0 @@ -#include "nix/util/otel.hh" - -#include "util-config-private.hh" - -#include - -#if HAVE_OTEL -# include "nix/util/environment-variables.hh" -# include "nix/util/terminal.hh" - -# include - -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -#endif - -namespace nix::otel { - -#if HAVE_OTEL - -struct SpanImpl -{ - opentelemetry::nostd::shared_ptr span; -}; - -namespace { - -struct State -{ - std::unique_ptr provider; - opentelemetry::nostd::shared_ptr tracer; -}; - -/* Owned here rather than via opentelemetry's global Provider - singleton, and deliberately leaked: the provider's destructor joins - the batch exporter's worker thread, which must never happen from - static destructors at exit() time (cf. the OPENSSL_INIT_NO_ATEXIT - note in util.cc; in the daemon's forked children that thread does - not even exist). Teardown happens only via an explicit - forceFlushAndShutdown(). */ -std::atomic theState{nullptr}; - -opentelemetry::trace::SpanKind mapKind(SpanKind kind) -{ - switch (kind) { - case SpanKind::Client: - return opentelemetry::trace::SpanKind::kClient; - case SpanKind::Server: - return opentelemetry::trace::SpanKind::kServer; - case SpanKind::Internal: - default: - return opentelemetry::trace::SpanKind::kInternal; - } -} - -inline opentelemetry::nostd::string_view toNostd(std::string_view sv) noexcept -{ - return {sv.data(), sv.size()}; -} - -struct ExtractCarrier : opentelemetry::context::propagation::TextMapCarrier -{ - opentelemetry::nostd::string_view traceparent, tracestate; - - opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view key) const noexcept override - { - if (key == "traceparent") - return traceparent; - if (key == "tracestate") - return tracestate; - return {}; - } - - void Set(opentelemetry::nostd::string_view, opentelemetry::nostd::string_view) noexcept override {} -}; - -struct InjectCarrier : opentelemetry::context::propagation::TextMapCarrier -{ - Headers headers; - - opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view) const noexcept override - { - return {}; - } - - void Set(opentelemetry::nostd::string_view key, opentelemetry::nostd::string_view value) noexcept override - { - /* Copy immediately: `value` may point into a stack buffer of - the propagator. */ - headers.emplace_back(std::string(key.data(), key.size()), std::string(value.data(), value.size())); - } -}; - -} // namespace - -void init(std::string_view serviceName) -{ - if (theState.load(std::memory_order_acquire)) - return; - - /* Without an explicitly configured endpoint, stay off: the SDK's - default would silently export to http://localhost:4318. */ - if (!getEnv("OTEL_EXPORTER_OTLP_ENDPOINT") && !getEnv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")) - return; - - namespace sdktrace = opentelemetry::sdk::trace; - - /* Default-constructed options read the OTEL_EXPORTER_OTLP_* - environment variables (endpoint, headers, TLS, compression). */ - auto exporter = opentelemetry::exporter::otlp::OtlpHttpExporterFactory::Create(); - auto processor = - sdktrace::BatchSpanProcessorFactory::Create(std::move(exporter), sdktrace::BatchSpanProcessorOptions{}); - auto resource = opentelemetry::sdk::resource::Resource::Create({ - {opentelemetry::semconv::service::kServiceName, std::string(serviceName)}, - }); - - auto state = std::make_unique(); - state->provider = sdktrace::TracerProviderFactory::Create(std::move(processor), resource); - state->tracer = state->provider->GetTracer("nix"); - - State * expected = nullptr; - if (theState.compare_exchange_strong(expected, state.get())) - state.release(); -} - -bool isEnabled() noexcept -{ - return theState.load(std::memory_order_acquire) != nullptr; -} - -void resetAfterFork() -{ - /* Deliberately leak the old state: it references a worker thread - that does not exist in this process, so it can neither flush - nor be destroyed safely. */ - theState.exchange(nullptr); - setRootSpan(Span()); -} - -void forceFlushAndShutdown(std::chrono::milliseconds timeout) -{ - auto * state = theState.exchange(nullptr); - if (!state) - return; - /* Bound the timeouts: the SDK defaults are microseconds::max(), - and a hung collector must not hang process exit. */ - auto us = std::chrono::duration_cast(timeout); - state->provider->ForceFlush(us); - state->provider->Shutdown(us); - /* Deliberately leak `state`: outstanding Span handles may still - reference the tracer; after Shutdown() their spans are simply - dropped. */ -} - -Span startSpan(std::string_view name, SpanKind kind) -{ - auto * state = theState.load(std::memory_order_acquire); - if (!state) - return Span(); - opentelemetry::trace::StartSpanOptions options; - options.kind = mapKind(kind); - return Span(std::make_shared(SpanImpl{state->tracer->StartSpan(toNostd(name), options)})); -} - -Span startSpan(std::string_view name, const Span & parent, SpanKind kind) -{ - auto * state = theState.load(std::memory_order_acquire); - if (!state) - return Span(); - opentelemetry::trace::StartSpanOptions options; - options.kind = mapKind(kind); - if (parent.impl) - options.parent = parent.impl->span->GetContext(); - return Span(std::make_shared(SpanImpl{state->tracer->StartSpan(toNostd(name), options)})); -} - -Span startSpanFromRemoteParent( - std::string_view name, std::string_view traceparent, std::string_view tracestate, SpanKind kind) -{ - auto * state = theState.load(std::memory_order_acquire); - if (!state) - return Span(); - opentelemetry::trace::StartSpanOptions options; - options.kind = mapKind(kind); - ExtractCarrier carrier; - carrier.traceparent = toNostd(traceparent); - carrier.tracestate = toNostd(tracestate); - /* Extract() returns the input context unchanged on a parse - failure, leaving an invalid SpanContext, i.e. a root span. */ - opentelemetry::context::Context emptyCtx; - auto ctx = opentelemetry::trace::propagation::HttpTraceContext{}.Extract(carrier, emptyCtx); - auto spanContext = opentelemetry::trace::GetSpan(ctx)->GetContext(); - if (spanContext.IsValid()) - options.parent = spanContext; - return Span(std::make_shared(SpanImpl{state->tracer->StartSpan(toNostd(name), options)})); -} - -bool Span::isRecording() const noexcept -{ - return impl && impl->span->IsRecording(); -} - -void Span::setAttribute(std::string_view key, std::string_view value) -{ - if (impl) - impl->span->SetAttribute(toNostd(key), toNostd(value)); -} - -void Span::setAttribute(std::string_view key, int64_t value) -{ - if (impl) - impl->span->SetAttribute(toNostd(key), value); -} - -void Span::setAttribute(std::string_view key, bool value) -{ - if (impl) - impl->span->SetAttribute(toNostd(key), value); -} - -void Span::setError(std::string_view description) -{ - if (impl) { - auto filtered = filterANSIEscapes(description, true); - impl->span->SetStatus(opentelemetry::trace::StatusCode::kError, toNostd(filtered)); - } -} - -void Span::end() -{ - if (impl) - impl->span->End(); -} - -Headers Span::injectContext() const -{ - if (!impl || !impl->span->GetContext().IsValid()) - return {}; - InjectCarrier carrier; - opentelemetry::context::Context ctx; - auto withSpan = opentelemetry::trace::SetSpan(ctx, impl->span); - opentelemetry::trace::propagation::HttpTraceContext{}.Inject(carrier, withSpan); - return std::move(carrier.headers); -} - -#else - -struct SpanImpl -{}; - -void init(std::string_view) {} - -bool isEnabled() noexcept -{ - return false; -} - -void resetAfterFork() {} - -void forceFlushAndShutdown(std::chrono::milliseconds) {} - -Span startSpan(std::string_view, SpanKind) -{ - return Span(); -} - -Span startSpan(std::string_view, const Span &, SpanKind) -{ - return Span(); -} - -Span startSpanFromRemoteParent(std::string_view, std::string_view, std::string_view, SpanKind) -{ - return Span(); -} - -bool Span::isRecording() const noexcept -{ - return false; -} - -void Span::setAttribute(std::string_view, std::string_view) {} - -void Span::setAttribute(std::string_view, int64_t) {} - -void Span::setAttribute(std::string_view, bool) {} - -void Span::setError(std::string_view) {} - -void Span::end() {} - -Headers Span::injectContext() const -{ - return {}; -} - -#endif // HAVE_OTEL - -Span::Span() noexcept = default; -Span::Span(const Span &) = default; -Span::Span(Span &&) noexcept = default; -Span & Span::operator=(const Span &) = default; -Span & Span::operator=(Span &&) noexcept = default; -Span::~Span() = default; - -Span::Span(std::shared_ptr impl) - : impl(std::move(impl)) -{ -} - -static std::mutex rootSpanMutex; -static std::weak_ptr rootSpanImpl; - -void setRootSpan(const Span & span) -{ - std::lock_guard lock(rootSpanMutex); - rootSpanImpl = span.impl; -} - -Span rootSpan() -{ - std::lock_guard lock(rootSpanMutex); - return Span(rootSpanImpl.lock()); -} - -} // namespace nix::otel diff --git a/src/libutil/package.nix b/src/libutil/package.nix index fc6a95eb2c..fe6fcb75f4 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -11,14 +11,11 @@ libsodium, nlohmann_json, openssl, - opentelemetry-cpp, zstd, # Configuration Options version, - - withOtel ? !stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform, }: let @@ -58,8 +55,7 @@ mkMesonLibrary (finalAttrs: { openssl zstd ] - ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid - ++ lib.optional withOtel opentelemetry-cpp; + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; propagatedBuildInputs = [ boost @@ -69,7 +65,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ (lib.mesonEnable "cpuid" stdenv.hostPlatform.isx86_64) - (lib.mesonEnable "otel" withOtel) ]; meta = { diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index f914bdf40d..5450c42afd 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -74,6 +74,21 @@ class TeeLogger final : public Logger logger->result(act, type, json); } + Headers getTraceContext(ActivityId act) override + { + for (auto & logger : loggers) { + auto headers = logger->getTraceContext(act); + if (!headers.empty()) + return headers; + } + return {}; + } + + void addLogger(std::unique_ptr logger) + { + loggers.push_back(std::move(logger)); + } + void writeToStdout(std::string_view s) override { for (auto & logger : loggers) { @@ -115,4 +130,20 @@ makeTeeLogger(std::unique_ptr mainLogger, std::vector(std::move(allLoggers)); } +void applyExtraLogger(std::unique_ptr extraLogger) +{ + if (auto teeLogger = dynamic_cast(logger)) + teeLogger->addLogger(std::move(extraLogger)); + else { + std::vector> loggers; + loggers.push_back(std::move(extraLogger)); + try { + logger = makeTeeLogger(std::unique_ptr(logger), std::move(loggers)).release(); + } catch (...) { + // `logger` is now gone so give up. + abort(); + } + } +} + } // namespace nix diff --git a/src/nix/main.cc b/src/nix/main.cc index e530831dfb..44004f9323 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -24,11 +24,11 @@ #include "nix/flake/flake.hh" #include "nix/flake/settings.hh" #include "nix/util/exit.hh" -#include "nix/util/otel.hh" #include "nix/util/sentry.hh" #include "self-exe.hh" #include "crash-handler.hh" +#include "otel-logger.hh" #include "cli-config-private.hh" #include @@ -535,6 +535,11 @@ void mainWrapped(int argc, char ** argv) tryEnterPrivateMountNamespace(); #endif + /* Note: registered before the logger->stop() Finally below, so + that at exit, the loggers are stopped (ending any open spans) + before the spans are exported. */ + Finally flushOtel([] { flushOtelAndShutdown(); }); + Finally f([] { logger->stop(); }); programPath = argv[0]; @@ -551,9 +556,7 @@ void mainWrapped(int argc, char ** argv) /* No-op unless OTEL_EXPORTER_OTLP_ENDPOINT is set in the environment. */ - otel::init(programName); - - Finally flushOtel([] { otel::forceFlushAndShutdown(); }); + initOtel(programName); { if (auto legacy = get(RegisterLegacyCommand::commands(), programName)) @@ -714,9 +717,13 @@ void mainWrapped(int argc, char ** argv) setSentryTag("nix_subcommand", concatStringsSep(" ", subcommand).c_str()); - /* Ended by RAII, including on exceptions. */ - auto rootSpan = otel::startSpan("nix " + concatStringsSep(" ", subcommand)); - otel::setRootSpan(rootSpan); + /* Map activities to OpenTelemetry spans, under a root span named + after the subcommand. */ + OpenTelemetryLogger * otelLogger = nullptr; + if (auto l = makeOpenTelemetryLogger("nix " + concatStringsSep(" ", subcommand))) { + otelLogger = l.get(); + applyExtraLogger(std::move(l)); + } try { args.command->second->run(); @@ -728,8 +735,9 @@ void mainWrapped(int argc, char ** argv) } catch (Exit &) { throw; } catch (std::exception & e) { - // FIXME: privacy - rootSpan.setError(e.what()); + if (otelLogger) + // FIXME: privacy + otelLogger->setRootError(e.what()); throw; } } diff --git a/src/nix/meson.build b/src/nix/meson.build index fc232a8154..13b98512a7 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -82,6 +82,12 @@ if sentry_required.enabled() configdata.set_quoted('CRASHPAD_HANDLER_PATH', crashpad_handler_path) endif +otel_required = get_option('otel') +otel = dependency('opentelemetry_exporter_otlp_http', required : otel_required) +otel_trace = dependency('opentelemetry_trace', required : otel_required) +configdata.set('HAVE_OTEL', (otel.found() and otel_trace.found()).to_int()) +deps_private += [ otel, otel_trace ] + config_priv_h = configure_file( configuration : configdata, output : 'cli-config-private.hh', @@ -124,6 +130,7 @@ nix_sources = [ config_priv_h ] + files( 'nar.cc', 'nario.cc', 'optimise-store.cc', + 'otel-logger.cc', 'path-from-hash-part.cc', 'path-info.cc', 'prefetch.cc', diff --git a/src/nix/meson.options b/src/nix/meson.options index 57219e3eb3..eefdddbce3 100644 --- a/src/nix/meson.options +++ b/src/nix/meson.options @@ -29,6 +29,12 @@ option( description : 'Enable Sentry crash reporting', ) +option( + 'otel', + type : 'feature', + description : 'distributed tracing via OpenTelemetry (OTLP/HTTP exporter)', +) + option( 'crashpad-handler', type : 'string', diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc new file mode 100644 index 0000000000..9f30e04696 --- /dev/null +++ b/src/nix/otel-logger.cc @@ -0,0 +1,409 @@ +#include "otel-logger.hh" + +#include "cli-config-private.hh" + +#if HAVE_OTEL +# include "nix/util/environment-variables.hh" +# include "nix/util/sync.hh" +# include "nix/util/terminal.hh" + +# include +# include + +# include + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + +namespace nix { + +#if HAVE_OTEL + +namespace { + +struct OtelState +{ + std::unique_ptr provider; + opentelemetry::nostd::shared_ptr tracer; +}; + +/* Owned here rather than via opentelemetry's global Provider + singleton, and deliberately leaked: the provider's destructor joins + the batch exporter's worker thread, which must never happen from + static destructors at exit() time (cf. the OPENSSL_INIT_NO_ATEXIT + note in util.cc; in the daemon's forked children that thread does + not even exist). Teardown happens only via an explicit + flushOtelAndShutdown(). */ +std::atomic otelState{nullptr}; + +inline opentelemetry::nostd::string_view toNostd(std::string_view sv) noexcept +{ + return {sv.data(), sv.size()}; +} + +struct ExtractCarrier : opentelemetry::context::propagation::TextMapCarrier +{ + opentelemetry::nostd::string_view traceparent, tracestate; + + opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view key) const noexcept override + { + if (key == "traceparent") + return traceparent; + if (key == "tracestate") + return tracestate; + return {}; + } + + void Set(opentelemetry::nostd::string_view, opentelemetry::nostd::string_view) noexcept override {} +}; + +struct InjectCarrier : opentelemetry::context::propagation::TextMapCarrier +{ + Headers headers; + + opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view) const noexcept override + { + return {}; + } + + void Set(opentelemetry::nostd::string_view key, opentelemetry::nostd::string_view value) noexcept override + { + /* Copy immediately: `value` may point into a stack buffer of + the propagator. */ + headers.emplace_back(std::string(key.data(), key.size()), std::string(value.data(), value.size())); + } +}; + +/** + * The span name for an activity. An empty result means the activity's + * text should be used instead. + */ +std::string_view activityName(ActivityType type) +{ + switch (type) { + case actUnknown: + return {}; + case actCopyPath: + return "copy path"; + case actFileTransfer: + return "file transfer"; + case actRealise: + return "realise"; + case actCopyPaths: + return "copy paths"; + case actBuilds: + return "builds"; + case actBuild: + return "build"; + case actOptimiseStore: + return "optimise store"; + case actVerifyPaths: + return "verify paths"; + case actSubstitute: + return "substitute"; + case actQueryPathInfo: + return "query path info"; + case actPostBuildHook: + return "post-build hook"; + case actBuildWaiting: + return "build waiting"; + case actFetchTree: + return "fetch tree"; + case actFileTransferAttempt: + return "http request"; + } + return {}; +} + +/* Defensive field accessors, like the progress bar's. */ +std::string_view getS(const Logger::Fields & fields, size_t n) +{ + if (n >= fields.size() || fields[n].type != Logger::Field::tString) + return {}; + return fields[n].s; +} + +uint64_t getI(const Logger::Fields & fields, size_t n) +{ + if (n >= fields.size() || fields[n].type != Logger::Field::tInt) + return 0; + return fields[n].i; +} + +class OpenTelemetryLoggerImpl : public OpenTelemetryLogger +{ + using SpanPtr = opentelemetry::nostd::shared_ptr; + + opentelemetry::nostd::shared_ptr tracer; + + SpanPtr rootSpan; + + Sync> spans_; + + static Headers injectContext(const SpanPtr & span) + { + if (!span->GetContext().IsValid()) + return {}; + InjectCarrier carrier; + opentelemetry::context::Context ctx; + auto withSpan = opentelemetry::trace::SetSpan(ctx, span); + opentelemetry::trace::propagation::HttpTraceContext{}.Inject(carrier, withSpan); + return std::move(carrier.headers); + } + +public: + OpenTelemetryLoggerImpl(OtelState & state, std::string_view rootSpanName, std::string_view remoteParentTraceparent) + : tracer(state.tracer) + { + opentelemetry::trace::StartSpanOptions options; + if (!remoteParentTraceparent.empty()) { + options.kind = opentelemetry::trace::SpanKind::kServer; + /* Extract() returns the input context unchanged on a + parse failure, leaving an invalid SpanContext, i.e. an + unparented root span. */ + ExtractCarrier carrier; + carrier.traceparent = toNostd(remoteParentTraceparent); + opentelemetry::context::Context emptyCtx; + auto ctx = opentelemetry::trace::propagation::HttpTraceContext{}.Extract(carrier, emptyCtx); + auto spanContext = opentelemetry::trace::GetSpan(ctx)->GetContext(); + if (spanContext.IsValid()) + options.parent = spanContext; + } + rootSpan = tracer->StartSpan(toNostd(rootSpanName), options); + } + + void log(Verbosity lvl, std::string_view s) noexcept override {} + + void logEI(const ErrorInfo & ei) noexcept override {} + + void startActivity( + ActivityId act, + Verbosity lvl, + ActivityType type, + const std::string & s, + const Fields & fields, + ActivityId parent) noexcept override + { + try { + if (isRemoteLogSource()) + return; + + opentelemetry::trace::StartSpanOptions options; + if (type == actFileTransferAttempt) + options.kind = opentelemetry::trace::SpanKind::kClient; + + auto name = activityName(type); + bool textIsName = name.empty(); + if (textIsName) + name = s.empty() ? "activity" : std::string_view(s); + + auto spans(spans_.lock()); + + if (auto i = spans->find(parent); i != spans->end()) + options.parent = i->second->GetContext(); + else + options.parent = rootSpan->GetContext(); + + auto span = tracer->StartSpan(toNostd(name), options); + + if (!s.empty() && !textIsName) + span->SetAttribute("nix.activity.text", toNostd(filterANSIEscapes(s, true))); + +// Allow handling a subset of enum values +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wswitch-enum" + switch (type) { + case actFileTransfer: + case actFileTransferAttempt: + span->SetAttribute("url.full", toNostd(getS(fields, 0))); + if (type == actFileTransferAttempt) { + if (auto attempt = getI(fields, 1)) + span->SetAttribute("http.request.resend_count", (int64_t) attempt); + } + break; + case actBuild: + case actPostBuildHook: + span->SetAttribute("nix.drv.path", toNostd(getS(fields, 0))); + if (auto machine = getS(fields, 1); !machine.empty()) + span->SetAttribute("nix.machine", toNostd(machine)); + break; + case actSubstitute: + case actQueryPathInfo: + span->SetAttribute("nix.store.path", toNostd(getS(fields, 0))); + span->SetAttribute("nix.substituter", toNostd(getS(fields, 1))); + break; + case actCopyPath: + span->SetAttribute("nix.store.path", toNostd(getS(fields, 0))); + span->SetAttribute("nix.src.store", toNostd(getS(fields, 1))); + span->SetAttribute("nix.dst.store", toNostd(getS(fields, 2))); + break; + default: + break; + } +# pragma GCC diagnostic pop + + spans->emplace(act, std::move(span)); + } catch (...) { + } + } + + void stopActivity(ActivityId act) noexcept override + { + try { + if (isRemoteLogSource()) + return; + auto spans(spans_.lock()); + if (auto i = spans->find(act); i != spans->end()) { + i->second->End(); + spans->erase(i); + } + } catch (...) { + } + } + + void result(ActivityId act, ResultType type, const nlohmann::json & json) noexcept override + { + try { + if (isRemoteLogSource()) + return; + if (type == resHttpStatus) { + auto spans(spans_.lock()); + if (auto i = spans->find(act); i != spans->end()) + if (auto status = json.find("httpStatus"); status != json.end() && status->is_number()) + i->second->SetAttribute("http.response.status_code", status->get()); + } + } catch (...) { + } + } + + Headers getTraceContext(ActivityId act) override + { + try { + if (act) { + auto spans(spans_.lock()); + if (auto i = spans->find(act); i != spans->end()) + return injectContext(i->second); + } + return injectContext(rootSpan); + } catch (...) { + return {}; + } + } + + void setRootError(std::string_view description) noexcept override + { + try { + rootSpan->SetStatus( + opentelemetry::trace::StatusCode::kError, toNostd(filterANSIEscapes(description, true))); + } catch (...) { + } + } + + void stop() override + { + try { + auto spans(spans_.lock()); + for (auto & [_, span] : *spans) + span->End(); + spans->clear(); + rootSpan->End(); + } catch (...) { + } + } +}; + +} // namespace + +void initOtel(std::string_view serviceName) +{ + if (otelState.load(std::memory_order_acquire)) + return; + + /* Without an explicitly configured endpoint, stay off: the SDK's + default would silently export to http://localhost:4318. */ + if (!getEnv("OTEL_EXPORTER_OTLP_ENDPOINT") && !getEnv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")) + return; + + namespace sdktrace = opentelemetry::sdk::trace; + + /* Default-constructed options read the OTEL_EXPORTER_OTLP_* + environment variables (endpoint, headers, TLS, compression). */ + auto exporter = opentelemetry::exporter::otlp::OtlpHttpExporterFactory::Create(); + auto processor = + sdktrace::BatchSpanProcessorFactory::Create(std::move(exporter), sdktrace::BatchSpanProcessorOptions{}); + auto resource = opentelemetry::sdk::resource::Resource::Create({ + {opentelemetry::semconv::service::kServiceName, std::string(serviceName)}, + }); + + auto state = std::make_unique(); + state->provider = sdktrace::TracerProviderFactory::Create(std::move(processor), resource); + state->tracer = state->provider->GetTracer("nix"); + + OtelState * expected = nullptr; + if (otelState.compare_exchange_strong(expected, state.get())) + state.release(); +} + +void flushOtelAndShutdown() +{ + auto * state = otelState.exchange(nullptr); + if (!state) + return; + /* Bound the timeouts: the SDK defaults are microseconds::max(), + and a hung collector must not hang process exit. */ + auto timeout = std::chrono::microseconds(std::chrono::seconds(5)); + state->provider->ForceFlush(timeout); + state->provider->Shutdown(timeout); + /* Deliberately leak `state`: outstanding span handles may still + reference the tracer; after Shutdown() their spans are simply + dropped. */ +} + +void resetOtelAfterFork() +{ + /* Deliberately leak the old state: it references a worker thread + that does not exist in this process, so it can be neither + flushed nor destroyed safely. */ + otelState.exchange(nullptr); +} + +std::unique_ptr +makeOpenTelemetryLogger(std::string_view rootSpanName, std::string_view remoteParentTraceparent) +{ + auto * state = otelState.load(std::memory_order_acquire); + if (!state) + return nullptr; + return std::make_unique(*state, rootSpanName, remoteParentTraceparent); +} + +#else + +void initOtel(std::string_view) {} + +void flushOtelAndShutdown() {} + +void resetOtelAfterFork() {} + +std::unique_ptr makeOpenTelemetryLogger(std::string_view, std::string_view) +{ + return nullptr; +} + +#endif // HAVE_OTEL + +} // namespace nix diff --git a/src/nix/otel-logger.hh b/src/nix/otel-logger.hh new file mode 100644 index 0000000000..e56770e665 --- /dev/null +++ b/src/nix/otel-logger.hh @@ -0,0 +1,74 @@ +#pragma once +///@file + +#include "nix/util/logging.hh" + +#include +#include + +namespace nix { + +/** + * A `Logger` that maps Nix activities onto OpenTelemetry spans and + * exports them via OTLP/HTTP. Intended to be added to the global + * logger using `applyExtraLogger()`. + * + * Activities replayed from another process (cf. `RemoteLogSource`) + * are ignored, since the originating process is responsible for + * exporting them. + */ +class OpenTelemetryLogger : public Logger +{ +public: + /** + * Record an error on the root span. + */ + virtual void setRootError(std::string_view description) noexcept = 0; +}; + +/** + * Create an `OpenTelemetryLogger` whose root span is named + * `rootSpanName`. Activities without a known parent become children + * of the root span. + * + * If `remoteParentTraceparent` is non-empty, the root span is created + * as a server span whose parent is the given W3C trace context (used + * for daemon connections, where the client sends its trace context + * during the handshake). An invalid value yields an unparented root + * span. + * + * Returns null if tracing support is not compiled in or `initOtel()` + * did not enable tracing; the caller should then not attach a logger. + */ +std::unique_ptr +makeOpenTelemetryLogger(std::string_view rootSpanName, std::string_view remoteParentTraceparent = {}); + +/** + * Initialize OpenTelemetry tracing for this process. Does nothing + * (and is cheap) if `OTEL_EXPORTER_OTLP_ENDPOINT` / + * `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is not set in the environment, + * if tracing support is not compiled in, or if tracing is already + * initialized. + */ +void initOtel(std::string_view serviceName); + +/** + * Flush all pending spans and shut down the exporter, with bounded + * timeouts. Safe to call if tracing was never initialized, and safe + * to call more than once. Must be called explicitly before process + * exit: nothing is flushed from static destructors (cf. the + * OPENSSL_INIT_NO_ATEXIT note in util.cc). Spans that should be + * included must be ended first (e.g. via `logger->stop()`). + */ +void flushOtelAndShutdown(); + +/** + * Discard all tracing state inherited from the parent process after a + * fork(): the exporter's worker thread does not exist in the child, + * so the inherited state can be neither used nor destroyed safely. + * Afterwards `initOtel()` can be called again to start fresh tracing + * in the child. + */ +void resetOtelAfterFork(); + +} // namespace nix diff --git a/src/nix/package.nix b/src/nix/package.nix index be9a253e6f..58c7591514 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -8,6 +8,7 @@ nix-expr, nix-main, nix-cmd, + opentelemetry-cpp, sentry-native, libmicrohttpd, @@ -45,6 +46,7 @@ let inherit (lib) fileset; enableSentry = !stdenv.hostPlatform.isStatic; + withOtel = !stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform; in mkMesonExecutable (finalAttrs: { @@ -116,12 +118,14 @@ mkMesonExecutable (finalAttrs: { && stdenv.cc.libcxx != null && stdenv.cc.libcxx.isLLVM ) llvmPackages.libunwind - ++ lib.optional enableSentry sentry-native; + ++ lib.optional enableSentry sentry-native + ++ lib.optional withOtel opentelemetry-cpp; mesonFlags = [ (lib.mesonEnable "mimalloc" withMimalloc) (lib.mesonBool "plugin-c-api" withPluginCApi) (lib.mesonEnable "sentry" enableSentry) + (lib.mesonEnable "otel" withOtel) ] ++ lib.optional enableSentry ( lib.mesonOption "crashpad-handler" "${sentry-native}/bin/crashpad_handler" diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 2ecd7f139d..3205f80723 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -8,7 +8,6 @@ #include "nix/store/remote-store.hh" #include "nix/store/remote-store-connection.hh" #include "nix/store/store-open.hh" -#include "nix/util/otel.hh" #include "nix/util/serialise.hh" #include "nix/store/globals.hh" #include "nix/util/config-global.hh" @@ -17,6 +16,7 @@ #include "nix/cmd/unix-socket-server.hh" #include "nix/store/daemon.hh" #include "man-pages.hh" +#include "otel-logger.hh" #include "nix/util/socket.hh" #include @@ -41,6 +41,18 @@ namespace nix { +/** + * Set up distributed tracing for a daemon connection: create a span + * covering the connection's lifetime, parented under the trace + * context received from the client, if any. No-op unless + * OpenTelemetry export is configured. + */ +static void setupConnectionTelemetry(std::string_view traceparent) +{ + if (auto otelLogger = makeOpenTelemetryLogger("daemon connection", traceparent)) + applyExtraLogger(std::move(otelLogger)); +} + /** * Settings related to authenticating clients for the Nix daemon. * @@ -370,9 +382,8 @@ static void daemonLoop( /* The OpenTelemetry exporter's worker thread does not survive the fork, so set up tracing afresh. */ - otel::resetAfterFork(); - otel::init("nix-daemon"); - Finally flushOtel([] { otel::forceFlushAndShutdown(); }); + resetOtelAfterFork(); + initOtel("nix-daemon"); closeListeners(); @@ -397,12 +408,14 @@ static void daemonLoop( FdSource(remote.get()), FdSink(remote.get()), trusted, - RecursiveFlag::NotRecursive); + RecursiveFlag::NotRecursive, + setupConnectionTelemetry); - /* Flush explicitly, since exit() does not - unwind the stack and so won't run - `flushOtel`. */ - otel::forceFlushAndShutdown(); + /* End the connection span and export all + telemetry. This has to be done explicitly, + since exit() does not unwind the stack. */ + logger->stop(); + flushOtelAndShutdown(); exit(0); }, @@ -468,7 +481,13 @@ static void forwardStdioConnection(RemoteStore & store) */ static void processStdioConnection(ref store, TrustedFlag trustClient) { - processConnection(store, FdSource(STDIN_FILENO), FdSink(STDOUT_FILENO), trustClient, daemon::NotRecursive); + processConnection( + store, + FdSource(STDIN_FILENO), + FdSink(STDOUT_FILENO), + trustClient, + daemon::NotRecursive, + setupConnectionTelemetry); } /** From 9e67d78ec5bf69a8e0c12e0f692b44487b0f6b07 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 13:59:47 +0200 Subject: [PATCH 679/927] libutil: Simplify applyExtraLogger() Turn the global logger into a TeeLogger if it isn't one already, then unconditionally append the extra logger to it. Assisted-by: Claude Fable 5 --- src/libutil/tee-logger.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index 5450c42afd..087a3fb199 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -132,18 +132,19 @@ makeTeeLogger(std::unique_ptr mainLogger, std::vector extraLogger) { - if (auto teeLogger = dynamic_cast(logger)) - teeLogger->addLogger(std::move(extraLogger)); - else { - std::vector> loggers; - loggers.push_back(std::move(extraLogger)); + auto teeLogger = dynamic_cast(logger); + if (!teeLogger) { try { - logger = makeTeeLogger(std::unique_ptr(logger), std::move(loggers)).release(); + std::vector> loggers; + loggers.push_back(std::unique_ptr(logger)); + teeLogger = new TeeLogger(std::move(loggers)); } catch (...) { // `logger` is now gone so give up. abort(); } + logger = teeLogger; } + teeLogger->addLogger(std::move(extraLogger)); } } // namespace nix From 8446dd2e8b8302c5959c5787e03f33f7b534bb49 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 14:07:17 +0200 Subject: [PATCH 680/927] libutil: Remove makeTeeLogger() It no longer has any callers: applyExtraLogger() constructs the TeeLogger directly. Assisted-by: Claude Fable 5 --- src/libutil/include/nix/util/logging.hh | 8 -------- src/libutil/tee-logger.cc | 10 ---------- 2 files changed, 18 deletions(-) diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index 79cbb11c26..a2016472de 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -311,14 +311,6 @@ extern Logger * logger; std::unique_ptr makeSimpleLogger(bool printBuildLogs = true); -/** - * Create a logger that sends log messages to `mainLogger` and the - * list of loggers in `extraLoggers`. Only `mainLogger` is used for - * writing to stdout and getting user input. - */ -std::unique_ptr -makeTeeLogger(std::unique_ptr mainLogger, std::vector> && extraLoggers); - std::unique_ptr makeJSONLogger(Descriptor fd, bool includeNixPrefix = true); std::unique_ptr makeJSONLogger(const std::filesystem::path & path, bool includeNixPrefix = true); diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index 087a3fb199..7230e725cd 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -120,16 +120,6 @@ class TeeLogger final : public Logger } // namespace -std::unique_ptr -makeTeeLogger(std::unique_ptr mainLogger, std::vector> && extraLoggers) -{ - std::vector> allLoggers; - allLoggers.push_back(std::move(mainLogger)); - for (auto & l : extraLoggers) - allLoggers.push_back(std::move(l)); - return std::make_unique(std::move(allLoggers)); -} - void applyExtraLogger(std::unique_ptr extraLogger) { auto teeLogger = dynamic_cast(logger); From 38d5d02980cd9b17eaf325049267f398d14d2d33 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 14:40:38 +0200 Subject: [PATCH 681/927] libstore: Add the request method and body size to resHttpStatus The JSON payload now carries `method` (e.g. `GET`) and `bodySize` (the number of body bytes received) alongside `httpStatus`. The OpenTelemetryLogger maps them onto the `http.request.method` and `http.response.body.size` span attributes. Assisted-by: Claude Fable 5 --- src/libstore/filetransfer.cc | 23 ++++++++++++++++++++++- src/libutil/include/nix/util/logging.hh | 8 +++++--- src/nix/otel-logger.cc | 7 ++++++- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index f3aa5e2976..2b278bd8ec 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -62,6 +62,23 @@ constexpr bool operator==(long lhs, HttpStatus rhs) noexcept return lhs == static_cast(rhs); } +const char * httpMethodName(HttpMethod method) +{ + switch (method) { + case HttpMethod::Get: + return "GET"; + case HttpMethod::Put: + return "PUT"; + case HttpMethod::Head: + return "HEAD"; + case HttpMethod::Post: + return "POST"; + case HttpMethod::Delete: + return "DELETE"; + } + unreachable(); +} + } // namespace std::chrono::milliseconds computeRetryDelayMs(const RetryDelayParams & p, std::mt19937 & rng) @@ -810,8 +827,12 @@ struct curlFileTransfer : public FileTransfer } if (attemptAct) { + nlohmann::json json; + json["method"] = httpMethodName(request.method); if (httpStatus) - logger->result(attemptAct->id, resHttpStatus, nlohmann::json{{"httpStatus", httpStatus}}); + json["httpStatus"] = httpStatus; + json["bodySize"] = result.bodySize; + logger->result(attemptAct->id, resHttpStatus, json); attemptAct.reset(); } diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index a2016472de..c01a3bea26 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -50,9 +50,11 @@ typedef enum { resHashMismatch = 109, resBuildResult = 110, /** - * Emitted via the JSON `result()` overload: an object with an - * `httpStatus` field containing the status code of an HTTP - * response. More fields may be added in the future. + * Emitted via the JSON `result()` overload: an object describing + * the response to an HTTP request, with the fields `method` (the + * request method, e.g. `GET`), `httpStatus` (the status code, + * absent for non-HTTP protocols) and `bodySize` (the number of + * body bytes received). More fields may be added in the future. */ resHttpStatus = 111, } ResultType; diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 9f30e04696..80288a7a01 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -283,9 +283,14 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger return; if (type == resHttpStatus) { auto spans(spans_.lock()); - if (auto i = spans->find(act); i != spans->end()) + if (auto i = spans->find(act); i != spans->end()) { + if (auto method = json.find("method"); method != json.end() && method->is_string()) + i->second->SetAttribute("http.request.method", toNostd(method->get_ref())); if (auto status = json.find("httpStatus"); status != json.end() && status->is_number()) i->second->SetAttribute("http.response.status_code", status->get()); + if (auto bodySize = json.find("bodySize"); bodySize != json.end() && bodySize->is_number()) + i->second->SetAttribute("http.response.body.size", bodySize->get()); + } } } catch (...) { } From 8a541f479f88ce2724cd0016645b010b7e49430b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 15:01:52 +0200 Subject: [PATCH 682/927] Name HTTP request spans after the request method Per the OpenTelemetry semantic conventions, HTTP client spans are named after the request method (e.g. `GET`). Pass the method as a field of actFileTransferAttempt so the OpenTelemetryLogger can use it as the span name and the `http.request.method` attribute, and drop it from resHttpStatus where it is now superfluous. Assisted-by: Claude Fable 5 --- src/libstore/filetransfer.cc | 3 +-- src/libutil/include/nix/util/logging.hh | 11 ++++++----- src/nix/otel-logger.cc | 10 ++++++++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 2b278bd8ec..437abd6ee8 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -633,7 +633,7 @@ struct curlFileTransfer : public FileTransfer lvlDebug, actFileTransferAttempt, "", - Logger::Fields{request.displayUri(), (uint64_t) attempt}, + Logger::Fields{request.displayUri(), (uint64_t) attempt, httpMethodName(request.method)}, act().id); /* (Re)build the request headers, since the traceparent @@ -828,7 +828,6 @@ struct curlFileTransfer : public FileTransfer if (attemptAct) { nlohmann::json json; - json["method"] = httpMethodName(request.method); if (httpStatus) json["httpStatus"] = httpStatus; json["bodySize"] = result.bodySize; diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index c01a3bea26..a64ad76ad7 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -32,7 +32,8 @@ typedef enum { /** * A single HTTP request within an `actFileTransfer` activity * (there can be several due to retries). Fields: [0] the URI - * (string), [1] the number of preceding attempts (int). + * (string), [1] the number of preceding attempts (int), [2] the + * request method (string, e.g. `GET`). */ actFileTransferAttempt = 113, } ActivityType; @@ -51,10 +52,10 @@ typedef enum { resBuildResult = 110, /** * Emitted via the JSON `result()` overload: an object describing - * the response to an HTTP request, with the fields `method` (the - * request method, e.g. `GET`), `httpStatus` (the status code, - * absent for non-HTTP protocols) and `bodySize` (the number of - * body bytes received). More fields may be added in the future. + * the response to an HTTP request, with the fields `httpStatus` + * (the status code, absent for non-HTTP protocols) and `bodySize` + * (the number of body bytes received). More fields may be added + * in the future. */ resHttpStatus = 111, } ResultType; diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 80288a7a01..05ce9d9f61 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -212,6 +212,12 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger if (textIsName) name = s.empty() ? "activity" : std::string_view(s); + /* Per the OpenTelemetry semantic conventions, HTTP client + spans are named after the request method. */ + if (type == actFileTransferAttempt) + if (auto method = getS(fields, 2); !method.empty()) + name = method; + auto spans(spans_.lock()); if (auto i = spans->find(parent); i != spans->end()) @@ -234,6 +240,8 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger if (type == actFileTransferAttempt) { if (auto attempt = getI(fields, 1)) span->SetAttribute("http.request.resend_count", (int64_t) attempt); + if (auto method = getS(fields, 2); !method.empty()) + span->SetAttribute("http.request.method", toNostd(method)); } break; case actBuild: @@ -284,8 +292,6 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger if (type == resHttpStatus) { auto spans(spans_.lock()); if (auto i = spans->find(act); i != spans->end()) { - if (auto method = json.find("method"); method != json.end() && method->is_string()) - i->second->SetAttribute("http.request.method", toNostd(method->get_ref())); if (auto status = json.find("httpStatus"); status != json.end() && status->is_number()) i->second->SetAttribute("http.response.status_code", status->get()); if (auto bodySize = json.find("bodySize"); bodySize != json.end() && bodySize->is_number()) From 9e837fddf5f39f42854e9ee81f81b153c54d5fc2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 15:06:34 +0200 Subject: [PATCH 683/927] otel-logger: Derive span names from the ActivityType enum values Return the name of the enum value without the `act` prefix (e.g. `OptimiseStore`), generated with a macro so the names cannot drift from the enum. In the future, C++26 reflection can make this fully generic. Assisted-by: Claude Fable 5 --- src/nix/otel-logger.cc | 54 +++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 05ce9d9f61..76df2b6a09 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -90,42 +90,36 @@ struct InjectCarrier : opentelemetry::context::propagation::TextMapCarrier }; /** - * The span name for an activity. An empty result means the activity's - * text should be used instead. + * The span name for an activity: the name of the enum value without + * the `act` prefix, e.g. `OptimiseStore`. An empty result means the + * activity's text should be used instead. + * + * TODO: Use C++26 reflection to derive this from the enum + * definition generically instead of enumerating the values here. */ std::string_view activityName(ActivityType type) { switch (type) { case actUnknown: return {}; - case actCopyPath: - return "copy path"; - case actFileTransfer: - return "file transfer"; - case actRealise: - return "realise"; - case actCopyPaths: - return "copy paths"; - case actBuilds: - return "builds"; - case actBuild: - return "build"; - case actOptimiseStore: - return "optimise store"; - case actVerifyPaths: - return "verify paths"; - case actSubstitute: - return "substitute"; - case actQueryPathInfo: - return "query path info"; - case actPostBuildHook: - return "post-build hook"; - case actBuildWaiting: - return "build waiting"; - case actFetchTree: - return "fetch tree"; - case actFileTransferAttempt: - return "http request"; +# define ACTIVITY_NAME(name) \ + case act##name: \ + return #name; + ACTIVITY_NAME(CopyPath) + ACTIVITY_NAME(FileTransfer) + ACTIVITY_NAME(Realise) + ACTIVITY_NAME(CopyPaths) + ACTIVITY_NAME(Builds) + ACTIVITY_NAME(Build) + ACTIVITY_NAME(OptimiseStore) + ACTIVITY_NAME(VerifyPaths) + ACTIVITY_NAME(Substitute) + ACTIVITY_NAME(QueryPathInfo) + ACTIVITY_NAME(PostBuildHook) + ACTIVITY_NAME(BuildWaiting) + ACTIVITY_NAME(FetchTree) + ACTIVITY_NAME(FileTransferAttempt) +# undef ACTIVITY_NAME } return {}; } From a0516759ff8c314a447f3ba39eeb6ede577239b3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 16:09:11 +0200 Subject: [PATCH 684/927] logging: Turn Field into an std::variant --- src/libmain/progress-bar.cc | 16 ++++++++++------ src/libstore/daemon.cc | 13 +++++++------ src/libstore/worker-protocol-connection.cc | 6 +++--- src/libutil-c/nix_api_util.cc | 5 +++-- src/libutil/include/nix/util/logging.hh | 15 ++++----------- src/libutil/logging.cc | 19 ++++++++----------- src/nix/otel-logger.cc | 20 ++++++++++++-------- 7 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 48d44a3a4b..4d2a36672a 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -19,16 +19,20 @@ namespace { static std::string_view getS(const std::vector & fields, size_t n) { - if (n >= fields.size() || fields[n].type != Logger::Field::tString) - throw Error("could not get expected log field of type 'string' at index %d", n); - return fields[n].s; + if (n < fields.size()) { + if (auto p = std::get_if(&fields[n].raw)) + return *p; + } + throw Error("could not get expected log field of type 'string' at index %d", n); } static uint64_t getI(const std::vector & fields, size_t n) { - if (n >= fields.size() || fields[n].type != Logger::Field::tInt) - throw Error("could not get expected log field of type 'int' at index %d", n); - return fields[n].i; + if (n < fields.size()) { + if (auto p = std::get_if(&fields[n].raw)) + return *p; + } + throw Error("could not get expected log field of type 'int' at index %d", n); } static std::string_view storePathToName(std::string_view path) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e3e268db2e..84dd68e3ec 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -35,12 +35,13 @@ Sink & operator<<(Sink & sink, const Logger::Fields & fields) { sink << fields.size(); for (auto & f : fields) { - sink << f.type; - if (f.type == Logger::Field::tInt) - sink << f.i; - else if (f.type == Logger::Field::tString) - sink << f.s; - else + if (auto p = std::get_if(&f.raw)) { + sink << 0; + sink << *p; + } else if (auto p = std::get_if(&f.raw)) { + sink << 1; + sink << *p; + } else unreachable(); } return sink; diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index c7fdf9736f..7a0ed2330d 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -19,10 +19,10 @@ static Logger::Fields readFields(Source & from) Logger::Fields fields; size_t size = readInt(from); for (size_t n = 0; n < size; n++) { - auto type = (decltype(Logger::Field::type)) readInt(from); - if (type == Logger::Field::tInt) + auto type = readInt(from); + if (type == 0) fields.push_back(readNum(from)); - else if (type == Logger::Field::tString) + else if (type == 1) fields.push_back(readString(from)); else throw Error("got unsupported field type %x from Nix daemon", (int) type); diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index 5fd1b8510e..a3fe7614d5 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -75,9 +75,10 @@ class CallbackLogger : public nix::Logger { if (!vtable.result_string) return; - if (fields.empty() || fields[0].type != nix::Logger::Field::tString) + if (fields.empty() || !std::get_if(&fields[0].raw)) return; - vtable.result_string(userdata, act, static_cast(type), fields[0].s.c_str()); + vtable.result_string( + userdata, act, static_cast(type), std::get(fields[0].raw).c_str()); } virtual void anchor(); diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index a64ad76ad7..409dfcea21 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -108,27 +108,20 @@ public: struct Field { - // FIXME: use std::variant. - enum { tInt = 0, tString = 1 } type; - - uint64_t i = 0; - std::string s; + std::variant raw; Field(const std::string & s) - : type(tString) - , s(s) + : raw(s) { } Field(const char * s) - : type(tString) - , s(s) + : raw(std::string(s)) { } Field(const uint64_t & i) - : type(tInt) - , i(i) + : raw(i) { } }; diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index b231b573e8..95b532d805 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -167,13 +167,10 @@ class SimpleLogger : public Logger void result(ActivityId act, ResultType type, const Fields & fields) noexcept override { - if (type == resBuildLogLine && printBuildLogs) { - auto lastLine = fields[0].s; - printError(lastLine); - } else if (type == resPostBuildLogLine && printBuildLogs) { - auto lastLine = fields[0].s; - printError("post-build-hook: " + lastLine); - } + if (type == resBuildLogLine && printBuildLogs) + printError(std::get(fields[0].raw)); + else if (type == resPostBuildLogLine && printBuildLogs) + printError("post-build-hook: " + std::get(fields[0].raw)); } }; @@ -276,10 +273,10 @@ struct JSONLogger : Logger return; auto & arr = json["fields"] = nlohmann::json::array(); for (auto & f : fields) - if (f.type == Logger::Field::tInt) - arr.push_back(f.i); - else if (f.type == Logger::Field::tString) - arr.push_back(f.s); + if (auto p = std::get_if(&f.raw)) + arr.push_back(*p); + else if (auto p = std::get_if(&f.raw)) + arr.push_back(*p); else unreachable(); } diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 76df2b6a09..96c9ebdb5f 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -103,8 +103,8 @@ std::string_view activityName(ActivityType type) case actUnknown: return {}; # define ACTIVITY_NAME(name) \ - case act##name: \ - return #name; + case act##name: \ + return #name; ACTIVITY_NAME(CopyPath) ACTIVITY_NAME(FileTransfer) ACTIVITY_NAME(Realise) @@ -127,16 +127,20 @@ std::string_view activityName(ActivityType type) /* Defensive field accessors, like the progress bar's. */ std::string_view getS(const Logger::Fields & fields, size_t n) { - if (n >= fields.size() || fields[n].type != Logger::Field::tString) - return {}; - return fields[n].s; + if (n < fields.size()) { + if (auto p = std::get_if(&fields[n].raw)) + return *p; + } + return {}; } uint64_t getI(const Logger::Fields & fields, size_t n) { - if (n >= fields.size() || fields[n].type != Logger::Field::tInt) - return 0; - return fields[n].i; + if (n < fields.size()) { + if (auto p = std::get_if(&fields[n].raw)) + return *p; + } + return 0; } class OpenTelemetryLoggerImpl : public OpenTelemetryLogger From f478ed4cbbd56c1b23060c074db99470d57e1ed3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 16:54:03 +0200 Subject: [PATCH 685/927] libutil: Support string-named activities with structured metadata ActivityType is an enum, so adding a new type of activity is expensive (it requires changing logging.hh) and not self-describing. Also, activity fields are positional, with their meaning living in per-type conventions. So add a new Activity constructor that takes a string activity name (e.g. `FileTransferAttempt`) and key/value meta-information about the activity. Metadata keys follow the OpenTelemetry attribute naming conventions where applicable (e.g. `http.request.method`); Nix-specific keys use a `nix.` prefix. This is backwards compatible: the default implementation of the new name-based Logger::startActivity() reports the activity through the old ActivityType-based interface as the new type `actStringly`, discarding the name and metadata. So legacy loggers (including the daemon's TunnelLogger, and therefore old clients) work unchanged. Loggers that know about the new interface get the rich form: JSONLogger emits the name and a `payload` object with the metadata, and the OpenTelemetryLogger uses the name as the span name and the metadata as span attributes, without needing per-type knowledge. FileTransfer's per-HTTP-request activity becomes the first string-named activity, replacing the actFileTransferAttempt enum value introduced recently. Assisted-by: Claude Fable 5 --- src/libstore/filetransfer.cc | 8 ++- src/libutil/include/nix/util/logging.hh | 49 ++++++++++++++++-- src/libutil/logging.cc | 36 +++++++++++++ src/libutil/tee-logger.cc | 12 +++++ src/nix/otel-logger.cc | 68 +++++++++++++++++++------ 5 files changed, 150 insertions(+), 23 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 437abd6ee8..683e56ca54 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -631,9 +631,13 @@ struct curlFileTransfer : public FileTransfer attemptAct = std::make_unique( *logger, lvlDebug, - actFileTransferAttempt, + "FileTransferAttempt", + std::to_array>({ + {"url.full", request.displayUri()}, + {"http.request.method", httpMethodName(request.method)}, + {"http.request.resend_count", (uint64_t) attempt}, + }), "", - Logger::Fields{request.displayUri(), (uint64_t) attempt, httpMethodName(request.method)}, act().id); /* (Re)build the request headers, since the traceparent diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index 409dfcea21..4d77e6baea 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -9,6 +9,7 @@ #include "nix/util/fun.hh" #include +#include #include @@ -30,12 +31,12 @@ typedef enum { actBuildWaiting = 111, actFetchTree = 112, /** - * A single HTTP request within an `actFileTransfer` activity - * (there can be several due to retries). Fields: [0] the URI - * (string), [1] the number of preceding attempts (int), [2] the - * request method (string, e.g. `GET`). + * The type under which string-named activities (created via the + * name-based `Activity` constructor) are reported on legacy code + * paths, cf. the name-based `Logger::startActivity()`. Do not use + * directly. */ - actFileTransferAttempt = 113, + actStringly = 113, } ActivityType; typedef enum { @@ -128,6 +129,12 @@ public: typedef std::vector Fields; + /** + * Key/value meta-information about a string-named activity, cf. + * the name-based `startActivity()`. + */ + using ActivityMetadata = std::span>; + virtual ~Logger(); virtual void stop() {}; @@ -183,6 +190,26 @@ public: const Fields & fields, ActivityId parent) noexcept {}; + /** + * Start a string-named activity carrying key/value + * meta-information. The default implementation reports it via the + * `ActivityType`-based overload as `actStringly`, discarding the + * name and metadata, so legacy loggers work unchanged. Metadata + * keys follow the OpenTelemetry attribute naming conventions + * where applicable (e.g. `http.request.method`); Nix-specific + * keys use a `nix.` prefix. + */ + virtual void startActivity( + ActivityId act, + Verbosity lvl, + std::string_view name, + ActivityMetadata metadata, + std::string_view s, + ActivityId parent) noexcept + { + startActivity(act, lvl, actStringly, std::string(s), {}, parent); + }; + virtual void stopActivity(ActivityId act) noexcept {}; virtual void result(ActivityId act, ResultType type, const Fields & fields) noexcept {}; @@ -252,6 +279,18 @@ struct Activity Logger & logger, ActivityType type, const Logger::Fields & fields = {}, ActivityId parent = getCurActivity()) : Activity(logger, lvlError, type, "", fields, parent) {}; + /** + * Start a string-named activity carrying key/value + * meta-information, cf. the name-based `Logger::startActivity()`. + */ + Activity( + Logger & logger, + Verbosity lvl, + std::string_view name, + Logger::ActivityMetadata metadata = {}, + std::string_view s = {}, + ActivityId parent = getCurActivity()); + Activity(const Activity & act) = delete; ~Activity(); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 95b532d805..daab62b08c 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -224,6 +224,19 @@ Activity::Activity( logger.startActivity(id, lvl, type, s, fields, parent); } +Activity::Activity( + Logger & logger, + Verbosity lvl, + std::string_view name, + Logger::ActivityMetadata metadata, + std::string_view s, + ActivityId parent) + : logger(logger) + , id(nextId++ + (((uint64_t) getPid()) << 32)) +{ + logger.startActivity(id, lvl, name, metadata, s, parent); +} + void to_json(nlohmann::json & json, std::shared_ptr pos) { if (pos) { @@ -366,6 +379,29 @@ struct JSONLogger : Logger write(std::move(json)); } + void startActivity( + ActivityId act, + Verbosity lvl, + std::string_view name, + ActivityMetadata metadata, + std::string_view s, + ActivityId parent) noexcept override + { + nlohmann::json json; + json["action"] = "start"; + json["id"] = act; + json["level"] = lvl; + json["type"] = actStringly; + json["text"] = s; + json["parent"] = parent; + json["name"] = name; + auto payload = nlohmann::json::object(); + for (auto & [key, value] : metadata) + std::visit([&](auto & v) { payload[std::string(key)] = v; }, value.raw); + json["payload"] = std::move(payload); + write(std::move(json)); + } + void stopActivity(ActivityId act) noexcept override { nlohmann::json json; diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index 7230e725cd..f4adaf86ca 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -56,6 +56,18 @@ class TeeLogger final : public Logger logger->startActivity(act, lvl, type, s, fields, parent); } + void startActivity( + ActivityId act, + Verbosity lvl, + std::string_view name, + ActivityMetadata metadata, + std::string_view s, + ActivityId parent) noexcept override + { + for (auto & logger : loggers) + logger->startActivity(act, lvl, name, metadata, s, parent); + } + void stopActivity(ActivityId act) noexcept override { for (auto & logger : loggers) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 96c9ebdb5f..8331f5fa88 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -118,8 +118,11 @@ std::string_view activityName(ActivityType type) ACTIVITY_NAME(PostBuildHook) ACTIVITY_NAME(BuildWaiting) ACTIVITY_NAME(FetchTree) - ACTIVITY_NAME(FileTransferAttempt) # undef ACTIVITY_NAME + case actStringly: + /* Only reached via a Logger that flattens string-named + activities; the text fallback is the best we can do. */ + return {}; } return {}; } @@ -202,20 +205,12 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger return; opentelemetry::trace::StartSpanOptions options; - if (type == actFileTransferAttempt) - options.kind = opentelemetry::trace::SpanKind::kClient; auto name = activityName(type); bool textIsName = name.empty(); if (textIsName) name = s.empty() ? "activity" : std::string_view(s); - /* Per the OpenTelemetry semantic conventions, HTTP client - spans are named after the request method. */ - if (type == actFileTransferAttempt) - if (auto method = getS(fields, 2); !method.empty()) - name = method; - auto spans(spans_.lock()); if (auto i = spans->find(parent); i != spans->end()) @@ -233,14 +228,7 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger # pragma GCC diagnostic ignored "-Wswitch-enum" switch (type) { case actFileTransfer: - case actFileTransferAttempt: span->SetAttribute("url.full", toNostd(getS(fields, 0))); - if (type == actFileTransferAttempt) { - if (auto attempt = getI(fields, 1)) - span->SetAttribute("http.request.resend_count", (int64_t) attempt); - if (auto method = getS(fields, 2); !method.empty()) - span->SetAttribute("http.request.method", toNostd(method)); - } break; case actBuild: case actPostBuildHook: @@ -268,6 +256,54 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger } } + void startActivity( + ActivityId act, + Verbosity lvl, + std::string_view name, + ActivityMetadata metadata, + std::string_view s, + ActivityId parent) noexcept override + { + try { + if (isRemoteLogSource()) + return; + + opentelemetry::trace::StartSpanOptions options; + + /* Per the OpenTelemetry semantic conventions, HTTP client + spans are named after the request method. */ + auto spanName = name; + for (auto & [key, value] : metadata) + if (key == "http.request.method") + if (auto method = std::get_if(&value.raw)) { + options.kind = opentelemetry::trace::SpanKind::kClient; + spanName = *method; + } + + auto spans(spans_.lock()); + + if (auto i = spans->find(parent); i != spans->end()) + options.parent = i->second->GetContext(); + else + options.parent = rootSpan->GetContext(); + + auto span = tracer->StartSpan(toNostd(spanName), options); + + if (!s.empty()) + span->SetAttribute("nix.activity.text", toNostd(filterANSIEscapes(s, true))); + + for (auto & [key, value] : metadata) { + if (auto str = std::get_if(&value.raw)) + span->SetAttribute(toNostd(key), toNostd(*str)); + else if (auto n = std::get_if(&value.raw)) + span->SetAttribute(toNostd(key), (int64_t) *n); + } + + spans->emplace(act, std::move(span)); + } catch (...) { + } + } + void stopActivity(ActivityId act) noexcept override { try { From 192748ab47ee4b380443aaa74ea4ef79f01efdb6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 16:58:11 +0200 Subject: [PATCH 686/927] libcmd: Turn flake output evaluation into a string-named activity Rename the anonymous "evaluating derivation" activity in InstallableFlake::toDerivedPaths() to `EvaluateFlakeDerivationOutput`, carrying the installable as metadata. This makes flake evaluation show up as a properly named span in OpenTelemetry traces. Assisted-by: Claude Fable 5 --- src/libcmd/installable-flake.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index d2022aeb09..6c65fad10f 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -54,7 +54,15 @@ InstallableFlake::InstallableFlake( DerivedPathsWithInfo InstallableFlake::toDerivedPaths() { - Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); + Activity act( + *logger, + lvlTalkative, + "EvaluateFlakeDerivationOutput", + std::to_array>({ + {"nix.installable", what()}, + }), + fmt("evaluating derivation '%s'", what())); + PushActivity pact(act.id); auto attr = getCursor(*state); From bda8d5aff292231aab126fd98f50c6eb8a65844a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 17:01:21 +0200 Subject: [PATCH 687/927] otel-logger: Print the trace ID on stderr if NIX_OTEL_DEBUG is set Handy for finding the trace of a particular Nix invocation in the tracing backend. Assisted-by: Claude Fable 5 --- src/nix/otel-logger.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 8331f5fa88..d531e4f069 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -186,6 +186,12 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger options.parent = spanContext; } rootSpan = tracer->StartSpan(toNostd(rootSpanName), options); + + if (getEnv("NIX_OTEL_DEBUG")) { + char buf[2 * opentelemetry::trace::TraceId::kSize]; + rootSpan->GetContext().trace_id().ToLowerBase16(buf); + writeToStderr(fmt("OpenTelemetry trace ID: %s\n", std::string_view(buf, sizeof(buf)))); + } } void log(Verbosity lvl, std::string_view s) noexcept override {} From b079c5a9721810bd37e4c57f014864d70d694c0e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 17:44:33 +0200 Subject: [PATCH 688/927] libfetchers: Turn source path hashing/copying into structured activities The anonymous activity in fetchToStore2() becomes `CopySourcePath` or `HashSourcePath` (depending on the fetch mode), carrying the source path as metadata. Assisted-by: Claude Fable 5 --- src/libfetchers/fetch-to-store.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 4062d5899a..c1f136880a 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -97,7 +97,10 @@ std::pair fetchToStore2( Activity act( *logger, lvlChatty, - actUnknown, + mode == FetchMode::DryRun ? "HashSourcePath" : "CopySourcePath", + std::to_array>({ + {"nix.source.path", path.to_string()}, + }), fmt(mode == FetchMode::DryRun ? "hashing '%s'" : "copying '%s' to the store", path)); auto filter2 = filter ? *filter : defaultPathFilter; From 92a4335f266a66fc52672ecce1c41170d99a6e51 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 17:52:46 +0200 Subject: [PATCH 689/927] libstore: Name the queryMissing() activity "QueryMissing" Also register it as the current activity (via PushActivity), so that the QueryPathInfo activities triggered by it are parented under it instead of dangling from the root. (The asio coroutines involved all run on the calling thread via ctx.run(), so the thread-local current activity is visible inside them; only the push was missing.) Assisted-by: Claude Fable 5 --- src/libstore/misc.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 4e2a35a046..7d42acb9bd 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -219,7 +219,8 @@ static void collectDerivedPaths( MissingPaths Store::queryMissing(const std::vector & targets) { - Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); + Activity act(*logger, lvlDebug, "QueryMissing", {}, "querying info about missing paths"); + PushActivity pact(act.id); MissingPaths res; From 2e2230024e0b038317f329cb2f05c7c6d0d42be5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 18:21:17 +0200 Subject: [PATCH 690/927] libstore: Fix parenting of the worker's activities Parent the worker's aggregate activities (actBuilds, actCopyPaths) under its actRealise activity, parent actSubstitute under actCopyPaths, and parent the actBuild activities (both the local build and build hook cases) under actBuilds. The actSubstitute case requires passing the parent activity ID into the substitution thread explicitly, since the current activity is thread-local and the substitution runs on a fresh thread. Assisted-by: Claude Fable 5 --- src/libstore/build/derivation-building-goal.cc | 10 ++++++++-- src/libstore/build/substitution-goal.cc | 6 ++++-- src/libstore/build/worker.cc | 4 ++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 19f1cbadcc..d7edca2a99 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -708,7 +708,8 @@ Goal::Co DerivationBuildingGoal::buildWithHook( lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook->machineName, 1, 1})); + Logger::Fields{worker.store.printStorePath(drvPath), hook->machineName, 1, 1}, + worker.actDerivations.id)); mcRunningBuilds = std::make_unique>(worker.runningBuilds); worker.updateProgress(); @@ -876,7 +877,12 @@ Goal::Co DerivationBuildingGoal::buildLocally( : "building '%s'", worker.store.printStorePath(drvPath)); auto act = make_ref( - *logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), "", 1, 1}); + *logger, + lvlInfo, + actBuild, + msg, + Logger::Fields{worker.store.printStorePath(drvPath), "", 1, 1}, + worker.actDerivations.id); std::unique_ptr buildLog; std::unique_ptr logFile; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index de49582cd4..88f57cacae 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -237,7 +237,8 @@ Goal::Co PathSubstitutionGoal::tryToRun( repair = repair, sub, maybeWaker = worker.getCrossThreadWaker(), - maybeWorkerStore = worker.store.weak_from_this()]() mutable { + maybeWorkerStore = worker.store.weak_from_this(), + parentAct = worker.actSubstitutions.id]() mutable { try { ReceiveInterrupts receiveInterrupts; @@ -249,7 +250,8 @@ Goal::Co PathSubstitutionGoal::tryToRun( Activity act( *logger, actSubstitute, - Logger::Fields{workerStore->printStorePath(storePath), sub->config.getHumanReadableURI()}); + Logger::Fields{workerStore->printStorePath(storePath), sub->config.getHumanReadableURI()}, + parentAct); PushActivity pact(act.id); promise.set_value( diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index fbf9b8e62f..b812d5c34b 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -20,8 +20,8 @@ Worker::Worker(Store & store, Store & evalStore) /* Can't use make_ref, because the constructor is private. */ : wakerState(ref(new Waker{})) , act(*logger, actRealise) - , actDerivations(*logger, actBuilds) - , actSubstitutions(*logger, actCopyPaths) + , actDerivations(*logger, actBuilds, {}, act.id) + , actSubstitutions(*logger, actCopyPaths, {}, act.id) #ifdef _WIN32 , ioport{CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0)} #endif From f51115c839bfc8a49e790e7a924a1a20bd619b77 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 19:08:45 +0200 Subject: [PATCH 691/927] Add BuildInstallables activity --- src/libcmd/installables.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index cdb11914cc..f2edad3a27 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -556,6 +556,9 @@ std::vector Installable::build2( } } + Activity act(*logger, lvlTalkative, "BuildInstallables", {}); + PushActivity pact(act.id); + std::vector res; switch (mode) { From a5d2c4a7431b4c7fd9e15282231e7f273e1b81e9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 19:50:26 +0200 Subject: [PATCH 692/927] Propagate the client's trace context per operation to the daemon Previously all daemon activity was parented under a single "daemon connection" span. Instead, the opcode send is now factored into WorkerProto::BasicClientConnection::startOp(), which appends the trace context of the client's current activity when the "open-telemetry" protocol feature has been negotiated (safe to redefine since it was never released). On the daemon side, the traceparent is read centrally in the processConnection() loop, which wraps each operation in a generic "daemon operation" activity carrying the numeric opcode as `nix.daemon.op` metadata and the received trace context as `traceparent` metadata. The OpenTelemetryLogger treats such a metadata field as a remote parent, taking precedence over the local parent. Everything the daemon does on behalf of an operation then nests under the client's span via the current-activity mechanism. Note that in the daemon, the per-operation activity must be created on the global logger rather than the TunnelLogger: sending it only to the TunnelLogger would forward it to the client (which ignores remote activities) without ever reaching the daemon's own OpenTelemetryLogger. Also add getTraceparent() to extract the traceparent header from Logger::getTraceContext() results. Assisted-by: Claude Fable 5 --- src/libstore/daemon.cc | 16 ++++ .../nix/store/worker-protocol-connection.hh | 8 ++ src/libstore/remote-store.cc | 78 ++++++++++--------- src/libstore/ssh-store.cc | 2 +- src/libstore/uds-remote-store.cc | 3 +- src/libstore/worker-protocol-connection.cc | 25 ++++-- src/libutil/include/nix/util/logging.hh | 7 ++ src/libutil/logging.cc | 8 ++ src/nix/otel-logger.cc | 44 ++++++++--- 9 files changed, 138 insertions(+), 53 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 84dd68e3ec..920fe63498 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1175,12 +1175,28 @@ void processConnection( break; } + std::string traceparent; + if (conn.protoVersion.features.contains(WorkerProto::featureOpenTelemetry)) + traceparent = readString(conn.from); + printMsgUsing(prevLogger, lvlDebug, "received daemon op %d", op); opCount++; debug("performing daemon worker op: %d", op); + /* Parent our work under the client activity that + initiated this operation. */ + Activity act( + *logger, + lvlDebug, + "daemon operation", + std::to_array>({ + {"nix.daemon.op", (uint64_t) op}, + {"traceparent", traceparent}, + })); + PushActivity pact(act.id); + try { performOp(tunnelLogger, store, trusted, recursive, conn, op); } catch (Error & e) { diff --git a/src/libstore/include/nix/store/worker-protocol-connection.hh b/src/libstore/include/nix/store/worker-protocol-connection.hh index 49de3fe673..d49110ba06 100644 --- a/src/libstore/include/nix/store/worker-protocol-connection.hh +++ b/src/libstore/include/nix/store/worker-protocol-connection.hh @@ -94,6 +94,14 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection */ ClientHandshakeInfo postHandshake(const StoreDirConfig & store); + /** + * Start sending an operation: write the opcode, followed by the + * trace context of the current activity if the "open-telemetry" + * protocol feature has been negotiated, so that the daemon can + * parent its work under it. + */ + void startOp(WorkerProto::Op op); + void addTempRoot(const StoreDirConfig & remoteStore, bool * daemonException, const StorePath & path); StorePathSet queryValidPaths( diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f02d1c6e16..54a26f1648 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -115,11 +115,7 @@ void RemoteStore::initConnection(Connection & conn) if (conn.protoVersion.features.contains(WorkerProto::featureOpenTelemetry)) { /* Send our trace context so that the daemon can parent its spans under ours. */ - std::string traceparent; - for (auto & [name, value] : logger->getTraceContext(0)) - if (name == "traceparent") - traceparent = value; - conn.to << traceparent; + conn.to << getTraceparent(logger->getTraceContext(0)); conn.to.flush(); } @@ -142,7 +138,8 @@ void RemoteStore::initConnection(Connection & conn) void RemoteStore::setOptions(Connection & conn) { - conn.to << WorkerProto::Op::SetOptions << settings.keepFailed << settings.getWorkerSettings().keepGoing + conn.startOp(WorkerProto::Op::SetOptions); + conn.to << settings.keepFailed << settings.getWorkerSettings().keepGoing << settings.getWorkerSettings().tryFallback << verbosity << settings.getWorkerSettings().maxBuildJobs << settings.getWorkerSettings().maxSilentTime << true << (settings.verboseBuild ? lvlError : lvlVomit) << 0 // obsolete log type @@ -197,7 +194,7 @@ void RemoteStore::setOptions() bool RemoteStore::isValidPathUncached(const StorePath & path) { auto conn(getConnection()); - conn->to << WorkerProto::Op::IsValidPath; + conn->startOp(WorkerProto::Op::IsValidPath); WorkerProto::write(*this, *conn, path); conn.processStderr(); return readInt(conn->from); @@ -212,7 +209,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute StorePathSet RemoteStore::queryAllValidPaths() { auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryAllValidPaths; + conn->startOp(WorkerProto::Op::QueryAllValidPaths); conn.processStderr(); return WorkerProto::Serialise::read(*this, *conn); } @@ -220,7 +217,7 @@ StorePathSet RemoteStore::queryAllValidPaths() StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) { auto conn(getConnection()); - conn->to << WorkerProto::Op::QuerySubstitutablePaths; + conn->startOp(WorkerProto::Op::QuerySubstitutablePaths); WorkerProto::write(*this, *conn, paths); conn.processStderr(); return WorkerProto::Serialise::read(*this, *conn); @@ -233,7 +230,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto conn(getConnection()); - conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; + conn->startOp(WorkerProto::Op::QuerySubstitutablePathInfos); if (conn->protoVersion.number < WorkerProto::Version::Number{1, 22}) { StorePathSet paths; for (auto & path : pathsMap) @@ -298,7 +295,7 @@ asio::awaitable RemoteStore::queryPathInfos( pathInfoCache->lock()->upsert(path, PathInfoCacheValue{.value = info}); }; - conn->to << WorkerProto::Op::QueryPathInfos; + conn->startOp(WorkerProto::Op::QueryPathInfos); WorkerProto::write(*this, *conn, uncached); conn.processStderr(); @@ -337,7 +334,7 @@ asio::awaitable RemoteStore::queryPathInfos( void RemoteStore::queryReferrers(const StorePath & path, StorePathSet & referrers) { auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryReferrers; + conn->startOp(WorkerProto::Op::QueryReferrers); WorkerProto::write(*this, *conn, path); conn.processStderr(); for (auto & i : WorkerProto::Serialise::read(*this, *conn)) @@ -347,7 +344,7 @@ void RemoteStore::queryReferrers(const StorePath & path, StorePathSet & referrer StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) { auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryValidDerivers; + conn->startOp(WorkerProto::Op::QueryValidDerivers); WorkerProto::write(*this, *conn, path); conn.processStderr(); return WorkerProto::Serialise::read(*this, *conn); @@ -359,7 +356,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) return Store::queryDerivationOutputs(path); } auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryDerivationOutputs; + conn->startOp(WorkerProto::Op::QueryDerivationOutputs); WorkerProto::write(*this, *conn, path); conn.processStderr(); return WorkerProto::Serialise::read(*this, *conn); @@ -371,7 +368,7 @@ RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * eva if (WorkerProto::Version::Number::fromWire(getProtocol()) >= WorkerProto::Version::Number{1, 22}) { if (!evalStore_) { auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryDerivationOutputMap; + conn->startOp(WorkerProto::Op::QueryDerivationOutputMap); WorkerProto::write(*this, *conn, path); conn.processStderr(); return WorkerProto::Serialise>>::read(*this, *conn); @@ -403,7 +400,8 @@ RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * eva std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryPathFromHashPart << hashPart; + conn->startOp(WorkerProto::Op::QueryPathFromHashPart); + conn->to << hashPart; conn.processStderr(); return WorkerProto::Serialise>::read(*this, *conn); } @@ -422,7 +420,9 @@ ref RemoteStore::addCAToStore( if (conn->protoVersion >= WorkerProto::Version{.number = {1, 25}}) { - conn->to << WorkerProto::Op::AddToStore << name << caMethod.renderWithAlgo(hashAlgo); + conn->startOp(WorkerProto::Op::AddToStore); + + conn->to << name << caMethod.renderWithAlgo(hashAlgo); WorkerProto::write(*this, *conn, references); conn->to << repair; if (conn->protoVersion.features.contains(WorkerProto::featureProvenance)) @@ -448,7 +448,8 @@ ref RemoteStore::addCAToStore( name, printHashAlgo(hashAlgo)); std::string s = dump.drain(); - conn->to << WorkerProto::Op::AddTextToStore << name << s; + conn->startOp(WorkerProto::Op::AddTextToStore); + conn->to << name << s; WorkerProto::write(*this, *conn, references); conn.processStderr(); break; @@ -458,7 +459,8 @@ ref RemoteStore::addCAToStore( case ContentAddressMethod::Raw::Git: default: { auto fim = caMethod.getFileIngestionMethod(); - conn->to << WorkerProto::Op::AddToStore << name + conn->startOp(WorkerProto::Op::AddToStore); + conn->to << name << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::NixArchive) ? 0 : 1) /* backwards compatibility hack */ @@ -533,7 +535,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, Repair { auto conn(getConnection()); - conn->to << WorkerProto::Op::AddToStoreNar; + conn->startOp(WorkerProto::Op::AddToStoreNar); WorkerProto::write(*this, *conn, info.path); WorkerProto::write(*this, *conn, info.deriver); conn->to << info.narHash.to_string(HashFormat::Base16, false); @@ -595,14 +597,16 @@ void RemoteStore::addMultipleToStore( } }); - conn->to << WorkerProto::Op::AddMultipleToStore << repair << !checkSigs; + conn->startOp(WorkerProto::Op::AddMultipleToStore); + + conn->to << repair << !checkSigs; conn.withFramedSink([&](Sink & sink) { source->drainInto(sink); }); } void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); - conn->to << WorkerProto::Op::RegisterDrvOutput; + conn->startOp(WorkerProto::Op::RegisterDrvOutput); WorkerProto::write(*this, *conn, info); conn.processStderr(); } @@ -620,7 +624,7 @@ void RemoteStore::queryRealisationUncached( return callback(nullptr); } - conn->to << WorkerProto::Op::QueryRealisation; + conn->startOp(WorkerProto::Op::QueryRealisation); WorkerProto::write(*this, *conn, id); conn.processStderr(); @@ -661,7 +665,7 @@ void RemoteStore::buildPaths( copyDrvsFromEvalStore(drvPaths, evalStore); auto conn(getConnection()); - conn->to << WorkerProto::Op::BuildPaths; + conn->startOp(WorkerProto::Op::BuildPaths); WorkerProto::write(*this, *conn, drvPaths); conn->to << buildMode; conn.processStderr(); @@ -677,7 +681,7 @@ std::vector RemoteStore::buildPathsWithResults( auto & conn = *conn_; if (conn->protoVersion >= WorkerProto::Version{.number = {1, 34}}) { - conn->to << WorkerProto::Op::BuildPathsWithResults; + conn->startOp(WorkerProto::Op::BuildPathsWithResults); WorkerProto::write(*this, *conn, paths); conn->to << buildMode; conn.processStderr(); @@ -752,7 +756,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD void RemoteStore::ensurePath(const StorePath & path) { auto conn(getConnection()); - conn->to << WorkerProto::Op::EnsurePath; + conn->startOp(WorkerProto::Op::EnsurePath); WorkerProto::write(*this, *conn, path); conn.processStderr(); readInt(conn->from); @@ -766,7 +770,7 @@ void RemoteStore::addTempRoots(const StorePathSet & paths, bool skipIfSlow) auto conn(getConnection()); if (conn->protoVersion.features.contains(WorkerProto::featureAddTempRoots)) { - conn->to << WorkerProto::Op::AddTempRoots; + conn->startOp(WorkerProto::Op::AddTempRoots); WorkerProto::write(*this, *conn, paths); conn.processStderr(); readInt(conn->from); @@ -785,7 +789,7 @@ void RemoteStore::addTempRoots(const StorePathSet & paths, bool skipIfSlow) Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); - conn->to << WorkerProto::Op::FindRoots; + conn->startOp(WorkerProto::Op::FindRoots); conn.processStderr(); size_t count = readNum(conn->from); Roots result; @@ -804,7 +808,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) conn->protoVersion.features.contains(WorkerProto::featureDeleteDeadSpecificReferrers); if (supportsDeleteSpecificReferrers) { - conn->to << WorkerProto::Op::CollectGarbage; + conn->startOp(WorkerProto::Op::CollectGarbage); WorkerProto::write(*this, *conn, options.action); WorkerProto::write(*this, *conn, options.pathsToDelete); } else { @@ -821,7 +825,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) [](const GCOptions::WholeStore & _) { return StorePathSet{}; }, }, options.pathsToDelete); - conn->to << WorkerProto::Op::CollectGarbage; + conn->startOp(WorkerProto::Op::CollectGarbage); WorkerProto::write(*this, *conn, options.action); WorkerProto::write(*this, *conn, paths); } @@ -843,7 +847,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) void RemoteStore::optimiseStore() { auto conn(getConnection()); - conn->to << WorkerProto::Op::OptimiseStore; + conn->startOp(WorkerProto::Op::OptimiseStore); conn.processStderr(); readInt(conn->from); } @@ -851,7 +855,8 @@ void RemoteStore::optimiseStore() bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) { auto conn(getConnection()); - conn->to << WorkerProto::Op::VerifyStore << checkContents << repair; + conn->startOp(WorkerProto::Op::VerifyStore); + conn->to << checkContents << repair; conn.processStderr(); return readInt(conn->from); } @@ -859,7 +864,7 @@ bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) void RemoteStore::addSignatures(const StorePath & storePath, const std::set & sigs) { auto conn(getConnection()); - conn->to << WorkerProto::Op::AddSignatures; + conn->startOp(WorkerProto::Op::AddSignatures); WorkerProto::write(*this, *conn, storePath); WorkerProto::write(*this, *conn, sigs); conn.processStderr(); @@ -874,7 +879,7 @@ MissingPaths RemoteStore::queryMissing(const std::vector & targets) // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; - conn->to << WorkerProto::Op::QueryMissing; + conn->startOp(WorkerProto::Op::QueryMissing); WorkerProto::write(*this, *conn, targets); conn.processStderr(); MissingPaths res; @@ -892,7 +897,8 @@ MissingPaths RemoteStore::queryMissing(const std::vector & targets) void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log) { auto conn(getConnection()); - conn->to << WorkerProto::Op::AddBuildLog << drvPath.to_string(); + conn->startOp(WorkerProto::Op::AddBuildLog); + conn->to << drvPath.to_string(); StringSource source(log); conn.withFramedSink([&](Sink & sink) { source.drainInto(sink); }); readInt(conn->from); @@ -903,7 +909,7 @@ std::vector RemoteStore::queryActiveBuilds() auto conn(getConnection()); if (!conn->protoVersion.features.count(WorkerProto::featureQueryActiveBuilds)) throw Error("remote store does not support querying active builds"); - conn->to << WorkerProto::Op::QueryActiveBuilds; + conn->startOp(WorkerProto::Op::QueryActiveBuilds); conn.processStderr(); return nlohmann::json::parse(readString(conn->from)).get>(); } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 2f6ae86790..5303fb82f4 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -206,7 +206,7 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore std::filesystem::path addPermRoot(const StorePath & path, const std::filesystem::path & gcRoot) override { auto conn(getConnection()); - conn->to << WorkerProto::Op::AddPermRoot; + conn->startOp(WorkerProto::Op::AddPermRoot); WorkerProto::write(*this, *conn, path); WorkerProto::write(*this, *conn, gcRoot.string()); conn.processStderr(); diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index c4137df7c6..6133ec7b57 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -106,7 +106,8 @@ ref UDSRemoteStore::openConnection() void UDSRemoteStore::addIndirectRoot(const std::filesystem::path & path) { auto conn(getConnection()); - conn->to << WorkerProto::Op::AddIndirectRoot << path.string(); + conn->startOp(WorkerProto::Op::AddIndirectRoot); + conn->to << path.string(); conn.processStderr(); readInt(conn->from); } diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 7a0ed2330d..84a1a96fdf 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -1,5 +1,6 @@ #include "nix/store/worker-protocol-connection.hh" #include "nix/store/worker-protocol-impl.hh" +#include "nix/util/logging.hh" #include "nix/store/build-result.hh" #include "nix/store/derivations.hh" @@ -236,6 +237,16 @@ WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandsha return WorkerProto::Serialise::read(store, *this); } + +void WorkerProto::BasicClientConnection::startOp(WorkerProto::Op op) +{ + to << op; + if (protoVersion.features.contains(WorkerProto::featureOpenTelemetry)) + /* Send the trace context of the current activity, so that the + daemon can parent its work under it. */ + to << getTraceparent(logger->getTraceContext(getCurActivity())); +} + void WorkerProto::BasicServerConnection::postHandshake(const StoreDirConfig & store, const ClientHandshakeInfo & info) { if (protoVersion >= WorkerProto::Version{.number = {1, 14}} && readInt(from)) { @@ -252,7 +263,8 @@ void WorkerProto::BasicServerConnection::postHandshake(const StoreDirConfig & st std::optional WorkerProto::BasicClientConnection::queryPathInfo( const StoreDirConfig & store, bool * daemonException, const StorePath & path) { - to << WorkerProto::Op::QueryPathInfo << store.printStorePath(path); + startOp(WorkerProto::Op::QueryPathInfo); + to << store.printStorePath(path); try { processStderr(daemonException); } catch (Error & e) { @@ -274,7 +286,7 @@ StorePathSet WorkerProto::BasicClientConnection::queryValidPaths( const StoreDirConfig & store, bool * daemonException, const StorePathSet & paths, SubstituteFlag maybeSubstitute) { assert((protoVersion >= WorkerProto::Version{.number = {1, 12}})); - to << WorkerProto::Op::QueryValidPaths; + startOp(WorkerProto::Op::QueryValidPaths); WorkerProto::write(store, *this, paths); if (protoVersion >= WorkerProto::Version{.number = {1, 27}}) { to << maybeSubstitute; @@ -286,7 +298,8 @@ StorePathSet WorkerProto::BasicClientConnection::queryValidPaths( void WorkerProto::BasicClientConnection::addTempRoot( const StoreDirConfig & store, bool * daemonException, const StorePath & path) { - to << WorkerProto::Op::AddTempRoot << store.printStorePath(path); + startOp(WorkerProto::Op::AddTempRoot); + to << store.printStorePath(path); processStderr(daemonException); readInt(from); } @@ -298,7 +311,8 @@ void WorkerProto::BasicClientConnection::putBuildDerivationRequest( const BasicDerivation & drv, BuildMode buildMode) { - to << WorkerProto::Op::BuildDerivation << store.printStorePath(drvPath); + startOp(WorkerProto::Op::BuildDerivation); + to << store.printStorePath(drvPath); writeDerivation(to, store, drv); to << buildMode; } @@ -312,7 +326,8 @@ WorkerProto::BasicClientConnection::getBuildDerivationResponse(const StoreDirCon void WorkerProto::BasicClientConnection::narFromPath( const StoreDirConfig & store, bool * daemonException, const StorePath & path, fun receiveNar) { - to << WorkerProto::Op::NarFromPath << store.printStorePath(path); + startOp(WorkerProto::Op::NarFromPath); + to << store.printStorePath(path); processStderr(daemonException); receiveNar(from); diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index 4d77e6baea..05ada6fdce 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -359,6 +359,13 @@ void applyExtraLogger(std::unique_ptr extraLogger); void applyJSONLogger(); +/** + * Extract the value of the `traceparent` header from trace context + * headers returned by `Logger::getTraceContext()`, or the empty + * string if there is none. + */ +std::string getTraceparent(const Headers & headers); + /** * Marks, for the duration of its existence, Logger calls made on this * thread as replaying messages that originated in another process diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index daab62b08c..b0e5b22faa 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -466,6 +466,14 @@ std::unique_ptr makeJSONLogger(const std::filesystem::path & path, bool return std::make_unique(std::move(fd), includeNixPrefix); } +std::string getTraceparent(const Headers & headers) +{ + for (auto & [name, value] : headers) + if (name == "traceparent") + return value; + return ""; +} + void applyJSONLogger() { if (auto & opt = loggerSettings.jsonLogPath.get()) { diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index d531e4f069..6f434203a3 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -72,6 +72,26 @@ struct ExtractCarrier : opentelemetry::context::propagation::TextMapCarrier void Set(opentelemetry::nostd::string_view, opentelemetry::nostd::string_view) noexcept override {} }; +/** + * Parse a W3C `traceparent` value into a span context usable as a + * remote parent. Returns std::nullopt on an empty or invalid value. + * (Extract() returns the input context unchanged on a parse failure, + * leaving an invalid SpanContext.) + */ +std::optional parseTraceparent(std::string_view traceparent) +{ + if (traceparent.empty()) + return std::nullopt; + ExtractCarrier carrier; + carrier.traceparent = toNostd(traceparent); + opentelemetry::context::Context emptyCtx; + auto ctx = opentelemetry::trace::propagation::HttpTraceContext{}.Extract(carrier, emptyCtx); + auto spanContext = opentelemetry::trace::GetSpan(ctx)->GetContext(); + if (!spanContext.IsValid()) + return std::nullopt; + return spanContext; +} + struct InjectCarrier : opentelemetry::context::propagation::TextMapCarrier { Headers headers; @@ -174,16 +194,8 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger opentelemetry::trace::StartSpanOptions options; if (!remoteParentTraceparent.empty()) { options.kind = opentelemetry::trace::SpanKind::kServer; - /* Extract() returns the input context unchanged on a - parse failure, leaving an invalid SpanContext, i.e. an - unparented root span. */ - ExtractCarrier carrier; - carrier.traceparent = toNostd(remoteParentTraceparent); - opentelemetry::context::Context emptyCtx; - auto ctx = opentelemetry::trace::propagation::HttpTraceContext{}.Extract(carrier, emptyCtx); - auto spanContext = opentelemetry::trace::GetSpan(ctx)->GetContext(); - if (spanContext.IsValid()) - options.parent = spanContext; + if (auto spanContext = parseTraceparent(remoteParentTraceparent)) + options.parent = *spanContext; } rootSpan = tracer->StartSpan(toNostd(rootSpanName), options); @@ -293,12 +305,24 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger else options.parent = rootSpan->GetContext(); + /* A `traceparent` metadata field denotes a parent span in + another process (e.g. the client activity on whose + behalf the daemon is performing this activity), which + takes precedence over the local parent. */ + for (auto & [key, value] : metadata) + if (key == "traceparent") + if (auto str = std::get_if(&value.raw)) + if (auto spanContext = parseTraceparent(*str)) + options.parent = *spanContext; + auto span = tracer->StartSpan(toNostd(spanName), options); if (!s.empty()) span->SetAttribute("nix.activity.text", toNostd(filterANSIEscapes(s, true))); for (auto & [key, value] : metadata) { + if (key == "traceparent") + continue; if (auto str = std::get_if(&value.raw)) span->SetAttribute(toNostd(key), toNostd(*str)); else if (auto n = std::get_if(&value.raw)) From 7f8d2097771bd74e6f08f4d75dd080714962cfe9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 20:13:17 +0200 Subject: [PATCH 693/927] Don't emit spans for the per-operation link activities A typical Nix invocation performs thousands of daemon operations, so emitting a span per "daemon operation" activity is very spammy. Instead, an activity carrying a `traceparent` metadata field now produces no span at all: the OpenTelemetryLogger records the remote context (as a non-recording DefaultSpan) so that child activities resolve their parent to the client's span, linking the daemon's work directly under the client activity that initiated the operation. Also, the daemon only creates the link activity when the client actually sent a trace context. Assisted-by: Claude Fable 5 --- src/libstore/daemon.cc | 22 +++++++----- src/libstore/remote-store.cc | 9 +++-- src/libstore/worker-protocol-connection.cc | 1 - src/nix/otel-logger.cc | 39 +++++++++++++++------- 4 files changed, 44 insertions(+), 27 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 920fe63498..d677a1f11c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1187,15 +1187,19 @@ void processConnection( /* Parent our work under the client activity that initiated this operation. */ - Activity act( - *logger, - lvlDebug, - "daemon operation", - std::to_array>({ - {"nix.daemon.op", (uint64_t) op}, - {"traceparent", traceparent}, - })); - PushActivity pact(act.id); + std::optional act; + std::optional pact; + if (!traceparent.empty()) { + act.emplace( + *logger, + lvlDebug, + "daemon operation", + std::to_array>({ + {"nix.daemon.op", (uint64_t) op}, + {"traceparent", traceparent}, + })); + pact.emplace(act->id); + } try { performOp(tunnelLogger, store, trusted, recursive, conn, op); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 54a26f1648..579c3551dd 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -139,11 +139,10 @@ void RemoteStore::initConnection(Connection & conn) void RemoteStore::setOptions(Connection & conn) { conn.startOp(WorkerProto::Op::SetOptions); - conn.to << settings.keepFailed << settings.getWorkerSettings().keepGoing - << settings.getWorkerSettings().tryFallback << verbosity << settings.getWorkerSettings().maxBuildJobs - << settings.getWorkerSettings().maxSilentTime << true << (settings.verboseBuild ? lvlError : lvlVomit) - << 0 // obsolete log type - << 0 /* obsolete print build trace */ + conn.to << settings.keepFailed << settings.getWorkerSettings().keepGoing << settings.getWorkerSettings().tryFallback + << verbosity << settings.getWorkerSettings().maxBuildJobs << settings.getWorkerSettings().maxSilentTime + << true << (settings.verboseBuild ? lvlError : lvlVomit) << 0 // obsolete log type + << 0 /* obsolete print build trace */ << settings.getLocalSettings().buildCores << settings.getWorkerSettings().useSubstitutes; std::map overrides; diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 84a1a96fdf..b2999c3246 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -237,7 +237,6 @@ WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandsha return WorkerProto::Serialise::read(store, *this); } - void WorkerProto::BasicClientConnection::startOp(WorkerProto::Op op) { to << op; diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 6f434203a3..f4a5804d0e 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -23,6 +23,7 @@ # include # include # include +# include # include # include # include @@ -286,6 +287,32 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger if (isRemoteLogSource()) return; + /* An activity carrying a `traceparent` metadata field + exists only to link its child activities to a span in + another process (e.g. the client activity on whose + behalf the daemon is performing an operation). Don't + emit a span for the activity itself — there can be very + many of them (one per daemon operation) — but record + the remote context, or the local parent if the trace + context is absent or invalid, for parent lookups by + child activities. */ + for (auto & [key, value] : metadata) { + if (key != "traceparent") + continue; + std::optional spanContext; + if (auto str = std::get_if(&value.raw)) + spanContext = parseTraceparent(*str); + auto spans(spans_.lock()); + if (!spanContext) { + if (auto i = spans->find(parent); i != spans->end()) + spanContext = i->second->GetContext(); + else + spanContext = rootSpan->GetContext(); + } + spans->emplace(act, SpanPtr(new opentelemetry::trace::DefaultSpan(*spanContext))); + return; + } + opentelemetry::trace::StartSpanOptions options; /* Per the OpenTelemetry semantic conventions, HTTP client @@ -305,24 +332,12 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger else options.parent = rootSpan->GetContext(); - /* A `traceparent` metadata field denotes a parent span in - another process (e.g. the client activity on whose - behalf the daemon is performing this activity), which - takes precedence over the local parent. */ - for (auto & [key, value] : metadata) - if (key == "traceparent") - if (auto str = std::get_if(&value.raw)) - if (auto spanContext = parseTraceparent(*str)) - options.parent = *spanContext; - auto span = tracer->StartSpan(toNostd(spanName), options); if (!s.empty()) span->SetAttribute("nix.activity.text", toNostd(filterANSIEscapes(s, true))); for (auto & [key, value] : metadata) { - if (key == "traceparent") - continue; if (auto str = std::get_if(&value.raw)) span->SetAttribute(toNostd(key), toNostd(*str)); else if (auto n = std::get_if(&value.raw)) From 908af9cc614b54bb4e37f9271dd831fb20487026 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 20:15:38 +0200 Subject: [PATCH 694/927] otel-logger: Print the NIX_OTEL_DEBUG trace ID after uploading Print the trace ID at the end, in flushOtelAndShutdown(), so that when the user sees it, the trace has actually been uploaded to the collector. Assisted-by: Claude Fable 5 --- src/nix/otel-logger.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index f4a5804d0e..07fb87e33d 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -41,6 +41,12 @@ struct OtelState { std::unique_ptr provider; opentelemetry::nostd::shared_ptr tracer; + + /** + * The root span's trace ID, to be printed on shutdown if + * NIX_OTEL_DEBUG is set. + */ + std::string debugTraceId; }; /* Owned here rather than via opentelemetry's global Provider @@ -203,7 +209,9 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger if (getEnv("NIX_OTEL_DEBUG")) { char buf[2 * opentelemetry::trace::TraceId::kSize]; rootSpan->GetContext().trace_id().ToLowerBase16(buf); - writeToStderr(fmt("OpenTelemetry trace ID: %s\n", std::string_view(buf, sizeof(buf)))); + /* Printed by flushOtelAndShutdown() once the trace has + been uploaded. */ + state.debugTraceId = std::string(buf, sizeof(buf)); } } @@ -459,6 +467,10 @@ void flushOtelAndShutdown() auto timeout = std::chrono::microseconds(std::chrono::seconds(5)); state->provider->ForceFlush(timeout); state->provider->Shutdown(timeout); + + if (!state->debugTraceId.empty()) + writeToStderr(fmt("OpenTelemetry trace ID: %s\n", state->debugTraceId)); + /* Deliberately leak `state`: outstanding span handles may still reference the tracer; after Shutdown() their spans are simply dropped. */ From b3cab19e50f3e3732e9d801549f5766213e11630 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Sep 2026 20:31:53 +0200 Subject: [PATCH 695/927] libstore: Add a SubstitutionGoal activity Add a string-named activity covering the lifetime of a PathSubstitutionGoal, starting as soon as init() runs. The substituter queryPathInfo() activities and the actSubstitute activity (which is left alone, since the progress bar uses it to indicate that the actual download is happening) are now parented under it, instead of dangling from actCopyPaths or the root. Note that the PushActivity around queryPathInfo() is narrowly scoped to the synchronous part of the call, so that it does not extend across a coroutine suspension point (the thread-local current activity would leak into unrelated interleaved coroutines). Assisted-by: Claude Fable 5 --- src/libstore/build/substitution-goal.cc | 31 ++++++++++++++++--- .../nix/store/build/substitution-goal.hh | 6 +++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 88f57cacae..b122299a60 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -60,6 +60,20 @@ Goal::Co PathSubstitutionGoal::init() throw Error( "cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); + /* Covers the lifetime of this goal, including querying + substituters and the actual substitution. Note that the + progress bar only shows the `actSubstitute` activity, which + denotes that the actual substitution is happening. */ + Activity act( + *logger, + lvlDebug, + "SubstitutionGoal", + std::to_array>({ + {"nix.store.path", worker.store.printStorePath(storePath)}, + }), + "", + worker.actSubstitutions.id); + auto subs = worker.getSubstituters(); bool substituterFailed = false; @@ -92,7 +106,12 @@ Goal::Co PathSubstitutionGoal::init() try { info = co_await AsyncCallback>( - [sub, path = subPath.value_or(storePath)](auto cb) { sub->queryPathInfo(path, std::move(cb)); }); + [&act, sub, path = subPath.value_or(storePath)](auto cb) { + /* Note: narrowly scoped so that the `PushActivity` + does not extend across a suspension point. */ + PushActivity pact(act.id); + sub->queryPathInfo(path, std::move(cb)); + }); } catch (InvalidPath &) { continue; } catch (SubstituterDisabled & e) { @@ -152,7 +171,7 @@ Goal::Co PathSubstitutionGoal::init() // FIXME: consider returning boolean instead of passing in reference bool out = false; // is mutated by tryToRun - co_await tryToRun(subPath ? *subPath : storePath, sub, info, out); + co_await tryToRun(subPath ? *subPath : storePath, sub, info, out, act.id); substituterFailed = substituterFailed || out; } @@ -184,7 +203,11 @@ Goal::Co PathSubstitutionGoal::init() } Goal::Co PathSubstitutionGoal::tryToRun( - StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed) + StorePath subPath, + nix::ref sub, + std::shared_ptr info, + bool & substituterFailed, + ActivityId parentAct) { trace("all references realised"); @@ -238,7 +261,7 @@ Goal::Co PathSubstitutionGoal::tryToRun( sub, maybeWaker = worker.getCrossThreadWaker(), maybeWorkerStore = worker.store.weak_from_this(), - parentAct = worker.actSubstitutions.id]() mutable { + parentAct]() mutable { try { ReceiveInterrupts receiveInterrupts; diff --git a/src/libstore/include/nix/store/build/substitution-goal.hh b/src/libstore/include/nix/store/build/substitution-goal.hh index ab09ef4a7e..841147eae4 100644 --- a/src/libstore/include/nix/store/build/substitution-goal.hh +++ b/src/libstore/include/nix/store/build/substitution-goal.hh @@ -60,7 +60,11 @@ public: Co init(); Co gotInfo(); Co tryToRun( - StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed); + StorePath subPath, + nix::ref sub, + std::shared_ptr info, + bool & substituterFailed, + ActivityId parentAct); Co finished(); /* Called by destructor, can't be overridden */ From 29f34c50b9ef6745d368ccdf32d5bb4a99f80a66 Mon Sep 17 00:00:00 2001 From: carl Date: Thu, 3 Sep 2026 23:07:45 +0200 Subject: [PATCH 696/927] tests/functional: skip the sentry test when Yama restricts ptrace The crashpad handler writes a crash's minidump by ptrace-attaching to the crashing process. With kernel.yama.ptrace_scope >= 2 ptrace is reserved for CAP_SYS_PTRACE, which an unprivileged test run does not have (PR_SET_PTRACER only helps at scope 1), so no dump can ever be captured and every crash type fails with 'No crash dump found'. Skip the test on such kernels, matching the existing per-platform skips. Observed as a deterministic nix-functional-tests:sentry failure when building determinate-nix inside an unprivileged CI sandbox on a hardened host (Talos Linux, ptrace_scope=2), while the same derivation builds clean on a workstation with ptrace_scope=1. --- tests/functional/sentry.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/functional/sentry.sh b/tests/functional/sentry.sh index 35eb93c732..da523b97ec 100644 --- a/tests/functional/sentry.sh +++ b/tests/functional/sentry.sh @@ -15,6 +15,15 @@ if ! [[ -d $sentryDir ]]; then skipTest "not built with sentry support" fi +# The crashpad handler captures a crash by ptrace-attaching to the crashing +# process. Yama ptrace_scope >= 2 restricts ptrace to CAP_SYS_PTRACE, which an +# unprivileged test run does not have (and PR_SET_PTRACER only helps at scope +# 1), so no minidump can ever be written on such kernels — e.g. hardened hosts +# or CI sandboxes. +if [[ -r /proc/sys/kernel/yama/ptrace_scope && $(< /proc/sys/kernel/yama/ptrace_scope) -ge 2 ]]; then + skipTest "ptrace is restricted by Yama (kernel.yama.ptrace_scope >= 2)" +fi + waitForCrashDump() { local i for ((i = 0; i < 10; i++)); do From 3b696cdf2d40044762e9266d6576c833d96140a2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Sep 2026 12:56:01 +0200 Subject: [PATCH 697/927] otel-logger: Mark spans as failed when stopped during unwinding stopActivity() carries no success/failure status, so spans generally ended without one. However, if an Activity is destroyed by stack unwinding (std::uncaught_exceptions() is non-zero in stopActivity(), which runs on the unwinding thread), we can infer that the activity failed and set an error status on its span. Assisted-by: Claude Fable 5 --- src/nix/otel-logger.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 07fb87e33d..0885421b17 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -8,6 +8,7 @@ # include "nix/util/terminal.hh" # include +# include # include # include @@ -364,6 +365,13 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger return; auto spans(spans_.lock()); if (auto i = spans->find(act); i != spans->end()) { + /* If the activity is being stopped while an exception + is in flight, i.e. the `Activity` is being destroyed + by stack unwinding, assume that the activity + failed. */ + if (std::uncaught_exceptions()) + i->second->SetStatus( + opentelemetry::trace::StatusCode::kError, "activity terminated by an exception"); i->second->End(); spans->erase(i); } From 63320be73348a7b31e05ed04c4002fcd934858b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Sep 2026 13:14:17 +0200 Subject: [PATCH 698/927] otel-logger: Support OTEL_TRACES_SAMPLER MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Honor the standard OTEL_TRACES_SAMPLER / OTEL_TRACES_SAMPLER_ARG environment variables (which the C++ SDK does not read itself), supporting the sampler names from the OpenTelemetry specification: always_on, always_off, traceidratio and their parentbased_* variants. The default remains parentbased_always_on, i.e. record everything. The parentbased_* samplers follow the sampling decision of the parent span, which propagates in the sampled flag of the W3C trace context — so e.g. the daemon and binary cache servers follow the client's decision, keeping traces complete or absent as a whole. Assisted-by: Claude Fable 5 --- src/nix/otel-logger.cc | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 0885421b17..87912a0ebf 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -20,6 +20,10 @@ # include # include # include +# include +# include +# include +# include # include # include # include @@ -456,8 +460,44 @@ void initOtel(std::string_view serviceName) {opentelemetry::semconv::service::kServiceName, std::string(serviceName)}, }); + /* Support the standard OTEL_TRACES_SAMPLER / OTEL_TRACES_SAMPLER_ARG + environment variables, which the C++ SDK does not read itself. + "parentbased" samplers follow the sampling decision of the + parent span, which propagates in the sampled flag of the W3C + trace context — so the daemon follows the client's decision. */ + auto sampler = [&]() -> std::unique_ptr { + auto ratio = [&]() -> double { + auto arg = getEnv("OTEL_TRACES_SAMPLER_ARG"); + if (!arg) + return 1.0; + try { + return std::stod(*arg); + } catch (...) { + warn("invalid OTEL_TRACES_SAMPLER_ARG '%s'; assuming 1.0", *arg); + return 1.0; + } + }; + auto parentBased = [](std::shared_ptr delegate) -> std::unique_ptr { + return std::make_unique(std::move(delegate)); + }; + auto name = getEnv("OTEL_TRACES_SAMPLER").value_or("parentbased_always_on"); + if (name == "always_on") + return std::make_unique(); + if (name == "always_off") + return std::make_unique(); + if (name == "traceidratio") + return std::make_unique(ratio()); + if (name == "parentbased_always_off") + return parentBased(std::make_shared()); + if (name == "parentbased_traceidratio") + return parentBased(std::make_shared(ratio())); + if (name != "parentbased_always_on") + warn("unknown OTEL_TRACES_SAMPLER '%s'; assuming 'parentbased_always_on'", name); + return parentBased(std::make_shared()); + }(); + auto state = std::make_unique(); - state->provider = sdktrace::TracerProviderFactory::Create(std::move(processor), resource); + state->provider = sdktrace::TracerProviderFactory::Create(std::move(processor), resource, std::move(sampler)); state->tracer = state->provider->GetTracer("nix"); OtelState * expected = nullptr; From 27f102f9a437f065cde580163a847b68a00aa2dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Sep 2026 13:40:44 +0200 Subject: [PATCH 699/927] manual: Document the OpenTelemetry environment variables Assisted-by: Claude Fable 5 --- doc/manual/source/command-ref/env-common.md | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index cc7fe77eae..aac4b20499 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -167,6 +167,35 @@ Likewise for the state and cache directories. Indicator that tells if the current environment was set up by `nix-shell`. It can have the values `pure` or `impure`. +## OpenTelemetry tracing environment variables + +Nix can emit [OpenTelemetry](https://opentelemetry.io/) traces of its activities (such as evaluation, builds, substitutions and HTTP requests), exported using the OTLP/HTTP protocol. +Tracing is a no-op unless one of the endpoint variables below is set. +When talking to the Nix daemon or to a binary cache server, Nix propagates the [W3C trace context](https://www.w3.org/TR/trace-context/), so that work done on your behalf by other processes or servers shows up in the same trace. + +- [`OTEL_EXPORTER_OTLP_ENDPOINT`](#env-OTEL_EXPORTER_OTLP_ENDPOINT) + + The base URL of the OpenTelemetry collector to which traces are sent, e.g. `https://otel.example.org`. + The path `/v1/traces` is appended automatically. + Note: the URL should *not* have a trailing slash, since the resulting double slash confuses some collectors. + +- [`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`](#env-OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) + + The full URL to which traces are sent, used as-is (nothing is appended). + Takes precedence over `OTEL_EXPORTER_OTLP_ENDPOINT`. + +- Other standard [OTLP exporter variables](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/) are also honored, such as `OTEL_EXPORTER_OTLP_HEADERS` (e.g. to pass an authorization header) and `OTEL_EXPORTER_OTLP_COMPRESSION` (set to `gzip` to compress uploads). + +- [`OTEL_TRACES_SAMPLER`](#env-OTEL_TRACES_SAMPLER) + + Selects the [sampler](https://opentelemetry.io/docs/languages/sdk-configuration/general/#otel_traces_sampler) used to decide which traces are recorded. + One of `always_on`, `always_off`, `traceidratio`, `parentbased_always_on` (the default), `parentbased_always_off` or `parentbased_traceidratio`. + The `parentbased_*` samplers follow the sampling decision of the parent span, which propagates in the W3C trace context — so the Nix daemon and binary cache servers follow the client's decision, keeping traces complete or absent as a whole. + +- [`OTEL_TRACES_SAMPLER_ARG`](#env-OTEL_TRACES_SAMPLER_ARG) + + For the `traceidratio` samplers: the fraction of traces to record, e.g. `0.01`. Defaults to `1.0`. + ## Appendix: OS-specific conventions This information is not Nix-specific, but is referenced above. From 1bf76b485fe6c245fb1eb9f2c2838aec1c53d1ea Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Sep 2026 15:35:27 +0200 Subject: [PATCH 700/927] Fix logging test --- tests/functional/logging.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/logging.sh b/tests/functional/logging.sh index 56e76893b4..3ebd51fe52 100755 --- a/tests/functional/logging.sh +++ b/tests/functional/logging.sh @@ -35,7 +35,7 @@ fi # Test json-log-path. clearStore nix build -vv --file dependencies.nix --no-link --json-log-path "$TEST_ROOT/log.json" 2>&1 | grepQuiet 'building.*dependencies-top.drv' -grep '{"action":"start","fields":\[".*-dependencies-top.drv","",1,1\],"id":.*,"level":3,"parent":0' "$TEST_ROOT/log.json" >&2 +grep '{"action":"start","fields":\[".*-dependencies-top.drv","",1,1\],"id":.*,"level":3,"parent":[1-9]' "$TEST_ROOT/log.json" >&2 grep -E '{"action":"result","id":[^,]+,"payload":{"builtOutputs":{"out":{"outPath":"[^-]+-dependencies-top".*"status":"Built".*"success":true' "$TEST_ROOT/log.json" >&2 (( $(grep -c '{"action":"msg","level":5,"msg":"executing builder .*"}' "$TEST_ROOT/log.json" ) == 5 )) From ba22f34b821f93c8d45efa616df2a3a892776168 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Sep 2026 16:54:34 +0200 Subject: [PATCH 701/927] protobuf: Use multiple outputs --- packaging/dependencies.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 3449443bc0..859282b430 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -224,6 +224,13 @@ scope: { # The default build has no OTLP exporters at all; we need the # OTLP/HTTP one (and specifically not the gRPC one). enableHttp = true; + protobuf = pkgs.protobuf.overrideAttrs { + outputs = [ + "out" + "lib" + "dev" + ]; + }; }).overrideAttrs (old: { # Support OTEL_EXPORTER_OTLP_COMPRESSION=gzip. From 39c8590a0d4baf5aa7d8793b536fb05dcaf7e14e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 7 Sep 2026 16:15:28 +0200 Subject: [PATCH 702/927] Export OpenTelemetry traces as OTLP/JSON via FileTransfer The OTLP/HTTP exporter that comes with opentelemetry-cpp requires protobuf, which is by far the most expensive part of the OpenTelemetry dependency: protobuf itself (18.6 MiB), abseil (7.4 MiB, including a runtime reference to abseil's `dev` output) and gtest (842 KiB, pulled in by abseil's `libabsl_scoped_mock_log.so`). However, OTLP also has a JSON encoding, and Nix already has a capable HTTP client. So serialize spans to OTLP/JSON ourselves and upload them using FileTransfer. Note that opentelemetry-cpp cannot help us here: its JSON support serializes a protobuf message using protobuf reflection, so it needs protobuf all the same. We keep using the OpenTelemetry SDK for everything else (the tracer provider, the batch span processor, samplers, span data and the W3C propagator), none of which needs protobuf. So opentelemetry-cpp can now be used as it comes from Nixpkgs, without an override. This reduces the cost of the OpenTelemetry dependency from ~31 MiB to ~2.9 MiB. Since the upload creates activities of its own, which would be exported as spans, which would create more activities, ad infinitum, we do it inside an activity named `UploadOpenTelemetry` that the OpenTelemetryLogger ignores, along with its children. (Note that suppressing this only at shutdown would not be enough: the batch span processor exports every 5 seconds, so this is a steady-state feedback loop.) The activities themselves are still created as usual, so uploads show up in the progress bar like any other transfer. Assisted-by: Claude Opus 5 --- packaging/dependencies.nix | 27 ---- src/nix/meson.build | 7 +- src/nix/otel-logger.cc | 303 ++++++++++++++++++++++++++++++++++++- src/nix/unix/daemon.cc | 12 ++ 4 files changed, 312 insertions(+), 37 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 859282b430..183ec3fb6a 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -219,33 +219,6 @@ scope: { inherit (scope) curl; }; - opentelemetry-cpp = - (pkgs.opentelemetry-cpp.override { - # The default build has no OTLP exporters at all; we need the - # OTLP/HTTP one (and specifically not the gRPC one). - enableHttp = true; - protobuf = pkgs.protobuf.overrideAttrs { - outputs = [ - "out" - "lib" - "dev" - ]; - }; - }).overrideAttrs - (old: { - # Support OTEL_EXPORTER_OTLP_COMPRESSION=gzip. - cmakeFlags = old.cmakeFlags or [ ] ++ [ (pkgs.lib.cmakeBool "WITH_OTLP_HTTP_COMPRESSION" true) ]; - buildInputs = old.buildInputs or [ ] ++ [ pkgs.zlib ]; - # The gzip tests enabled by WITH_OTLP_HTTP_COMPRESSION do an - # HTTP round trip against a local test server and time out - # flakily in the build sandbox. - checkPhase = '' - runHook preCheck - ctest --output-on-failure -E 'BasicCurlHttpTests.Gzip' - runHook postCheck - ''; - }); - libmicrohttpd = pkgs.libmicrohttpd.overrideDerivation (old: { # Don't pull in gnutls since it's pretty big and we don't need it. configureFlags = old.configureFlags or [ ] ++ [ "--without-gnutls" ]; diff --git a/src/nix/meson.build b/src/nix/meson.build index 13b98512a7..03e03f93c0 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -83,10 +83,11 @@ if sentry_required.enabled() endif otel_required = get_option('otel') -otel = dependency('opentelemetry_exporter_otlp_http', required : otel_required) +# Note: we don't need the OTLP exporter from opentelemetry-cpp (which +# would pull in protobuf); we serialize spans to OTLP/JSON ourselves. otel_trace = dependency('opentelemetry_trace', required : otel_required) -configdata.set('HAVE_OTEL', (otel.found() and otel_trace.found()).to_int()) -deps_private += [ otel, otel_trace ] +configdata.set('HAVE_OTEL', otel_trace.found().to_int()) +deps_private += otel_trace config_priv_h = configure_file( configuration : configdata, diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 87912a0ebf..5b8d8ecb35 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -3,20 +3,30 @@ #include "cli-config-private.hh" #if HAVE_OTEL +# include "nix/store/filetransfer.hh" +# include "nix/util/base-n.hh" +# include "nix/util/compression.hh" # include "nix/util/environment-variables.hh" +# include "nix/util/serialise.hh" # include "nix/util/sync.hh" # include "nix/util/terminal.hh" +# include "nix/util/url.hh" # include # include # include +# include + # include # include # include -# include # include +# include +# include +# include +# include # include # include # include @@ -68,6 +78,14 @@ inline opentelemetry::nostd::string_view toNostd(std::string_view sv) noexcept return {sv.data(), sv.size()}; } +/** + * The name of the activity under which we upload spans. Since the + * upload itself creates activities (namely the file transfer), we + * must not create spans for this activity or its children, since + * that would create an infinite regress. + */ +constexpr std::string_view uploadActivityName = "UploadOpenTelemetry"; + struct ExtractCarrier : opentelemetry::context::propagation::TextMapCarrier { opentelemetry::nostd::string_view traceparent, tracestate; @@ -188,6 +206,25 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger Sync> spans_; + /** + * Activities for which we don't create spans, namely the + * `UploadOpenTelemetry` activities and their children. + */ + boost::concurrent_flat_set ignoredActs; + + /** + * Whether this activity should be ignored, i.e. whether it's an + * `UploadOpenTelemetry` activity or a child of one. If so, record + * it so that its children are ignored as well. + */ + bool ignoreActivity(ActivityId act, std::string_view name, ActivityId parent) + { + if (name != uploadActivityName && !ignoredActs.contains(parent)) + return false; + ignoredActs.insert(act); + return true; + } + static Headers injectContext(const SpanPtr & span) { if (!span->GetContext().IsValid()) @@ -243,6 +280,9 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger if (textIsName) name = s.empty() ? "activity" : std::string_view(s); + if (ignoreActivity(act, name, parent)) + return; + auto spans(spans_.lock()); if (auto i = spans->find(parent); i != spans->end()) @@ -300,6 +340,9 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger if (isRemoteLogSource()) return; + if (ignoreActivity(act, name, parent)) + return; + /* An activity carrying a `traceparent` metadata field exists only to link its child activities to a span in another process (e.g. the client activity on whose @@ -367,6 +410,8 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger try { if (isRemoteLogSource()) return; + if (ignoredActs.erase(act)) + return; auto spans(spans_.lock()); if (auto i = spans->find(act); i != spans->end()) { /* If the activity is being stopped while an exception @@ -437,6 +482,241 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger } }; +/** + * Serialize an attribute value to its OTLP/JSON representation, i.e. + * an `AnyValue` object such as `{"stringValue": "foo"}`. Note that + * 64-bit integers are represented as strings in OTLP/JSON. + */ +nlohmann::json toAnyValue(const opentelemetry::sdk::common::OwnedAttributeValue & value) +{ + return opentelemetry::nostd::visit( + [](const auto & v) -> nlohmann::json { + using T = std::decay_t; + if constexpr (std::is_same_v) + return {{"boolValue", v}}; + else if constexpr (std::is_same_v) + return {{"doubleValue", v}}; + else if constexpr (std::is_same_v) + return {{"stringValue", v}}; + else if constexpr (std::is_integral_v) + return {{"intValue", std::to_string(v)}}; + else if constexpr (std::is_same_v>) + return {{"bytesValue", base64::encode(std::as_bytes(std::span{v}))}}; + else { + /* Any other vector: an OTLP array of AnyValues. Note + that we can't take the elements by reference, since + `std::vector` yields proxy references. */ + auto values = nlohmann::json::array(); + for (auto x : v) { + using E = std::decay_t; + if constexpr (std::is_same_v) + values.push_back({{"boolValue", (bool) x}}); + else if constexpr (std::is_same_v) + values.push_back({{"doubleValue", x}}); + else if constexpr (std::is_same_v) + values.push_back({{"stringValue", x}}); + else + values.push_back({{"intValue", std::to_string(x)}}); + } + return {{"arrayValue", {{"values", std::move(values)}}}}; + } + }, + value); +} + +/** + * Serialize an attribute map to an OTLP/JSON `KeyValue` array. + */ +template +nlohmann::json toAttributes(const Map & map) +{ + auto res = nlohmann::json::array(); + for (auto & [key, value] : map) + res.push_back({{"key", key}, {"value", toAnyValue(value)}}); + return res; +} + +template +std::string toHex(const Id & id) +{ + char buf[2 * Id::kSize]; + id.ToLowerBase16(buf); + return std::string(buf, sizeof(buf)); +} + +std::string toUnixNano(std::chrono::nanoseconds t) +{ + return std::to_string(t.count()); +} + +/** + * A span exporter that serializes spans to OTLP/JSON and uploads them + * using Nix's own `FileTransfer`. Compared to the exporter that comes + * with opentelemetry-cpp, this avoids a dependency on protobuf (which + * is very large), and it reuses the HTTP client that Nix already has. + */ +class OtlpJsonSpanExporter final : public opentelemetry::sdk::trace::SpanExporter +{ + std::string endpoint; + Headers headers; + bool compress; + + std::atomic isShutdown{false}; + +public: + + OtlpJsonSpanExporter(std::string endpoint, Headers headers, bool compress) + : endpoint(std::move(endpoint)) + , headers(std::move(headers)) + , compress(compress) + { + } + + std::unique_ptr MakeRecordable() noexcept override + { + return std::make_unique(); + } + + opentelemetry::sdk::common::ExportResult Export( + const opentelemetry::nostd::span> & recordables) noexcept + override + { + using opentelemetry::sdk::common::ExportResult; + + if (isShutdown.load(std::memory_order_acquire)) + return ExportResult::kFailure; + + try { + if (recordables.empty()) + return ExportResult::kSuccess; + + /* All spans in a batch come from the same tracer provider, + so they share a resource and (in our case) a scope. */ + auto spans = nlohmann::json::array(); + const opentelemetry::sdk::resource::Resource * resource = nullptr; + const opentelemetry::sdk::trace::InstrumentationScope * scope = nullptr; + + for (auto & recordable : recordables) { + /* Safe: `MakeRecordable()` only ever returns `SpanData`. */ + auto & span = static_cast(*recordable); + + resource = &span.GetResource(); + scope = &span.GetInstrumentationScope(); + + nlohmann::json json{ + {"traceId", toHex(span.GetTraceId())}, + {"spanId", toHex(span.GetSpanId())}, + {"name", std::string(span.GetName().data(), span.GetName().size())}, + /* `SpanKind` is declared in the same order as in + OTLP, which starts counting at `unspecified`. */ + {"kind", (int) span.GetSpanKind() + 1}, + {"startTimeUnixNano", toUnixNano(span.GetStartTime().time_since_epoch())}, + {"endTimeUnixNano", toUnixNano(span.GetStartTime().time_since_epoch() + span.GetDuration())}, + {"attributes", toAttributes(span.GetAttributes())}, + {"flags", span.GetFlags().flags()}, + }; + + if (span.GetParentSpanId().IsValid()) + json["parentSpanId"] = toHex(span.GetParentSpanId()); + + if (span.GetStatus() != opentelemetry::trace::StatusCode::kUnset) { + auto description = span.GetDescription(); + json["status"] = { + {"code", (int) span.GetStatus()}, + {"message", std::string(description.data(), description.size())}, + }; + } + + spans.push_back(std::move(json)); + } + + nlohmann::json doc{ + {"resourceSpans", + {{ + {"resource", {{"attributes", toAttributes(resource->GetAttributes())}}}, + {"scopeSpans", + {{ + {"scope", {{"name", scope->GetName()}, {"version", scope->GetVersion()}}}, + {"spans", std::move(spans)}, + }}}, + }}}, + }; + + upload(doc.dump()); + + return ExportResult::kSuccess; + } catch (...) { + /* Note that nothing retries a failed export, so all we can + do is drop the spans. Don't let the exception escape, + since this method is noexcept. Also don't bother the + user about it: failing to export telemetry should not + be noise on top of whatever they're actually doing. */ + ignoreExceptionInDestructor(lvlDebug); + return ExportResult::kFailure; + } + } + + void upload(std::string payload) + { + /* The upload creates activities of its own, which would be + exported as spans, which would create more activities, ad + infinitum. So do it inside an activity that the + OpenTelemetryLogger ignores, along with its children. */ + Activity act(*logger, lvlDebug, uploadActivityName, {}, "", 0); + PushActivity pact(act.id); + + FileTransferRequest request(parseURL(endpoint)); + request.method = HttpMethod::Post; + request.mimeType = "application/json"; + request.headers = headers; + + if (compress) { + payload = nix::compress(CompressionAlgo::gzip, payload); + request.headers.emplace_back("Content-Encoding", "gzip"); + } + + StringSource source{payload}; + request.data = {source}; + + /* Don't hold up the process at exit retrying telemetry. */ + request.retryAttempts = 0; + + getFileTransfer()->upload(request); + } + + bool ForceFlush(std::chrono::microseconds) noexcept override + { + /* We upload synchronously in `Export()`, so there is never + anything buffered here. */ + return true; + } + + bool Shutdown(std::chrono::microseconds) noexcept override + { + isShutdown.store(true, std::memory_order_release); + return true; + } +}; + +/** + * Parse `OTEL_EXPORTER_OTLP_HEADERS`, a comma-separated list of + * percent-encoded `name=value` pairs. + */ +Headers parseOtlpHeaders(std::string_view s) +{ + Headers headers; + for (auto & item : tokenizeString(s, ",")) { + auto eq = item.find('='); + if (eq == std::string::npos) + continue; + auto name = trim(item.substr(0, eq)); + auto value = trim(item.substr(eq + 1)); + if (!name.empty()) + headers.emplace_back(percentDecode(name), percentDecode(value)); + } + return headers; +} + } // namespace void initOtel(std::string_view serviceName) @@ -444,16 +724,25 @@ void initOtel(std::string_view serviceName) if (otelState.load(std::memory_order_acquire)) return; - /* Without an explicitly configured endpoint, stay off: the SDK's - default would silently export to http://localhost:4318. */ - if (!getEnv("OTEL_EXPORTER_OTLP_ENDPOINT") && !getEnv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")) + /* Without an explicitly configured endpoint, stay off; we don't + want to export to some default endpoint behind the user's + back. */ + auto endpoint = [&]() -> std::string { + if (auto s = getEnv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")) + return *s; + if (auto s = getEnv("OTEL_EXPORTER_OTLP_ENDPOINT")) + return *s + "/v1/traces"; + return ""; + }(); + if (endpoint.empty()) return; namespace sdktrace = opentelemetry::sdk::trace; - /* Default-constructed options read the OTEL_EXPORTER_OTLP_* - environment variables (endpoint, headers, TLS, compression). */ - auto exporter = opentelemetry::exporter::otlp::OtlpHttpExporterFactory::Create(); + auto exporter = std::make_unique( + endpoint, + parseOtlpHeaders(getEnv("OTEL_EXPORTER_OTLP_HEADERS").value_or("")), + getEnv("OTEL_EXPORTER_OTLP_COMPRESSION").value_or("gzip") == "gzip"); auto processor = sdktrace::BatchSpanProcessorFactory::Create(std::move(exporter), sdktrace::BatchSpanProcessorOptions{}); auto resource = opentelemetry::sdk::resource::Resource::Create({ diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 3205f80723..809a1ea948 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -531,6 +531,10 @@ static void runDaemon( std::visit( overloaded{ [&](StdIO) { + /* FIXME: we don't currently trace connections served + over stdio. */ + resetOtelAfterFork(); + auto store = storeConfig->openStore(); store->init(); @@ -550,6 +554,14 @@ static void runDaemon( processStdioConnection(store, forceTrustClientOpt.value_or(Trusted)); }, [&](UnixSocket socketPathOverride) { + /* We fork a child per connection, and the exporter's + worker threads don't survive a fork(), so don't + trace in this process; the children set up tracing + themselves. (This discards any tracing state set up + by `main()`, without touching the exporter, which + is what we want.) */ + resetOtelAfterFork(); + auto socketPath = std::move(socketPathOverride) .or_else([&]() -> std::optional { return getDaemonSocketPath(*storeConfig); From af6885be7b0244bf7d2bb52fad2f61d204892b5f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 7 Sep 2026 17:24:31 +0200 Subject: [PATCH 703/927] nix: Trace legacy commands as well Legacy commands (`nix-instantiate`, `nix-store`, ...) return from `mainWrapped()` long before the point where the OpenTelemetryLogger is attached, so while tracing was initialized for them, they never produced any spans. Attach the logger before dispatching to them, using the program name as the root span name, and record failures on the root span like we do for the new CLI. Assisted-by: Claude Opus 5 --- src/nix/main.cc | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 44004f9323..586b19199c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -559,8 +559,25 @@ void mainWrapped(int argc, char ** argv) initOtel(programName); { - if (auto legacy = get(RegisterLegacyCommand::commands(), programName)) - return (*legacy)(argc, argv); + if (auto legacy = get(RegisterLegacyCommand::commands(), programName)) { + /* Legacy commands don't have subcommands, so we can set up + the root span right away. */ + OpenTelemetryLogger * otelLogger = nullptr; + if (auto l = makeOpenTelemetryLogger(programName)) { + otelLogger = l.get(); + applyExtraLogger(std::move(l)); + } + try { + return (*legacy)(argc, argv); + } catch (Exit &) { + throw; + } catch (std::exception & e) { + if (otelLogger) + // FIXME: privacy + otelLogger->setRootError(e.what()); + throw; + } + } } evalSettings.pureEval = true; From b97ab9603358c049155eb56f596c4864098a5dc4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Sep 2026 12:41:20 +0200 Subject: [PATCH 704/927] Move exception printing into the Logger Previously, `handleExceptions()` printed uncaught exceptions itself, so loggers had no way to observe them. The OpenTelemetry logger worked around this with `setRootError()`, which had to be called manually from every place that caught an exception. Instead, add a virtual `Logger::printException()`. Its default implementation contains the printing logic moved out of `handleExceptions()`, `TeeLogger` forwards it to all its loggers, and the OpenTelemetry logger overrides it to mark the root span as failed with the exception's message. Thus `setRootError()` and its call sites can go away. Also move `logger->stop()` into `handleExceptions()`, where it must run *after* `printException()` so that loggers can still record the exception. Assisted-by: Claude Opus 5 --- src/libutil/error.cc | 47 +++++++++++++------------ src/libutil/include/nix/util/logging.hh | 8 +++++ src/libutil/logging.cc | 33 +++++++++++++++++ src/libutil/tee-logger.cc | 6 ++++ src/nix/main.cc | 40 +++++---------------- src/nix/otel-logger.cc | 34 +++++++++--------- src/nix/otel-logger.hh | 8 +---- 7 files changed, 96 insertions(+), 80 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 4f9c9c96f4..e7d3ae8701 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -496,32 +496,33 @@ int handleExceptions(const std::string & programName, fun body) ErrorInfo::programName = baseNameOf(programName); - auto doLog = [&](BaseError & e) { - try { - logError(e.info()); - } catch (...) { - printError(ANSI_RED "error:" ANSI_NORMAL " Exception while printing an exception."); - } - }; + /* Note: this must happen after `printException()` below, so that + loggers that record the exception (like the OpenTelemetry + logger) can still do so. */ + Finally stopLogger([]() { logger->stop(); }); - std::string error = ANSI_RED "error:" ANSI_NORMAL " "; try { body(); - } catch (Exit & e) { - return e.status; - } catch (UsageError & e) { - doLog(e); - printError("\nTry '%1% --help' for more information.", programName); - return 1; - } catch (BaseError & e) { - doLog(e); - return e.info().status; - } catch (std::bad_alloc & e) { - printError(error + "out of memory"); - return 1; - } catch (std::exception & e) { - printError(error + e.what()); - return 1; + } catch (...) { + auto ex = std::current_exception(); + + /* Report the exception to the logger, which prints it and, in + the case of the OpenTelemetry logger, records it on the + root span. */ + logger->printException(ex, programName); + + /* Determine the exit status. */ + try { + std::rethrow_exception(ex); + } catch (Exit & e) { + return e.status; + } catch (UsageError & e) { + return 1; + } catch (BaseError & e) { + return e.info().status; + } catch (...) { + return 1; + } } return 0; diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index 05ada6fdce..5517499f91 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -174,6 +174,14 @@ public: virtual void logEI(const ErrorInfo & ei) noexcept = 0; + /** + * Report the exception that terminated the program (see + * `handleExceptions()`). The default implementation prints it; + * loggers that report to a monitoring system can override this to + * record it there as well (or instead). + */ + virtual void printException(const std::exception_ptr & ex, std::string_view programName) noexcept; + void logEI(Verbosity lvl, ErrorInfo ei) noexcept { ei.level = lvl; diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index b0e5b22faa..323de64105 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -7,6 +7,7 @@ #include "nix/util/position.hh" #include "nix/util/sync.hh" #include "nix/util/unix-domain-socket.hh" +#include "nix/util/exit.hh" #include #include @@ -66,6 +67,38 @@ void Logger::warn(const std::string & msg) noexcept log(lvlWarn, ANSI_WARNING "warning:" ANSI_NORMAL " " + msg); } +void Logger::printException(const std::exception_ptr & ex, std::string_view programName) noexcept +{ + /* Note: we log to `this` rather than to the global `logger`, so + that a `TeeLogger` can forward this to each of its loggers + without the message being printed once per logger. */ + auto doLog = [&](const BaseError & e) { + try { + logEI(e.info()); + } catch (...) { + log(lvlError, ANSI_RED "error:" ANSI_NORMAL " Exception while printing an exception."); + } + }; + + constexpr std::string_view error = ANSI_RED "error:" ANSI_NORMAL " "; + + try { + std::rethrow_exception(ex); + } catch (Exit &) { + } catch (UsageError & e) { + doLog(e); + log(lvlError, fmt("\nTry '%1% --help' for more information.", programName)); + } catch (BaseError & e) { + doLog(e); + } catch (std::bad_alloc & e) { + log(lvlError, std::string(error) + "out of memory"); + } catch (std::exception & e) { + log(lvlError, std::string(error) + e.what()); + } catch (...) { + log(lvlError, std::string(error) + "unknown exception"); + } +} + void Logger::writeToStdout(std::string_view s) { Descriptor standard_out = getStandardOutput(); diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index f4adaf86ca..f8fc7abe5b 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -44,6 +44,12 @@ class TeeLogger final : public Logger logger->logEI(ei); } + void printException(const std::exception_ptr & ex, std::string_view programName) noexcept override + { + for (auto & logger : loggers) + logger->printException(ex, programName); + } + void startActivity( ActivityId act, Verbosity lvl, diff --git a/src/nix/main.cc b/src/nix/main.cc index 586b19199c..f8b818c69f 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -535,13 +535,6 @@ void mainWrapped(int argc, char ** argv) tryEnterPrivateMountNamespace(); #endif - /* Note: registered before the logger->stop() Finally below, so - that at exit, the loggers are stopped (ending any open spans) - before the spans are exported. */ - Finally flushOtel([] { flushOtelAndShutdown(); }); - - Finally f([] { logger->stop(); }); - programPath = argv[0]; auto programName = std::string(baseNameOf(programPath)); auto extensionPos = programName.find_last_of("."); @@ -562,21 +555,9 @@ void mainWrapped(int argc, char ** argv) if (auto legacy = get(RegisterLegacyCommand::commands(), programName)) { /* Legacy commands don't have subcommands, so we can set up the root span right away. */ - OpenTelemetryLogger * otelLogger = nullptr; - if (auto l = makeOpenTelemetryLogger(programName)) { - otelLogger = l.get(); + if (auto l = makeOpenTelemetryLogger(programName)) applyExtraLogger(std::move(l)); - } - try { - return (*legacy)(argc, argv); - } catch (Exit &) { - throw; - } catch (std::exception & e) { - if (otelLogger) - // FIXME: privacy - otelLogger->setRootError(e.what()); - throw; - } + return (*legacy)(argc, argv); } } @@ -736,11 +717,8 @@ void mainWrapped(int argc, char ** argv) /* Map activities to OpenTelemetry spans, under a root span named after the subcommand. */ - OpenTelemetryLogger * otelLogger = nullptr; - if (auto l = makeOpenTelemetryLogger("nix " + concatStringsSep(" ", subcommand))) { - otelLogger = l.get(); + if (auto l = makeOpenTelemetryLogger("nix " + concatStringsSep(" ", subcommand))) applyExtraLogger(std::move(l)); - } try { args.command->second->run(); @@ -749,13 +727,6 @@ void mainWrapped(int argc, char ** argv) cached error so that we can show the original error to the user. */ e.force(); - } catch (Exit &) { - throw; - } catch (std::exception & e) { - if (otelLogger) - // FIXME: privacy - otelLogger->setRootError(e.what()); - throw; } } @@ -767,5 +738,10 @@ int main(int argc, char ** argv) // The CLI has a more detailed version than the libraries; see nixVersion. nix::nixVersion = NIX_CLI_VERSION; + + /* Note: this must happen after `handleExceptions()`, which stops + the logger, ending any open spans. */ + Finally flushOtel([] { flushOtelAndShutdown(); }); + return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv); }); } diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 5b8d8ecb35..3d0486352e 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -7,6 +7,7 @@ # include "nix/util/base-n.hh" # include "nix/util/compression.hh" # include "nix/util/environment-variables.hh" +# include "nix/util/exit.hh" # include "nix/util/serialise.hh" # include "nix/util/sync.hh" # include "nix/util/terminal.hh" @@ -187,15 +188,6 @@ std::string_view getS(const Logger::Fields & fields, size_t n) return {}; } -uint64_t getI(const Logger::Fields & fields, size_t n) -{ - if (n < fields.size()) { - if (auto p = std::get_if(&fields[n].raw)) - return *p; - } - return 0; -} - class OpenTelemetryLoggerImpl : public OpenTelemetryLogger { using SpanPtr = opentelemetry::nostd::shared_ptr; @@ -261,6 +253,21 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger void logEI(const ErrorInfo & ei) noexcept override {} + void printException(const std::exception_ptr & ex, std::string_view programName) noexcept override + { + try { + std::rethrow_exception(ex); + } catch (Exit &) { + /* Not a failure: this is how commands like `--version` + return. */ + } catch (std::exception & e) { + // FIXME: privacy + rootSpan->SetStatus(opentelemetry::trace::StatusCode::kError, toNostd(filterANSIEscapes(e.what(), true))); + } catch (...) { + rootSpan->SetStatus(opentelemetry::trace::StatusCode::kError, "unknown exception"); + } + } + void startActivity( ActivityId act, Verbosity lvl, @@ -460,15 +467,6 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger } } - void setRootError(std::string_view description) noexcept override - { - try { - rootSpan->SetStatus( - opentelemetry::trace::StatusCode::kError, toNostd(filterANSIEscapes(description, true))); - } catch (...) { - } - } - void stop() override { try { diff --git a/src/nix/otel-logger.hh b/src/nix/otel-logger.hh index e56770e665..8d261bbfab 100644 --- a/src/nix/otel-logger.hh +++ b/src/nix/otel-logger.hh @@ -18,13 +18,7 @@ namespace nix { * exporting them. */ class OpenTelemetryLogger : public Logger -{ -public: - /** - * Record an error on the root span. - */ - virtual void setRootError(std::string_view description) noexcept = 0; -}; +{}; /** * Create an `OpenTelemetryLogger` whose root span is named From c176f5812dc57bad283fff37729398cae4997375 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Sep 2026 15:45:08 +0200 Subject: [PATCH 705/927] Add a Logger::flush() method The OpenTelemetry export was triggered by a `Finally` in `main()` calling `flushOtelAndShutdown()`, which required `main()` and the daemon to know about OpenTelemetry specifically. Instead, add a virtual `Logger::flush()` for loggers that need to push buffered state somewhere before the process exits, and call it from `handleExceptions()` right after `logger->stop()` (which ends any open spans). The OpenTelemetry logger overrides it to do the actual export, so `flushOtelAndShutdown()` can go away. Note that this only exports; it no longer shuts the exporter down or clears the tracing state. Both were only needed to make the old global function safe to call repeatedly from unrelated places: `ForceFlush()` already drains the queue synchronously, and the state has to outlive the logger anyway, since spans may still reference the tracer. Assisted-by: Claude Opus 5 --- src/libutil/error.cc | 8 +++-- src/libutil/include/nix/util/logging.hh | 7 ++++ src/libutil/tee-logger.cc | 6 ++++ src/nix/main.cc | 4 --- src/nix/otel-logger.cc | 43 +++++++++++-------------- src/nix/otel-logger.hh | 15 +++------ src/nix/unix/daemon.cc | 2 +- 7 files changed, 43 insertions(+), 42 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e7d3ae8701..dcd0bcb162 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -498,8 +498,12 @@ int handleExceptions(const std::string & programName, fun body) /* Note: this must happen after `printException()` below, so that loggers that record the exception (like the OpenTelemetry - logger) can still do so. */ - Finally stopLogger([]() { logger->stop(); }); + logger) can still do so. `flush()` must come after `stop()`, + which ends any open spans. */ + Finally stopLogger([]() { + logger->stop(); + logger->flush(); + }); try { body(); diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index 5517499f91..609a2774eb 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -139,6 +139,13 @@ public: virtual void stop() {}; + /** + * Flush any buffered state to its final destination. Loggers that + * upload to a remote service (such as the OpenTelemetry logger) use + * this to perform the upload on exit. + */ + virtual void flush() {}; + /** * Guard object to resume the logger when done. */ diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index f8fc7abe5b..7b40a036ad 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -20,6 +20,12 @@ class TeeLogger final : public Logger logger->stop(); }; + void flush() override + { + for (auto & logger : loggers) + logger->flush(); + }; + void pause() override { for (auto & logger : loggers) diff --git a/src/nix/main.cc b/src/nix/main.cc index f8b818c69f..af2f4e4d01 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -739,9 +739,5 @@ int main(int argc, char ** argv) // The CLI has a more detailed version than the libraries; see nixVersion. nix::nixVersion = NIX_CLI_VERSION; - /* Note: this must happen after `handleExceptions()`, which stops - the logger, ending any open spans. */ - Finally flushOtel([] { flushOtelAndShutdown(); }); - return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv); }); } diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 3d0486352e..4259adc429 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -70,8 +70,8 @@ struct OtelState the batch exporter's worker thread, which must never happen from static destructors at exit() time (cf. the OPENSSL_INIT_NO_ATEXIT note in util.cc; in the daemon's forked children that thread does - not even exist). Teardown happens only via an explicit - flushOtelAndShutdown(). */ + not even exist). So it's never destroyed; pending spans are + exported by `OpenTelemetryLogger::flush()` instead. */ std::atomic otelState{nullptr}; inline opentelemetry::nostd::string_view toNostd(std::string_view sv) noexcept @@ -243,8 +243,8 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger if (getEnv("NIX_OTEL_DEBUG")) { char buf[2 * opentelemetry::trace::TraceId::kSize]; rootSpan->GetContext().trace_id().ToLowerBase16(buf); - /* Printed by flushOtelAndShutdown() once the trace has - been uploaded. */ + /* Printed by `flush()` once the trace has been + uploaded. */ state.debugTraceId = std::string(buf, sizeof(buf)); } } @@ -478,6 +478,20 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger } catch (...) { } } + + void flush() override + { + auto * state = otelState.load(std::memory_order_acquire); + if (!state) + return; + + /* Bound the timeout: the SDK default is microseconds::max(), + and a hung collector must not hang process exit. */ + state->provider->ForceFlush(std::chrono::microseconds(std::chrono::seconds(5))); + + if (!state->debugTraceId.empty()) + writeToStderr(fmt("OpenTelemetry trace ID: %s\n", state->debugTraceId)); + } }; /** @@ -792,25 +806,6 @@ void initOtel(std::string_view serviceName) state.release(); } -void flushOtelAndShutdown() -{ - auto * state = otelState.exchange(nullptr); - if (!state) - return; - /* Bound the timeouts: the SDK defaults are microseconds::max(), - and a hung collector must not hang process exit. */ - auto timeout = std::chrono::microseconds(std::chrono::seconds(5)); - state->provider->ForceFlush(timeout); - state->provider->Shutdown(timeout); - - if (!state->debugTraceId.empty()) - writeToStderr(fmt("OpenTelemetry trace ID: %s\n", state->debugTraceId)); - - /* Deliberately leak `state`: outstanding span handles may still - reference the tracer; after Shutdown() their spans are simply - dropped. */ -} - void resetOtelAfterFork() { /* Deliberately leak the old state: it references a worker thread @@ -832,8 +827,6 @@ makeOpenTelemetryLogger(std::string_view rootSpanName, std::string_view remotePa void initOtel(std::string_view) {} -void flushOtelAndShutdown() {} - void resetOtelAfterFork() {} std::unique_ptr makeOpenTelemetryLogger(std::string_view, std::string_view) diff --git a/src/nix/otel-logger.hh b/src/nix/otel-logger.hh index 8d261bbfab..2c7ae5c4bf 100644 --- a/src/nix/otel-logger.hh +++ b/src/nix/otel-logger.hh @@ -16,6 +16,11 @@ namespace nix { * Activities replayed from another process (cf. `RemoteLogSource`) * are ignored, since the originating process is responsible for * exporting them. + * + * `flush()` exports all pending spans and shuts down the exporter, + * with bounded timeouts. Since loggers are generally not destroyed, + * it has to be called explicitly before the process exits; + * `handleExceptions()` does so. */ class OpenTelemetryLogger : public Logger {}; @@ -46,16 +51,6 @@ makeOpenTelemetryLogger(std::string_view rootSpanName, std::string_view remotePa */ void initOtel(std::string_view serviceName); -/** - * Flush all pending spans and shut down the exporter, with bounded - * timeouts. Safe to call if tracing was never initialized, and safe - * to call more than once. Must be called explicitly before process - * exit: nothing is flushed from static destructors (cf. the - * OPENSSL_INIT_NO_ATEXIT note in util.cc). Spans that should be - * included must be ended first (e.g. via `logger->stop()`). - */ -void flushOtelAndShutdown(); - /** * Discard all tracing state inherited from the parent process after a * fork(): the exporter's worker thread does not exist in the child, diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 809a1ea948..5ddc9dfd58 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -415,7 +415,7 @@ static void daemonLoop( telemetry. This has to be done explicitly, since exit() does not unwind the stack. */ logger->stop(); - flushOtelAndShutdown(); + logger->flush(); exit(0); }, From a3888f1009e8c440a2dfc0dfa8a410baecf1ac67 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Sep 2026 16:01:09 +0200 Subject: [PATCH 706/927] Add a Logger::resetAfterFork() method Like `flush()`, this replaces an OpenTelemetry-specific free function (`resetOtelAfterFork()`) with a virtual method on `Logger`, so that the daemon no longer needs to know about OpenTelemetry specifically. Assisted-by: Claude Opus 5 --- src/libutil/include/nix/util/logging.hh | 8 ++++++++ src/libutil/tee-logger.cc | 6 ++++++ src/nix/otel-logger.cc | 20 ++++++++++---------- src/nix/otel-logger.hh | 14 +++++--------- src/nix/unix/daemon.cc | 6 +++--- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index 609a2774eb..af6ed20176 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -146,6 +146,14 @@ public: */ virtual void flush() {}; + /** + * Discard any state inherited from the parent process after a + * `fork()`, such as worker threads, which do not exist in the + * child. Note that this discards rather than flushes, since the + * parent remains responsible for its own state. + */ + virtual void resetAfterFork() {}; + /** * Guard object to resume the logger when done. */ diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index 7b40a036ad..6413f525d1 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -26,6 +26,12 @@ class TeeLogger final : public Logger logger->flush(); }; + void resetAfterFork() override + { + for (auto & logger : loggers) + logger->resetAfterFork(); + }; + void pause() override { for (auto & logger : loggers) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 4259adc429..f91ca3ac74 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -492,6 +492,16 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger if (!state->debugTraceId.empty()) writeToStderr(fmt("OpenTelemetry trace ID: %s\n", state->debugTraceId)); } + + void resetAfterFork() override + { + /* Deliberately leak the old state: it references a worker + thread that does not exist in this process, so it can be + neither flushed nor destroyed safely. Afterwards + `initOtel()` can be called again to start fresh tracing in + the child. */ + otelState.exchange(nullptr); + } }; /** @@ -806,14 +816,6 @@ void initOtel(std::string_view serviceName) state.release(); } -void resetOtelAfterFork() -{ - /* Deliberately leak the old state: it references a worker thread - that does not exist in this process, so it can be neither - flushed nor destroyed safely. */ - otelState.exchange(nullptr); -} - std::unique_ptr makeOpenTelemetryLogger(std::string_view rootSpanName, std::string_view remoteParentTraceparent) { @@ -827,8 +829,6 @@ makeOpenTelemetryLogger(std::string_view rootSpanName, std::string_view remotePa void initOtel(std::string_view) {} -void resetOtelAfterFork() {} - std::unique_ptr makeOpenTelemetryLogger(std::string_view, std::string_view) { return nullptr; diff --git a/src/nix/otel-logger.hh b/src/nix/otel-logger.hh index 2c7ae5c4bf..40b6c9b44a 100644 --- a/src/nix/otel-logger.hh +++ b/src/nix/otel-logger.hh @@ -21,6 +21,11 @@ namespace nix { * with bounded timeouts. Since loggers are generally not destroyed, * it has to be called explicitly before the process exits; * `handleExceptions()` does so. + * + * `resetAfterFork()` discards the tracing state inherited from the + * parent process, since the exporter's worker thread does not exist + * in the child. Afterwards `initOtel()` can be called again to start + * fresh tracing in the child. */ class OpenTelemetryLogger : public Logger {}; @@ -51,13 +56,4 @@ makeOpenTelemetryLogger(std::string_view rootSpanName, std::string_view remotePa */ void initOtel(std::string_view serviceName); -/** - * Discard all tracing state inherited from the parent process after a - * fork(): the exporter's worker thread does not exist in the child, - * so the inherited state can be neither used nor destroyed safely. - * Afterwards `initOtel()` can be called again to start fresh tracing - * in the child. - */ -void resetOtelAfterFork(); - } // namespace nix diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 5ddc9dfd58..cb30206cb9 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -382,7 +382,7 @@ static void daemonLoop( /* The OpenTelemetry exporter's worker thread does not survive the fork, so set up tracing afresh. */ - resetOtelAfterFork(); + logger->resetAfterFork(); initOtel("nix-daemon"); closeListeners(); @@ -533,7 +533,7 @@ static void runDaemon( [&](StdIO) { /* FIXME: we don't currently trace connections served over stdio. */ - resetOtelAfterFork(); + logger->resetAfterFork(); auto store = storeConfig->openStore(); store->init(); @@ -560,7 +560,7 @@ static void runDaemon( themselves. (This discards any tracing state set up by `main()`, without touching the exporter, which is what we want.) */ - resetOtelAfterFork(); + logger->resetAfterFork(); auto socketPath = std::move(socketPathOverride) .or_else([&]() -> std::optional { From 2afd5cb7f226f0eef22cefd034c06f1449f2b9cd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Sep 2026 16:26:19 +0200 Subject: [PATCH 707/927] Make initOtel() reset any previously initialized tracing `initOtel()` used to be idempotent, returning early if tracing was already initialized. That made the daemon's `resetAfterFork()` call in the fork child load-bearing in a subtle way: without it, the child's `initOtel("nix-daemon")` would silently reuse the parent's provider, whose exporter thread doesn't exist after the fork, so the child produced no spans at all. (The call was in fact a no-op anyway, since `startProcess()` replaces the child's logger with a simple one before running the child function.) Instead, have `initOtel()` discard any existing state, leaking rather than destroying it since we may be in a child process where its worker thread no longer exists. The child then only needs `initOtel()`. Also reset the inherited `FileTransfer` in the child, for the same reason: its curl worker thread doesn't survive the fork, but it looks healthy otherwise, so `getFileTransfer()` won't replace it by itself. This isn't currently reachable, since the daemon parent never performs a file transfer before forking, but it's a trap for future changes. Note that the parent's `resetAfterFork()` calls in `runDaemon()` are still needed: they turn tracing off in the process that forks, which is not something `initOtel()` can do. Assisted-by: Claude Opus 5 --- src/nix/otel-logger.cc | 16 ++++++---------- src/nix/otel-logger.hh | 15 +++++++++------ src/nix/unix/daemon.cc | 16 +++++++++++++--- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index f91ca3ac74..914bb46d02 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -495,11 +495,9 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger void resetAfterFork() override { - /* Deliberately leak the old state: it references a worker + /* Deliberately leak the old state: it may reference a worker thread that does not exist in this process, so it can be - neither flushed nor destroyed safely. Afterwards - `initOtel()` can be called again to start fresh tracing in - the child. */ + neither flushed nor destroyed safely. */ otelState.exchange(nullptr); } }; @@ -743,9 +741,6 @@ Headers parseOtlpHeaders(std::string_view s) void initOtel(std::string_view serviceName) { - if (otelState.load(std::memory_order_acquire)) - return; - /* Without an explicitly configured endpoint, stay off; we don't want to export to some default endpoint behind the user's back. */ @@ -811,9 +806,10 @@ void initOtel(std::string_view serviceName) state->provider = sdktrace::TracerProviderFactory::Create(std::move(processor), resource, std::move(sampler)); state->tracer = state->provider->GetTracer("nix"); - OtelState * expected = nullptr; - if (otelState.compare_exchange_strong(expected, state.get())) - state.release(); + /* Any previously initialized state is deliberately leaked rather + than destroyed, since we may be in a child process where its + worker thread doesn't exist. */ + otelState.exchange(state.release(), std::memory_order_release); } std::unique_ptr diff --git a/src/nix/otel-logger.hh b/src/nix/otel-logger.hh index 40b6c9b44a..7fcb1098b9 100644 --- a/src/nix/otel-logger.hh +++ b/src/nix/otel-logger.hh @@ -22,10 +22,10 @@ namespace nix { * it has to be called explicitly before the process exits; * `handleExceptions()` does so. * - * `resetAfterFork()` discards the tracing state inherited from the - * parent process, since the exporter's worker thread does not exist - * in the child. Afterwards `initOtel()` can be called again to start - * fresh tracing in the child. + * `resetAfterFork()` discards the tracing state without exporting + * anything, turning tracing off in this process. Note that a process + * that wants to trace after a `fork()` doesn't need this: `initOtel()` + * discards the inherited state by itself. */ class OpenTelemetryLogger : public Logger {}; @@ -51,8 +51,11 @@ makeOpenTelemetryLogger(std::string_view rootSpanName, std::string_view remotePa * Initialize OpenTelemetry tracing for this process. Does nothing * (and is cheap) if `OTEL_EXPORTER_OTLP_ENDPOINT` / * `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is not set in the environment, - * if tracing support is not compiled in, or if tracing is already - * initialized. + * or if tracing support is not compiled in. + * + * Any previously initialized tracing state is discarded rather than + * reused, so this can be called in a child process after a `fork()`, + * where the exporter's worker thread no longer exists. */ void initOtel(std::string_view serviceName); diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index cb30206cb9..652668c6f8 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -15,6 +15,7 @@ #include "nix/cmd/legacy.hh" #include "nix/cmd/unix-socket-server.hh" #include "nix/store/daemon.hh" +#include "nix/store/filetransfer.hh" #include "man-pages.hh" #include "otel-logger.hh" #include "nix/util/socket.hh" @@ -379,10 +380,19 @@ static void daemonLoop( [&, storeConfig, closeListeners = std::move(closeListeners)]() { setInterrupted(false); + /* Don't use the inherited `FileTransfer`: its + curl worker thread does not exist in this + process, so using it would hang. (It looks + healthy otherwise, so `getFileTransfer()` + won't replace it by itself.) We must not + destroy it either, since that would join a + thread that doesn't exist, so leak it. */ + static auto prevFileTransfer = resetFileTransfer(); + /* The OpenTelemetry exporter's worker thread - does not survive the fork, so set up - tracing afresh. */ - logger->resetAfterFork(); + does not survive the fork, so set up tracing + afresh. (`initOtel()` discards any state + inherited from the parent.) */ initOtel("nix-daemon"); closeListeners(); From 08336a1b11c560cfc07f70b95d6268910402cb82 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Sep 2026 16:38:43 +0200 Subject: [PATCH 708/927] Make resetFileTransfer() leak the previous FileTransfer itself `resetFileTransfer()` returned the previous singleton, leaving it to the caller to keep it alive: destroying it joins its curl worker thread, which doesn't exist after a `fork()`. Both callers did so with a function-local `static`, which doesn't actually accomplish that, since its destructor still runs via `__cxa_atexit` when the child calls `exit()`. So do the leaking in `resetFileTransfer()` itself, and document why. Callers now just call it. Also call it in the daemon's fork child, which didn't before: it never performs a file transfer before forking today, so nothing breaks, but it's a trap for future changes. Assisted-by: Claude Opus 5 --- src/libfetchers/builtin.cc | 5 ++--- src/libstore/filetransfer.cc | 9 +++++---- src/libstore/include/nix/store/filetransfer.hh | 10 +++++++++- src/nix/unix/daemon.cc | 12 ++++-------- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc index 44b3baf0b1..5509e5a230 100644 --- a/src/libfetchers/builtin.cc +++ b/src/libfetchers/builtin.cc @@ -32,10 +32,9 @@ static void builtinFetchTree(const BuiltinBuilderContext & ctx) myFetchSettings.accessTokens = fetchSettings.accessTokens.get(); // Make sure we don't use the FileTransfer object of the parent - // since it's in a broken state after the fork. We also must not - // delete it, so hang on to the shared_ptr. + // since it's in a broken state after the fork. // FIXME: move FileTransfer into fetchers::Settings. - static auto prevFileTransfer = resetFileTransfer(); + resetFileTransfer(); // FIXME: disable use of the git/tarball cache diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 683e56ca54..b5ced0699e 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1372,12 +1372,13 @@ ref getFileTransfer() return ref(*fileTransfer); } -std::shared_ptr resetFileTransfer() +void resetFileTransfer() { auto fileTransfer(_fileTransfer->lock()); - std::shared_ptr prev; - fileTransfer->swap(prev); - return prev; + /* Deliberately leak the previous object: destroying it would join + its worker thread, which doesn't exist in this process. */ + new std::shared_ptr(std::move(*fileTransfer)); + fileTransfer->reset(); } ref makeFileTransfer(const FileTransferSettings & settings) diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 606c075f3e..c3e2d99e04 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -488,7 +488,15 @@ ref getFileTransfer(); */ ref makeFileTransfer(const FileTransferSettings & settings = fileTransferSettings); -std::shared_ptr resetFileTransfer(); +/** + * Clear the `FileTransfer` singleton, so that the next + * `getFileTransfer()` creates a fresh one. Used after a `fork()`, + * since the inherited object's curl worker thread doesn't exist in + * the child, yet it looks healthy enough that `getFileTransfer()` + * won't replace it by itself. The previous object is deliberately + * leaked, since destroying it would join that thread. + */ +void resetFileTransfer(); class FileTransferError final : public CloneableError { diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 652668c6f8..effe6cde1f 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -380,14 +380,10 @@ static void daemonLoop( [&, storeConfig, closeListeners = std::move(closeListeners)]() { setInterrupted(false); - /* Don't use the inherited `FileTransfer`: its - curl worker thread does not exist in this - process, so using it would hang. (It looks - healthy otherwise, so `getFileTransfer()` - won't replace it by itself.) We must not - destroy it either, since that would join a - thread that doesn't exist, so leak it. */ - static auto prevFileTransfer = resetFileTransfer(); + /* Don't use the `FileTransfer` object of the + parent, since it's in a broken state after + the fork. */ + resetFileTransfer(); /* The OpenTelemetry exporter's worker thread does not survive the fork, so set up tracing From da13a1c7bf0956f762e21cd6ee4a5453222f9332 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Sep 2026 17:06:44 +0200 Subject: [PATCH 709/927] Add RegisterForkCallback Various state doesn't survive a `fork()`, in particular objects owning a thread: the thread doesn't exist in the child, so such objects can be neither used nor destroyed there. So far each such object was reset by hand at every place that forks, which is easy to forget and to get wrong. Instead, let code register a callback to be run by `startProcess()` in the forked child, in the manner of `RegisterCommand` and friends: static RegisterForkCallback resetFoo([]() { ... }); `filetransfer.cc` and `otel-logger.cc` now register their own callbacks, so the places that fork don't have to reset them explicitly. (`resetFileTransfer()` therefore no longer needs to be public.) Assisted-by: Claude Opus 5 --- src/libfetchers/builtin.cc | 6 ----- src/libstore/filetransfer.cc | 9 ++++--- .../include/nix/store/filetransfer.hh | 10 -------- src/libutil/include/nix/util/processes.hh | 25 +++++++++++++++++++ src/libutil/processes.cc | 6 +++++ src/libutil/unix/processes.cc | 9 +++++++ src/nix/otel-logger.cc | 7 ++++++ src/nix/unix/daemon.cc | 13 +++------- 8 files changed, 56 insertions(+), 29 deletions(-) diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc index 5509e5a230..ccd7cc845a 100644 --- a/src/libfetchers/builtin.cc +++ b/src/libfetchers/builtin.cc @@ -3,7 +3,6 @@ #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/util/archive.hh" -#include "nix/store/filetransfer.hh" #include "nix/store/store-open.hh" #include @@ -31,11 +30,6 @@ static void builtinFetchTree(const BuiltinBuilderContext & ctx) fetchers::Settings myFetchSettings; myFetchSettings.accessTokens = fetchSettings.accessTokens.get(); - // Make sure we don't use the FileTransfer object of the parent - // since it's in a broken state after the fork. - // FIXME: move FileTransfer into fetchers::Settings. - resetFileTransfer(); - // FIXME: disable use of the git/tarball cache auto input = Input::fromAttrs(myFetchSettings, jsonToAttrs(ctx.drv.structuredAttrs->structuredAttrs.at("input"))); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index b5ced0699e..24714d2623 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -4,6 +4,7 @@ #include "nix/util/config-global.hh" #include "nix/util/finally.hh" #include "nix/util/callback.hh" +#include "nix/util/processes.hh" #include "nix/util/signals.hh" #include "nix/util/util.hh" @@ -1372,14 +1373,16 @@ ref getFileTransfer() return ref(*fileTransfer); } -void resetFileTransfer() -{ +/* The curl worker thread doesn't exist in a forked child, so discard + the singleton there. Note that it looks healthy otherwise, so + `getFileTransfer()` wouldn't replace it by itself. */ +static RegisterForkCallback resetFileTransfer([]() { auto fileTransfer(_fileTransfer->lock()); /* Deliberately leak the previous object: destroying it would join its worker thread, which doesn't exist in this process. */ new std::shared_ptr(std::move(*fileTransfer)); fileTransfer->reset(); -} +}); ref makeFileTransfer(const FileTransferSettings & settings) { diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index c3e2d99e04..1456b79fb4 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -488,16 +488,6 @@ ref getFileTransfer(); */ ref makeFileTransfer(const FileTransferSettings & settings = fileTransferSettings); -/** - * Clear the `FileTransfer` singleton, so that the next - * `getFileTransfer()` creates a fresh one. Used after a `fork()`, - * since the inherited object's curl worker thread doesn't exist in - * the child, yet it looks healthy enough that `getFileTransfer()` - * won't replace it by itself. The previous object is deliberately - * leaked, since destroying it would join that thread. - */ -void resetFileTransfer(); - class FileTransferError final : public CloneableError { private: diff --git a/src/libutil/include/nix/util/processes.hh b/src/libutil/include/nix/util/processes.hh index 3927807b44..06e40ecf44 100644 --- a/src/libutil/include/nix/util/processes.hh +++ b/src/libutil/include/nix/util/processes.hh @@ -111,6 +111,31 @@ struct ProcessOptions int cloneFlags = 0; }; +/** + * Register a callback to be run by `startProcess()` in the forked + * child, before the child's main function. This is for state that + * doesn't survive a `fork()`, in particular objects owning a thread: + * the thread doesn't exist in the child, so such objects can be + * neither used nor destroyed there. Typical usage: + * + * static RegisterForkCallback resetFoo([]() { ... }); + * + * Note that callbacks are not run for `vfork()`ed children, since + * those share the parent's memory. Exceptions thrown by a callback + * are ignored, since there's not much the child can do about them. + */ +struct RegisterForkCallback +{ + typedef std::vector> Callbacks; + + static Callbacks & callbacks(); + + RegisterForkCallback(fun callback) + { + callbacks().push_back(std::move(callback)); + } +}; + #ifndef _WIN32 pid_t startProcess(fun processMain, const ProcessOptions & options = ProcessOptions()); #endif diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc index 0211557034..4c79e55b92 100644 --- a/src/libutil/processes.cc +++ b/src/libutil/processes.cc @@ -5,6 +5,12 @@ namespace nix { void ExecError::anchor() {} +RegisterForkCallback::Callbacks & RegisterForkCallback::callbacks() +{ + static Callbacks callbacks; + return callbacks; +} + Pid & Pid::operator=(Pid && other) noexcept { swap(*this, other); diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 40f4785da7..4084716c30 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -263,6 +263,15 @@ pid_t startProcess(fun processMain, const ProcessOptions & options) ~ProgressBar() tries to join a thread that doesn't exist. */ logger = newLogger; + + /* Discard other state that doesn't survive the fork, + such as objects owning a thread. */ + for (auto & callback : RegisterForkCallback::callbacks()) { + try { + callback(); + } catch (...) { + } + } } try { #ifdef __linux__ diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 914bb46d02..802fae74ac 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -8,6 +8,7 @@ # include "nix/util/compression.hh" # include "nix/util/environment-variables.hh" # include "nix/util/exit.hh" +# include "nix/util/processes.hh" # include "nix/util/serialise.hh" # include "nix/util/sync.hh" # include "nix/util/terminal.hh" @@ -74,6 +75,12 @@ struct OtelState exported by `OpenTelemetryLogger::flush()` instead. */ std::atomic otelState{nullptr}; +/* The exporter's worker thread doesn't exist in a forked child, so + don't trace there unless the child sets up tracing itself (i.e. by + calling `initOtel()`). The old state is deliberately leaked, since + it can be neither flushed nor destroyed safely. */ +static RegisterForkCallback resetOtel([]() { otelState.exchange(nullptr); }); + inline opentelemetry::nostd::string_view toNostd(std::string_view sv) noexcept { return {sv.data(), sv.size()}; diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index effe6cde1f..787c7897cc 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -15,7 +15,6 @@ #include "nix/cmd/legacy.hh" #include "nix/cmd/unix-socket-server.hh" #include "nix/store/daemon.hh" -#include "nix/store/filetransfer.hh" #include "man-pages.hh" #include "otel-logger.hh" #include "nix/util/socket.hh" @@ -380,15 +379,9 @@ static void daemonLoop( [&, storeConfig, closeListeners = std::move(closeListeners)]() { setInterrupted(false); - /* Don't use the `FileTransfer` object of the - parent, since it's in a broken state after - the fork. */ - resetFileTransfer(); - - /* The OpenTelemetry exporter's worker thread - does not survive the fork, so set up tracing - afresh. (`initOtel()` discards any state - inherited from the parent.) */ + /* Set up tracing afresh; `startProcess()` has + discarded the state inherited from the + parent. */ initOtel("nix-daemon"); closeListeners(); From 4ec1ec3d805c364e1fcabc5a344f09d4c11d5cd1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Sep 2026 14:09:16 +0200 Subject: [PATCH 710/927] Add nix.conf settings for OpenTelemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tracing could so far only be configured through the standard `OTEL_*` environment variables, which is awkward for a machine-wide setup: it means exporting variables from every shell profile, systemd unit and build hook. So add the corresponding `nix.conf` settings: `otlp`, `otlp-endpoint`, `otlp-headers` and `otlp-compression`. The environment variables take precedence, so existing setups keep working unchanged. In particular, setting an endpoint variable enables tracing regardless of `otlp`, while the `otlp-endpoint` setting is only used if `otlp` is enabled — that way tracing can be turned off without removing the endpoint from the configuration. Note that `otlp-headers` uses Nix's usual whitespace-separated `name=value` syntax rather than the comma-separated, percent-encoded syntax of `OTEL_EXPORTER_OTLP_HEADERS`. Also move `initOtel()` for the new CLI to after `parseCmdline()`, so that `--option otlp-endpoint ...` works. Legacy commands parse their arguments themselves, so they keep initializing tracing before that and can only be configured through `nix.conf` and the environment. Assisted-by: Claude Opus 5 --- doc/manual/source/command-ref/env-common.md | 6 +- src/nix/main.cc | 13 +-- src/nix/otel-logger.cc | 98 +++++++++++++++++++-- 3 files changed, 104 insertions(+), 13 deletions(-) diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index aac4b20499..52b1864b32 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -170,7 +170,8 @@ Likewise for the state and cache directories. ## OpenTelemetry tracing environment variables Nix can emit [OpenTelemetry](https://opentelemetry.io/) traces of its activities (such as evaluation, builds, substitutions and HTTP requests), exported using the OTLP/HTTP protocol. -Tracing is a no-op unless one of the endpoint variables below is set. +Tracing is a no-op unless one of the endpoint variables below is set, or the [`otlp`](@docroot@/command-ref/conf-file.md#conf-otlp) and [`otlp-endpoint`](@docroot@/command-ref/conf-file.md#conf-otlp-endpoint) settings are configured. +The variables below take precedence over the corresponding settings; in particular, setting an endpoint variable enables tracing even if `otlp` is disabled. When talking to the Nix daemon or to a binary cache server, Nix propagates the [W3C trace context](https://www.w3.org/TR/trace-context/), so that work done on your behalf by other processes or servers shows up in the same trace. - [`OTEL_EXPORTER_OTLP_ENDPOINT`](#env-OTEL_EXPORTER_OTLP_ENDPOINT) @@ -178,6 +179,7 @@ When talking to the Nix daemon or to a binary cache server, Nix propagates the [ The base URL of the OpenTelemetry collector to which traces are sent, e.g. `https://otel.example.org`. The path `/v1/traces` is appended automatically. Note: the URL should *not* have a trailing slash, since the resulting double slash confuses some collectors. + Overrides the [`otlp-endpoint`](@docroot@/command-ref/conf-file.md#conf-otlp-endpoint) setting. - [`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`](#env-OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) @@ -185,6 +187,8 @@ When talking to the Nix daemon or to a binary cache server, Nix propagates the [ Takes precedence over `OTEL_EXPORTER_OTLP_ENDPOINT`. - Other standard [OTLP exporter variables](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/) are also honored, such as `OTEL_EXPORTER_OTLP_HEADERS` (e.g. to pass an authorization header) and `OTEL_EXPORTER_OTLP_COMPRESSION` (set to `gzip` to compress uploads). + These override the [`otlp-headers`](@docroot@/command-ref/conf-file.md#conf-otlp-headers) and [`otlp-compression`](@docroot@/command-ref/conf-file.md#conf-otlp-compression) settings respectively. + Note that `OTEL_EXPORTER_OTLP_HEADERS` uses the OpenTelemetry syntax (comma-separated and percent-encoded), whereas `otlp-headers` uses Nix's usual whitespace-separated `name=value` syntax. - [`OTEL_TRACES_SAMPLER`](#env-OTEL_TRACES_SAMPLER) diff --git a/src/nix/main.cc b/src/nix/main.cc index af2f4e4d01..9db8c59b5d 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -547,14 +547,13 @@ void mainWrapped(int argc, char ** argv) argc--; } - /* No-op unless OTEL_EXPORTER_OTLP_ENDPOINT is set in the - environment. */ - initOtel(programName); - { if (auto legacy = get(RegisterLegacyCommand::commands(), programName)) { /* Legacy commands don't have subcommands, so we can set up - the root span right away. */ + the root span right away. Note that they parse their + arguments themselves, so unlike for the commands below, + `--option` cannot configure tracing here. */ + initOtel(programName); if (auto l = makeOpenTelemetryLogger(programName)) applyExtraLogger(std::move(l)); return (*legacy)(argc, argv); @@ -716,7 +715,9 @@ void mainWrapped(int argc, char ** argv) setSentryTag("nix_subcommand", concatStringsSep(" ", subcommand).c_str()); /* Map activities to OpenTelemetry spans, under a root span named - after the subcommand. */ + after the subcommand. Note: this must happen after + `parseCmdline()`, so that `--option` can configure tracing. */ + initOtel(programName); if (auto l = makeOpenTelemetryLogger("nix " + concatStringsSep(" ", subcommand))) applyExtraLogger(std::move(l)); diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 802fae74ac..812a17707f 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -6,6 +6,7 @@ # include "nix/store/filetransfer.hh" # include "nix/util/base-n.hh" # include "nix/util/compression.hh" +# include "nix/util/config-global.hh" # include "nix/util/environment-variables.hh" # include "nix/util/exit.hh" # include "nix/util/processes.hh" @@ -52,6 +53,68 @@ namespace nix { #if HAVE_OTEL +struct OtelSettings : Config +{ + Setting enable{ + this, + false, + "otlp", + R"( + Whether to export [OpenTelemetry](https://opentelemetry.io/) traces of + Nix's activities (such as evaluation, builds, substitutions and HTTP + requests). Requires `otlp-endpoint` to be set as well. + + Setting the `OTEL_EXPORTER_OTLP_ENDPOINT` or + `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` environment variable enables + tracing regardless of this setting. + )"}; + + Setting endpoint{ + this, + "", + "otlp-endpoint", + R"( + The base URL of the OpenTelemetry collector to which traces are sent, + e.g. `https://otel.example.org`. The path `/v1/traces` is appended + automatically. + + Only used if `otlp` is enabled. Overridden by the + `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable. + )"}; + + Setting headers{ + this, + {}, + "otlp-headers", + R"( + Extra HTTP headers to send to the collector, specified as a string + made up of whitespace-separated `name=value` pairs (e.g. to pass an + authorization token). + + Overridden by the `OTEL_EXPORTER_OTLP_HEADERS` environment variable. + Note that the environment variable uses the OpenTelemetry syntax + instead, i.e. comma-separated and percent-encoded. + )"}; + + Setting compression{ + this, + "gzip", + "otlp-compression", + R"( + The compression to apply to exported traces: `gzip` (the default) or + `none`. + + Overridden by the `OTEL_EXPORTER_OTLP_COMPRESSION` environment + variable. + )"}; +}; + +/* Note: deliberately not applied from the client's `setOptions` in the + daemon, so that a client can't redirect the daemon's telemetry. */ +static OtelSettings otelSettings; + +static GlobalConfig::Register rOtelSettings(&otelSettings); + namespace { struct OtelState @@ -748,25 +811,48 @@ Headers parseOtlpHeaders(std::string_view s) void initOtel(std::string_view serviceName) { - /* Without an explicitly configured endpoint, stay off; we don't - want to export to some default endpoint behind the user's - back. */ + /* An endpoint in the environment enables tracing by itself, and + takes precedence over the settings. */ auto endpoint = [&]() -> std::string { if (auto s = getEnv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")) return *s; if (auto s = getEnv("OTEL_EXPORTER_OTLP_ENDPOINT")) return *s + "/v1/traces"; + /* Unlike the environment variables, the setting is subject to + `otlp`, so that tracing can be turned off without removing + the endpoint from the configuration. */ + if (otelSettings.enable && !otelSettings.endpoint.get().empty()) + return otelSettings.endpoint.get() + "/v1/traces"; return ""; }(); - if (endpoint.empty()) + + /* Without an explicitly configured endpoint, stay off; we don't + want to export to some default endpoint behind the user's + back. */ + if (endpoint.empty()) { + if (otelSettings.enable) + warn("OpenTelemetry tracing is enabled but no endpoint is configured; see the 'otlp-endpoint' setting"); return; + } namespace sdktrace = opentelemetry::sdk::trace; + /* Note that the environment variable uses the OpenTelemetry + syntax, while the setting uses Nix's usual `name=value` + syntax. */ + auto headers = [&]() -> Headers { + if (auto s = getEnv("OTEL_EXPORTER_OTLP_HEADERS")) + return parseOtlpHeaders(*s); + Headers res; + for (auto & [name, value] : otelSettings.headers.get()) + res.emplace_back(name, value); + return res; + }(); + auto exporter = std::make_unique( endpoint, - parseOtlpHeaders(getEnv("OTEL_EXPORTER_OTLP_HEADERS").value_or("")), - getEnv("OTEL_EXPORTER_OTLP_COMPRESSION").value_or("gzip") == "gzip"); + std::move(headers), + getEnv("OTEL_EXPORTER_OTLP_COMPRESSION").value_or(otelSettings.compression) == "gzip"); auto processor = sdktrace::BatchSpanProcessorFactory::Create(std::move(exporter), sdktrace::BatchSpanProcessorOptions{}); auto resource = opentelemetry::sdk::resource::Resource::Create({ From a7f8d1dccc804602e98dbf6853129f85c7156b11 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Sep 2026 14:48:50 +0200 Subject: [PATCH 711/927] fixup! Add nix.conf settings for OpenTelemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the OpenTelemetry syntax for `otlp-headers` `Setting` splits on whitespace and has no quoting or escaping, so it cannot express a header value containing a space — which rules out `authorization: Bearer `, the main reason to set a header in the first place. So make `otlp-headers` a plain string parsed by `parseOtlpHeaders()`, i.e. the same comma-separated, percent-encoded syntax as `OTEL_EXPORTER_OTLP_HEADERS`. That also means a value can simply be copied from the environment variable to `nix.conf` and vice versa, rather than silently sending a literal `%20` to the collector. Assisted-by: Claude Opus 5 --- doc/manual/source/command-ref/env-common.md | 3 +- src/nix/otel-logger.cc | 36 +++++++++------------ 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index 52b1864b32..497797ff8f 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -187,8 +187,7 @@ When talking to the Nix daemon or to a binary cache server, Nix propagates the [ Takes precedence over `OTEL_EXPORTER_OTLP_ENDPOINT`. - Other standard [OTLP exporter variables](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/) are also honored, such as `OTEL_EXPORTER_OTLP_HEADERS` (e.g. to pass an authorization header) and `OTEL_EXPORTER_OTLP_COMPRESSION` (set to `gzip` to compress uploads). - These override the [`otlp-headers`](@docroot@/command-ref/conf-file.md#conf-otlp-headers) and [`otlp-compression`](@docroot@/command-ref/conf-file.md#conf-otlp-compression) settings respectively. - Note that `OTEL_EXPORTER_OTLP_HEADERS` uses the OpenTelemetry syntax (comma-separated and percent-encoded), whereas `otlp-headers` uses Nix's usual whitespace-separated `name=value` syntax. + These override the [`otlp-headers`](@docroot@/command-ref/conf-file.md#conf-otlp-headers) and [`otlp-compression`](@docroot@/command-ref/conf-file.md#conf-otlp-compression) settings respectively, and use the same syntax. - [`OTEL_TRACES_SAMPLER`](#env-OTEL_TRACES_SAMPLER) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 812a17707f..58819b2250 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -82,18 +82,24 @@ struct OtelSettings : Config `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable. )"}; - Setting headers{ + Setting headers{ this, - {}, + "", "otlp-headers", R"( - Extra HTTP headers to send to the collector, specified as a string - made up of whitespace-separated `name=value` pairs (e.g. to pass an - authorization token). + Extra HTTP headers to send to the collector, e.g. to pass an + authorization token. + + This uses the same syntax as the `OTEL_EXPORTER_OTLP_HEADERS` + environment variable (which overrides it): a comma-separated list of + `name=value` pairs, where the values are percent-encoded. For + instance, + + ``` + otlp-headers = authorization=Bearer%20secret123 + ``` - Overridden by the `OTEL_EXPORTER_OTLP_HEADERS` environment variable. - Note that the environment variable uses the OpenTelemetry syntax - instead, i.e. comma-separated and percent-encoded. + sends the header `authorization: Bearer secret123`. )"}; Setting compression{ @@ -837,21 +843,9 @@ void initOtel(std::string_view serviceName) namespace sdktrace = opentelemetry::sdk::trace; - /* Note that the environment variable uses the OpenTelemetry - syntax, while the setting uses Nix's usual `name=value` - syntax. */ - auto headers = [&]() -> Headers { - if (auto s = getEnv("OTEL_EXPORTER_OTLP_HEADERS")) - return parseOtlpHeaders(*s); - Headers res; - for (auto & [name, value] : otelSettings.headers.get()) - res.emplace_back(name, value); - return res; - }(); - auto exporter = std::make_unique( endpoint, - std::move(headers), + parseOtlpHeaders(getEnv("OTEL_EXPORTER_OTLP_HEADERS").value_or(otelSettings.headers)), getEnv("OTEL_EXPORTER_OTLP_COMPRESSION").value_or(otelSettings.compression) == "gzip"); auto processor = sdktrace::BatchSpanProcessorFactory::Create(std::move(exporter), sdktrace::BatchSpanProcessorOptions{}); From 1ef4ceca3ab1fa090471d66d31e899d952e1071f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Sep 2026 14:51:09 +0200 Subject: [PATCH 712/927] Don't propagate trace context to the OpenTelemetry collector `getTraceContext()` looked up the activity's span and, not finding one, fell back to the root span. But activities under `UploadOpenTelemetry` deliberately have no span, so the upload of the telemetry ended up carrying a `traceparent` header pointing at the very trace it was carrying. Return no context for ignored activities instead. Assisted-by: Claude Opus 5 --- src/nix/otel-logger.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 58819b2250..857e25193a 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -533,6 +533,11 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger { try { if (act) { + /* Don't propagate any context from an ignored + activity: the telemetry upload must not be part of + the trace it's carrying. */ + if (ignoredActs.contains(act)) + return {}; auto spans(spans_.lock()); if (auto i = spans->find(act); i != spans->end()) return injectContext(i->second); From 80aec31b6fbee2af1d4f5b63a596d46783dd7c17 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Sep 2026 15:52:06 +0200 Subject: [PATCH 713/927] NIX_OTEL_DEBUG -> NIX_DEBUG_OTEL --- src/nix/otel-logger.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 857e25193a..47fe1a6f42 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -130,7 +130,7 @@ struct OtelState /** * The root span's trace ID, to be printed on shutdown if - * NIX_OTEL_DEBUG is set. + * NIX_DEBUG_OTEL is set. */ std::string debugTraceId; }; @@ -316,7 +316,7 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger } rootSpan = tracer->StartSpan(toNostd(rootSpanName), options); - if (getEnv("NIX_OTEL_DEBUG")) { + if (getEnv("NIX_DEBUG_OTEL")) { char buf[2 * opentelemetry::trace::TraceId::kSize]; rootSpan->GetContext().trace_id().ToLowerBase16(buf); /* Printed by `flush()` once the trace has been From 5781bdbf0da0aecf24c09154ef07660c8387324b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Sep 2026 18:04:00 +0200 Subject: [PATCH 714/927] Add a functional test for OpenTelemetry tracing Run a few commands against a local OTLP/HTTP collector (a small Python script that saves what it receives, without any processing) and check the resulting OTLP/JSON: the root span's name and status, the legacy command path, that headers from the configuration arrive percent-decoded, that the upload of the telemetry doesn't carry a `traceparent` header, and that an endpoint in the environment takes precedence over `otlp = false`. The test is skipped if Nix was built without OpenTelemetry support or if Python isn't available. Assisted-by: Claude Fable 5.1 --- tests/functional/meson.build | 1 + tests/functional/otel-sink.py | 40 +++++++++++++++++++ tests/functional/otel.sh | 74 +++++++++++++++++++++++++++++++++++ tests/functional/package.nix | 3 ++ 4 files changed, 118 insertions(+) create mode 100644 tests/functional/otel-sink.py create mode 100644 tests/functional/otel.sh diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 07350619e4..c77fef34f7 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -144,6 +144,7 @@ suites = [ 'nix_path.sh', 'no-url-literals.sh', 'optimise-store.sh', + 'otel.sh', 'output-normalization.sh', 'pass-as-file.sh', 'path-from-hash-part.sh', diff --git a/tests/functional/otel-sink.py b/tests/functional/otel-sink.py new file mode 100644 index 0000000000..b6c4603399 --- /dev/null +++ b/tests/functional/otel-sink.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""A minimal OTLP/HTTP collector for testing: saves every POST body it +receives, unmodified, to a numbered file in the given directory, along +with the request headers. Listens on an ephemeral port on 127.0.0.1 +and writes the port number to `/port` once it's ready.""" + +import http.server +import json +import os +import sys + +out_dir = sys.argv[1] +count = 0 + + +class Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + global count + body = self.rfile.read(int(self.headers.get("content-length", 0))) + with open(os.path.join(out_dir, f"{count}.body"), "wb") as f: + f.write(body) + with open(os.path.join(out_dir, f"{count}.headers"), "w") as f: + json.dump({k.lower(): v for k, v in self.headers.items()}, f) + count += 1 + self.send_response(200) + self.end_headers() + + def log_message(self, *args): + pass + + +server = http.server.HTTPServer(("127.0.0.1", 0), Handler) + +# Write the port atomically, so the test never sees a partial file. +port_file = os.path.join(out_dir, "port") +with open(port_file + ".tmp", "w") as f: + f.write(str(server.server_address[1])) +os.rename(port_file + ".tmp", port_file) + +server.serve_forever() diff --git a/tests/functional/otel.sh b/tests/functional/otel.sh new file mode 100644 index 0000000000..cc58551215 --- /dev/null +++ b/tests/functional/otel.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# Test OpenTelemetry trace export: run a few commands against a local +# collector that just saves whatever it receives, then check the +# resulting OTLP/JSON. + +source common.sh + +# Only meaningful if Nix was built with OpenTelemetry support. +nix config show | grepQuiet '^otlp ' || skipTest "Nix was built without OpenTelemetry support" + +[[ $(type -p python3) ]] || skipTest "Python is not installed" + +sinkDir=$TEST_ROOT/otel +mkdir -p "$sinkDir" + +python3 ./otel-sink.py "$sinkDir" & +sinkPid=$! +trap 'kill "$sinkPid"' EXIT + +for ((i = 0; i < 100; i++)); do + [[ -e $sinkDir/port ]] && break + kill -0 "$sinkPid" || fail "the collector died unexpectedly" + sleep 0.1 +done +[[ -e $sinkDir/port ]] || fail "the collector didn't start" + +cat >> "$test_nix_conf" < Date: Sat, 22 Aug 2026 16:04:18 +0300 Subject: [PATCH 715/927] libfetchers: Support building against libgit2 2.0 (release candidate) Tested against the interface from https://github.com/libgit2/libgit2/pull/7337. Also adds a configuration for clang-tidy to disable formatting on just a single line, because if doesn't really format nicely with arguments which need a comma in an ifdef. Co-authored-by: John Ericson (cherry picked from commit 08fdb6eb184a112c937cd4ddc9e6b33e0f861ae2) --- .clang-format | 1 + src/libfetchers/git-utils.cc | 66 +++++++++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/.clang-format b/.clang-format index 1aadf2cadc..44f493019a 100644 --- a/.clang-format +++ b/.clang-format @@ -33,3 +33,4 @@ PPIndentWidth: 2 BinPackArguments: false BreakBeforeTernaryOperators: true SeparateDefinitionBlocks: Always +OneLineFormatOffRegex: "// NOFORMAT$" diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 52986a6b45..8f2e9bb1f8 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -16,6 +16,7 @@ #include "nix/util/executable-path.hh" #include "nix/util/deleter.hh" +#include #include #include #include @@ -123,10 +124,22 @@ typedef std::unique_ptr> Index; static Hash toHash(const git_oid & oid) { -#ifdef GIT_EXPERIMENTAL_SHA256 - assert(oid.type == GIT_OID_SHA1); + HashAlgorithm algo; +#if LIBGIT2_VERSION_CHECK(2, 0, 0) + switch (oid.type) { + case GIT_OID_SHA1: + algo = HashAlgorithm::SHA1; + break; + case GIT_OID_SHA256: + algo = HashAlgorithm::SHA256; + break; + default: + unreachable(); + } +#else + algo = HashAlgorithm::SHA1; #endif - Hash hash(HashAlgorithm::SHA1); + Hash hash(algo); memcpy(hash.hash, oid.id, hash.hashSize); return hash; } @@ -143,8 +156,29 @@ static void initLibGit2() static git_oid hashToOID(const Hash & hash) { git_oid oid; +#if LIBGIT2_VERSION_CHECK(2, 0, 0) + git_oid_t t; +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wswitch-enum" + switch (hash.algo) { + case HashAlgorithm::SHA1: + t = GIT_OID_SHA1; + break; + case HashAlgorithm::SHA256: + t = GIT_OID_SHA256; + break; + default: + throw Error("unsupported hash algorithm for Git: %s", printHashAlgo(hash.algo)); + } +# pragma GCC diagnostic pop + if (git_oid_from_raw(&oid, hash.hash, t)) + /* This can really never happen, since libgit2 just reads out our raw bytes. + The only failure mode is us specifying an invalid `type` parameter. */ + unreachable(); +#else if (git_oid_fromstr(&oid, hash.gitRev().c_str())) throw GitError("cannot convert '%s' to a Git OID", hash.gitRev()); +#endif return oid; } @@ -310,7 +344,16 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_odb_new(Setter(odb))) throw GitError("creating Git object database"); - if (git_odb_backend_pack(&packBackend, (path / "objects").string().c_str())) +#if LIBGIT2_VERSION_CHECK(2, 0, 0) + git_odb_backend_pack_options packOpts = GIT_ODB_OPTIONS_INIT; +#endif + if (git_odb_backend_pack( + &packBackend, + (path / "objects").string().c_str() +#if LIBGIT2_VERSION_CHECK(2, 0, 0) + , &packOpts // NOFORMAT +#endif + )) throw GitError("creating pack backend"); if (git_odb_add_backend(odb.get(), packBackend, 1)) @@ -368,7 +411,20 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this // (synchronously on the git_packbuilder_write_buf thread) Indexer indexer; git_indexer_progress stats; - if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr)) +#if LIBGIT2_VERSION_CHECK(2, 0, 0) + git_indexer_options indexerOpts = GIT_INDEXER_OPTIONS_INIT; +#endif + if (git_indexer_new( + Setter(indexer), + pack_dir_path.c_str(), +#if LIBGIT2_VERSION_CHECK(2, 0, 0) + &indexerOpts +#else + 0, + nullptr, + nullptr +#endif + )) throw GitError("creating git packfile indexer"); // TODO: provide index callback for checkInterrupt() termination From 8899a4ceaaaf61dc57b7d54f3ced624eb0b46722 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 4 Sep 2026 23:36:48 +0300 Subject: [PATCH 716/927] libfetchers: Disable GIT_OPT_ENABLE_STRICT_HASH_VERIFICATION See the comment for reasoning. Re-validiating all objects we read is probably out of scope for nix, just to catch occasional odb corruption. (cherry picked from commit d690cbfb584a2d2067957b5ae36e7780a51ecd3d) --- src/libfetchers/git-utils.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 8f2e9bb1f8..6027682a86 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -150,6 +150,12 @@ static void initLibGit2() std::call_once(initialized, []() { if (git_libgit2_init() < 0) throw GitError("initialising libgit2"); + + /* Nuke the "hashing on all reads" behavior, since that can lead to bad + performance https://github.com/libgit2/libgit2/issues/4951. It's a + compromise of course, but one that is mostly in line with git cli and + like how we don't recalculate narHash when reading from a store. */ + git_libgit2_opts(GIT_OPT_ENABLE_STRICT_HASH_VERIFICATION, 0); }); } From 7c92a1f804807950fbdedc0da2a9f3584965a721 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Fri, 4 Sep 2026 23:03:09 +0000 Subject: [PATCH 717/927] libfetchers: Don't try to find deltas when unpacking to the tarball cache This is by far the most expensive part of unpacking now: 49.35% nix libgit2.so.2.0.0 [.] git_delta_create_from_index 8.56% nix libgit2.so.2.0.0 [.] sha1_compression_states 5.89% nix libz.so.1.3.2 [.] longest_match 5.24% nix libz.so.1.3.2 [.] inflate_fast 4.80% nix libz.so.1.3.2 [.] deflate_slow 4.44% nix libgit2.so.2.0.0 [.] ubc_check 1.78% nix libz.so.1.3.2 [.] pqdownheap.constprop.0 1.53% nix libz.so.1.3.2 [.] compress_block 1.27% nix libgit2.so.2.0.0 [.] git_delta_index_init (cherry picked from commit 52bff94dc2693c555d695c8e104c64cb0ea5ba62) --- packaging/dependencies.nix | 49 +++++++++---------- packaging/patches/0002-memory-config.patch | 22 +++++++++ src/libfetchers/git-utils.cc | 39 ++++++++++++++- .../include/nix/fetchers/git-utils.hh | 6 +++ 4 files changed, 88 insertions(+), 28 deletions(-) create mode 100644 packaging/patches/0002-memory-config.patch diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 183ec3fb6a..2b589d4f04 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -164,33 +164,28 @@ scope: { ]; }); - libgit2 = - ( - if lib.versionAtLeast pkgs.libgit2.version "1.9.4" then - pkgs.libgit2 - else - # Grab newer libgit2. - pkgs.libgit2.overrideAttrs rec { - version = "1.9.4"; - src = pkgs.fetchFromGitHub { - owner = "libgit2"; - repo = "libgit2"; - tag = "v${version}"; - hash = "sha256-ZKUiz3pdFE2SKxh53X2oyr7hs32Njj5YVA0OXDXz7h0="; - }; - } - ).overrideAttrs - (old: { - separateDebugInfo = true; - - patches = old.patches or [ ] ++ [ - # Fix a use-after-free crash when `git_thread_create` fails during - # pack building (e.g. with EAGAIN under thread pressure), leaving - # orphaned delta-search worker threads running while the - # packbuilder is freed. - ./patches/libgit2-packbuilder-dont-fail-on-thread-create-error.patch - ]; - }); + libgit2 = pkgs.libgit2.overrideAttrs ( + finalAttrs: prevAttrs: { + version = "2.0.0-rc.1"; + src = pkgs.fetchFromGitHub { + owner = "libgit2"; + repo = "libgit2"; + rev = "ae45d0d168f7e8dbfdb8c623589cb51caac96ab3"; + hash = "sha256-3sbqHm37SOwBeFgtjI2DLN6kx1F7G2N1m6rRIkqDXNI="; + }; + patches = prevAttrs.patches or [ ] ++ [ + ./patches/0002-memory-config.patch + + # Fix a use-after-free crash when `git_thread_create` fails during + # pack building (e.g. with EAGAIN under thread pressure), leaving + # orphaned delta-search worker threads running while the + # packbuilder is freed. + # TODO: we can probably drop this patch since we're not finding deltas anymore. + ./patches/libgit2-packbuilder-dont-fail-on-thread-create-error.patch + ]; + separateDebugInfo = true; + } + ); # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. boost = diff --git a/packaging/patches/0002-memory-config.patch b/packaging/patches/0002-memory-config.patch new file mode 100644 index 0000000000..ce1ff61aca --- /dev/null +++ b/packaging/patches/0002-memory-config.patch @@ -0,0 +1,22 @@ +diff --git a/include/git2/sys/config.h b/include/git2/sys/config.h +index dcce18e6f..8945a3285 100644 +--- a/include/git2/sys/config.h ++++ b/include/git2/sys/config.h +@@ -200,7 +200,7 @@ GIT_EXTERN(int) git_config_backend_memory_options_init( + * @param opts the options to initialize this backend with, or NULL + * @return 0 on success or an error code + */ +-extern int git_config_backend_from_string( ++GIT_EXTERN(int) git_config_backend_from_string( + git_config_backend **out, + const char *cfg, + size_t len, +@@ -216,7 +216,7 @@ extern int git_config_backend_from_string( + * @param opts the options to initialize this backend with, or NULL + * @return 0 on success or an error code + */ +-extern int git_config_backend_from_values( ++GIT_EXTERN(int) git_config_backend_from_values( + git_config_backend **out, + const char **values, + size_t len, diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 6027682a86..c192318e86 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -116,6 +117,11 @@ typedef std::unique_ptr> typedef std::unique_ptr> StatusList; typedef std::unique_ptr> Remote; typedef std::unique_ptr> GitConfig; +typedef std::unique_ptrfree(backend); + })> + GitConfigBackend; typedef std::unique_ptr> ConfigIterator; typedef std::unique_ptr> ObjectDb; typedef std::unique_ptr> PackBuilder; @@ -337,6 +343,28 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_repository_open(Setter(repo), path.string().c_str())) throw GitError("opening Git repository %s", PathFmt(path)); + GitConfig config; + if (git_repository_config(Setter(config), *this)) + throw GitError("getting Git repository config"); + + /* Create an in-memory configuration so that we can set config options without modifying the + config file on-disk. */ + git_config_backend_memory_options configOpts = GIT_CONFIG_BACKEND_MEMORY_OPTIONS_INIT; + configOpts.backend_type = "nix"; + + std::vector configValues; + if (options.dontFindDeltas) + configValues.push_back("pack.deltacachesize=1"); + + GitConfigBackend memBackend; + if (git_config_backend_from_values(Setter(memBackend), configValues.data(), configValues.size(), &configOpts)) + throw GitError("creating an in-memory Git config"); + + if (git_config_add_backend(config.get(), memBackend.get(), GIT_CONFIG_LEVEL_APP, *this, /*force=*/false)) + throw GitError("adding the in-memory Git configuration backend"); + + memBackend.release(); + ObjectDb odb; if (options.packfilesOnly) { /* Create a fresh object database because by default the repo also @@ -1615,7 +1643,16 @@ ref Settings::getTarballCache() const static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache-v2"; auto tarballCache(_tarballCache.lock()); if (!*tarballCache) - *tarballCache = GitRepo::openRepo(repoDir, {.create = true, .bare = true, .packfilesOnly = true}); + *tarballCache = GitRepo::openRepo( + repoDir, + { + .create = true, + .bare = true, + .packfilesOnly = true, + /* Tarball unpacking is not expected to benefit from deltas much, + compared to how much CPU times it takes to find. */ + .dontFindDeltas = true, + }); return ref(*tarballCache); } diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 6ffc7372df..0d037bedb0 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -44,6 +44,12 @@ struct GitRepo bool create = false; bool bare = false; bool packfilesOnly = false; + /** + * Whether to avoid finding deltas when writing packfiles. It's an + * expensive operation, which should be avoided if no benefit is + * expected from possible deduplication in the same packfile. + */ + bool dontFindDeltas = false; }; static ref openRepo(const std::filesystem::path & path, Options options); From a257437bb9c2590bd7bd5ff2ce7cf7733f8c1831 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 5 Sep 2026 12:44:38 +0300 Subject: [PATCH 718/927] libfetchers: fix libgit2 pack window/bigfilethreshold config (cherry picked from commit 36bdf2ab83431794a8740e4bb61d1d8c45b91843) --- packaging/dependencies.nix | 1 + .../0003-packbuilder-correct-config.patch | 45 +++++++++++++++++++ src/libfetchers/git-utils.cc | 2 +- 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 packaging/patches/0003-packbuilder-correct-config.patch diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 2b589d4f04..b28042b560 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -175,6 +175,7 @@ scope: { }; patches = prevAttrs.patches or [ ] ++ [ ./patches/0002-memory-config.patch + ./patches/0003-packbuilder-correct-config.patch # Fix a use-after-free crash when `git_thread_create` fails during # pack building (e.g. with EAGAIN under thread pressure), leaving diff --git a/packaging/patches/0003-packbuilder-correct-config.patch b/packaging/patches/0003-packbuilder-correct-config.patch new file mode 100644 index 0000000000..8d1ab47920 --- /dev/null +++ b/packaging/patches/0003-packbuilder-correct-config.patch @@ -0,0 +1,45 @@ +diff --git a/src/libgit2/pack-objects.c b/src/libgit2/pack-objects.c +index 0da84b657..d540795f9 100644 +--- a/src/libgit2/pack-objects.c ++++ b/src/libgit2/pack-objects.c +@@ -115,8 +115,9 @@ static int packbuilder_config(git_packbuilder *pb) + GIT_PACK_DELTA_CACHE_SIZE); + config_get("pack.deltaCacheLimit", pb->cache_max_small_delta_size, + GIT_PACK_DELTA_CACHE_LIMIT); +- config_get("pack.deltaCacheSize", pb->big_file_threshold, ++ config_get("core.bigFileThreshold", pb->big_file_threshold, + GIT_PACK_BIG_FILE_THRESHOLD); ++ config_get("pack.window", pb->window_size, GIT_PACK_WINDOW); + config_get("pack.windowMemory", pb->window_memory_limit, 0); + + #undef config_get +@@ -1341,7 +1342,7 @@ int git_packbuilder__prepare(git_packbuilder *pb) + size_t i, n = 0; + int error; + +- if (pb->nr_objects == 0 || pb->done) ++ if (pb->nr_objects == 0 || pb->done || pb->window_size == 0) + return 0; /* nothing to do */ + + /* +@@ -1369,7 +1370,7 @@ int git_packbuilder__prepare(git_packbuilder *pb) + if (n > 1) { + git__tsort((void **)delta_list, n, type_size_sort); + if ((error = ll_find_deltas(pb, delta_list, n, +- GIT_PACK_WINDOW + 1, ++ pb->window_size + 1, + GIT_PACK_DEPTH)) < 0) { + git__free(delta_list); + return error; +diff --git a/src/libgit2/pack-objects.h b/src/libgit2/pack-objects.h +index ad04fb0ab..d9e03df60 100644 +--- a/src/libgit2/pack-objects.h ++++ b/src/libgit2/pack-objects.h +@@ -94,6 +94,7 @@ struct git_packbuilder { + size_t cache_max_small_delta_size; + size_t big_file_threshold; + size_t window_memory_limit; ++ size_t window_size; + + unsigned int nr_threads; /* nr of threads to use */ + diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index c192318e86..fe74979d8d 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -354,7 +354,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::vector configValues; if (options.dontFindDeltas) - configValues.push_back("pack.deltacachesize=1"); + configValues.push_back("pack.window=0"); GitConfigBackend memBackend; if (git_config_backend_from_values(Setter(memBackend), configValues.data(), configValues.size(), &configOpts)) From 587d96f21b7c9bd32a9db4e6b78254f3708eae7e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Sep 2026 12:07:45 +0200 Subject: [PATCH 719/927] Record store path names and versions on OpenTelemetry spans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spans for builds, substitutions, path info queries and path copies already carry the store path, but that's awkward to filter on. So also record the name and version separately, e.g. `nix.drv.name = patchelf` and `nix.drv.version = 0.18.0` for the derivation `…-patchelf-0.18.0.drv` (and likewise `nix.store.name` / `nix.store.version` for output paths), using `DrvName` to split them. Assisted-by: Claude Fable 5.1 --- src/nix/otel-logger.cc | 32 +++++++++++++++++++++++++++++--- tests/functional/otel.sh | 15 +++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 47fe1a6f42..7869885993 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -3,7 +3,9 @@ #include "cli-config-private.hh" #if HAVE_OTEL +# include "nix/store/derivations.hh" # include "nix/store/filetransfer.hh" +# include "nix/store/names.hh" # include "nix/util/base-n.hh" # include "nix/util/compression.hh" # include "nix/util/config-global.hh" @@ -264,6 +266,30 @@ std::string_view getS(const Logger::Fields & fields, size_t n) return {}; } +/** + * Set the `.path` attribute to a store path, along with + * `.name` and `.version` giving its name and version + * separately, e.g. `patchelf` and `0.18.0` for + * `/nix/store/…-patchelf-0.18.0` or `…-patchelf-0.18.0.drv`. Those are + * much easier to filter on than the path itself. A path that cannot + * be parsed just doesn't get these extra attributes. + */ +void setPathAttributes(opentelemetry::trace::Span & span, std::string_view prefix, std::string_view path) +{ + auto key = [&](std::string_view suffix) { return std::string(prefix) + "." + std::string(suffix); }; + + span.SetAttribute(toNostd(key("path")), toNostd(path)); + + try { + StorePath storePath(baseNameOf(path)); + DrvName drvName(storePath.isDerivation() ? BasicDerivation::nameFromPath(storePath) : storePath.name()); + span.SetAttribute(toNostd(key("name")), toNostd(drvName.name)); + if (!drvName.version.empty()) + span.SetAttribute(toNostd(key("version")), toNostd(drvName.version)); + } catch (...) { + } +} + class OpenTelemetryLoggerImpl : public OpenTelemetryLogger { using SpanPtr = opentelemetry::nostd::shared_ptr; @@ -387,17 +413,17 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger break; case actBuild: case actPostBuildHook: - span->SetAttribute("nix.drv.path", toNostd(getS(fields, 0))); + setPathAttributes(*span, "nix.drv", getS(fields, 0)); if (auto machine = getS(fields, 1); !machine.empty()) span->SetAttribute("nix.machine", toNostd(machine)); break; case actSubstitute: case actQueryPathInfo: - span->SetAttribute("nix.store.path", toNostd(getS(fields, 0))); + setPathAttributes(*span, "nix.store", getS(fields, 0)); span->SetAttribute("nix.substituter", toNostd(getS(fields, 1))); break; case actCopyPath: - span->SetAttribute("nix.store.path", toNostd(getS(fields, 0))); + setPathAttributes(*span, "nix.store", getS(fields, 0)); span->SetAttribute("nix.src.store", toNostd(getS(fields, 1))); span->SetAttribute("nix.dst.store", toNostd(getS(fields, 2))); break; diff --git a/tests/functional/otel.sh b/tests/functional/otel.sh index cc58551215..06081d92e8 100644 --- a/tests/functional/otel.sh +++ b/tests/functional/otel.sh @@ -42,6 +42,12 @@ span() { body "$1" | jq -r ".resourceSpans[0].scopeSpans[0].spans[0]$2" } +# Return the value of a string attribute of the span with the given +# name in the n-th upload. +attr() { + body "$1" | jq -r ".resourceSpans[0].scopeSpans[0].spans[] | select(.name == \"$2\") | .attributes[] | select(.key == \"$3\") | .value.stringValue" +} + # A successful command produces a root span named after it, with no # status. [[ $(nix eval --expr '1 + 2') = 3 ]] @@ -72,3 +78,12 @@ span 1 .status.message | grepQuiet "cannot add a string to an integer" [[ ! -e $sinkDir/3.body ]] [[ $(OTEL_EXPORTER_OTLP_ENDPOINT="http://127.0.0.1:$(cat "$sinkDir/port")" NIX_CONFIG="otlp = false" nix eval --expr '1 + 2') = 3 ]] [[ $(span 3 .name) = "nix eval" ]] + +# A build produces a `Build` span carrying the derivation's path, name +# and version as separate attributes. +# shellcheck disable=SC2016 # `$out` is for the Nix builder, not the shell. +drvPath=$(nix-instantiate --expr 'with import ./config.nix; mkDerivation { name = "foo-1.2"; buildCommand = "echo > $out"; }') +nix build --no-link "$drvPath^*" +[[ $(attr 5 Build nix.drv.path) = "$drvPath" ]] +[[ $(attr 5 Build nix.drv.name) = foo ]] +[[ $(attr 5 Build nix.drv.version) = 1.2 ]] From a18fc1c7b398bc81bb69e81ec2acd83137fd8444 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Sep 2026 12:39:01 +0200 Subject: [PATCH 720/927] Build opentelemetry-cpp against the standard library opentelemetry-cpp defaults to its own `nostd::` back-ports of `string_view`, `shared_ptr`, `span` and `variant` in its API, which required converting at every call site (`toNostd()`). Building it with `WITH_STL` instead makes those plain aliases of the standard types. This is an ABI switch, so it has to happen when building the library: nixpkgs exposes it as the `cxxStandard` override argument. The resulting package exports `-DOPENTELEMETRY_STL_VERSION` through pkg-config, so nothing else is needed on our side. Assisted-by: Claude Fable 5.1 --- packaging/dependencies.nix | 6 ++++ src/nix/otel-logger.cc | 71 +++++++++++++++++--------------------- 2 files changed, 37 insertions(+), 40 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 183ec3fb6a..63e35678ef 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -212,6 +212,12 @@ scope: { installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; }); + # Build opentelemetry-cpp against the standard library, so that its + # API uses `std::string_view`, `std::shared_ptr` etc. instead of its + # own `nostd::` back-ports. This is an ABI switch, so it has to be + # done when building the library, not just on our side. + opentelemetry-cpp = pkgs.opentelemetry-cpp.override { cxxStandard = "20"; }; + wasmtime = pkgs.callPackage ./wasmtime.nix { }; sentry-native = (pkgs.callPackage ./sentry-native.nix { }).override { diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 7869885993..5b140ad5e0 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -27,8 +27,6 @@ # include # include -# include -# include # include # include # include @@ -128,7 +126,7 @@ namespace { struct OtelState { std::unique_ptr provider; - opentelemetry::nostd::shared_ptr tracer; + std::shared_ptr tracer; /** * The root span's trace ID, to be printed on shutdown if @@ -152,11 +150,6 @@ std::atomic otelState{nullptr}; it can be neither flushed nor destroyed safely. */ static RegisterForkCallback resetOtel([]() { otelState.exchange(nullptr); }); -inline opentelemetry::nostd::string_view toNostd(std::string_view sv) noexcept -{ - return {sv.data(), sv.size()}; -} - /** * The name of the activity under which we upload spans. Since the * upload itself creates activities (namely the file transfer), we @@ -167,9 +160,9 @@ constexpr std::string_view uploadActivityName = "UploadOpenTelemetry"; struct ExtractCarrier : opentelemetry::context::propagation::TextMapCarrier { - opentelemetry::nostd::string_view traceparent, tracestate; + std::string_view traceparent, tracestate; - opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view key) const noexcept override + std::string_view Get(std::string_view key) const noexcept override { if (key == "traceparent") return traceparent; @@ -178,7 +171,7 @@ struct ExtractCarrier : opentelemetry::context::propagation::TextMapCarrier return {}; } - void Set(opentelemetry::nostd::string_view, opentelemetry::nostd::string_view) noexcept override {} + void Set(std::string_view, std::string_view) noexcept override {} }; /** @@ -192,7 +185,7 @@ std::optional parseTraceparent(std::string_vi if (traceparent.empty()) return std::nullopt; ExtractCarrier carrier; - carrier.traceparent = toNostd(traceparent); + carrier.traceparent = traceparent; opentelemetry::context::Context emptyCtx; auto ctx = opentelemetry::trace::propagation::HttpTraceContext{}.Extract(carrier, emptyCtx); auto spanContext = opentelemetry::trace::GetSpan(ctx)->GetContext(); @@ -205,16 +198,16 @@ struct InjectCarrier : opentelemetry::context::propagation::TextMapCarrier { Headers headers; - opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view) const noexcept override + std::string_view Get(std::string_view) const noexcept override { return {}; } - void Set(opentelemetry::nostd::string_view key, opentelemetry::nostd::string_view value) noexcept override + void Set(std::string_view key, std::string_view value) noexcept override { /* Copy immediately: `value` may point into a stack buffer of the propagator. */ - headers.emplace_back(std::string(key.data(), key.size()), std::string(value.data(), value.size())); + headers.emplace_back(std::string(key), std::string(value)); } }; @@ -278,23 +271,23 @@ void setPathAttributes(opentelemetry::trace::Span & span, std::string_view prefi { auto key = [&](std::string_view suffix) { return std::string(prefix) + "." + std::string(suffix); }; - span.SetAttribute(toNostd(key("path")), toNostd(path)); + span.SetAttribute(key("path"), path); try { StorePath storePath(baseNameOf(path)); DrvName drvName(storePath.isDerivation() ? BasicDerivation::nameFromPath(storePath) : storePath.name()); - span.SetAttribute(toNostd(key("name")), toNostd(drvName.name)); + span.SetAttribute(key("name"), drvName.name); if (!drvName.version.empty()) - span.SetAttribute(toNostd(key("version")), toNostd(drvName.version)); + span.SetAttribute(key("version"), drvName.version); } catch (...) { } } class OpenTelemetryLoggerImpl : public OpenTelemetryLogger { - using SpanPtr = opentelemetry::nostd::shared_ptr; + using SpanPtr = std::shared_ptr; - opentelemetry::nostd::shared_ptr tracer; + std::shared_ptr tracer; SpanPtr rootSpan; @@ -340,7 +333,7 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger if (auto spanContext = parseTraceparent(remoteParentTraceparent)) options.parent = *spanContext; } - rootSpan = tracer->StartSpan(toNostd(rootSpanName), options); + rootSpan = tracer->StartSpan(rootSpanName, options); if (getEnv("NIX_DEBUG_OTEL")) { char buf[2 * opentelemetry::trace::TraceId::kSize]; @@ -364,7 +357,7 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger return. */ } catch (std::exception & e) { // FIXME: privacy - rootSpan->SetStatus(opentelemetry::trace::StatusCode::kError, toNostd(filterANSIEscapes(e.what(), true))); + rootSpan->SetStatus(opentelemetry::trace::StatusCode::kError, filterANSIEscapes(e.what(), true)); } catch (...) { rootSpan->SetStatus(opentelemetry::trace::StatusCode::kError, "unknown exception"); } @@ -399,33 +392,33 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger else options.parent = rootSpan->GetContext(); - auto span = tracer->StartSpan(toNostd(name), options); + auto span = tracer->StartSpan(name, options); if (!s.empty() && !textIsName) - span->SetAttribute("nix.activity.text", toNostd(filterANSIEscapes(s, true))); + span->SetAttribute("nix.activity.text", filterANSIEscapes(s, true)); // Allow handling a subset of enum values # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wswitch-enum" switch (type) { case actFileTransfer: - span->SetAttribute("url.full", toNostd(getS(fields, 0))); + span->SetAttribute("url.full", getS(fields, 0)); break; case actBuild: case actPostBuildHook: setPathAttributes(*span, "nix.drv", getS(fields, 0)); if (auto machine = getS(fields, 1); !machine.empty()) - span->SetAttribute("nix.machine", toNostd(machine)); + span->SetAttribute("nix.machine", machine); break; case actSubstitute: case actQueryPathInfo: setPathAttributes(*span, "nix.store", getS(fields, 0)); - span->SetAttribute("nix.substituter", toNostd(getS(fields, 1))); + span->SetAttribute("nix.substituter", getS(fields, 1)); break; case actCopyPath: setPathAttributes(*span, "nix.store", getS(fields, 0)); - span->SetAttribute("nix.src.store", toNostd(getS(fields, 1))); - span->SetAttribute("nix.dst.store", toNostd(getS(fields, 2))); + span->SetAttribute("nix.src.store", getS(fields, 1)); + span->SetAttribute("nix.dst.store", getS(fields, 2)); break; default: break; @@ -497,16 +490,16 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger else options.parent = rootSpan->GetContext(); - auto span = tracer->StartSpan(toNostd(spanName), options); + auto span = tracer->StartSpan(spanName, options); if (!s.empty()) - span->SetAttribute("nix.activity.text", toNostd(filterANSIEscapes(s, true))); + span->SetAttribute("nix.activity.text", filterANSIEscapes(s, true)); for (auto & [key, value] : metadata) { if (auto str = std::get_if(&value.raw)) - span->SetAttribute(toNostd(key), toNostd(*str)); + span->SetAttribute(key, *str); else if (auto n = std::get_if(&value.raw)) - span->SetAttribute(toNostd(key), (int64_t) *n); + span->SetAttribute(key, (int64_t) *n); } spans->emplace(act, std::move(span)); @@ -616,7 +609,7 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger */ nlohmann::json toAnyValue(const opentelemetry::sdk::common::OwnedAttributeValue & value) { - return opentelemetry::nostd::visit( + return std::visit( [](const auto & v) -> nlohmann::json { using T = std::decay_t; if constexpr (std::is_same_v) @@ -704,9 +697,8 @@ class OtlpJsonSpanExporter final : public opentelemetry::sdk::trace::SpanExporte return std::make_unique(); } - opentelemetry::sdk::common::ExportResult Export( - const opentelemetry::nostd::span> & recordables) noexcept - override + opentelemetry::sdk::common::ExportResult + Export(const std::span> & recordables) noexcept override { using opentelemetry::sdk::common::ExportResult; @@ -733,7 +725,7 @@ class OtlpJsonSpanExporter final : public opentelemetry::sdk::trace::SpanExporte nlohmann::json json{ {"traceId", toHex(span.GetTraceId())}, {"spanId", toHex(span.GetSpanId())}, - {"name", std::string(span.GetName().data(), span.GetName().size())}, + {"name", std::string(span.GetName())}, /* `SpanKind` is declared in the same order as in OTLP, which starts counting at `unspecified`. */ {"kind", (int) span.GetSpanKind() + 1}, @@ -747,10 +739,9 @@ class OtlpJsonSpanExporter final : public opentelemetry::sdk::trace::SpanExporte json["parentSpanId"] = toHex(span.GetParentSpanId()); if (span.GetStatus() != opentelemetry::trace::StatusCode::kUnset) { - auto description = span.GetDescription(); json["status"] = { {"code", (int) span.GetStatus()}, - {"message", std::string(description.data(), description.size())}, + {"message", std::string(span.GetDescription())}, }; } From 2d5bb11a5ed5cfba03056eb9dc09d9d671b1c47c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Sep 2026 16:54:31 +0200 Subject: [PATCH 721/927] Warn when exporting OpenTelemetry traces fails Failed uploads were only logged at debug level, which made problems like a rejected authorization header invisible unless running with `-vvvvv`. Warn instead, but only about the first failure: the batch processor exports every few seconds, so an unreachable collector would otherwise repeat the same warning throughout a long build. Assisted-by: Claude Fable 5.1 --- src/nix/otel-logger.cc | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 5b140ad5e0..66edd10d3b 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -13,6 +13,7 @@ # include "nix/util/exit.hh" # include "nix/util/processes.hh" # include "nix/util/serialise.hh" +# include "nix/util/signals.hh" # include "nix/util/sync.hh" # include "nix/util/terminal.hh" # include "nix/util/url.hh" @@ -683,6 +684,9 @@ class OtlpJsonSpanExporter final : public opentelemetry::sdk::trace::SpanExporte std::atomic isShutdown{false}; + /* Only ever touched from the batch processor's worker thread. */ + bool warned = false; + public: OtlpJsonSpanExporter(std::string endpoint, Headers headers, bool compress) @@ -763,13 +767,19 @@ class OtlpJsonSpanExporter final : public opentelemetry::sdk::trace::SpanExporte upload(doc.dump()); return ExportResult::kSuccess; - } catch (...) { + } catch (Interrupted & e) { + /* Not a problem with the collector, so not worth a + warning. */ + printMsg(lvlDebug, "OpenTelemetry export interrupted: %s", e.what()); + return ExportResult::kFailure; + } catch (std::exception & e) { /* Note that nothing retries a failed export, so all we can do is drop the spans. Don't let the exception escape, - since this method is noexcept. Also don't bother the - user about it: failing to export telemetry should not - be noise on top of whatever they're actually doing. */ - ignoreExceptionInDestructor(lvlDebug); + since this method is noexcept. Only warn about the first + failure: the batch processor exports every few seconds, + so an unreachable collector would otherwise flood the + output with the same warning. */ + warnOnce(warned, "unable to export OpenTelemetry traces: %s", e.what()); return ExportResult::kFailure; } } From 9d2b15576251b31efb487c1be251a87099f71e09 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Sep 2026 11:11:22 +0200 Subject: [PATCH 722/927] packaging/secure-packages/flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix/nixpkgs': 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013502%2Brev-88369aab48f8d2b0f69126d306c5af06e997849b/01a05deb-f862-7c32-a065-599b037b2e53/source.tar.gz' (2026-09-01) → 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013551%2Brev-07b3c48788b0deb9ee48c40ed5ff1dff7f4643e7/01a08dbc-1a6d-7e60-8dd0-bc2bf8ecb4fb/source.tar.gz' (2026-09-10) --- packaging/secure-packages/flake.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index ff01da9861..e420ac4ef9 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -77,12 +77,12 @@ }, "nixpkgs": { "locked": { - "lastModified": 1788280191, - "narHash": "sha256-mB3qK5ql+QFtajxSI/qwCPRPaJMuWyqIKnTZvADKwU8=", - "rev": "88369aab48f8d2b0f69126d306c5af06e997849b", - "revCount": 1013502, + "lastModified": 1789082161, + "narHash": "sha256-E9iYEGs3Jx9xmYsP4nbVAK+7TXfMjGjGAazuyYLpivU=", + "rev": "07b3c48788b0deb9ee48c40ed5ff1dff7f4643e7", + "revCount": 1013551, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013502%2Brev-88369aab48f8d2b0f69126d306c5af06e997849b/01a05deb-f862-7c32-a065-599b037b2e53/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013551%2Brev-07b3c48788b0deb9ee48c40ed5ff1dff7f4643e7/01a08dbc-1a6d-7e60-8dd0-bc2bf8ecb4fb/source.tar.gz" }, "original": { "type": "tarball", From a73c8bbec438876d6e92fe3bc411422a6b149f21 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Sep 2026 15:39:11 +0200 Subject: [PATCH 723/927] packaging: Fix libgit2 eval error with provenance-enabled nixpkgs Nixpkgs derives libgit2's `meta.changelog` from `src.tag`, which is null for our override since we fetch an untagged 2.0.0-rc.1 commit. This was harmless with upstream nixpkgs, but nixpkgs variants with provenance support (`derivationWithMeta`) force `meta.changelog` at instantiation time, causing error: cannot coerce null to a string: null when evaluating packaging/secure-packages. Assisted-by: Claude Fable 5.1 --- packaging/dependencies.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index b28042b560..2a435133a3 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -185,6 +185,12 @@ scope: { ./patches/libgit2-packbuilder-dont-fail-on-thread-create-error.patch ]; separateDebugInfo = true; + # Nixpkgs derives `meta.changelog` from `src.tag`, which is null + # here since we fetch an untagged commit. This would be harmless + # except that nixpkgs variants with provenance support + # (`derivationWithMeta`) force `meta.changelog` at derivation + # instantiation time, causing an eval error. + meta = builtins.removeAttrs prevAttrs.meta [ "changelog" ]; } ); From a8adabddcd9404b802d43b1c3e2dcb0e7cc9e936 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Sep 2026 11:01:31 +0200 Subject: [PATCH 724/927] Mark the spans of failed builds and substitutions as failed The OpenTelemetry logger only marked a span as failed if its activity was stopped during stack unwinding. But a failed build doesn't throw: the goal returns normally via `doneFailure()`, so its `Build` span ended with no error status. Same for a failed substitution. The information is already there: the goals report a `resBuildResult` whose payload has `success`, `status` and `errorMsg`. The problem was that they reported it on `getCurActivity()`, which is always 0 in the goal code (nothing there does `PushActivity`), so the result wasn't attached to any span. Report it on the goal's own `actBuild` / `SubstitutionGoal` activity instead, via an optional `ActivityId` parameter to the `done*()` helpers, and have the OpenTelemetry logger set the span's status from it. It also records the status as a `nix.build.status` attribute (`Built`, `PermanentFailure`, ...). A substitution goal that "fails" with `NoSubstituters` is the normal prelude to building the path, so that isn't treated as an error. The only other consumer of `resBuildResult` is the JSON logger, whose records now carry the real activity id rather than 0. (The result isn't forwarded over the daemon protocol at all.) The outer `DerivationGoal` has no build activity and keeps reporting on activity 0. Assisted-by: Claude Fable 5.1 --- .../build/derivation-building-goal.cc | 39 +++++++++++-------- src/libstore/build/substitution-goal.cc | 15 ++++--- .../store/build/derivation-building-goal.hh | 3 +- .../nix/store/build/substitution-goal.hh | 2 +- src/nix/otel-logger.cc | 18 +++++++++ tests/functional/otel.sh | 22 +++++++++++ 6 files changed, 72 insertions(+), 27 deletions(-) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index d7edca2a99..4c2c4b9c74 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -771,7 +771,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook( break; } else if (auto * timeout = std::get_if>(&event)) { hook.reset(); - co_return doneFailure(std::move(**timeout)); + co_return doneFailure(std::move(**timeout), buildLog->act->id); } } @@ -806,7 +806,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook( /* TODO (once again) support fine-grained error codes, see issue #12641. */ - co_return doneFailure(std::move(e)); + co_return doneFailure(std::move(e), buildLog->act->id); } /* Compute the FS closure of the outputs and register them as @@ -855,7 +855,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook( outputLocks.setDeletion(true); outputLocks.unlock(); - co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs)); + co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs), buildLog->act->id); #endif } @@ -979,7 +979,7 @@ Goal::Co DerivationBuildingGoal::buildLocally( desugaredEnv = DesugaredEnv::create(worker.store, *drv, drvOptions, inputPaths); } catch (BuildError & e) { outputLocks.unlock(); - co_return doneFailure(std::move(e)); + co_return doneFailure(std::move(e), act->id); } DerivationBuilderParams params{ @@ -1053,7 +1053,7 @@ Goal::Co DerivationBuildingGoal::buildLocally( break; } else if (auto * timeout = std::get_if>(&event)) { builder->killChild(); - co_return doneFailure(std::move(**timeout)); + co_return doneFailure(std::move(**timeout), act->id); } } @@ -1065,11 +1065,11 @@ Goal::Co DerivationBuildingGoal::buildLocally( } catch (BuilderFailureError & e) { builder.reset(); outputLocks.unlock(); - co_return doneFailure(fixupBuilderFailureErrorMessage(std::move(e), *buildLog)); + co_return doneFailure(fixupBuilderFailureErrorMessage(std::move(e), *buildLog), act->id); } catch (BuildError & e) { builder.reset(); outputLocks.unlock(); - co_return doneFailure(std::move(e)); + co_return doneFailure(std::move(e), act->id); } { builder.reset(); @@ -1109,7 +1109,7 @@ Goal::Co DerivationBuildingGoal::buildLocally( (unlinked) lock files. */ outputLocks.setDeletion(true); outputLocks.unlock(); - co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs), provenance); + co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs), act->id, provenance); } #endif } @@ -1314,11 +1314,13 @@ LogFile::~LogFile() Goal::Done DerivationBuildingGoal::doneFailureLogTooLong(BuildLog & buildLog) { - return doneFailure(BuildError( - BuildResult::Failure::LogLimitExceeded, - "%s killed after writing more than %d bytes of log output", - getName(), - worker.settings.maxLogSize)); + return doneFailure( + BuildError( + BuildResult::Failure::LogLimitExceeded, + "%s killed after writing more than %d bytes of log output", + getName(), + worker.settings.maxLogSize), + buildLog.act->id); } std::map> DerivationBuildingGoal::queryPartialDerivationOutputMap() @@ -1405,7 +1407,10 @@ DerivationBuildingGoal::checkPathValidity(std::map & } Goal::Done DerivationBuildingGoal::doneSuccess( - BuildResult::Success::Status status, SingleDrvOutputs builtOutputs, std::shared_ptr provenance) + BuildResult::Success::Status status, + SingleDrvOutputs builtOutputs, + ActivityId act, + std::shared_ptr provenance) { mcRunningBuilds.reset(); @@ -1422,7 +1427,7 @@ Goal::Done DerivationBuildingGoal::doneSuccess( }); logger->result( - getCurActivity(), + act, resBuildResult, nlohmann::json(KeyedBuildResult( buildResult, @@ -1431,7 +1436,7 @@ Goal::Done DerivationBuildingGoal::doneSuccess( return res; } -Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex) +Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex, ActivityId act) { mcRunningBuilds.reset(); @@ -1444,7 +1449,7 @@ Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex) auto res = Goal::doneFailure(ecFailed, std::move(ex)); logger->result( - getCurActivity(), + act, resBuildResult, nlohmann::json(KeyedBuildResult( buildResult, diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index b122299a60..45556a5699 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -33,14 +33,11 @@ PathSubstitutionGoal::~PathSubstitutionGoal() cleanup(); } -Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure failure) +Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure failure, ActivityId act) { auto res = Goal::doneFailure(result, std::move(failure)); - logger->result( - getCurActivity(), - resBuildResult, - nlohmann::json(KeyedBuildResult(buildResult, DerivedPath::Opaque{storePath}))); + logger->result(act, resBuildResult, nlohmann::json(KeyedBuildResult(buildResult, DerivedPath::Opaque{storePath}))); return res; } @@ -199,7 +196,8 @@ Goal::Co PathSubstitutionGoal::init() .msg = HintFmt( "path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)), - }}); + }}, + act.id); } Goal::Co PathSubstitutionGoal::tryToRun( @@ -218,7 +216,8 @@ Goal::Co PathSubstitutionGoal::tryToRun( .status = BuildResult::Failure::DependencyFailed, .msg = HintFmt( "some references of path '%s' could not be realised", worker.store.printStorePath(storePath)), - }}); + }}, + parentAct); } for (auto & i : info->references) @@ -350,7 +349,7 @@ Goal::Co PathSubstitutionGoal::tryToRun( auto success = BuildResult::Success{.status = BuildResult::Success::Substituted, .provenance = provenance}; logger->result( - getCurActivity(), resBuildResult, nlohmann::json(KeyedBuildResult({success}, DerivedPath::Opaque{storePath}))); + parentAct, resBuildResult, nlohmann::json(KeyedBuildResult({success}, DerivedPath::Opaque{storePath}))); co_return doneSuccess(std::move(success)); } diff --git a/src/libstore/include/nix/store/build/derivation-building-goal.hh b/src/libstore/include/nix/store/build/derivation-building-goal.hh index 4d366f648c..7d8c489719 100644 --- a/src/libstore/include/nix/store/build/derivation-building-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-building-goal.hh @@ -118,9 +118,10 @@ private: Done doneSuccess( BuildResult::Success::Status status, SingleDrvOutputs builtOutputs, + ActivityId act = 0, std::shared_ptr provenance = nullptr); - Done doneFailure(BuildError ex); + Done doneFailure(BuildError ex, ActivityId act = 0); BuildError fixupBuilderFailureErrorMessage(BuilderFailureError msg, BuildLog & buildLog); diff --git a/src/libstore/include/nix/store/build/substitution-goal.hh b/src/libstore/include/nix/store/build/substitution-goal.hh index 841147eae4..73c54819bf 100644 --- a/src/libstore/include/nix/store/build/substitution-goal.hh +++ b/src/libstore/include/nix/store/build/substitution-goal.hh @@ -75,7 +75,7 @@ public: return JobCategory::Substitution; }; - Done doneFailure(ExitCode result, BuildResult::Failure failure); + Done doneFailure(ExitCode result, BuildResult::Failure failure, ActivityId act = 0); }; } // namespace nix diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 66edd10d3b..fd5a36d0b9 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -544,6 +544,24 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger if (auto bodySize = json.find("bodySize"); bodySize != json.end() && bodySize->is_number()) i->second->SetAttribute("http.response.body.size", bodySize->get()); } + } else if (type == resBuildResult) { + /* A failed build or substitution doesn't throw, so + `stopActivity()` can't tell; the result tells us. */ + auto spans(spans_.lock()); + if (auto i = spans->find(act); i != spans->end()) { + auto status = json.value("status", ""); + if (!status.empty()) + i->second->SetAttribute("nix.build.status", status); + /* Note: a substitution goal "failing" because there + is no substituter for the path is the normal + prelude to building it, so don't call that an + error. */ + if (!json.value("success", true) && status != "NoSubstituters") { + // FIXME: privacy (the message can include the build log tail) + auto msg = json.value("errorMsg", "build failed"); + i->second->SetStatus(opentelemetry::trace::StatusCode::kError, filterANSIEscapes(msg, true)); + } + } } } catch (...) { } diff --git a/tests/functional/otel.sh b/tests/functional/otel.sh index 06081d92e8..18fa6e5b36 100644 --- a/tests/functional/otel.sh +++ b/tests/functional/otel.sh @@ -48,6 +48,12 @@ attr() { body "$1" | jq -r ".resourceSpans[0].scopeSpans[0].spans[] | select(.name == \"$2\") | .attributes[] | select(.key == \"$3\") | .value.stringValue" } +# Return a field of the status of the span with the given name in the +# n-th upload, e.g. `.code`. +spanStatus() { + body "$1" | jq -r ".resourceSpans[0].scopeSpans[0].spans[] | select(.name == \"$2\") | .status$3" +} + # A successful command produces a root span named after it, with no # status. [[ $(nix eval --expr '1 + 2') = 3 ]] @@ -87,3 +93,19 @@ nix build --no-link "$drvPath^*" [[ $(attr 5 Build nix.drv.path) = "$drvPath" ]] [[ $(attr 5 Build nix.drv.name) = foo ]] [[ $(attr 5 Build nix.drv.version) = 1.2 ]] +[[ $(attr 5 Build nix.build.status) = Built ]] +[[ $(spanStatus 5 Build .code) = null ]] +# Not being able to substitute the path is the normal prelude to +# building it, not an error. +[[ $(attr 5 SubstitutionGoal nix.build.status) = NoSubstituters ]] +[[ $(spanStatus 5 SubstitutionGoal .code) = null ]] + +# A failed build marks its `Build` span as failed, with the build's +# status and error message. Note that the failure doesn't throw, so +# this relies on the build result rather than on stack unwinding. +# shellcheck disable=SC2016 # `$out` is for the Nix builder, not the shell. +drvPath=$(nix-instantiate --expr 'with import ./config.nix; mkDerivation { name = "bar-1.2"; buildCommand = "echo something went wrong >&2; exit 1"; }') +expect 1 nix build --no-link "$drvPath^*" +[[ $(attr 7 Build nix.build.status) = PermanentFailure ]] +[[ $(spanStatus 7 Build .code) = 2 ]] +spanStatus 7 Build .message | grepQuiet "builder failed with exit code 1" From c312f76ea8ea9d2935baa3fd1aab131715cc3063 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Sep 2026 18:23:08 +0200 Subject: [PATCH 725/927] Propagate traces to the daemon over ssh-ng:// A daemon serving a connection over stdio (`nix-daemon --stdio`, as used by `ssh-ng://`) didn't export any spans. `main()` had already set up tracing for it with a root span named `nix-daemon`, so the daemon ended up with two OpenTelemetry loggers: that one, and the per-connection one parented to the client's trace, and the stdio code path then discarded the tracing state to avoid the duplication, losing both. Instead, `main()` no longer sets up tracing for the daemon at all: the daemon does it itself, per connection, parented to the client's trace. The stdio path initializes tracing where the forked children of the socket path already did. For `ssh-ng://localhost`, which runs the daemon directly rather than through SSH, the client killed the daemon on disconnect before it could export anything. Give it a chance to exit by itself after it sees EOF, with a bounded wait, since unlike an `ssh` client process it *is* the remote side. Assisted-by: Claude Fable 5.1 --- doc/manual/source/command-ref/env-common.md | 3 ++- src/libstore/ssh.cc | 11 ++++++++++ src/nix/main.cc | 24 ++++++++++++++------- src/nix/unix/daemon.cc | 18 ++++++---------- tests/functional/otel.sh | 22 +++++++++++++++++++ 5 files changed, 58 insertions(+), 20 deletions(-) diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index 497797ff8f..7f6293cba0 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -172,7 +172,8 @@ Likewise for the state and cache directories. Nix can emit [OpenTelemetry](https://opentelemetry.io/) traces of its activities (such as evaluation, builds, substitutions and HTTP requests), exported using the OTLP/HTTP protocol. Tracing is a no-op unless one of the endpoint variables below is set, or the [`otlp`](@docroot@/command-ref/conf-file.md#conf-otlp) and [`otlp-endpoint`](@docroot@/command-ref/conf-file.md#conf-otlp-endpoint) settings are configured. The variables below take precedence over the corresponding settings; in particular, setting an endpoint variable enables tracing even if `otlp` is disabled. -When talking to the Nix daemon or to a binary cache server, Nix propagates the [W3C trace context](https://www.w3.org/TR/trace-context/), so that work done on your behalf by other processes or servers shows up in the same trace. +When talking to the Nix daemon (including a remote one via `ssh-ng://`) or to a binary cache server, Nix propagates the [W3C trace context](https://www.w3.org/TR/trace-context/), so that work done on your behalf by other processes or servers shows up in the same trace. +Note that a daemon exports its own spans, so it needs its own OTLP configuration; in particular, environment variables aren't inherited over SSH, so a remote daemon has to be configured through the remote machine's `nix.conf`. - [`OTEL_EXPORTER_OTLP_ENDPOINT`](#env-OTEL_EXPORTER_OTLP_ENDPOINT) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index fd6f3b3243..35692f4287 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -236,6 +236,17 @@ std::unique_ptr SSHMaster::startCommand(OsStrings && comm conn->out = std::move(out.readSide); conn->in = std::move(in.writeSide); + /* Without real SSH, the child *is* the remote program (e.g. + `nix-daemon --stdio`), rather than an `ssh` client that we can + kill without affecting the remote side. So on teardown, give + it a chance to exit by itself after it sees EOF on its stdin, + e.g. to export its telemetry, instead of killing it right + away. */ + if (fakeSSH) { + conn->sshPid.setKillSignal(0); + conn->sshPid.setKillTimeout(std::chrono::seconds(10)); + } + return conn; #endif } diff --git a/src/nix/main.cc b/src/nix/main.cc index 9db8c59b5d..21978d6e54 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -552,10 +552,14 @@ void mainWrapped(int argc, char ** argv) /* Legacy commands don't have subcommands, so we can set up the root span right away. Note that they parse their arguments themselves, so unlike for the commands below, - `--option` cannot configure tracing here. */ - initOtel(programName); - if (auto l = makeOpenTelemetryLogger(programName)) - applyExtraLogger(std::move(l)); + `--option` cannot configure tracing here. The daemon is + the exception: it sets up tracing itself, per + connection, parented to the client's trace. */ + if (programName != "nix-daemon") { + initOtel(programName); + if (auto l = makeOpenTelemetryLogger(programName)) + applyExtraLogger(std::move(l)); + } return (*legacy)(argc, argv); } } @@ -716,10 +720,14 @@ void mainWrapped(int argc, char ** argv) /* Map activities to OpenTelemetry spans, under a root span named after the subcommand. Note: this must happen after - `parseCmdline()`, so that `--option` can configure tracing. */ - initOtel(programName); - if (auto l = makeOpenTelemetryLogger("nix " + concatStringsSep(" ", subcommand))) - applyExtraLogger(std::move(l)); + `parseCmdline()`, so that `--option` can configure tracing. The + daemon is the exception: it sets up tracing itself, per + connection, parented to the client's trace. */ + if (subcommand != std::vector{"daemon"}) { + initOtel(programName); + if (auto l = makeOpenTelemetryLogger("nix " + concatStringsSep(" ", subcommand))) + applyExtraLogger(std::move(l)); + } try { args.command->second->run(); diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 787c7897cc..f969cb35ac 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -530,9 +530,11 @@ static void runDaemon( std::visit( overloaded{ [&](StdIO) { - /* FIXME: we don't currently trace connections served - over stdio. */ - logger->resetAfterFork(); + /* Set up tracing; `processStdioConnection()` attaches + a logger parented to the client's trace. Note that + `main()` deliberately doesn't do this for the + daemon. */ + initOtel("nix-daemon"); auto store = storeConfig->openStore(); store->init(); @@ -553,14 +555,8 @@ static void runDaemon( processStdioConnection(store, forceTrustClientOpt.value_or(Trusted)); }, [&](UnixSocket socketPathOverride) { - /* We fork a child per connection, and the exporter's - worker threads don't survive a fork(), so don't - trace in this process; the children set up tracing - themselves. (This discards any tracing state set up - by `main()`, without touching the exporter, which - is what we want.) */ - logger->resetAfterFork(); - + /* Note: we don't trace in this process; the forked + children set up tracing themselves, per connection. */ auto socketPath = std::move(socketPathOverride) .or_else([&]() -> std::optional { return getDaemonSocketPath(*storeConfig); diff --git a/tests/functional/otel.sh b/tests/functional/otel.sh index 18fa6e5b36..c5422d6ceb 100644 --- a/tests/functional/otel.sh +++ b/tests/functional/otel.sh @@ -109,3 +109,25 @@ expect 1 nix build --no-link "$drvPath^*" [[ $(attr 7 Build nix.build.status) = PermanentFailure ]] [[ $(spanStatus 7 Build .code) = 2 ]] spanStatus 7 Build .message | grepQuiet "builder failed with exit code 1" + +# A daemon reached via `ssh-ng://` (here without real SSH, since the +# host is `localhost`) exports its own spans, in the client's trace. +# The client and the daemon upload independently, in no particular +# order, so wait for both and tell them apart by service name. +service() { + body "$1" | jq -r '.resourceSpans[0].resource.attributes[] | select(.key == "service.name") | .value.stringValue' +} +nix store info --store "ssh-ng://localhost?remote-store=$TEST_ROOT/other-store" > /dev/null +for ((i = 0; i < 100; i++)); do + [[ -e $sinkDir/9.body ]] && break + sleep 0.1 +done +[[ -e $sinkDir/9.body ]] +if [[ $(service 8) = nix ]]; then client=8; daemon=9; else client=9; daemon=8; fi +[[ $(service $client) = nix ]] +[[ $(service $daemon) = nix-daemon ]] +[[ $(span $client .name) = "nix store info" ]] +[[ $(span $daemon .name) = "daemon connection" ]] +[[ $(span $daemon .kind) = 2 ]] # SERVER +[[ $(span $daemon .traceId) = $(span $client .traceId) ]] +[[ $(span $daemon .parentSpanId) = $(span $client .spanId) ]] From 368a98227b4ab8d41526d35260ecb2bf1e5ac725 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Sep 2026 19:11:47 +0200 Subject: [PATCH 726/927] Remove Logger::resetAfterFork() Its only callers were in the daemon, which no longer needs it: the forked children discard the inherited tracing state through the fork callback registered by the OpenTelemetry logger, and `initOtel()` discards any previous state by itself. Assisted-by: Claude Fable 5.1 --- src/libutil/include/nix/util/logging.hh | 8 -------- src/libutil/tee-logger.cc | 6 ------ src/nix/otel-logger.cc | 8 -------- src/nix/otel-logger.hh | 5 ----- 4 files changed, 27 deletions(-) diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index af6ed20176..609a2774eb 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -146,14 +146,6 @@ public: */ virtual void flush() {}; - /** - * Discard any state inherited from the parent process after a - * `fork()`, such as worker threads, which do not exist in the - * child. Note that this discards rather than flushes, since the - * parent remains responsible for its own state. - */ - virtual void resetAfterFork() {}; - /** * Guard object to resume the logger when done. */ diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index 6413f525d1..7b40a036ad 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -26,12 +26,6 @@ class TeeLogger final : public Logger logger->flush(); }; - void resetAfterFork() override - { - for (auto & logger : loggers) - logger->resetAfterFork(); - }; - void pause() override { for (auto & logger : loggers) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index fd5a36d0b9..09e83fc8a6 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -611,14 +611,6 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger if (!state->debugTraceId.empty()) writeToStderr(fmt("OpenTelemetry trace ID: %s\n", state->debugTraceId)); } - - void resetAfterFork() override - { - /* Deliberately leak the old state: it may reference a worker - thread that does not exist in this process, so it can be - neither flushed nor destroyed safely. */ - otelState.exchange(nullptr); - } }; /** diff --git a/src/nix/otel-logger.hh b/src/nix/otel-logger.hh index 7fcb1098b9..9d882d706b 100644 --- a/src/nix/otel-logger.hh +++ b/src/nix/otel-logger.hh @@ -21,11 +21,6 @@ namespace nix { * with bounded timeouts. Since loggers are generally not destroyed, * it has to be called explicitly before the process exits; * `handleExceptions()` does so. - * - * `resetAfterFork()` discards the tracing state without exporting - * anything, turning tracing off in this process. Note that a process - * that wants to trace after a `fork()` doesn't need this: `initOtel()` - * discards the inherited state by itself. */ class OpenTelemetryLogger : public Logger {}; From a5a93f8e81e0cb77884d20d92260ae675eca274a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 25 Jun 2026 10:57:14 -0600 Subject: [PATCH 727/927] libstore: detect Rosetta via runtime file instead of spawning arch getDefaultExtraPlatforms() ran `arch -arch x86_64 /usr/bin/true` to decide whether to add x86_64-darwin to extra-platforms. Because this is the default value of the extra-platforms setting, it runs during static initialization of the global `settings` object, so every libstore linked process on Apple silicon paid roughly 13ms of subprocess startup, even ones that never read extra-platforms (devenv version, hook-should-activate, nix --version). Check for the Rosetta runtime file instead. It is present exactly when Rosetta 2 is installed, which is exactly when the exec probe succeeded, and a stat costs about 0.01ms versus 12.5ms for the subprocess. Assisted-by: Claude Code (Claude Opus 4.8) (cherry picked from commit a8419ef8e770401fd6e6ce4bfeacdbe0d7392b4f) --- src/libstore/globals.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ea76326bf5..5e62408847 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -4,6 +4,7 @@ #include "nix/util/config-global.hh" #include "nix/util/current-process.hh" #include "nix/util/executable-path.hh" +#include "nix/util/file-system.hh" #include "nix/util/args.hh" #include "nix/util/abstract-setting-to-json.hh" #include "nix/util/compute-levels.hh" @@ -254,11 +255,11 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. + // + // The runtime file exists iff Rosetta 2 is installed; checking it avoids + // spawning a subprocess during static initialization of `settings`. if (std::string{NIX_LOCAL_SYSTEM} == "aarch64-darwin" - && runProgram( - RunOptions{.program = "arch", .args = {"-arch", "x86_64", "/usr/bin/true"}, .mergeStderrToStdout = true}) - .first - == 0) + && pathExists("/Library/Apple/usr/libexec/oah/libRosettaRuntime")) extraPlatforms.insert("x86_64-darwin"); #endif From 282e2ac26c6342bfbbce158acfb64c21f2d04731 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 13 Sep 2026 17:05:56 +0200 Subject: [PATCH 728/927] Don't let the client hanging up interrupt the daemon's telemetry export `processConnection()` cleared the interrupt flag in a `Finally` that ran *before* the `MonitorFdHup` was destroyed. So if the monitor thread noticed the client hanging up after that, it set the flag again, and the daemon's subsequent OpenTelemetry export failed with `Interrupted` (which is deliberately not reported). Whether the monitor thread won that race is why the daemon's spans sometimes went missing. Clear the flag only once the monitor is gone, i.e. its thread joined. Assisted-by: Claude Fable 5.1 --- src/libstore/daemon.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d677a1f11c..9e0b54ebdf 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1094,6 +1094,15 @@ void processConnection( std::function setupTelemetry) { #ifndef _WIN32 // TODO need graceful async exit support on Windows? + /* The client hanging up triggers an interrupt (via `MonitorFdHup` + below), which is how we abort whatever we were doing for it. + Once the connection is over, that interrupt has served its + purpose, so clear it, e.g. so that we can still export our + telemetry. Note: this has to run *after* the monitor has been + destroyed, i.e. its thread joined, since it might otherwise + still trigger the interrupt after we've cleared it. */ + Finally clearInterrupt([]() { setInterrupted(false); }); + auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; (void) monitor; // suppress warning ReceiveInterrupts receiveInterrupts; @@ -1142,10 +1151,7 @@ void processConnection( unsigned int opCount = 0; - Finally finally([&]() { - setInterrupted(false); - printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount); - }); + Finally finally([&]() { printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount); }); conn.postHandshake( *store, From ad3750a8e2d59368bb9900755db17d0e22804d71 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 13 Sep 2026 17:05:56 +0200 Subject: [PATCH 729/927] Propagate traces to build-remote Pass the trace context to the build hook in the `TRACEPARENT` environment variable, and have the root span of any Nix process be parented to that, if set. So a remote build now shows up in the trace of the `nix` invocation that requested it: `nix` -> `build-remote` -> the remote daemon (the latter via the existing `ssh-ng://` handshake). Two things stood in the way. `build-remote` replaces the global logger with a JSON logger, discarding the OpenTelemetry logger that `main()` had attached, so like the daemon it now sets up tracing itself. And `HookInstance` sent the hook `SIGTERM` on teardown, which interrupted its export; since the hook exits by itself on EOF, just give it a chance to do so. A root span parented to a remote context is no longer implicitly a server span; the daemon says so explicitly. Assisted-by: Claude Fable 5.1 --- src/libstore/unix/build/hook-instance.cc | 17 +++++++++++++++-- src/nix/build-remote/build-remote.cc | 10 ++++++++++ src/nix/main.cc | 19 ++++++++++++------- src/nix/otel-logger.cc | 16 ++++++++-------- src/nix/otel-logger.hh | 15 ++++++++------- src/nix/unix/daemon.cc | 2 +- tests/functional/otel.sh | 12 ++++++++++++ 7 files changed, 66 insertions(+), 25 deletions(-) diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index 64c541e3a9..61e14a105d 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -44,6 +44,12 @@ HookInstance::HookInstance(const Strings & _buildHook) /* Create a pipe to get the output of the builder. */ builderOut.create(); + /* Propagate our trace context to the hook, so that its trace (and, + through it, the remote builder's) is part of ours. Note: this + has to be obtained before forking, since the child gets a + different logger. */ + auto traceContext = logger->getTraceContext(getCurActivity()); + /* Fork the hook. */ pid = startProcess([&]() { if (dup2(fromHook.writeSide.get(), STDERR_FILENO) == -1) @@ -51,6 +57,10 @@ HookInstance::HookInstance(const Strings & _buildHook) commonChildInit(); + for (auto & [name, value] : traceContext) + if (name == "traceparent") + setenv("TRACEPARENT", value.c_str(), 1); + if (chdir("/") == -1) throw SysError("changing into /"); @@ -73,8 +83,11 @@ HookInstance::HookInstance(const Strings & _buildHook) }); /* Give custom build hooks the chance to cleanup. */ - pid.setKillSignal(SIGTERM); - pid.setKillTimeout(500ms); + /* The hook exits by itself when it sees EOF on its stdin, so on + teardown, give it a chance to do so (e.g. to export its + telemetry) instead of signalling it right away. */ + pid.setKillSignal(0); + pid.setKillTimeout(10s); pid.setSeparatePG(true); fromHook.writeSide = -1; diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index 8f8f21a8c3..f33a9527f2 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -18,6 +18,9 @@ #include "nix/util/serialise.hh" #include "nix/store/build-result.hh" #include "nix/store/store-open.hh" +#include "nix/util/environment-variables.hh" + +#include "otel-logger.hh" #include "nix/util/strings.hh" #include "nix/store/derivations.hh" #include "nix/store/local-store.hh" @@ -72,6 +75,13 @@ static int main_build_remote(int argc, char ** argv) logger = makeJSONLogger(getStandardError()).release(); + /* Set up tracing now that we have our logger. Our root span + is parented to the trace of the `nix` process that runs us, + which it passes in `TRACEPARENT`. */ + initOtel("nix-build-remote"); + if (auto l = makeOpenTelemetryLogger("build-remote", getEnv("TRACEPARENT").value_or(""))) + applyExtraLogger(std::move(l)); + /* Ensure we don't get any SSH passphrase or host key popups. */ unsetenv("DISPLAY"); unsetenv("SSH_ASKPASS"); diff --git a/src/nix/main.cc b/src/nix/main.cc index 21978d6e54..d82fd46d87 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -552,12 +552,12 @@ void mainWrapped(int argc, char ** argv) /* Legacy commands don't have subcommands, so we can set up the root span right away. Note that they parse their arguments themselves, so unlike for the commands below, - `--option` cannot configure tracing here. The daemon is - the exception: it sets up tracing itself, per - connection, parented to the client's trace. */ - if (programName != "nix-daemon") { + `--option` cannot configure tracing here. The daemon and + the build hook are the exceptions: they install their + own logger, so they set up tracing themselves. */ + if (programName != "nix-daemon" && programName != "build-remote") { initOtel(programName); - if (auto l = makeOpenTelemetryLogger(programName)) + if (auto l = makeOpenTelemetryLogger(programName, getEnv("TRACEPARENT").value_or(""))) applyExtraLogger(std::move(l)); } return (*legacy)(argc, argv); @@ -722,10 +722,15 @@ void mainWrapped(int argc, char ** argv) after the subcommand. Note: this must happen after `parseCmdline()`, so that `--option` can configure tracing. The daemon is the exception: it sets up tracing itself, per - connection, parented to the client's trace. */ + connection, parented to the client's trace. + + The root span is parented to the trace context in `TRACEPARENT`, + if any, so that a parent process (such as `nix` running + `build-remote`) can include us in its trace. */ if (subcommand != std::vector{"daemon"}) { initOtel(programName); - if (auto l = makeOpenTelemetryLogger("nix " + concatStringsSep(" ", subcommand))) + if (auto l = + makeOpenTelemetryLogger("nix " + concatStringsSep(" ", subcommand), getEnv("TRACEPARENT").value_or(""))) applyExtraLogger(std::move(l)); } diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 09e83fc8a6..88d8e17ec8 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -325,15 +325,15 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger } public: - OpenTelemetryLoggerImpl(OtelState & state, std::string_view rootSpanName, std::string_view remoteParentTraceparent) + OpenTelemetryLoggerImpl( + OtelState & state, std::string_view rootSpanName, std::string_view remoteParentTraceparent, bool isServer) : tracer(state.tracer) { opentelemetry::trace::StartSpanOptions options; - if (!remoteParentTraceparent.empty()) { + if (isServer) options.kind = opentelemetry::trace::SpanKind::kServer; - if (auto spanContext = parseTraceparent(remoteParentTraceparent)) - options.parent = *spanContext; - } + if (auto spanContext = parseTraceparent(remoteParentTraceparent)) + options.parent = *spanContext; rootSpan = tracer->StartSpan(rootSpanName, options); if (getEnv("NIX_DEBUG_OTEL")) { @@ -942,19 +942,19 @@ void initOtel(std::string_view serviceName) } std::unique_ptr -makeOpenTelemetryLogger(std::string_view rootSpanName, std::string_view remoteParentTraceparent) +makeOpenTelemetryLogger(std::string_view rootSpanName, std::string_view remoteParentTraceparent, bool isServer) { auto * state = otelState.load(std::memory_order_acquire); if (!state) return nullptr; - return std::make_unique(*state, rootSpanName, remoteParentTraceparent); + return std::make_unique(*state, rootSpanName, remoteParentTraceparent, isServer); } #else void initOtel(std::string_view) {} -std::unique_ptr makeOpenTelemetryLogger(std::string_view, std::string_view) +std::unique_ptr makeOpenTelemetryLogger(std::string_view, std::string_view, bool) { return nullptr; } diff --git a/src/nix/otel-logger.hh b/src/nix/otel-logger.hh index 9d882d706b..52c8f6cc3a 100644 --- a/src/nix/otel-logger.hh +++ b/src/nix/otel-logger.hh @@ -30,17 +30,18 @@ class OpenTelemetryLogger : public Logger * `rootSpanName`. Activities without a known parent become children * of the root span. * - * If `remoteParentTraceparent` is non-empty, the root span is created - * as a server span whose parent is the given W3C trace context (used - * for daemon connections, where the client sends its trace context - * during the handshake). An invalid value yields an unparented root - * span. + * If `remoteParentTraceparent` is non-empty, the root span's parent is + * the given W3C trace context, e.g. received from the client during + * the daemon handshake, or from a parent process via the `TRACEPARENT` + * environment variable. An invalid value yields an unparented root + * span. If `isServer` is set, the root span is a server span (i.e. it + * handles a request from another process, as in the daemon). * * Returns null if tracing support is not compiled in or `initOtel()` * did not enable tracing; the caller should then not attach a logger. */ -std::unique_ptr -makeOpenTelemetryLogger(std::string_view rootSpanName, std::string_view remoteParentTraceparent = {}); +std::unique_ptr makeOpenTelemetryLogger( + std::string_view rootSpanName, std::string_view remoteParentTraceparent = {}, bool isServer = false); /** * Initialize OpenTelemetry tracing for this process. Does nothing diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index f969cb35ac..e62a0fae1c 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -49,7 +49,7 @@ namespace nix { */ static void setupConnectionTelemetry(std::string_view traceparent) { - if (auto otelLogger = makeOpenTelemetryLogger("daemon connection", traceparent)) + if (auto otelLogger = makeOpenTelemetryLogger("daemon connection", traceparent, /*isServer=*/true)) applyExtraLogger(std::move(otelLogger)); } diff --git a/tests/functional/otel.sh b/tests/functional/otel.sh index c5422d6ceb..f6ce2184f7 100644 --- a/tests/functional/otel.sh +++ b/tests/functional/otel.sh @@ -30,6 +30,9 @@ otlp = true otlp-endpoint = http://127.0.0.1:$(cat "$sinkDir/port") otlp-compression = none otlp-headers = authorization=Bearer%20secret +# The build hook is traced too, so it would add uploads of its own to +# the ones counted below. +build-hook = EOF # Return the body of the n-th upload, as JSON. @@ -131,3 +134,12 @@ if [[ $(service 8) = nix ]]; then client=8; daemon=9; else client=9; daemon=8; f [[ $(span $daemon .kind) = 2 ]] # SERVER [[ $(span $daemon .traceId) = $(span $client .traceId) ]] [[ $(span $daemon .parentSpanId) = $(span $client .spanId) ]] + +# The root span is parented to the trace context in `TRACEPARENT`, +# which is how remote builds via `build-remote` are linked to the +# trace of the `nix` process that runs it. +[[ $(TRACEPARENT=00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01 nix eval --expr '1 + 2') = 3 ]] +[[ $(span 10 .name) = "nix eval" ]] +[[ $(span 10 .traceId) = 0af7651916cd43dd8448eb211c80319c ]] +[[ $(span 10 .parentSpanId) = b7ad6b7169203331 ]] +[[ $(span 10 .kind) = 1 ]] # INTERNAL, not SERVER From 069c4d5b69efba550dc4e4f61e1abcf96225c3a2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 10:20:46 +0200 Subject: [PATCH 730/927] build-remote: Set up tracing lazily The build hook is started for every build, but usually ends up declining (e.g. because there are no builders for the required system). It set up tracing at startup regardless, so every build uploaded a trace consisting of nothing but the hook's root span. Instead, set up tracing when we're actually about to connect to a remote builder. That also makes the workaround in the OpenTelemetry test unnecessary, which disabled the hook to keep its uploads out of the count. Assisted-by: Claude Fable 5.1 --- src/nix/build-remote/build-remote.cc | 22 +++++++++++++++------- tests/functional/otel.sh | 3 --- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index f33a9527f2..6e332080b7 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -75,13 +75,6 @@ static int main_build_remote(int argc, char ** argv) logger = makeJSONLogger(getStandardError()).release(); - /* Set up tracing now that we have our logger. Our root span - is parented to the trace of the `nix` process that runs us, - which it passes in `TRACEPARENT`. */ - initOtel("nix-build-remote"); - if (auto l = makeOpenTelemetryLogger("build-remote", getEnv("TRACEPARENT").value_or(""))) - applyExtraLogger(std::move(l)); - /* Ensure we don't get any SSH passphrase or host key popups. */ unsetenv("DISPLAY"); unsetenv("SSH_ASKPASS"); @@ -119,6 +112,7 @@ static int main_build_remote(int argc, char ** argv) std::shared_ptr sshStore; AutoCloseFD bestSlotLock; + bool tracing = false; auto machines = Machine::parseConfig({settings.thisSystem}, settings.getWorkerSettings().builders); debug("got %d remote builders", machines.size()); @@ -255,6 +249,20 @@ static int main_build_remote(int argc, char ** argv) lock = -1; + /* Set up tracing only now that we're actually going to + use a remote builder, so that a build hook that + turns out to be unused (which is the common case, + since we're started for every build) doesn't upload + a trace. Our root span is parented to the trace of + the `nix` process that runs us, which it passes in + `TRACEPARENT`. */ + if (!tracing) { + tracing = true; + initOtel("nix-build-remote"); + if (auto l = makeOpenTelemetryLogger("build-remote", getEnv("TRACEPARENT").value_or(""))) + applyExtraLogger(std::move(l)); + } + try { storeUri = bestMachine->storeUri.render(); diff --git a/tests/functional/otel.sh b/tests/functional/otel.sh index f6ce2184f7..1159498ff3 100644 --- a/tests/functional/otel.sh +++ b/tests/functional/otel.sh @@ -30,9 +30,6 @@ otlp = true otlp-endpoint = http://127.0.0.1:$(cat "$sinkDir/port") otlp-compression = none otlp-headers = authorization=Bearer%20secret -# The build hook is traced too, so it would add uploads of its own to -# the ones counted below. -build-hook = EOF # Return the body of the n-th upload, as JSON. From 7a3ea50cd83fe63acd9b9427b56fa7e9a2fba694 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 12:30:48 +0200 Subject: [PATCH 731/927] build-remote -> nix-build-remote --- src/nix/build-remote/build-remote.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index 6e332080b7..ba28da4111 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -259,7 +259,7 @@ static int main_build_remote(int argc, char ** argv) if (!tracing) { tracing = true; initOtel("nix-build-remote"); - if (auto l = makeOpenTelemetryLogger("build-remote", getEnv("TRACEPARENT").value_or(""))) + if (auto l = makeOpenTelemetryLogger("nix-build-remote", getEnv("TRACEPARENT").value_or(""))) applyExtraLogger(std::move(l)); } From ffc0f6835f6927abeb7a9542bfb3f11a35696904 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 12:33:38 +0200 Subject: [PATCH 732/927] TODO --- src/libstore/unix/build/hook-instance.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index 61e14a105d..3a8fa843c7 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -61,6 +61,8 @@ HookInstance::HookInstance(const Strings & _buildHook) if (name == "traceparent") setenv("TRACEPARENT", value.c_str(), 1); + // TODO: Send our settings to the hook (so that e.g. `--otlp` takes effect). + if (chdir("/") == -1) throw SysError("changing into /"); From 607449f66f0de24221a5874c2b29f52186c2fb4d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 12:35:12 +0200 Subject: [PATCH 733/927] nix-daemon: Initialize tracing in setupConnectionTelemetry() Rather than in the two places (the forked child of the socket daemon, and the stdio daemon) that lead to it. It's a no-op unless configured, and a connection is served once per process, so this is exactly the point at which to set up tracing. Assisted-by: Claude Fable 5.1 --- src/nix/unix/daemon.cc | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index e62a0fae1c..f5fa3c0aa6 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -46,9 +46,15 @@ namespace nix { * covering the connection's lifetime, parented under the trace * context received from the client, if any. No-op unless * OpenTelemetry export is configured. + * + * Note that `main()` deliberately doesn't set up tracing for the + * daemon, so this is where it happens, once per connection (which is + * once per process, since connections are served by forked children + * or over stdio). */ static void setupConnectionTelemetry(std::string_view traceparent) { + initOtel("nix-daemon"); if (auto otelLogger = makeOpenTelemetryLogger("daemon connection", traceparent, /*isServer=*/true)) applyExtraLogger(std::move(otelLogger)); } @@ -379,11 +385,6 @@ static void daemonLoop( [&, storeConfig, closeListeners = std::move(closeListeners)]() { setInterrupted(false); - /* Set up tracing afresh; `startProcess()` has - discarded the state inherited from the - parent. */ - initOtel("nix-daemon"); - closeListeners(); // Background the daemon. @@ -530,12 +531,6 @@ static void runDaemon( std::visit( overloaded{ [&](StdIO) { - /* Set up tracing; `processStdioConnection()` attaches - a logger parented to the client's trace. Note that - `main()` deliberately doesn't do this for the - daemon. */ - initOtel("nix-daemon"); - auto store = storeConfig->openStore(); store->init(); From 3fca637421ebdabac6417edca9831eb4f0573479 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 12:59:31 +0200 Subject: [PATCH 734/927] Fold makeOpenTelemetryLogger() into initOtel() Every caller of `initOtel()` immediately created the logger and added it to the global logger, so do that in `initOtel()` itself, which now takes the root span's name and parent. That leaves `initOtel()` as the only thing in the header, so the `OpenTelemetryLogger` class there (an empty base of the actual implementation) goes away too; the implementation takes its name. Assisted-by: Claude Fable 5.1 --- src/nix/build-remote/build-remote.cc | 4 +-- src/nix/main.cc | 9 ++--- src/nix/otel-logger.cc | 38 ++++++++++---------- src/nix/otel-logger.hh | 52 +++++++++------------------- src/nix/unix/daemon.cc | 4 +-- 5 files changed, 41 insertions(+), 66 deletions(-) diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index ba28da4111..adc2d6b004 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -258,9 +258,7 @@ static int main_build_remote(int argc, char ** argv) `TRACEPARENT`. */ if (!tracing) { tracing = true; - initOtel("nix-build-remote"); - if (auto l = makeOpenTelemetryLogger("nix-build-remote", getEnv("TRACEPARENT").value_or(""))) - applyExtraLogger(std::move(l)); + initOtel("nix-build-remote", "nix-build-remote", getEnv("TRACEPARENT").value_or("")); } try { diff --git a/src/nix/main.cc b/src/nix/main.cc index d82fd46d87..51872007aa 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -556,9 +556,7 @@ void mainWrapped(int argc, char ** argv) the build hook are the exceptions: they install their own logger, so they set up tracing themselves. */ if (programName != "nix-daemon" && programName != "build-remote") { - initOtel(programName); - if (auto l = makeOpenTelemetryLogger(programName, getEnv("TRACEPARENT").value_or(""))) - applyExtraLogger(std::move(l)); + initOtel(programName, programName, getEnv("TRACEPARENT").value_or("")); } return (*legacy)(argc, argv); } @@ -728,10 +726,7 @@ void mainWrapped(int argc, char ** argv) if any, so that a parent process (such as `nix` running `build-remote`) can include us in its trace. */ if (subcommand != std::vector{"daemon"}) { - initOtel(programName); - if (auto l = - makeOpenTelemetryLogger("nix " + concatStringsSep(" ", subcommand), getEnv("TRACEPARENT").value_or(""))) - applyExtraLogger(std::move(l)); + initOtel(programName, "nix " + concatStringsSep(" ", subcommand), getEnv("TRACEPARENT").value_or("")); } try { diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 88d8e17ec8..7997e6e593 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -284,7 +284,16 @@ void setPathAttributes(opentelemetry::trace::Span & span, std::string_view prefi } } -class OpenTelemetryLoggerImpl : public OpenTelemetryLogger +/** + * A `Logger` that maps Nix activities onto OpenTelemetry spans, + * under a root span created in the constructor. It's added to the + * global logger by `initOtel()`. + * + * Activities replayed from another process (cf. `RemoteLogSource`) + * are ignored, since the originating process is responsible for + * exporting them. + */ +class OpenTelemetryLogger : public Logger { using SpanPtr = std::shared_ptr; @@ -325,7 +334,7 @@ class OpenTelemetryLoggerImpl : public OpenTelemetryLogger } public: - OpenTelemetryLoggerImpl( + OpenTelemetryLogger( OtelState & state, std::string_view rootSpanName, std::string_view remoteParentTraceparent, bool isServer) : tracer(state.tracer) { @@ -857,7 +866,11 @@ Headers parseOtlpHeaders(std::string_view s) } // namespace -void initOtel(std::string_view serviceName) +void initOtel( + std::string_view serviceName, + std::string_view rootSpanName, + std::string_view remoteParentTraceparent, + bool isServer) { /* An endpoint in the environment enables tracing by itself, and takes precedence over the settings. */ @@ -938,26 +951,15 @@ void initOtel(std::string_view serviceName) /* Any previously initialized state is deliberately leaked rather than destroyed, since we may be in a child process where its worker thread doesn't exist. */ - otelState.exchange(state.release(), std::memory_order_release); -} + auto * statePtr = state.release(); + otelState.exchange(statePtr, std::memory_order_release); -std::unique_ptr -makeOpenTelemetryLogger(std::string_view rootSpanName, std::string_view remoteParentTraceparent, bool isServer) -{ - auto * state = otelState.load(std::memory_order_acquire); - if (!state) - return nullptr; - return std::make_unique(*state, rootSpanName, remoteParentTraceparent, isServer); + applyExtraLogger(std::make_unique(*statePtr, rootSpanName, remoteParentTraceparent, isServer)); } #else -void initOtel(std::string_view) {} - -std::unique_ptr makeOpenTelemetryLogger(std::string_view, std::string_view, bool) -{ - return nullptr; -} +void initOtel(std::string_view, std::string_view, std::string_view, bool) {} #endif // HAVE_OTEL diff --git a/src/nix/otel-logger.hh b/src/nix/otel-logger.hh index 52c8f6cc3a..aa26c2e8bc 100644 --- a/src/nix/otel-logger.hh +++ b/src/nix/otel-logger.hh @@ -1,34 +1,19 @@ #pragma once ///@file -#include "nix/util/logging.hh" - -#include #include namespace nix { /** - * A `Logger` that maps Nix activities onto OpenTelemetry spans and - * exports them via OTLP/HTTP. Intended to be added to the global - * logger using `applyExtraLogger()`. - * - * Activities replayed from another process (cf. `RemoteLogSource`) - * are ignored, since the originating process is responsible for - * exporting them. + * Initialize OpenTelemetry tracing for this process, and add a logger + * to the global logger (using `applyExtraLogger()`) that maps Nix + * activities onto OpenTelemetry spans, under a root span named + * `rootSpanName`, and exports them via OTLP/HTTP. * - * `flush()` exports all pending spans and shuts down the exporter, - * with bounded timeouts. Since loggers are generally not destroyed, - * it has to be called explicitly before the process exits; - * `handleExceptions()` does so. - */ -class OpenTelemetryLogger : public Logger -{}; - -/** - * Create an `OpenTelemetryLogger` whose root span is named - * `rootSpanName`. Activities without a known parent become children - * of the root span. + * Does nothing (and is cheap) if tracing isn't configured (see the + * `otlp` settings and the `OTEL_EXPORTER_OTLP_*` environment + * variables), or if tracing support is not compiled in. * * If `remoteParentTraceparent` is non-empty, the root span's parent is * the given W3C trace context, e.g. received from the client during @@ -37,22 +22,19 @@ class OpenTelemetryLogger : public Logger * span. If `isServer` is set, the root span is a server span (i.e. it * handles a request from another process, as in the daemon). * - * Returns null if tracing support is not compiled in or `initOtel()` - * did not enable tracing; the caller should then not attach a logger. - */ -std::unique_ptr makeOpenTelemetryLogger( - std::string_view rootSpanName, std::string_view remoteParentTraceparent = {}, bool isServer = false); - -/** - * Initialize OpenTelemetry tracing for this process. Does nothing - * (and is cheap) if `OTEL_EXPORTER_OTLP_ENDPOINT` / - * `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is not set in the environment, - * or if tracing support is not compiled in. - * * Any previously initialized tracing state is discarded rather than * reused, so this can be called in a child process after a `fork()`, * where the exporter's worker thread no longer exists. + * + * The logger's `flush()` exports all pending spans, with bounded + * timeouts. Since loggers are generally not destroyed, it has to be + * called explicitly before the process exits; `handleExceptions()` + * does so. */ -void initOtel(std::string_view serviceName); +void initOtel( + std::string_view serviceName, + std::string_view rootSpanName, + std::string_view remoteParentTraceparent = {}, + bool isServer = false); } // namespace nix diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index f5fa3c0aa6..2ad49cd9bb 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -54,9 +54,7 @@ namespace nix { */ static void setupConnectionTelemetry(std::string_view traceparent) { - initOtel("nix-daemon"); - if (auto otelLogger = makeOpenTelemetryLogger("daemon connection", traceparent, /*isServer=*/true)) - applyExtraLogger(std::move(otelLogger)); + initOtel("nix-daemon", "daemon connection", traceparent, /*isServer=*/true); } /** From 1314f3b2f3f27efb1007dae76d1c0d48f041b69f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 13:17:52 +0200 Subject: [PATCH 735/927] Move the tracing state into OpenTelemetryLogger Now that `initOtel()` creates the logger, there's exactly one tracer provider per logger, so the logger can own it (and the root span's debug trace ID) rather than keeping it in a global. The logger creates the provider itself; `initOtel()` is left with resolving the configuration and creating the exporter from it. That also makes the fork callback unnecessary: it discarded the global state in forked children, but `startProcess()` already gives them a fresh global logger (leaking the parent's), so nothing in a child references the inherited provider; a child that wants tracing calls `initOtel()` and gets its own. Assisted-by: Claude Fable 5.1 --- src/nix/otel-logger.cc | 166 ++++++++++++++++++----------------------- src/nix/otel-logger.hh | 6 +- 2 files changed, 75 insertions(+), 97 deletions(-) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 7997e6e593..2617db701d 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -11,7 +11,6 @@ # include "nix/util/config-global.hh" # include "nix/util/environment-variables.hh" # include "nix/util/exit.hh" -# include "nix/util/processes.hh" # include "nix/util/serialise.hh" # include "nix/util/signals.hh" # include "nix/util/sync.hh" @@ -124,33 +123,6 @@ static GlobalConfig::Register rOtelSettings(&otelSettings); namespace { -struct OtelState -{ - std::unique_ptr provider; - std::shared_ptr tracer; - - /** - * The root span's trace ID, to be printed on shutdown if - * NIX_DEBUG_OTEL is set. - */ - std::string debugTraceId; -}; - -/* Owned here rather than via opentelemetry's global Provider - singleton, and deliberately leaked: the provider's destructor joins - the batch exporter's worker thread, which must never happen from - static destructors at exit() time (cf. the OPENSSL_INIT_NO_ATEXIT - note in util.cc; in the daemon's forked children that thread does - not even exist). So it's never destroyed; pending spans are - exported by `OpenTelemetryLogger::flush()` instead. */ -std::atomic otelState{nullptr}; - -/* The exporter's worker thread doesn't exist in a forked child, so - don't trace there unless the child sets up tracing itself (i.e. by - calling `initOtel()`). The old state is deliberately leaked, since - it can be neither flushed nor destroyed safely. */ -static RegisterForkCallback resetOtel([]() { otelState.exchange(nullptr); }); - /** * The name of the activity under which we upload spans. Since the * upload itself creates activities (namely the file transfer), we @@ -284,6 +256,48 @@ void setPathAttributes(opentelemetry::trace::Span & span, std::string_view prefi } } +/** + * Create the sampler selected by the standard `OTEL_TRACES_SAMPLER` / + * `OTEL_TRACES_SAMPLER_ARG` environment variables, which the C++ SDK + * does not read itself. "parentbased" samplers follow the sampling + * decision of the parent span, which propagates in the sampled flag + * of the W3C trace context — so the daemon follows the client's + * decision. + */ +std::unique_ptr makeSampler() +{ + namespace sdktrace = opentelemetry::sdk::trace; + + auto ratio = [&]() -> double { + auto arg = getEnv("OTEL_TRACES_SAMPLER_ARG"); + if (!arg) + return 1.0; + try { + return std::stod(*arg); + } catch (...) { + warn("invalid OTEL_TRACES_SAMPLER_ARG '%s'; assuming 1.0", *arg); + return 1.0; + } + }; + auto parentBased = [](std::shared_ptr delegate) -> std::unique_ptr { + return std::make_unique(std::move(delegate)); + }; + auto name = getEnv("OTEL_TRACES_SAMPLER").value_or("parentbased_always_on"); + if (name == "always_on") + return std::make_unique(); + if (name == "always_off") + return std::make_unique(); + if (name == "traceidratio") + return std::make_unique(ratio()); + if (name == "parentbased_always_off") + return parentBased(std::make_shared()); + if (name == "parentbased_traceidratio") + return parentBased(std::make_shared(ratio())); + if (name != "parentbased_always_on") + warn("unknown OTEL_TRACES_SAMPLER '%s'; assuming 'parentbased_always_on'", name); + return parentBased(std::make_shared()); +} + /** * A `Logger` that maps Nix activities onto OpenTelemetry spans, * under a root span created in the constructor. It's added to the @@ -297,10 +311,18 @@ class OpenTelemetryLogger : public Logger { using SpanPtr = std::shared_ptr; + std::unique_ptr provider; + std::shared_ptr tracer; SpanPtr rootSpan; + /** + * The root span's trace ID, to be printed by `flush()` if + * NIX_DEBUG_OTEL is set. + */ + std::string debugTraceId; + Sync> spans_; /** @@ -335,9 +357,22 @@ class OpenTelemetryLogger : public Logger public: OpenTelemetryLogger( - OtelState & state, std::string_view rootSpanName, std::string_view remoteParentTraceparent, bool isServer) - : tracer(state.tracer) + std::string_view serviceName, + std::unique_ptr exporter, + std::string_view rootSpanName, + std::string_view remoteParentTraceparent, + bool isServer) { + namespace sdktrace = opentelemetry::sdk::trace; + + auto processor = + sdktrace::BatchSpanProcessorFactory::Create(std::move(exporter), sdktrace::BatchSpanProcessorOptions{}); + auto resource = opentelemetry::sdk::resource::Resource::Create({ + {opentelemetry::semconv::service::kServiceName, std::string(serviceName)}, + }); + provider = sdktrace::TracerProviderFactory::Create(std::move(processor), resource, makeSampler()); + tracer = provider->GetTracer("nix"); + opentelemetry::trace::StartSpanOptions options; if (isServer) options.kind = opentelemetry::trace::SpanKind::kServer; @@ -348,9 +383,7 @@ class OpenTelemetryLogger : public Logger if (getEnv("NIX_DEBUG_OTEL")) { char buf[2 * opentelemetry::trace::TraceId::kSize]; rootSpan->GetContext().trace_id().ToLowerBase16(buf); - /* Printed by `flush()` once the trace has been - uploaded. */ - state.debugTraceId = std::string(buf, sizeof(buf)); + debugTraceId = std::string(buf, sizeof(buf)); } } @@ -609,16 +642,12 @@ class OpenTelemetryLogger : public Logger void flush() override { - auto * state = otelState.load(std::memory_order_acquire); - if (!state) - return; - /* Bound the timeout: the SDK default is microseconds::max(), and a hung collector must not hang process exit. */ - state->provider->ForceFlush(std::chrono::microseconds(std::chrono::seconds(5))); + provider->ForceFlush(std::chrono::microseconds(std::chrono::seconds(5))); - if (!state->debugTraceId.empty()) - writeToStderr(fmt("OpenTelemetry trace ID: %s\n", state->debugTraceId)); + if (!debugTraceId.empty()) + writeToStderr(fmt("OpenTelemetry trace ID: %s\n", debugTraceId)); } }; @@ -896,65 +925,14 @@ void initOtel( return; } - namespace sdktrace = opentelemetry::sdk::trace; - auto exporter = std::make_unique( endpoint, parseOtlpHeaders(getEnv("OTEL_EXPORTER_OTLP_HEADERS").value_or(otelSettings.headers)), getEnv("OTEL_EXPORTER_OTLP_COMPRESSION").value_or(otelSettings.compression) == "gzip"); - auto processor = - sdktrace::BatchSpanProcessorFactory::Create(std::move(exporter), sdktrace::BatchSpanProcessorOptions{}); - auto resource = opentelemetry::sdk::resource::Resource::Create({ - {opentelemetry::semconv::service::kServiceName, std::string(serviceName)}, - }); - - /* Support the standard OTEL_TRACES_SAMPLER / OTEL_TRACES_SAMPLER_ARG - environment variables, which the C++ SDK does not read itself. - "parentbased" samplers follow the sampling decision of the - parent span, which propagates in the sampled flag of the W3C - trace context — so the daemon follows the client's decision. */ - auto sampler = [&]() -> std::unique_ptr { - auto ratio = [&]() -> double { - auto arg = getEnv("OTEL_TRACES_SAMPLER_ARG"); - if (!arg) - return 1.0; - try { - return std::stod(*arg); - } catch (...) { - warn("invalid OTEL_TRACES_SAMPLER_ARG '%s'; assuming 1.0", *arg); - return 1.0; - } - }; - auto parentBased = [](std::shared_ptr delegate) -> std::unique_ptr { - return std::make_unique(std::move(delegate)); - }; - auto name = getEnv("OTEL_TRACES_SAMPLER").value_or("parentbased_always_on"); - if (name == "always_on") - return std::make_unique(); - if (name == "always_off") - return std::make_unique(); - if (name == "traceidratio") - return std::make_unique(ratio()); - if (name == "parentbased_always_off") - return parentBased(std::make_shared()); - if (name == "parentbased_traceidratio") - return parentBased(std::make_shared(ratio())); - if (name != "parentbased_always_on") - warn("unknown OTEL_TRACES_SAMPLER '%s'; assuming 'parentbased_always_on'", name); - return parentBased(std::make_shared()); - }(); - - auto state = std::make_unique(); - state->provider = sdktrace::TracerProviderFactory::Create(std::move(processor), resource, std::move(sampler)); - state->tracer = state->provider->GetTracer("nix"); - - /* Any previously initialized state is deliberately leaked rather - than destroyed, since we may be in a child process where its - worker thread doesn't exist. */ - auto * statePtr = state.release(); - otelState.exchange(statePtr, std::memory_order_release); - applyExtraLogger(std::make_unique(*statePtr, rootSpanName, remoteParentTraceparent, isServer)); + applyExtraLogger( + std::make_unique( + serviceName, std::move(exporter), rootSpanName, remoteParentTraceparent, isServer)); } #else diff --git a/src/nix/otel-logger.hh b/src/nix/otel-logger.hh index aa26c2e8bc..8c30fa414f 100644 --- a/src/nix/otel-logger.hh +++ b/src/nix/otel-logger.hh @@ -22,9 +22,9 @@ namespace nix { * span. If `isServer` is set, the root span is a server span (i.e. it * handles a request from another process, as in the daemon). * - * Any previously initialized tracing state is discarded rather than - * reused, so this can be called in a child process after a `fork()`, - * where the exporter's worker thread no longer exists. + * The logger owns the exporter and its worker thread, so a forked + * child (which gets a fresh global logger, cf. `startProcess()`) + * doesn't trace unless it calls this itself. * * The logger's `flush()` exports all pending spans, with bounded * timeouts. Since loggers are generally not destroyed, it has to be From bbcdd0de5be1ba8c51487a4e80cea96b02020bb2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 13:53:36 +0200 Subject: [PATCH 736/927] Update TODO --- src/libstore/unix/build/hook-instance.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index 3a8fa843c7..61e14a105d 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -61,8 +61,6 @@ HookInstance::HookInstance(const Strings & _buildHook) if (name == "traceparent") setenv("TRACEPARENT", value.c_str(), 1); - // TODO: Send our settings to the hook (so that e.g. `--otlp` takes effect). - if (chdir("/") == -1) throw SysError("changing into /"); From 6a52c78ba91ac249659f2c7e05c4518ee4584e10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 14:38:18 +0200 Subject: [PATCH 737/927] build-remote: Apply all of the parent's settings, not just `Settings` The parent sends the settings of every registered `Config`, but the hook applied them via `settings.set()`, which only knows `Settings` and silently returns false for anything else. So settings living in other `Config`s never reached the hook: the OpenTelemetry ones (so `nix build --otlp ...` didn't trace the hook), but also e.g. `experimental-features` and the file transfer settings. Apply them via `globalConfig.set()` instead. Only apply those that differ from our own values, though, since we've read the same configuration files: setting a deprecated setting warns even when it's set to its default, which showed up in the output of every build. (Having the parent send only its overridden settings instead doesn't work: things like `--store` set the setting without marking it as overridden, so the hook would end up using the wrong store.) Assisted-by: Claude Fable 5.1 --- src/nix/build-remote/build-remote.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index adc2d6b004..5c3dea6830 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -18,6 +18,7 @@ #include "nix/util/serialise.hh" #include "nix/store/build-result.hh" #include "nix/store/store-open.hh" +#include "nix/util/config-global.hh" #include "nix/util/environment-variables.hh" #include "otel-logger.hh" @@ -89,11 +90,21 @@ static int main_build_remote(int argc, char ** argv) FdSource source(STDIN_FILENO); - /* Read the parent's settings. */ + /* Read the parent's settings. The parent sends the settings of + all `Config`s, not just `Settings`, so apply them + accordingly. Only apply those that differ from ours, though: + we've read the same configuration files, so the rest are + defaults, and setting a deprecated setting warns even if + it's to its default. (Note that we can't just have the + parent send its overridden settings: things like `--store` + set the setting without marking it as overridden.) */ + std::map ourSettings; + globalConfig.getSettings(ourSettings); while (readInt(source)) { auto name = readString(source); auto value = readString(source); - settings.set(name, value); + if (auto i = ourSettings.find(name); i == ourSettings.end() || i->second.value != value) + globalConfig.set(name, value); } auto maxBuildJobs = settings.getWorkerSettings().maxBuildJobs; From 40a704d7c4c1eb85c48945612490683653884484 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 15:59:23 +0200 Subject: [PATCH 738/927] Set sampling.priority on spans when NIX_DEBUG_OTEL is set A collector's probabilistic sampler may drop a trace regardless of the W3C "sampled" flag, which makes the trace ID printed under NIX_DEBUG_OTEL useless if the trace never arrives. A non-zero `sampling.priority` attribute is the documented override, so set it on all spans when debugging. Assisted-by: Claude Fable 5.1 --- src/nix/otel-logger.cc | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/nix/otel-logger.cc b/src/nix/otel-logger.cc index 2617db701d..71ac06fa79 100644 --- a/src/nix/otel-logger.cc +++ b/src/nix/otel-logger.cc @@ -317,9 +317,14 @@ class OpenTelemetryLogger : public Logger SpanPtr rootSpan; + /** + * Whether `NIX_DEBUG_OTEL` is set. + */ + bool debug = false; + /** * The root span's trace ID, to be printed by `flush()` if - * NIX_DEBUG_OTEL is set. + * `debug` is set. */ std::string debugTraceId; @@ -355,6 +360,20 @@ class OpenTelemetryLogger : public Logger return std::move(carrier.headers); } + /** + * Start a span. When debugging (`NIX_DEBUG_OTEL`), spans carry + * `sampling.priority = 1`, which tells a collector's probabilistic + * sampler to keep them, so that the trace whose ID we print can + * actually be found. + */ + SpanPtr startSpan(std::string_view name, const opentelemetry::trace::StartSpanOptions & options) + { + auto span = tracer->StartSpan(name, options); + if (debug) + span->SetAttribute("sampling.priority", 1); + return span; + } + public: OpenTelemetryLogger( std::string_view serviceName, @@ -373,14 +392,16 @@ class OpenTelemetryLogger : public Logger provider = sdktrace::TracerProviderFactory::Create(std::move(processor), resource, makeSampler()); tracer = provider->GetTracer("nix"); + debug = getEnv("NIX_DEBUG_OTEL").has_value(); + opentelemetry::trace::StartSpanOptions options; if (isServer) options.kind = opentelemetry::trace::SpanKind::kServer; if (auto spanContext = parseTraceparent(remoteParentTraceparent)) options.parent = *spanContext; - rootSpan = tracer->StartSpan(rootSpanName, options); + rootSpan = startSpan(rootSpanName, options); - if (getEnv("NIX_DEBUG_OTEL")) { + if (debug) { char buf[2 * opentelemetry::trace::TraceId::kSize]; rootSpan->GetContext().trace_id().ToLowerBase16(buf); debugTraceId = std::string(buf, sizeof(buf)); @@ -435,7 +456,7 @@ class OpenTelemetryLogger : public Logger else options.parent = rootSpan->GetContext(); - auto span = tracer->StartSpan(name, options); + auto span = startSpan(name, options); if (!s.empty() && !textIsName) span->SetAttribute("nix.activity.text", filterANSIEscapes(s, true)); @@ -533,7 +554,7 @@ class OpenTelemetryLogger : public Logger else options.parent = rootSpan->GetContext(); - auto span = tracer->StartSpan(spanName, options); + auto span = startSpan(spanName, options); if (!s.empty()) span->SetAttribute("nix.activity.text", filterANSIEscapes(s, true)); From 39a355edb931e6b1a16ac3433af524b4817d61bb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 16:06:46 +0200 Subject: [PATCH 739/927] Don't create spans for log messages received from the build hook The build hook exports its own telemetry (parented to ours via TRACEPARENT), so replaying its activities into our logger produced duplicate spans. Mark them as coming from a remote log source, like we do for messages forwarded by the daemon. Assisted-by: Claude Fable 5.1 --- src/libstore/build/derivation-building-goal.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 4c2c4b9c74..49c0fc40bc 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -735,6 +735,8 @@ Goal::Co DerivationBuildingGoal::buildWithHook( if (c == '\n') { auto json = parseJSONMessage(currentHookLine, "the derivation builder"); if (json) { + /* The hook exports its own telemetry. */ + RemoteLogSource remoteLogSource; auto s = handleJSONLogMessage( *json, worker.act, hook->activities, "the derivation builder", true); // ensure that logs from a builder using `ssh-ng://` as protocol @@ -1232,6 +1234,8 @@ HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptionsactivities, "the build hook", true)) ; else if (s.substr(0, 2) == "# ") { From 19c8c1b9f70940324c7a15a7a62398e794c34000 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 17:03:00 +0200 Subject: [PATCH 740/927] Export telemetry before exec() Commands that exec() another program (nix run/shell/develop/fmt, nix-shell, and `--help` running man) stopped the logger but never flushed it, so their traces were lost along with the exporter thread. Assisted-by: Claude Fable 5.1 --- src/nix/man-pages.cc | 4 ++++ src/nix/nix-build/nix-build.cc | 2 ++ src/nix/run.cc | 2 ++ 3 files changed, 8 insertions(+) diff --git a/src/nix/man-pages.cc b/src/nix/man-pages.cc index 9b01307a13..0641c89e46 100644 --- a/src/nix/man-pages.cc +++ b/src/nix/man-pages.cc @@ -3,6 +3,7 @@ #include "nix/util/file-system.hh" #include "nix/util/current-process.hh" #include "nix/util/environment-variables.hh" +#include "nix/util/logging.hh" namespace nix { @@ -15,6 +16,9 @@ void showManPage(const std::string & name) { restoreProcessContext(); setEnv("MANPATH", (getNixManDir().string() + ":").c_str()); + /* We're about to exec, so end and export any telemetry. */ + logger->stop(); + logger->flush(); execlp("man", "man", name.c_str(), nullptr); if (errno == ENOENT) { // Not SysError because we don't want to suffix the errno, aka No such file or directory. diff --git a/src/nix/nix-build/nix-build.cc b/src/nix/nix-build/nix-build.cc index d2e927a2d2..76081bcc81 100644 --- a/src/nix/nix-build/nix-build.cc +++ b/src/nix/nix-build/nix-build.cc @@ -678,7 +678,9 @@ static void main_nix_build(int argc, char ** argv) restoreProcessContext(); + /* We're about to exec, so end and export any telemetry. */ logger->stop(); + logger->flush(); execvp(shell->c_str(), argPtrs.data()); diff --git a/src/nix/run.cc b/src/nix/run.cc index 02473ed901..fc763802cf 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -43,7 +43,9 @@ void execProgramInStore( std::optional system, std::optional env) { + /* We're about to exec, so end and export any telemetry. */ logger->stop(); + logger->flush(); char ** envp; Strings envStrs; From cda573537955aba86ba139f5b658c3dcfcc35d81 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 17:44:49 +0200 Subject: [PATCH 741/927] otel.sh: Don't depend on the number of uploads The exporter uploads periodically as well as on exit, so a slow build (e.g. on a loaded CI machine) spreads its spans over several uploads, which shifted the upload numbers the test relied on. Instead, delete the uploads before each step and select spans by name across all the uploads received since. Assisted-by: Claude Fable 5.1 --- tests/functional/otel.sh | 114 +++++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 52 deletions(-) diff --git a/tests/functional/otel.sh b/tests/functional/otel.sh index 1159498ff3..ecb2ed00a1 100644 --- a/tests/functional/otel.sh +++ b/tests/functional/otel.sh @@ -32,111 +32,121 @@ otlp-compression = none otlp-headers = authorization=Bearer%20secret EOF -# Return the body of the n-th upload, as JSON. -body() { - cat "$sinkDir/$1.body" +# The helpers below look at all the uploads received since the last +# `clearUploads`. Note that a process may spread its spans over +# several uploads (the exporter uploads periodically as well as on +# exit), so spans are selected by name rather than by upload. + +clearUploads() { + rm -f "$sinkDir"/*.body "$sinkDir"/*.headers +} + +# Return the number of uploads. +uploads() { + find "$sinkDir" -name '*.body' | wc -l } -# Return the value of the n-th upload's `.[0]`-th span field. +# Apply a jq filter to every span of every upload. +spans() { + jq -r ".resourceSpans[0].scopeSpans[0].spans[] | $1" "$sinkDir"/*.body +} + +# Return a field of the span with the given name, e.g. `.kind`. span() { - body "$1" | jq -r ".resourceSpans[0].scopeSpans[0].spans[0]$2" + spans "select(.name == \"$1\") | $2" } # Return the value of a string attribute of the span with the given -# name in the n-th upload. +# name. attr() { - body "$1" | jq -r ".resourceSpans[0].scopeSpans[0].spans[] | select(.name == \"$2\") | .attributes[] | select(.key == \"$3\") | .value.stringValue" + span "$1" ".attributes[] | select(.key == \"$2\") | .value.stringValue" } -# Return a field of the status of the span with the given name in the -# n-th upload, e.g. `.code`. -spanStatus() { - body "$1" | jq -r ".resourceSpans[0].scopeSpans[0].spans[] | select(.name == \"$2\") | .status$3" +# Return the service names of the uploads, one per line. +services() { + jq -r '.resourceSpans[0].resource.attributes[] | select(.key == "service.name") | .value.stringValue' "$sinkDir"/*.body | sort } # A successful command produces a root span named after it, with no # status. +clearUploads [[ $(nix eval --expr '1 + 2') = 3 ]] -[[ $(body 0 | jq -r '.resourceSpans[0].resource.attributes[] | select(.key == "service.name") | .value.stringValue') = nix ]] -[[ $(span 0 .name) = "nix eval" ]] -[[ $(span 0 .status) = null ]] -[[ $(span 0 .kind) = 1 ]] +[[ $(services) = nix ]] +[[ $(span "nix eval" .status) = null ]] +[[ $(span "nix eval" .kind) = 1 ]] # The headers from the configuration should arrive percent-decoded, # and the upload of the trace itself must not be part of the trace. -[[ $(jq -r .authorization "$sinkDir/0.headers") = "Bearer secret" ]] -[[ $(jq -r .traceparent "$sinkDir/0.headers") = null ]] +[[ $(jq -r .authorization "$sinkDir"/*.headers) = "Bearer secret" ]] +[[ $(jq -r .traceparent "$sinkDir"/*.headers) = null ]] # A failing command marks the root span as failed, with the error # message. +clearUploads expect 1 nix eval --expr '1 + "x"' -[[ $(span 1 .name) = "nix eval" ]] -[[ $(span 1 .status.code) = 2 ]] -span 1 .status.message | grepQuiet "cannot add a string to an integer" +[[ $(span "nix eval" .status.code) = 2 ]] +span "nix eval" .status.message | grepQuiet "cannot add a string to an integer" # Legacy commands are traced as well, under their own name. +clearUploads [[ $(nix-instantiate --eval --expr '1 + 2') = 3 ]] -[[ $(span 2 .name) = "nix-instantiate" ]] +[[ $(span nix-instantiate .kind) = 1 ]] # Setting an endpoint in the environment enables tracing even if # `otlp` is disabled, and takes precedence over the configuration. +clearUploads [[ $(NIX_CONFIG="otlp = false" nix eval --expr '1 + 2') = 3 ]] -[[ ! -e $sinkDir/3.body ]] +[[ $(uploads) = 0 ]] [[ $(OTEL_EXPORTER_OTLP_ENDPOINT="http://127.0.0.1:$(cat "$sinkDir/port")" NIX_CONFIG="otlp = false" nix eval --expr '1 + 2') = 3 ]] -[[ $(span 3 .name) = "nix eval" ]] +[[ $(span "nix eval" .kind) = 1 ]] # A build produces a `Build` span carrying the derivation's path, name # and version as separate attributes. # shellcheck disable=SC2016 # `$out` is for the Nix builder, not the shell. drvPath=$(nix-instantiate --expr 'with import ./config.nix; mkDerivation { name = "foo-1.2"; buildCommand = "echo > $out"; }') +clearUploads nix build --no-link "$drvPath^*" -[[ $(attr 5 Build nix.drv.path) = "$drvPath" ]] -[[ $(attr 5 Build nix.drv.name) = foo ]] -[[ $(attr 5 Build nix.drv.version) = 1.2 ]] -[[ $(attr 5 Build nix.build.status) = Built ]] -[[ $(spanStatus 5 Build .code) = null ]] +[[ $(attr Build nix.drv.path) = "$drvPath" ]] +[[ $(attr Build nix.drv.name) = foo ]] +[[ $(attr Build nix.drv.version) = 1.2 ]] +[[ $(attr Build nix.build.status) = Built ]] +[[ $(span Build .status.code) = null ]] # Not being able to substitute the path is the normal prelude to # building it, not an error. -[[ $(attr 5 SubstitutionGoal nix.build.status) = NoSubstituters ]] -[[ $(spanStatus 5 SubstitutionGoal .code) = null ]] +[[ $(attr SubstitutionGoal nix.build.status) = NoSubstituters ]] +[[ $(span SubstitutionGoal .status.code) = null ]] # A failed build marks its `Build` span as failed, with the build's # status and error message. Note that the failure doesn't throw, so # this relies on the build result rather than on stack unwinding. # shellcheck disable=SC2016 # `$out` is for the Nix builder, not the shell. drvPath=$(nix-instantiate --expr 'with import ./config.nix; mkDerivation { name = "bar-1.2"; buildCommand = "echo something went wrong >&2; exit 1"; }') +clearUploads expect 1 nix build --no-link "$drvPath^*" -[[ $(attr 7 Build nix.build.status) = PermanentFailure ]] -[[ $(spanStatus 7 Build .code) = 2 ]] -spanStatus 7 Build .message | grepQuiet "builder failed with exit code 1" +[[ $(attr Build nix.build.status) = PermanentFailure ]] +[[ $(span Build .status.code) = 2 ]] +span Build .status.message | grepQuiet "builder failed with exit code 1" # A daemon reached via `ssh-ng://` (here without real SSH, since the # host is `localhost`) exports its own spans, in the client's trace. -# The client and the daemon upload independently, in no particular -# order, so wait for both and tell them apart by service name. -service() { - body "$1" | jq -r '.resourceSpans[0].resource.attributes[] | select(.key == "service.name") | .value.stringValue' -} +# The client and the daemon upload independently, so wait for both. +clearUploads nix store info --store "ssh-ng://localhost?remote-store=$TEST_ROOT/other-store" > /dev/null for ((i = 0; i < 100; i++)); do - [[ -e $sinkDir/9.body ]] && break + [[ $(uploads) -ge 2 ]] && break sleep 0.1 done -[[ -e $sinkDir/9.body ]] -if [[ $(service 8) = nix ]]; then client=8; daemon=9; else client=9; daemon=8; fi -[[ $(service $client) = nix ]] -[[ $(service $daemon) = nix-daemon ]] -[[ $(span $client .name) = "nix store info" ]] -[[ $(span $daemon .name) = "daemon connection" ]] -[[ $(span $daemon .kind) = 2 ]] # SERVER -[[ $(span $daemon .traceId) = $(span $client .traceId) ]] -[[ $(span $daemon .parentSpanId) = $(span $client .spanId) ]] +[[ $(services) = $'nix\nnix-daemon' ]] +[[ $(span "nix store info" .kind) = 1 ]] +[[ $(span "daemon connection" .kind) = 2 ]] # SERVER +[[ $(span "daemon connection" .traceId) = $(span "nix store info" .traceId) ]] +[[ $(span "daemon connection" .parentSpanId) = $(span "nix store info" .spanId) ]] # The root span is parented to the trace context in `TRACEPARENT`, # which is how remote builds via `build-remote` are linked to the # trace of the `nix` process that runs it. +clearUploads [[ $(TRACEPARENT=00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01 nix eval --expr '1 + 2') = 3 ]] -[[ $(span 10 .name) = "nix eval" ]] -[[ $(span 10 .traceId) = 0af7651916cd43dd8448eb211c80319c ]] -[[ $(span 10 .parentSpanId) = b7ad6b7169203331 ]] -[[ $(span 10 .kind) = 1 ]] # INTERNAL, not SERVER +[[ $(span "nix eval" .traceId) = 0af7651916cd43dd8448eb211c80319c ]] +[[ $(span "nix eval" .parentSpanId) = b7ad6b7169203331 ]] +[[ $(span "nix eval" .kind) = 1 ]] # INTERNAL, not SERVER From c6781d07a402ceee24343540a0589a04c47f1fcd Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Thu, 3 Sep 2026 23:32:14 +0300 Subject: [PATCH 742/927] tests/functional: Silence git warning about initial branch name To get rid of: hint: Using 'master' as the name for the initial branch. This default branch name hint: will change to "main" in Git 3.0. Changing that from master breaks the tests. Also dedups test to use createGitRepo (cherry picked from commit 242d09e49546d0edb386d109bfbe070631ab9aaa) --- tests/functional/common/functions.sh | 2 +- tests/functional/git/packed-refs-no-cache.sh | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index a180730d39..a515cb16ba 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -375,7 +375,7 @@ initGitRepo() { local extraArgs="${2-}" # shellcheck disable=SC2086 # word splitting of extraArgs is intended - git -C "$repo" init $extraArgs + git -C "$repo" init --initial-branch=master $extraArgs git -C "$repo" config user.email "foobar@example.com" git -C "$repo" config user.name "Foobar" } diff --git a/tests/functional/git/packed-refs-no-cache.sh b/tests/functional/git/packed-refs-no-cache.sh index 33ffd5fa33..fd369e6b1e 100644 --- a/tests/functional/git/packed-refs-no-cache.sh +++ b/tests/functional/git/packed-refs-no-cache.sh @@ -21,11 +21,7 @@ repo=$TEST_ROOT/./git export _NIX_FORCE_HTTP=1 -rm -rf "$repo" "${repo}-tmp" "$TEST_HOME/.cache/nix" - -git init --initial-branch="master" "$repo" -git -C "$repo" config user.email "nix-tests@example.com" -git -C "$repo" config user.name "Nix Tests" +createGitRepo "$repo" echo "hello world" > "$repo/hello_world" git -C "$repo" add hello_world From 8d6965d30c07993d749323b5f9631150e784ae9d Mon Sep 17 00:00:00 2001 From: tbereknyei Date: Sun, 6 Sep 2026 01:24:39 -0400 Subject: [PATCH 743/927] fetchers/git: bucket rev-pinned negotiation refs by hash prefix A rev-pinned fetch left the rev un-refed, so a later fetch of the same URL had nothing to negotiate from and re-downloaded shared history. Fetch into refs/nix/tip-<2 hex chars of rev> instead. Bucketing (256 slots) means unrelated revs don't keep evicting each other's tip the way a single shared ref would. Verified against a real ~7.7M-object repo: a third fetch that would've re-downloaded 67,656 objects with a single shared ref dropped to 7 with bucketing. (cherry picked from commit 74d9f993be1ca84e056984a092eb2836b321645c) --- src/libfetchers/git.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7f560e0d2b..908f51c8b6 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -997,8 +997,15 @@ struct GitInputScheme : InputScheme return std::nullopt; try { - auto fetchRef = getAllRefsAttr(input) ? "refs/*:refs/*" - : input.getRev() ? input.getRev()->gitRev() + static constexpr std::string_view reservedRefNamespace = + "__nix_internal_ref_namespace_reserved_for_fetchers_2c17c6393771ee3048ae34d6b380c5ec__"; + /* Fetch into a bucketed ref instead of leaving the rev + un-refed, so a later fetch of a nearby rev can + negotiate against it instead of re-downloading. */ + auto rev = input.getRev(); + auto revStr = rev ? rev->gitRev() : ""; + auto fetchRef = getAllRefsAttr(input) ? "refs/*:refs/*" + : rev ? fmt("%s:refs/%s/tip-%s", revStr, reservedRefNamespace, revStr.substr(0, 2)) : ref.compare(0, 5, "refs/") == 0 ? fmt("%1%:%1%", ref) : ref == "HEAD" ? "HEAD:HEAD" : fmt("%1%:%1%", "refs/heads/" + ref); From 8db752f02b713fbf1d1e90d23b169871369915a3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 18:06:26 +0200 Subject: [PATCH 744/927] otel.sh: Request substitution explicitly for the build Without a network (as in the Nix sandbox on CI), `nix` turns off substitution unless it's requested explicitly, so the build never started a substitution goal and the test couldn't find its span. Assisted-by: Claude Fable 5.1 --- tests/functional/otel.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/functional/otel.sh b/tests/functional/otel.sh index ecb2ed00a1..bbaa9eb2e4 100644 --- a/tests/functional/otel.sh +++ b/tests/functional/otel.sh @@ -105,7 +105,10 @@ clearUploads # shellcheck disable=SC2016 # `$out` is for the Nix builder, not the shell. drvPath=$(nix-instantiate --expr 'with import ./config.nix; mkDerivation { name = "foo-1.2"; buildCommand = "echo > $out"; }') clearUploads -nix build --no-link "$drvPath^*" +# Note: without a network (e.g. in the Nix sandbox), `nix` turns off +# substitution unless it's requested explicitly, and then there is no +# substitution span. +nix build --no-link --substitute "$drvPath^*" [[ $(attr Build nix.drv.path) = "$drvPath" ]] [[ $(attr Build nix.drv.name) = foo ]] [[ $(attr Build nix.drv.version) = 1.2 ]] From c57c990dc06c501e0ca63b01ce9570d7ecb5c456 Mon Sep 17 00:00:00 2001 From: tbereknyei Date: Sun, 6 Sep 2026 06:04:03 -0400 Subject: [PATCH 745/927] tests/functional/git: cover bucketed negotiation refs Fetches an exact rev, checks its bucket ref was created, fetches a second rev, and asserts via GIT_TRACE_PACKET that the second fetch negotiated against the first rev's ref instead of re-downloading. (cherry picked from commit 99b5af91148e0aa92e292ac38c50b48af552d950) --- .../git/exact-rev-fetch-bucketed-refs.sh | 70 +++++++++++++++++++ tests/functional/git/meson.build | 1 + 2 files changed, 71 insertions(+) create mode 100644 tests/functional/git/exact-rev-fetch-bucketed-refs.sh diff --git a/tests/functional/git/exact-rev-fetch-bucketed-refs.sh b/tests/functional/git/exact-rev-fetch-bucketed-refs.sh new file mode 100644 index 0000000000..b6c954dad3 --- /dev/null +++ b/tests/functional/git/exact-rev-fetch-bucketed-refs.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# Rev-pinned fetches used to leave the fetched commit un-refed (only in +# FETCH_HEAD), so a later rev-pinned fetch of the same URL had nothing to +# negotiate from and re-downloaded objects already on disk. Nix now fetches +# into refs//tip-, so a later fetch of a +# rev in the same bucket can negotiate against it. + +source ../common.sh + +requireGit + +repo=$TEST_ROOT/repo +trace=$TEST_ROOT/git-trace + +createGitRepo "$repo" + +echo first > "$repo/first" +git -C "$repo" add first +git -C "$repo" commit -m 'First commit.' +rev1=$(git -C "$repo" rev-parse HEAD) + +echo second > "$repo/second" +git -C "$repo" add second +git -C "$repo" commit -m 'Second commit.' +rev2=$(git -C "$repo" rev-parse HEAD) + +export _NIX_FORCE_HTTP=1 + +nix-instantiate --eval --raw -E "(builtins.fetchGit { url = \"file://$repo\"; rev = \"$rev1\"; }).outPath" >/dev/null + +reservedRefNamespace="__nix_internal_ref_namespace_reserved_for_fetchers_2c17c6393771ee3048ae34d6b380c5ec__" +cacheDir=$(find "$TEST_HOME/.cache/nix/gitv3" -maxdepth 1 -mindepth 1 -type d) + +bucket1=${rev1:0:2} +if ! git -C "$cacheDir" rev-parse --verify -q "refs/$reservedRefNamespace/tip-$bucket1" >/dev/null; then + echo "Expected refs/nix/tip-$bucket1 to exist after fetching $rev1." >&2 + exit 1 +fi +if [[ $(git -C "$cacheDir" rev-parse "refs/$reservedRefNamespace/tip-$bucket1") != "$rev1" ]]; then + echo "Expected refs/nix/tip-$bucket1 to point at $rev1." >&2 + exit 1 +fi + +export GIT_TRACE_PACKET=$trace +nix-instantiate --eval --raw -E "(builtins.fetchGit { url = \"file://$repo\"; rev = \"$rev2\"; }).outPath" >/dev/null +unset GIT_TRACE_PACKET + +if ! grep -q "fetch> have $rev1" "$trace"; then + echo "Expected the second fetch to advertise $rev1 (via refs/$reservedRefNamespace/tip-$bucket1) as a negotiation tip." >&2 + cat "$trace" >&2 + exit 1 +fi + +totals=$(grep -oE 'sideband< .*Total [0-9]+' "$trace" | sed -E 's/.*Total ([0-9]+)/\1/') +if [[ $totals -ge 6 ]]; then + echo "Expected the second fetch to receive only new objects (got Total $totals, same as a full re-fetch would)." >&2 + exit 1 +fi + +bucket2=${rev2:0:2} +if [[ $(git -C "$cacheDir" rev-parse "refs/$reservedRefNamespace/tip-$bucket2") != "$rev2" ]]; then + echo "Expected refs/nix/tip-$bucket2 to point at $rev2 after the second fetch." >&2 + exit 1 +fi + +if ! git -C "$cacheDir" cat-file -e "$rev1"; then + echo "Expected $rev1 to remain resolvable after the second fetch." >&2 + exit 1 +fi diff --git a/tests/functional/git/meson.build b/tests/functional/git/meson.build index 73c4bc7828..d1c5686f04 100644 --- a/tests/functional/git/meson.build +++ b/tests/functional/git/meson.build @@ -2,6 +2,7 @@ suites += { 'name' : 'git', 'deps' : [], 'tests' : [ + 'exact-rev-fetch-bucketed-refs.sh', 'packed-refs-no-cache.sh', 'symlinked-repo.sh', ], From 489b37295457ba3e42dbf307cbcfddddc27774e2 Mon Sep 17 00:00:00 2001 From: tbereknyei Date: Tue, 8 Sep 2026 08:02:00 -0400 Subject: [PATCH 746/927] fetchers/git: use shorter refspec in git cache (cherry picked from commit e20887e6a62567bdf124ea2412ac68bf498cbf71) --- src/libfetchers/git.cc | 2 +- tests/functional/git/exact-rev-fetch-bucketed-refs.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 908f51c8b6..a098fba109 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -998,7 +998,7 @@ struct GitInputScheme : InputScheme try { static constexpr std::string_view reservedRefNamespace = - "__nix_internal_ref_namespace_reserved_for_fetchers_2c17c6393771ee3048ae34d6b380c5ec__"; + "__nix_internal_fetchers_48ae34d6b380c5ec__"; /* Fetch into a bucketed ref instead of leaving the rev un-refed, so a later fetch of a nearby rev can negotiate against it instead of re-downloading. */ diff --git a/tests/functional/git/exact-rev-fetch-bucketed-refs.sh b/tests/functional/git/exact-rev-fetch-bucketed-refs.sh index b6c954dad3..ee01e64b00 100644 --- a/tests/functional/git/exact-rev-fetch-bucketed-refs.sh +++ b/tests/functional/git/exact-rev-fetch-bucketed-refs.sh @@ -29,7 +29,8 @@ export _NIX_FORCE_HTTP=1 nix-instantiate --eval --raw -E "(builtins.fetchGit { url = \"file://$repo\"; rev = \"$rev1\"; }).outPath" >/dev/null -reservedRefNamespace="__nix_internal_ref_namespace_reserved_for_fetchers_2c17c6393771ee3048ae34d6b380c5ec__" +reservedRefNamespace="__nix_internal_fetchers_48ae34d6b380c5ec__"; + cacheDir=$(find "$TEST_HOME/.cache/nix/gitv3" -maxdepth 1 -mindepth 1 -type d) bucket1=${rev1:0:2} From d617f7d3387175d2d887d513ce270f64840757f8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Sep 2026 12:35:33 +0200 Subject: [PATCH 747/927] baked-flake.nix: Don't fail on unknown flake outputs Outputs that no flake schema recognises are recorded in outputs.json as { "unknown": true } without an "output" attribute, which made evaluation of every baked flake fail with "attribute 'output' missing". Assisted-by: Claude Fable 5.1 --- src/nix/baked-flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix index 70649e717b..56eb401bac 100644 --- a/src/nix/baked-flake.nix +++ b/src/nix/baked-flake.nix @@ -27,5 +27,5 @@ { }; in - cleanup (builtins.mapAttrs (outputName: output: convert output.output) data); + cleanup (builtins.mapAttrs (outputName: output: convert (output.output or { })) data); } From 283e5688b7779cd2739fee90d8290397dab9a614 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Sep 2026 12:35:55 +0200 Subject: [PATCH 748/927] nix flake bake: Add --all-systems flag By default, only outputs for the current system are baked, matching 'nix flake show'. With --all-systems, outputs for all systems are included. Assisted-by: Claude Fable 5.1 --- src/nix/flake-bake.cc | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/nix/flake-bake.cc b/src/nix/flake-bake.cc index 51cb18082a..deb92405e5 100644 --- a/src/nix/flake-bake.cc +++ b/src/nix/flake-bake.cc @@ -11,6 +11,12 @@ struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas { std::filesystem::path destDir; + flake_schemas::FlakeInventoryOptions options{ + .showLegacy = true, + .showOutputPaths = true, + .showDrvNames = true, + }; + CmdFlakeBake() { addFlag({ @@ -21,6 +27,11 @@ struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas .completer = completePath, .required = true, }); + addFlag({ + .longName = "all-systems", + .description = "Bake the outputs for all systems, not just the current system.", + .handler = {&options.showAllSystems, true}, + }); } std::string description() override @@ -43,12 +54,7 @@ struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas auto cache = flake_schemas::call(*state, flake, getDefaultFlakeSchemas()); - auto inv = flake_schemas::getFlakeInventory( - *state, - *getEvalStore(), - *flake, - cache, - {.showLegacy = true, .showOutputPaths = true, .showDrvNames = true}); + auto inv = flake_schemas::getFlakeInventory(*state, *getEvalStore(), *flake, cache, options); std::filesystem::create_directories(destDir); writeFile(destDir / "outputs.json", inv.dump()); From 9ebedeb5da4f9be850b7eac7d8f197e68772d80d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Sep 2026 12:36:35 +0200 Subject: [PATCH 749/927] tests/functional/flakes: Add test for 'nix flake bake' Bakes the simple test flake and checks that the baked flake shows the same outputs, that the baked derivations are builtin:substitute derivations with the original output paths, that building them fails without a substituter and succeeds by substituting from a binary cache, and that --all-systems and legacyPackages work. Assisted-by: Claude Fable 5.1 --- tests/functional/flakes/bake.sh | 64 +++++++++++++++++++++++++++++ tests/functional/flakes/meson.build | 1 + 2 files changed, 65 insertions(+) create mode 100755 tests/functional/flakes/bake.sh diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh new file mode 100755 index 0000000000..9087bfcd99 --- /dev/null +++ b/tests/functional/flakes/bake.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +source ./common.sh + +flakeDir=$TEST_ROOT/flake +bakedDir=$TEST_ROOT/baked +cacheDir=$TEST_ROOT/binary-cache + +mkdir -p "$flakeDir" +writeSimpleFlake "$flakeDir" + +# Bake the flake. +nix flake bake "$flakeDir" --dest-dir "$bakedDir" +[[ -e $bakedDir/flake.nix ]] +[[ -e $bakedDir/outputs.json ]] + +# The baked flake has the same outputs as the original for the current +# system. Outputs for other systems are not included by default. +nix flake show --json "path:$bakedDir" > "$TEST_ROOT/show-baked.json" +[[ $(jq -r ".inventory.packages.output.children.\"$system\".children.foo.derivation.name" < "$TEST_ROOT/show-baked.json") = simple ]] +[[ $(jq -r ".inventory.packages.output.children.\"$system\".children.default.derivation.name" < "$TEST_ROOT/show-baked.json") = simple ]] +[[ $(jq -r ".inventory.packages.output.children | has(\"someOtherSystem\")" < "$TEST_ROOT/show-baked.json") = false ]] + +# The baked derivations have the same output paths as the originals. +fooPath=$(nix eval --raw "$flakeDir#foo.outPath") +[[ $(nix eval --raw "path:$bakedDir#foo.outPath") = "$fooPath" ]] +[[ $(nix eval --raw "path:$bakedDir#foo.system") = "$system" ]] +[[ $(nix eval --raw "path:$bakedDir#foo.name") = simple ]] + +# The baked derivations are `builtin:substitute` derivations without inputs. +drvPath=$(nix eval --raw "path:$bakedDir#foo.drvPath") +nix derivation show "$drvPath" > "$TEST_ROOT/baked-drv.json" +[[ $(jq -r '.derivations[].builder' < "$TEST_ROOT/baked-drv.json") = builtin:substitute ]] +[[ $(jq -r '.derivations[].system' < "$TEST_ROOT/baked-drv.json") = builtin ]] +[[ $(jq -r '.derivations[].inputs.drvs | length' < "$TEST_ROOT/baked-drv.json") = 0 ]] +[[ $(jq -r '.derivations[].outputs.out.path' < "$TEST_ROOT/baked-drv.json") = "$(basename "$fooPath")" ]] + +# Building a baked derivation fails if its outputs cannot be substituted. +expectStderr 1 nix build --no-link "path:$bakedDir#foo" | grepQuiet "failed to substitute" + +# Build the original, copy it to a binary cache, and delete it from the store. +nix build --no-link "$flakeDir#foo" +[[ -e $fooPath/hello ]] +nix copy --to "file://$cacheDir" "$fooPath" +nix store delete "$fooPath" +[[ ! -e $fooPath ]] + +# Now the baked derivation can be realised by substituting its output +# from the binary cache. +nix build --no-link --substituters "file://$cacheDir" --no-require-sigs "path:$bakedDir#foo" +[[ -e $fooPath/hello ]] + +# Once the output is valid, building the baked derivation succeeds even without substituters. +nix build --no-link "path:$bakedDir#foo" + +# With `--all-systems`, outputs for other systems are baked as well. +nix flake bake "$flakeDir" --dest-dir "$bakedDir-all" --all-systems +nix flake show --json --all-systems "path:$bakedDir-all" > "$TEST_ROOT/show-baked-all.json" +[[ $(jq -r ".inventory.packages.output.children.someOtherSystem.children.foo.derivation.name" < "$TEST_ROOT/show-baked-all.json") = simple ]] +[[ $(nix eval --raw "path:$bakedDir-all#packages.someOtherSystem.foo.outPath") = $(nix eval --raw "$flakeDir#packages.someOtherSystem.foo.outPath") ]] + +# `legacyPackages` is baked too. +nix flake show --json --legacy "path:$bakedDir" > "$TEST_ROOT/show-baked-legacy.json" +[[ $(jq -r ".inventory.legacyPackages.output.children.\"$system\".children.hello.derivation.name" < "$TEST_ROOT/show-baked-legacy.json") = simple ]] diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build index a47ba66a3e..a6f864f0b1 100644 --- a/tests/functional/flakes/meson.build +++ b/tests/functional/flakes/meson.build @@ -41,6 +41,7 @@ suites += { 'shallow.sh', 'provenance.sh', 'search.sh', + 'bake.sh', ], 'workdir' : meson.current_source_dir(), } From 9cc9d3ca1c98dd051e373bde9ea8670030a82691 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Sep 2026 12:45:30 +0200 Subject: [PATCH 750/927] builtins.fakeDerivation: Return outputs with string context Previously it returned only the derivation path, and the baked flake took the output paths from outputs.json as plain strings. As a result, a derivation using a baked output as an input didn't record the fake derivation in its inputDrvs, so the output was never substituted. Now it returns an attribute set of the same shape as derivationStrict: drvPath plus one string per output, with proper string context. The baked flake uses those for drvPath and outPath. Add a test for depending on a baked flake. Assisted-by: Claude Fable 5.1 --- src/libexpr/primops/fakeDerivation.cc | 28 ++++++++++++++++++++++++++- src/nix/baked-flake.nix | 13 ++++++++----- tests/functional/flakes/bake.sh | 28 +++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/src/libexpr/primops/fakeDerivation.cc b/src/libexpr/primops/fakeDerivation.cc index a5b7134b92..7be890c816 100644 --- a/src/libexpr/primops/fakeDerivation.cc +++ b/src/libexpr/primops/fakeDerivation.cc @@ -1,9 +1,12 @@ #include "nix/expr/primops.hh" #include "nix/expr/eval-inline.hh" #include "nix/store/derivations.hh" +#include "nix/store/derived-path.hh" #include "nix/store/store-api.hh" #include "nix/util/hash.hh" +#include + namespace nix { static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -119,7 +122,30 @@ static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** ar // FIXME state.waitForPath(drvPath); - state.mkStorePathString(drvPath, v); + /* As in `derivationStrict`, cache the derivation hash so that derivations depending on this one don't need to + read it back from the store. */ + drvHashes.insert_or_assign(drvPath, hashDerivationModulo(*state.store, drv, false)); + + /* Return an attribute set of the same shape as `derivationStrict`: the derivation path and one string per + output. The output strings carry string context, so that they can be used as inputs of other derivations. */ + auto result = state.buildBindings(1 + drv.outputs.size()); + result.alloc(state.s.drvPath) + .mkString( + state.store->printStorePath(drvPath), + { + NixStringContextElem::DrvDeep{.drvPath = drvPath}, + }, + state.mem); + for (auto & [outName, out] : outputs) + state.mkOutputString( + result.alloc(outName), + SingleDerivedPath::Built{ + .drvPath = makeConstantStorePathRef(drvPath), + .output = outName, + }, + out.path); + + v.mkAttrs(result); } static RegisterPrimOp primop_fakeDerivation({ diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix index 56eb401bac..f7ca16888c 100644 --- a/src/nix/baked-flake.nix +++ b/src/nix/baked-flake.nix @@ -11,16 +11,19 @@ if output ? children then cleanup (builtins.mapAttrs (childName: child: convert child) output.children) else if output ? "derivation" then + let + fake = builtins.fakeDerivation { + name = output.derivation.name; + outputs = builtins.mapAttrs (outputName: path: { inherit path; }) output.derivation.outputs; + }; + in { type = "derivation"; name = output.derivation.name; system = builtins.head output.forSystems; # FIXME meta.description = output.shortDescription; - drvPath = builtins.fakeDerivation { - name = output.derivation.name; - outputs = builtins.mapAttrs (outputName: output: { path = output; }) output.derivation.outputs; - }; - outPath = output.derivation.outputs.out; # FIXME + drvPath = fake.drvPath; + outPath = fake.out; # FIXME outputName = "out"; # FIXME } else diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index 9087bfcd99..a0dbcc0c71 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -59,6 +59,34 @@ nix flake show --json --all-systems "path:$bakedDir-all" > "$TEST_ROOT/show-bake [[ $(jq -r ".inventory.packages.output.children.someOtherSystem.children.foo.derivation.name" < "$TEST_ROOT/show-baked-all.json") = simple ]] [[ $(nix eval --raw "path:$bakedDir-all#packages.someOtherSystem.foo.outPath") = $(nix eval --raw "$flakeDir#packages.someOtherSystem.foo.outPath") ]] +# A derivation in another flake can depend on the outputs of a baked flake. +depDir=$TEST_ROOT/dep +mkdir -p "$depDir" +cp "${config_nix}" "$depDir/" +cat > "$depDir/flake.nix" < "$TEST_ROOT/show-baked-legacy.json" [[ $(jq -r ".inventory.legacyPackages.output.children.\"$system\".children.hello.derivation.name" < "$TEST_ROOT/show-baked-legacy.json") = simple ]] From 8420c0999a17869498479bbf1fe26cba51ea3de0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Sep 2026 16:32:38 +0200 Subject: [PATCH 751/927] tests/functional/flakes/bake.sh: Pass --substitute Substitution is disabled automatically when there is no network access (as in the CI sandbox), so the file:// substituter was ignored and the test failed. Pass --substitute explicitly, like substitution.sh does. Assisted-by: Claude Fable 5.1 --- tests/functional/flakes/bake.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index a0dbcc0c71..1098438743 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -46,8 +46,9 @@ nix store delete "$fooPath" [[ ! -e $fooPath ]] # Now the baked derivation can be realised by substituting its output -# from the binary cache. -nix build --no-link --substituters "file://$cacheDir" --no-require-sigs "path:$bakedDir#foo" +# from the binary cache. (`--substitute` is needed because substitution +# is disabled automatically when there is no network access.) +nix build --no-link --substitute --substituters "file://$cacheDir" --no-require-sigs "path:$bakedDir#foo" [[ -e $fooPath/hello ]] # Once the output is valid, building the baked derivation succeeds even without substituters. @@ -84,7 +85,7 @@ nix store delete "$fooPath" expectStderr 1 nix build --no-link "$depDir" | grepQuiet "failed to substitute" # ... and succeeds if it can. -nix build --substituters "file://$cacheDir" --no-require-sigs "$depDir" -o "$TEST_ROOT/dep-result" +nix build --substitute --substituters "file://$cacheDir" --no-require-sigs "$depDir" -o "$TEST_ROOT/dep-result" [[ $(cat "$TEST_ROOT/dep-result") = "Hello World!" ]] # `legacyPackages` is baked too. From 99297e60a99743a81c0b12089ca8439c18a2dc4b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Sep 2026 16:46:04 +0200 Subject: [PATCH 752/927] Rename builtins.fakeDerivation to builtins.bakedDerivation Assisted-by: Claude Fable 5.1 --- .../{fakeDerivation.cc => bakedDerivation.cc} | 32 +++++++++---------- src/libexpr/primops/meson.build | 2 +- src/nix/baked-flake.nix | 6 ++-- 3 files changed, 20 insertions(+), 20 deletions(-) rename src/libexpr/primops/{fakeDerivation.cc => bakedDerivation.cc} (85%) diff --git a/src/libexpr/primops/fakeDerivation.cc b/src/libexpr/primops/bakedDerivation.cc similarity index 85% rename from src/libexpr/primops/fakeDerivation.cc rename to src/libexpr/primops/bakedDerivation.cc index 7be890c816..708f04388d 100644 --- a/src/libexpr/primops/fakeDerivation.cc +++ b/src/libexpr/primops/bakedDerivation.cc @@ -9,23 +9,23 @@ namespace nix { -static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** args, Value & v) +static void prim_bakedDerivation(EvalState & state, const PosIdx pos, Value ** args, Value & v) { - state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fakeDerivation"); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.bakedDerivation"); std::optional name; - struct FakeDerivationOutput + struct BakedDerivationOutput { StorePath path; std::optional narHash; }; - std::map outputs; + std::map outputs; for (auto & attr : *args[0]->attrs()) { std::string_view attrName = state.symbols[attr.name]; - auto attrHint = fmt("while evaluating the attribute '%s' passed to builtins.fakeDerivation", attrName); + auto attrHint = fmt("while evaluating the attribute '%s' passed to builtins.bakedDerivation", attrName); if (attrName == "name") { name = state.forceStringNoCtx(*attr.value, attr.pos, attrHint); @@ -39,7 +39,7 @@ static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** ar state.forceAttrs( *outAttr.value, outAttr.pos, - fmt("while evaluating the output '%s' passed to builtins.fakeDerivation", outName)); + fmt("while evaluating the output '%s' passed to builtins.bakedDerivation", outName)); std::optional path; std::optional narHash; @@ -47,7 +47,7 @@ static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** ar for (auto & outField : *outAttr.value->attrs()) { std::string_view fieldName = state.symbols[outField.name]; auto fieldHint = - fmt("while evaluating the attribute '%s' of output '%s' passed to builtins.fakeDerivation", + fmt("while evaluating the attribute '%s' of output '%s' passed to builtins.bakedDerivation", fieldName, outName); @@ -64,7 +64,7 @@ static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** ar else state .error( - "attribute '%s' isn't supported in an output passed to 'builtins.fakeDerivation'", + "attribute '%s' isn't supported in an output passed to 'builtins.bakedDerivation'", fieldName) .atPos(outField.pos) .debugThrow(); @@ -73,27 +73,27 @@ static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** ar if (!path) state .error( - "attribute 'path' is missing in output '%s' passed to 'builtins.fakeDerivation'", outName) + "attribute 'path' is missing in output '%s' passed to 'builtins.bakedDerivation'", outName) .atPos(outAttr.pos) .debugThrow(); - outputs.emplace(std::string(outName), FakeDerivationOutput{std::move(*path), std::move(narHash)}); + outputs.emplace(std::string(outName), BakedDerivationOutput{std::move(*path), std::move(narHash)}); } } else - state.error("attribute '%s' isn't supported in call to 'builtins.fakeDerivation'", attrName) + state.error("attribute '%s' isn't supported in call to 'builtins.bakedDerivation'", attrName) .atPos(attr.pos) .debugThrow(); } if (!name) - state.error("attribute 'name' is missing in call to 'builtins.fakeDerivation'") + state.error("attribute 'name' is missing in call to 'builtins.bakedDerivation'") .atPos(pos) .debugThrow(); if (outputs.empty()) - state.error("attribute 'outputs' is missing or empty in call to 'builtins.fakeDerivation'") + state.error("attribute 'outputs' is missing or empty in call to 'builtins.bakedDerivation'") .atPos(pos) .debugThrow(); @@ -148,13 +148,13 @@ static void prim_fakeDerivation(EvalState & state, const PosIdx pos, Value ** ar v.mkAttrs(result); } -static RegisterPrimOp primop_fakeDerivation({ - .name = "__fakeDerivation", +static RegisterPrimOp primop_bakedDerivation({ + .name = "__bakedDerivation", .args = {"attrs"}, .doc = R"( Placeholder. )", - .impl = prim_fakeDerivation, + .impl = prim_bakedDerivation, }); } // namespace nix diff --git a/src/libexpr/primops/meson.build b/src/libexpr/primops/meson.build index 78f40d331e..9d15b4003f 100644 --- a/src/libexpr/primops/meson.build +++ b/src/libexpr/primops/meson.build @@ -4,8 +4,8 @@ generated_headers += gen_header.process( ) sources += files( + 'bakedDerivation.cc', 'context.cc', - 'fakeDerivation.cc', 'fetchClosure.cc', 'fetchMercurial.cc', 'fetchTree.cc', diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix index f7ca16888c..1526d7ff4e 100644 --- a/src/nix/baked-flake.nix +++ b/src/nix/baked-flake.nix @@ -12,7 +12,7 @@ cleanup (builtins.mapAttrs (childName: child: convert child) output.children) else if output ? "derivation" then let - fake = builtins.fakeDerivation { + baked = builtins.bakedDerivation { name = output.derivation.name; outputs = builtins.mapAttrs (outputName: path: { inherit path; }) output.derivation.outputs; }; @@ -22,8 +22,8 @@ name = output.derivation.name; system = builtins.head output.forSystems; # FIXME meta.description = output.shortDescription; - drvPath = fake.drvPath; - outPath = fake.out; # FIXME + drvPath = baked.drvPath; + outPath = baked.out; # FIXME outputName = "out"; # FIXME } else From b1fe63f656f556458540f113b8af0713709abf03 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Sep 2026 17:24:32 +0200 Subject: [PATCH 753/927] nix flake bake: Support outputs with a derivationAttrPath Flake schemas such as nixosConfigurations declare that the derivation lives at a sub-path of the output attribute (e.g. config.system.build.toplevel). The inventory didn't record this, so the baked flake put the derivation at nixosConfigurations. instead of nixosConfigurations..config.system.build.toplevel. The inventory JSON now includes a 'derivationAttrPath' attribute on leaves with a non-empty path, and the baked flake nests the derivation accordingly. Also don't require 'shortDescription', which these schemas don't set. Assisted-by: Claude Fable 5.1 --- src/libcmd/flake-schemas.cc | 11 ++++++ src/nix/baked-flake.nix | 32 ++++++++++------ tests/functional/flakes/bake.sh | 67 +++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 11 deletions(-) diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index 1771c47ae1..81f06cb3f6 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -394,6 +394,17 @@ nlohmann::json getFlakeInventory( if (auto shortDescription = leaf.shortDescription()) obj.emplace("shortDescription", *shortDescription); + /* Record the attribute path of the derivation relative to the output attribute (e.g. + `config.system.build.toplevel` for `nixosConfigurations`), so that consumers such as `nix flake + bake` can put the derivation in the right place. Omitted if the output attribute is the + derivation itself. */ + if (auto path = leaf.derivationAttrPath(); path && !path->empty()) { + auto attrPath = nlohmann::json::array(); + for (auto & attr : *path) + attrPath.push_back(std::string(state.symbols[attr])); + obj.emplace("derivationAttrPath", std::move(attrPath)); + } + if (auto drv = leaf.derivation(outputs)) { auto drvObj = nlohmann::json::object(); diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix index 1526d7ff4e..3fd8c5aa1c 100644 --- a/src/nix/baked-flake.nix +++ b/src/nix/baked-flake.nix @@ -6,6 +6,14 @@ cleanup = builtins.filterAttrs (name: value: value != { }); + # `setAttrByPath [ "a" "b" ] x` returns `{ a.b = x; }`. + setAttrByPath = + path: value: + if path == [ ] then + value + else + { ${builtins.head path} = setAttrByPath (builtins.tail path) value; }; + convert = output: if output ? children then @@ -16,19 +24,21 @@ name = output.derivation.name; outputs = builtins.mapAttrs (outputName: path: { inherit path; }) output.derivation.outputs; }; + drv = { + type = "derivation"; + name = output.derivation.name; + system = builtins.head output.forSystems; # FIXME + meta = if output ? shortDescription then { description = output.shortDescription; } else { }; + drvPath = baked.drvPath; + outPath = baked.out; # FIXME + outputName = "out"; # FIXME + }; in - { - type = "derivation"; - name = output.derivation.name; - system = builtins.head output.forSystems; # FIXME - meta.description = output.shortDescription; - drvPath = baked.drvPath; - outPath = baked.out; # FIXME - outputName = "out"; # FIXME - } + # The derivation may live at a sub-path of the output attribute + # (e.g. `config.system.build.toplevel` for `nixosConfigurations`). + setAttrByPath (output.derivationAttrPath or [ ]) drv else - { - }; + { }; in cleanup (builtins.mapAttrs (outputName: output: convert (output.output or { })) data); } diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index 1098438743..025fb9cffb 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -91,3 +91,70 @@ nix build --substitute --substituters "file://$cacheDir" --no-require-sigs "$dep # `legacyPackages` is baked too. nix flake show --json --legacy "path:$bakedDir" > "$TEST_ROOT/show-baked-legacy.json" [[ $(jq -r ".inventory.legacyPackages.output.children.\"$system\".children.hello.derivation.name" < "$TEST_ROOT/show-baked-legacy.json") = simple ]] + +# Outputs whose derivation lives at a sub-path of the output attribute +# (`derivationAttrPath` in the flake schema, e.g. +# `nixosConfigurations..config.system.build.toplevel`) are baked +# at that sub-path, not at the output attribute itself. +configsDir=$TEST_ROOT/configs +mkdir -p "$configsDir" +cp ../simple.nix ../simple.builder.sh "${config_nix}" "$configsDir/" +cat > "$configsDir/flake.nix" < "$TEST_ROOT/show-configs.json" +[[ $(jq -r '.inventory.nixosConfigurations.output.children.foo.derivation.name' < "$TEST_ROOT/show-configs.json") = simple ]] +[[ $(jq -c '.inventory.nixosConfigurations.output.children.foo.derivationAttrPath' < "$TEST_ROOT/show-configs.json") = '["config","system","build","toplevel"]' ]] +[[ $(jq -r '.inventory.homeConfigurations.output.children.bar.derivation.name' < "$TEST_ROOT/show-configs.json") = simple ]] +[[ $(jq -c '.inventory.homeConfigurations.output.children.bar.derivationAttrPath' < "$TEST_ROOT/show-configs.json") = '["activationPackage"]' ]] + +# Outputs that are derivations themselves don't get a `derivationAttrPath`. +[[ $(jq -r ".inventory.packages.output.children.\"$system\".children.foo | has(\"derivationAttrPath\")" < "$TEST_ROOT/show-baked.json") = false ]] + +# The inventory written by `nix flake bake` records the attribute path. +nix flake bake "$configsDir" --dest-dir "$bakedDir-configs" +[[ $(jq -c '.nixosConfigurations.output.children.foo.derivationAttrPath' < "$bakedDir-configs/outputs.json") = '["config","system","build","toplevel"]' ]] +[[ $(jq -c '.homeConfigurations.output.children.bar.derivationAttrPath' < "$bakedDir-configs/outputs.json") = '["activationPackage"]' ]] + +# The baked derivations live at the nested path and have the same +# output paths as the originals... +toplevelPath=$(nix eval --raw "$configsDir#nixosConfigurations.foo.config.system.build.toplevel.outPath") +[[ $(nix eval --raw "path:$bakedDir-configs#nixosConfigurations.foo.config.system.build.toplevel.outPath") = "$toplevelPath" ]] +[[ $(nix eval --raw "path:$bakedDir-configs#nixosConfigurations.foo.config.system.build.toplevel.system") = "$system" ]] +activationPath=$(nix eval --raw "$configsDir#homeConfigurations.bar.activationPackage.outPath") +[[ $(nix eval --raw "path:$bakedDir-configs#homeConfigurations.bar.activationPackage.outPath") = "$activationPath" ]] + +# ... and not at the output attribute itself. +[[ $(nix eval "path:$bakedDir-configs#nixosConfigurations.foo" --apply 'x: x ? drvPath') = false ]] +[[ $(nix eval "path:$bakedDir-configs#homeConfigurations.bar" --apply 'x: x ? drvPath') = false ]] + +# `nix flake show` on a baked flake evaluates the flake schemas against +# the baked values. This works for `homeConfigurations`, whose schema +# only needs `activationPackage.system`. It does not currently work for +# `nixosConfigurations`, whose schema reads `pkgs.stdenv.system`, which +# the baked flake does not provide, so we don't test that here. +homeDir=$TEST_ROOT/home +mkdir -p "$homeDir" +cp ../simple.nix ../simple.builder.sh "${config_nix}" "$homeDir/" +cat > "$homeDir/flake.nix" < "$TEST_ROOT/show-baked-home.json" +[[ $(jq -r '.inventory.homeConfigurations.output.children.bar.derivation.name' < "$TEST_ROOT/show-baked-home.json") = simple ]] +[[ $(jq -c '.inventory.homeConfigurations.output.children.bar.forSystems' < "$TEST_ROOT/show-baked-home.json") = "[\"$system\"]" ]] +[[ $(jq -c '.inventory.homeConfigurations.output.children.bar.derivationAttrPath' < "$TEST_ROOT/show-baked-home.json") = '["activationPackage"]' ]] From 81d0820cf10f2a2fb4ef6d5dbeb81c524923f65a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:10:11 +0000 Subject: [PATCH 754/927] Prepare release v3.22.4 From 167040cf266b5123f64c7956dad4aad43299154e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:10:14 +0000 Subject: [PATCH 755/927] packaging/secure-packages/flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'path:../..' → 'path:../..' • Updated input 'nix/nixpkgs': 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013551%2Brev-07b3c48788b0deb9ee48c40ed5ff1dff7f4643e7/01a08dbc-1a6d-7e60-8dd0-bc2bf8ecb4fb/source.tar.gz' (2026-09-10) → 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013583%2Brev-f041631d1ed8316e18423533f65925c81c3abce7/01a0a705-a838-70e1-90f7-6126cdd43de6/source.tar.gz' (2026-09-15) --- packaging/secure-packages/flake.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index e420ac4ef9..50a26d9036 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -77,12 +77,12 @@ }, "nixpkgs": { "locked": { - "lastModified": 1789082161, - "narHash": "sha256-E9iYEGs3Jx9xmYsP4nbVAK+7TXfMjGjGAazuyYLpivU=", - "rev": "07b3c48788b0deb9ee48c40ed5ff1dff7f4643e7", - "revCount": 1013551, + "lastModified": 1789504077, + "narHash": "sha256-IL+ZRmtgTJIMd2lpYVN/W3MmkfMQU3n1CRtDuHADzNI=", + "rev": "f041631d1ed8316e18423533f65925c81c3abce7", + "revCount": 1013583, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013551%2Brev-07b3c48788b0deb9ee48c40ed5ff1dff7f4643e7/01a08dbc-1a6d-7e60-8dd0-bc2bf8ecb4fb/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013583%2Brev-f041631d1ed8316e18423533f65925c81c3abce7/01a0a705-a838-70e1-90f7-6126cdd43de6/source.tar.gz" }, "original": { "type": "tarball", From 7e709330ea939ebe512a9ef1653751203bc84964 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:10:17 +0000 Subject: [PATCH 756/927] Set .version-determinate to 3.22.4 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index ce070a60eb..00f104afa6 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.22.3 +3.22.4 From ed26b9bfdff5044e847d4ac175a6ad6b3afa7326 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:10:22 +0000 Subject: [PATCH 757/927] Generate release notes for 3.22.4 --- doc/manual/source/SUMMARY.md.in | 1 + .../source/release-notes-determinate/changes.md | 17 ++++++++++++++++- .../source/release-notes-determinate/v3.22.4.md | 15 +++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/v3.22.4.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 308569fb90..c6ae4e2906 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -154,6 +154,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.22.4 (2026-09-16)](release-notes-determinate/v3.22.4.md) - [Release 3.22.3 (2026-09-02)](release-notes-determinate/v3.22.3.md) - [Release 3.22.2 (2026-08-21)](release-notes-determinate/v3.22.2.md) - [Release 3.22.1 (2026-08-18)](release-notes-determinate/v3.22.1.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 27aa8517a5..e85c661c07 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.35 and Determinate Nix 3.22.3. +This section lists the differences between upstream Nix 2.35 and Determinate Nix 3.22.4. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -234,3 +234,18 @@ This section lists the differences between upstream Nix 2.35 and Determinate Nix * `nix eval --json` has a flag `--drv-link` to create symlinks to top-level derivations. This prevents them from being garbage-collected. [DeterminateSystems/nix-src#599](https://github.com/DeterminateSystems/nix-src/pull/599) + + + +* tests/functional: skip the sentry test when Yama restricts ptrace by @Reasonable-Solutions in [DeterminateSystems/nix-src#625](https://github.com/DeterminateSystems/nix-src/pull/625) + +* Tarball cache performance improvements by @edolstra in [DeterminateSystems/nix-src#627](https://github.com/DeterminateSystems/nix-src/pull/627) + +* fetchers/git: negotiate rev-pinned fetches from bucketed refs by @edolstra in [DeterminateSystems/nix-src#628](https://github.com/DeterminateSystems/nix-src/pull/628) + +* Fix the Nix < 2.20 NAR hash fallback for inputs with submodules by @joshuaspence in [DeterminateSystems/nix-src#624](https://github.com/DeterminateSystems/nix-src/pull/624) + + +* @Reasonable-Solutions made their first contribution in [DeterminateSystems/nix-src#625](https://github.com/DeterminateSystems/nix-src/pull/625) + +* @joshuaspence made their first contribution in [DeterminateSystems/nix-src#624](https://github.com/DeterminateSystems/nix-src/pull/624) diff --git a/doc/manual/source/release-notes-determinate/v3.22.4.md b/doc/manual/source/release-notes-determinate/v3.22.4.md new file mode 100644 index 0000000000..dc3759ae1c --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.22.4.md @@ -0,0 +1,15 @@ +# Release 3.22.4 (2026-09-16) + +* Based on [upstream Nix 2.35.2](../release-notes/rl-2.35.md). + +## What's Changed +* tests/functional: skip the sentry test when Yama restricts ptrace by @Reasonable-Solutions in [DeterminateSystems/nix-src#625](https://github.com/DeterminateSystems/nix-src/pull/625) +* Tarball cache performance improvements by @edolstra in [DeterminateSystems/nix-src#627](https://github.com/DeterminateSystems/nix-src/pull/627) +* fetchers/git: negotiate rev-pinned fetches from bucketed refs by @edolstra in [DeterminateSystems/nix-src#628](https://github.com/DeterminateSystems/nix-src/pull/628) +* Fix the Nix < 2.20 NAR hash fallback for inputs with submodules by @joshuaspence in [DeterminateSystems/nix-src#624](https://github.com/DeterminateSystems/nix-src/pull/624) + +## New Contributors +* @Reasonable-Solutions made their first contribution in [DeterminateSystems/nix-src#625](https://github.com/DeterminateSystems/nix-src/pull/625) +* @joshuaspence made their first contribution in [DeterminateSystems/nix-src#624](https://github.com/DeterminateSystems/nix-src/pull/624) + +**Full Changelog**: [v3.22.3...v3.22.4](https://github.com/DeterminateSystems/nix-src/compare/v3.22.3...v3.22.4) From a90b4a992d8d29feffe2879b89ff0b568eb56b25 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Sep 2026 17:12:29 +0200 Subject: [PATCH 758/927] tests/functional/flakes/bake.sh: Skip on NixOS The test substitutes from a file:// cache using --no-require-sigs, which the daemon ignores for untrusted users such as the one used by the functional_user NixOS test. Assisted-by: Claude Fable 5.1 --- tests/functional/flakes/bake.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index 025fb9cffb..b82656dd4c 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -2,6 +2,10 @@ source ./common.sh +# Substituting from a file:// cache with --no-require-sigs requires a +# trusted user, which the NixOS tests don't provide. +TODO_NixOS + flakeDir=$TEST_ROOT/flake bakedDir=$TEST_ROOT/baked cacheDir=$TEST_ROOT/binary-cache From 605b312c4bb07ec1cd236d8be9905a7f5b91f407 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Sep 2026 18:43:02 +0200 Subject: [PATCH 759/927] Tweak release notes --- .../source/release-notes-determinate/changes.md | 13 ------------- .../source/release-notes-determinate/v3.22.4.md | 9 ++++++--- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index e85c661c07..d577990594 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -236,16 +236,3 @@ This section lists the differences between upstream Nix 2.35 and Determinate Nix - -* tests/functional: skip the sentry test when Yama restricts ptrace by @Reasonable-Solutions in [DeterminateSystems/nix-src#625](https://github.com/DeterminateSystems/nix-src/pull/625) - -* Tarball cache performance improvements by @edolstra in [DeterminateSystems/nix-src#627](https://github.com/DeterminateSystems/nix-src/pull/627) - -* fetchers/git: negotiate rev-pinned fetches from bucketed refs by @edolstra in [DeterminateSystems/nix-src#628](https://github.com/DeterminateSystems/nix-src/pull/628) - -* Fix the Nix < 2.20 NAR hash fallback for inputs with submodules by @joshuaspence in [DeterminateSystems/nix-src#624](https://github.com/DeterminateSystems/nix-src/pull/624) - - -* @Reasonable-Solutions made their first contribution in [DeterminateSystems/nix-src#625](https://github.com/DeterminateSystems/nix-src/pull/625) - -* @joshuaspence made their first contribution in [DeterminateSystems/nix-src#624](https://github.com/DeterminateSystems/nix-src/pull/624) diff --git a/doc/manual/source/release-notes-determinate/v3.22.4.md b/doc/manual/source/release-notes-determinate/v3.22.4.md index dc3759ae1c..61fe55400a 100644 --- a/doc/manual/source/release-notes-determinate/v3.22.4.md +++ b/doc/manual/source/release-notes-determinate/v3.22.4.md @@ -2,10 +2,13 @@ * Based on [upstream Nix 2.35.2](../release-notes/rl-2.35.md). -## What's Changed +## Performance improvements + +* Tarball cache performance improvements by @xokdvium in [DeterminateSystems/nix-src#627](https://github.com/DeterminateSystems/nix-src/pull/627) + +## Bug fixes * tests/functional: skip the sentry test when Yama restricts ptrace by @Reasonable-Solutions in [DeterminateSystems/nix-src#625](https://github.com/DeterminateSystems/nix-src/pull/625) -* Tarball cache performance improvements by @edolstra in [DeterminateSystems/nix-src#627](https://github.com/DeterminateSystems/nix-src/pull/627) -* fetchers/git: negotiate rev-pinned fetches from bucketed refs by @edolstra in [DeterminateSystems/nix-src#628](https://github.com/DeterminateSystems/nix-src/pull/628) +* fetchers/git: negotiate rev-pinned fetches from bucketed refs by @tomberek in [DeterminateSystems/nix-src#628](https://github.com/DeterminateSystems/nix-src/pull/628) * Fix the Nix < 2.20 NAR hash fallback for inputs with submodules by @joshuaspence in [DeterminateSystems/nix-src#624](https://github.com/DeterminateSystems/nix-src/pull/624) ## New Contributors From 25bbfd2aefa615ce27e3e26475fe3727be33c1e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Sep 2026 19:16:59 +0200 Subject: [PATCH 760/927] nix flake bake: Preserve meta.mainProgram The inventory now records 'meta.mainProgram' of derivations (when set), and the baked flake passes it through, so that 'nix run' works on baked packages whose binary isn't named after the package. This and 'derivationAttrPath' are only emitted when the new FlakeInventoryOptions::bake option is set, so the output of 'nix flake show --json' is unchanged. Assisted-by: Claude Fable 5.1 --- src/libcmd/flake-schemas.cc | 24 ++++++++---- src/libcmd/include/nix/cmd/flake-schemas.hh | 5 +++ src/nix/baked-flake.nix | 6 ++- src/nix/flake-bake.cc | 1 + tests/functional/flakes/bake.sh | 41 +++++++++++++++++---- 5 files changed, 60 insertions(+), 17 deletions(-) diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index 81f06cb3f6..2f3d63a99b 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -395,14 +395,15 @@ nlohmann::json getFlakeInventory( obj.emplace("shortDescription", *shortDescription); /* Record the attribute path of the derivation relative to the output attribute (e.g. - `config.system.build.toplevel` for `nixosConfigurations`), so that consumers such as `nix flake - bake` can put the derivation in the right place. Omitted if the output attribute is the - derivation itself. */ - if (auto path = leaf.derivationAttrPath(); path && !path->empty()) { - auto attrPath = nlohmann::json::array(); - for (auto & attr : *path) - attrPath.push_back(std::string(state.symbols[attr])); - obj.emplace("derivationAttrPath", std::move(attrPath)); + `config.system.build.toplevel` for `nixosConfigurations`), so that the baked flake can put the + derivation in the right place. Omitted if the output attribute is the derivation itself. */ + if (options.bake) { + if (auto path = leaf.derivationAttrPath(); path && !path->empty()) { + auto attrPath = nlohmann::json::array(); + for (auto & attr : *path) + attrPath.push_back(std::string(state.symbols[attr])); + obj.emplace("derivationAttrPath", std::move(attrPath)); + } } if (auto drv = leaf.derivation(outputs)) { @@ -411,6 +412,13 @@ nlohmann::json getFlakeInventory( if (options.showDrvNames) drvObj.emplace("name", drv->getAttr(state.s.name)->getString()); + if (options.bake) { + /* Record `meta.mainProgram` so that the baked flake can preserve it for `nix run`. */ + if (auto aMeta = drv->maybeGetAttr(state.s.meta)) + if (auto aMainProgram = aMeta->maybeGetAttr("mainProgram")) + drvObj.emplace("mainProgram", aMainProgram->getString()); + } + if (options.showDrvPaths) { auto drvPath = drv->forceDerivation(); drvObj.emplace("path", state.store->printStorePath(drvPath)); diff --git a/src/libcmd/include/nix/cmd/flake-schemas.hh b/src/libcmd/include/nix/cmd/flake-schemas.hh index 2084ab5036..0db8f96d54 100644 --- a/src/libcmd/include/nix/cmd/flake-schemas.hh +++ b/src/libcmd/include/nix/cmd/flake-schemas.hh @@ -104,6 +104,11 @@ struct FlakeInventoryOptions bool showOutputPaths = false; bool showDrvPaths = false; bool showDrvNames = false; + /** + * Include additional information needed by `nix flake bake` + * (e.g. `derivationAttrPath` and `mainProgram`). + */ + bool bake = false; }; nlohmann::json getFlakeInventory( diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix index 3fd8c5aa1c..e2b2ea493f 100644 --- a/src/nix/baked-flake.nix +++ b/src/nix/baked-flake.nix @@ -28,7 +28,11 @@ type = "derivation"; name = output.derivation.name; system = builtins.head output.forSystems; # FIXME - meta = if output ? shortDescription then { description = output.shortDescription; } else { }; + meta = + (if output ? shortDescription then { description = output.shortDescription; } else { }) + // ( + if output.derivation ? mainProgram then { mainProgram = output.derivation.mainProgram; } else { } + ); drvPath = baked.drvPath; outPath = baked.out; # FIXME outputName = "out"; # FIXME diff --git a/src/nix/flake-bake.cc b/src/nix/flake-bake.cc index deb92405e5..78c67a49b3 100644 --- a/src/nix/flake-bake.cc +++ b/src/nix/flake-bake.cc @@ -15,6 +15,7 @@ struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas .showLegacy = true, .showOutputPaths = true, .showDrvNames = true, + .bake = true, }; CmdFlakeBake() diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index b82656dd4c..0e16b1c8d1 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -96,6 +96,36 @@ nix build --substitute --substituters "file://$cacheDir" --no-require-sigs "$dep nix flake show --json --legacy "path:$bakedDir" > "$TEST_ROOT/show-baked-legacy.json" [[ $(jq -r ".inventory.legacyPackages.output.children.\"$system\".children.hello.derivation.name" < "$TEST_ROOT/show-baked-legacy.json") = simple ]] +# `meta.mainProgram` is preserved, so `nix run` works on baked packages +# whose binary isn't named after the package. +greeterDir=$TEST_ROOT/greeter +mkdir -p "$greeterDir" +cp "${config_nix}" "$greeterDir/" +cat > "$greeterDir/flake.nix" < \$out/bin/hi + echo 'echo hello from hi' >> \$out/bin/hi + chmod +x \$out/bin/hi + ''; + }; + }; +} +EOF + +nix flake bake "$greeterDir" --dest-dir "$bakedDir-greeter" +[[ $(jq -r ".packages.output.children.\"$system\".children.greeter.derivation.mainProgram" < "$bakedDir-greeter/outputs.json") = hi ]] +[[ $(nix eval --raw "path:$bakedDir-greeter#greeter.meta.mainProgram") = hi ]] + +# Build the original so that the baked package's output is valid, then run it. +nix build --no-link "$greeterDir#greeter" +[[ $(nix run "path:$bakedDir-greeter#greeter") = "hello from hi" ]] + # Outputs whose derivation lives at a sub-path of the output attribute # (`derivationAttrPath` in the flake schema, e.g. # `nixosConfigurations..config.system.build.toplevel`) are baked @@ -115,18 +145,14 @@ cat > "$configsDir/flake.nix" < "$TEST_ROOT/show-configs.json" [[ $(jq -r '.inventory.nixosConfigurations.output.children.foo.derivation.name' < "$TEST_ROOT/show-configs.json") = simple ]] -[[ $(jq -c '.inventory.nixosConfigurations.output.children.foo.derivationAttrPath' < "$TEST_ROOT/show-configs.json") = '["config","system","build","toplevel"]' ]] [[ $(jq -r '.inventory.homeConfigurations.output.children.bar.derivation.name' < "$TEST_ROOT/show-configs.json") = simple ]] -[[ $(jq -c '.inventory.homeConfigurations.output.children.bar.derivationAttrPath' < "$TEST_ROOT/show-configs.json") = '["activationPackage"]' ]] - -# Outputs that are derivations themselves don't get a `derivationAttrPath`. -[[ $(jq -r ".inventory.packages.output.children.\"$system\".children.foo | has(\"derivationAttrPath\")" < "$TEST_ROOT/show-baked.json") = false ]] -# The inventory written by `nix flake bake` records the attribute path. +# The inventory written by `nix flake bake` records the derivation attribute path. nix flake bake "$configsDir" --dest-dir "$bakedDir-configs" +[[ $(jq -r '.nixosConfigurations.output.children.foo.derivation.name' < "$bakedDir-configs/outputs.json") = simple ]] [[ $(jq -c '.nixosConfigurations.output.children.foo.derivationAttrPath' < "$bakedDir-configs/outputs.json") = '["config","system","build","toplevel"]' ]] [[ $(jq -c '.homeConfigurations.output.children.bar.derivationAttrPath' < "$bakedDir-configs/outputs.json") = '["activationPackage"]' ]] @@ -161,4 +187,3 @@ nix flake bake "$homeDir" --dest-dir "$bakedDir-home" nix flake show --json "path:$bakedDir-home" > "$TEST_ROOT/show-baked-home.json" [[ $(jq -r '.inventory.homeConfigurations.output.children.bar.derivation.name' < "$TEST_ROOT/show-baked-home.json") = simple ]] [[ $(jq -c '.inventory.homeConfigurations.output.children.bar.forSystems' < "$TEST_ROOT/show-baked-home.json") = "[\"$system\"]" ]] -[[ $(jq -c '.inventory.homeConfigurations.output.children.bar.derivationAttrPath' < "$TEST_ROOT/show-baked-home.json") = '["activationPackage"]' ]] From 30711c1a5bbb21d0e2dfc51ae7389d96cf9ca232 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Sep 2026 19:56:51 +0200 Subject: [PATCH 761/927] DerivationTrampolineGoal: Report built outputs for builtin:substitute The short-circuit for builtin:substitute derivations returned a success result with an empty 'builtOutputs'. As a result, 'nix build' didn't create result symlinks for baked derivations, and 'nix build --json' reported no outputs for them. Assisted-by: Claude Fable 5.1 --- .../build/derivation-trampoline-goal.cc | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index 8f07afe785..f676afcad4 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -153,7 +153,14 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation /* Short-circuit `builtin:substitute`. Since these are never actually built, just create substitution goals for the * outputs. */ if (drv.type() == DerivationType::Substituted{}) { - std::vector> outputGoals; + struct OutputGoal + { + OutputName outputName; + StorePath outPath; + GoalPtr goal; + }; + + std::vector outputGoals; Goals waitees; for (auto & outputName : resolvedWantedOutputs) { auto i = drv.outputs.find(outputName); @@ -169,7 +176,7 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation outputName, worker.store.printStorePath(drvPath)); auto g = upcast_goal(worker.makePathSubstitutionGoal(*outPath)); - outputGoals.emplace_back(*outPath, g); + outputGoals.push_back({outputName, *outPath, g}); waitees.insert(g); } @@ -177,9 +184,9 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation if (nrFailed != 0) { StringSet failedPaths; - for (auto & [path, g] : outputGoals) - if (g->exitCode != ecSuccess) - failedPaths.insert(worker.store.printStorePath(path)); + for (auto & og : outputGoals) + if (og.goal->exitCode != ecSuccess) + failedPaths.insert(worker.store.printStorePath(og.outPath)); co_return doneFailure( ecFailed, BuildResult::Failure{{ @@ -191,9 +198,14 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation }}); } + SingleDrvOutputs builtOutputs; + for (auto & og : outputGoals) + builtOutputs.emplace(og.outputName, UnkeyedRealisation{.outPath = og.outPath}); + co_return doneSuccess( BuildResult::Success{ .status = BuildResult::Success::Substituted, + .builtOutputs = std::move(builtOutputs), }); } From de72dca3fd0ab26b4902aaac36032b23c190afa5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Sep 2026 19:56:51 +0200 Subject: [PATCH 762/927] nix flake bake: Don't assume the default output is 'out' The inventory now records the output name that each attribute refers to, and the baked flake constructs derivation attribute sets the same way as 'derivation' does: with 'outputs', 'all', and one attribute per output. This fixes derivations whose default output isn't 'out' and attributes that refer to a specific output (e.g. 'foo.dev'). Assisted-by: Claude Fable 5.1 --- src/libcmd/flake-schemas.cc | 4 ++++ src/nix/baked-flake.nix | 28 ++++++++++++++++++++++++---- tests/functional/flakes/bake.sh | 29 +++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index 2f3d63a99b..361d47266d 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -417,6 +417,10 @@ nlohmann::json getFlakeInventory( if (auto aMeta = drv->maybeGetAttr(state.s.meta)) if (auto aMainProgram = aMeta->maybeGetAttr("mainProgram")) drvObj.emplace("mainProgram", aMainProgram->getString()); + + /* Record which output this attribute refers to, since it's not necessarily `out`. */ + if (auto aOutputName = drv->maybeGetAttr(state.s.outputName)) + drvObj.emplace("outputName", aOutputName->getString()); } if (options.showDrvPaths) { diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix index e2b2ea493f..16f0d0fc50 100644 --- a/src/nix/baked-flake.nix +++ b/src/nix/baked-flake.nix @@ -24,7 +24,14 @@ name = output.derivation.name; outputs = builtins.mapAttrs (outputName: path: { inherit path; }) output.derivation.outputs; }; - drv = { + outputNames = builtins.attrNames output.derivation.outputs; + + # The output that this attribute refers to. + outputName = output.derivation.outputName or (builtins.head outputNames); + + # Mirror the attribute set produced by `derivation`: the common + # attributes, plus one attribute per output. + commonAttrs = { type = "derivation"; name = output.derivation.name; system = builtins.head output.forSystems; # FIXME @@ -34,9 +41,22 @@ if output.derivation ? mainProgram then { mainProgram = output.derivation.mainProgram; } else { } ); drvPath = baked.drvPath; - outPath = baked.out; # FIXME - outputName = "out"; # FIXME - }; + outputs = outputNames; + all = map (outputName: outputAttrs.${outputName}) outputNames; + } + // outputAttrs; + + outputAttrs = builtins.listToAttrs ( + map (outputName: { + name = outputName; + value = commonAttrs // { + inherit outputName; + outPath = baked.${outputName}; + }; + }) outputNames + ); + + drv = outputAttrs.${outputName}; in # The derivation may live at a sub-path of the output attribute # (e.g. `config.system.build.toplevel` for `nixosConfigurations`). diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index 0e16b1c8d1..dfd26597f0 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -114,6 +114,12 @@ cat > "$greeterDir/flake.nix" <.config.system.build.toplevel`) are baked From e90ba1bdd564cc6375d55ecd43ea65f8ea0a6505 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Sep 2026 20:06:03 +0200 Subject: [PATCH 763/927] builtins.bakedDerivation: Don't wait for the derivation to be written Consumers that need the .drv file (e.g. the eval cache's forceDerivation()) already wait for it, as with derivationStrict. Assisted-by: Claude Fable 5.1 --- src/libexpr/primops/bakedDerivation.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libexpr/primops/bakedDerivation.cc b/src/libexpr/primops/bakedDerivation.cc index 708f04388d..0cc8675e09 100644 --- a/src/libexpr/primops/bakedDerivation.cc +++ b/src/libexpr/primops/bakedDerivation.cc @@ -119,9 +119,6 @@ static void prim_bakedDerivation(EvalState & state, const PosIdx pos, Value ** a auto drvPath = state.store->writeDerivation(*state.asyncPathWriter, drv, state.repair); - // FIXME - state.waitForPath(drvPath); - /* As in `derivationStrict`, cache the derivation hash so that derivations depending on this one don't need to read it back from the store. */ drvHashes.insert_or_assign(drvPath, hashDerivationModulo(*state.store, drv, false)); From 17347ee70081f94bc405d4544abfa2344efc1ad2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Sep 2026 20:11:30 +0200 Subject: [PATCH 764/927] builtins.bakedDerivation: Document Assisted-by: Claude Fable 5.1 --- src/libexpr/primops/bakedDerivation.cc | 46 +++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/bakedDerivation.cc b/src/libexpr/primops/bakedDerivation.cc index 0cc8675e09..154070d276 100644 --- a/src/libexpr/primops/bakedDerivation.cc +++ b/src/libexpr/primops/bakedDerivation.cc @@ -149,7 +149,51 @@ static RegisterPrimOp primop_bakedDerivation({ .name = "__bakedDerivation", .args = {"attrs"}, .doc = R"( - Placeholder. + Create a *baked derivation*: a derivation that is never built, but + whose outputs are obtained by substitution from a binary cache. + This is what [`nix flake bake`](@docroot@/command-ref/new-cli/nix3-flake-bake.md) + uses to represent the pre-evaluated outputs of a flake. + + The argument is an attribute set with the following attributes: + + - `name`: The name of the derivation. + + - `outputs`: An attribute set mapping output names to attribute + sets with a single attribute `path`, the store path of that + output. + + The result has the same shape as the result of + `builtins.derivationStrict`: an attribute set containing `drvPath` + (the store path of the derivation) and one attribute per output + containing that output's store path. These strings carry + [string context](@docroot@/language/string-context.md), so the + outputs of a baked derivation can be used as inputs of other + derivations. + + The resulting derivation uses the builder `builtin:substitute` + and has no inputs. Realising it does not run anything; instead, + Nix substitutes the given output paths. If they cannot be + substituted, the build fails. + + Example: + + ```nix + builtins.bakedDerivation { + name = "hello-2.12.1"; + outputs = { + out.path = "/nix/store/1q8w6grhdj6pn0cvlw18cw07nvjb7pj5-hello-2.12.1"; + }; + } + ``` + + evaluates to + + ```nix + { + drvPath = "/nix/store/1jczli5n8zgxl2vgfsc5vy3z0f5lfn3q-hello-2.12.1.drv"; + out = "/nix/store/1q8w6grhdj6pn0cvlw18cw07nvjb7pj5-hello-2.12.1"; + } + ``` )", .impl = prim_bakedDerivation, }); From 1a9c127b2e6f2705ef4c8928989bc25cf0ba819d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Sep 2026 21:46:43 +0200 Subject: [PATCH 765/927] Add 'baked-derivation' experimental feature It gates 'builtin:substitute' derivations, builtins.bakedDerivation and 'nix flake bake'. Note that 'nix flake bake' has to call require() explicitly, since only the top-level command's experimentalFeature() is checked. Assisted-by: Claude Fable 5.1 --- src/libexpr/primops/bakedDerivation.cc | 1 + src/libstore/derivations.cc | 1 + src/libutil/experimental-features.cc | 13 ++++++++++++- .../include/nix/util/experimental-features.hh | 1 + src/nix/flake-bake.cc | 7 +++++++ tests/functional/flakes/bake.sh | 4 ++++ 6 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/bakedDerivation.cc b/src/libexpr/primops/bakedDerivation.cc index 154070d276..f472f1ed34 100644 --- a/src/libexpr/primops/bakedDerivation.cc +++ b/src/libexpr/primops/bakedDerivation.cc @@ -196,6 +196,7 @@ static RegisterPrimOp primop_bakedDerivation({ ``` )", .impl = prim_bakedDerivation, + .experimentalFeature = Xp::BakedDerivation, }); } // namespace nix diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 000fdae0e8..17b33394dc 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -878,6 +878,7 @@ DerivationType BasicDerivation::type() const throw Error("must have at least one output"); if (builder == "builtin:substitute") { + experimentalFeatureSettings.require(Xp::BakedDerivation, "'builtin:substitute' derivation"); if (!std::holds_alternative(ty.value().raw)) throw Error("'builtin:substitute' derivation must have input-addressed outputs"); return DerivationType::Substituted{}; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 674deec7f8..3f3b31d690 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -27,7 +27,7 @@ void MissingExperimentalFeature::anchor() {} * feature, we either have no issue at all if few features are not added * at the end of the list, or a proper merge conflict if they are. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::CNSA); +constexpr size_t numXpFeatures = 1 + static_cast(Xp::BakedDerivation); constexpr std::array xpFeatureDetails = {{ { @@ -317,6 +317,17 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "", }, + { + .tag = Xp::BakedDerivation, + .name = "baked-derivation", + .description = R"( + Enable *baked derivations*: derivations with the builder + `builtin:substitute` that are never built, but whose outputs are + obtained by substitution. This enables the `builtins.bakedDerivation` + function and the `nix flake bake` command. + )", + .trackingUrl = "", + }, }}; static_assert( diff --git a/src/libutil/include/nix/util/experimental-features.hh b/src/libutil/include/nix/util/experimental-features.hh index f20024f038..7c2f922bac 100644 --- a/src/libutil/include/nix/util/experimental-features.hh +++ b/src/libutil/include/nix/util/experimental-features.hh @@ -42,6 +42,7 @@ enum struct ExperimentalFeature { WasmDerivations, Provenance, CNSA, + BakedDerivation, }; extern std::set stabilizedFeatures; diff --git a/src/nix/flake-bake.cc b/src/nix/flake-bake.cc index 78c67a49b3..83c1988a6d 100644 --- a/src/nix/flake-bake.cc +++ b/src/nix/flake-bake.cc @@ -40,6 +40,11 @@ struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas return "bake a flake"; } + std::optional experimentalFeature() override + { + return Xp::BakedDerivation; + } + std::string doc() override { return @@ -49,6 +54,8 @@ struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas void run(nix::ref store) override { + experimentalFeatureSettings.require(Xp::BakedDerivation); + auto state = getEvalState(); auto evalStore = getEvalStore(); auto flake = make_ref(lockFlake()); diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index dfd26597f0..dc2e6702d7 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -13,6 +13,10 @@ cacheDir=$TEST_ROOT/binary-cache mkdir -p "$flakeDir" writeSimpleFlake "$flakeDir" +# Baking requires the `baked-derivation` experimental feature. +expectStderr 1 nix flake bake "$flakeDir" --dest-dir "$bakedDir" | grepQuiet "experimental Nix feature 'baked-derivation' is disabled" +enableFeatures "baked-derivation" + # Bake the flake. nix flake bake "$flakeDir" --dest-dir "$bakedDir" [[ -e $bakedDir/flake.nix ]] From dd991ef4e1a1e22d9fb9f9567f2f39468007adae Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Wed, 4 Mar 2026 17:05:41 +0300 Subject: [PATCH 766/927] libutil/serialise: Use zero-copy copy_file_range for copying PosixFileSourceAccessor -> FdSink On modern filesystems like ZFS/BTRFS copy_file_range can do use block cloning for copying file contents. In the future we could use something similar via sendfile for sending data to a socket from a file descriptor (or even send the descriptor via SCM_RIGHTS). This is only beneficial for copying files wholesale (like we do with NAR unpacking). This cuts down the overhead of copying FOD outputs (for breaking file handles) significantly. These are numbers from building with store on ZFS of a simple FOD with a large output: (After) 79.03 1.752192 9 183511 pread64 8.42 0.186688 186688 1 copy_file_range 8.40 0.186341 46585 4 wait4 1.06 0.023536 7 3092 getdents64 User time (seconds): 4.69 System time (seconds): 3.80 Percent of CPU this job got: 63% Elapsed (wall clock) time (h:mm:ss or m:ss): 0:13.32 (Before) 74.80 3.831072 15 244547 pread64 20.07 1.027953 16 61730 write 3.16 0.161804 40451 4 wait4 0.59 0.030202 9 3092 getdents64 User time (seconds): 4.71 System time (seconds): 7.27 Percent of CPU this job got: 82% Elapsed (wall clock) time (h:mm:ss or m:ss): 0:14.47 Also adds tests for copyFdRange and fixes some windows specific bugs. --- src/libstore/unix/build/derivation-builder.cc | 6 +- src/libutil-tests/file-descriptor.cc | 123 ++++++++++++++++++ src/libutil/file-descriptor-private.hh | 23 ++++ src/libutil/file-descriptor.cc | 26 +++- src/libutil/fs-sink.cc | 1 + .../include/nix/util/file-descriptor.hh | 6 +- src/libutil/include/nix/util/serialise.hh | 14 ++ src/libutil/posix-source-accessor.cc | 2 +- src/libutil/unix/file-descriptor.cc | 70 ++++++++++ src/libutil/unix/meson.build | 17 +++ src/libutil/windows/file-descriptor.cc | 24 +++- 11 files changed, 299 insertions(+), 13 deletions(-) create mode 100644 src/libutil/file-descriptor-private.hh diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index ff3c382d03..4dc6dc4cbb 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -1467,10 +1467,8 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() auto tmpOutput = tempDir / "x"; - /* Serialise and create a fresh copy of the output to break - any stale writable file descriptors. Copy through the - serialisation/deserialisation. TODO: Use copyRecursive here and - make use of reflinking. */ + /* Copy files to break stale file descriptors. copyRecursive below will use + reflinking to optimise the copying overhead. */ auto pathAccessor = makeFSSourceAccessor(actualPath); RestoreSink restoreSink{store.config->getLocalSettings().fsyncStorePaths}; restoreSink.dstPath = tmpOutput; diff --git a/src/libutil-tests/file-descriptor.cc b/src/libutil-tests/file-descriptor.cc index c385d2960f..b6b13fd9ab 100644 --- a/src/libutil-tests/file-descriptor.cc +++ b/src/libutil-tests/file-descriptor.cc @@ -2,10 +2,14 @@ #include #include "nix/util/file-descriptor.hh" +#include "nix/util/file-system.hh" #include "nix/util/serialise.hh" #include "nix/util/signals.hh" #include +#include +#include +#include #ifndef _WIN32 # include @@ -322,4 +326,123 @@ TEST(WriteFull, RespectsAllowInterrupts) EXPECT_EQ(source.readLine(/*eofOk=*/true), "hello"); } +class CopyFdRangeTest : public ::testing::TestWithParam> +{ +protected: + static std::string makeData() + { + std::mt19937 rng{42}; + std::string data(192 * 1024 + 1024, '\0'); + std::ranges::generate(data, rng); + return data; + } + + bool tryCoW() + { + return std::get<0>(GetParam()); + } + + bool isRegularFile() + { + return std::get<1>(GetParam()); + } +}; + +TEST_P(CopyFdRangeTest, regularToRegular) +{ + auto source = createAnonymousTempFile(); + auto dest = createAnonymousTempFile(); + auto input = makeData(); + writeFull(source.get(), input); + + FdSink sink{dest.get()}; + sink.isRegularFile = isRegularFile(); + copyFdRange(source.get(), 0, input.size(), sink, tryCoW()); + sink.flush(); + ASSERT_EQ(sink.written, input.size()); + ASSERT_EQ(lseek(dest.get(), 0, SEEK_SET), 0); + ASSERT_EQ(readFile(dest.get()), input); +} + +TEST_P(CopyFdRangeTest, offsetWorks) +{ + auto source = createAnonymousTempFile(); + auto dest = createAnonymousTempFile(); + auto input = makeData(); + writeFull(source.get(), input); + + static constexpr off_t offset = 4096; + FdSink sink{dest.get()}; + sink.isRegularFile = isRegularFile(); + copyFdRange(source.get(), offset, input.size() - offset, sink, tryCoW()); + sink.flush(); + ASSERT_EQ(sink.written, input.size() - offset); + ASSERT_EQ(lseek(dest.get(), 0, SEEK_SET), 0); + ASSERT_EQ(readFile(dest.get()), input.substr(offset)); +} + +TEST_P(CopyFdRangeTest, endOfFile) +{ + auto source = createAnonymousTempFile(); + Descriptor dest; + AutoCloseFD dest_ = createAnonymousTempFile(); + Pipe pipe; + if (isRegularFile()) { + dest = dest_.get(); + } else { + pipe.create(); + dest = pipe.writeSide.get(); + } + FdSink sink(dest); + sink.isRegularFile = isRegularFile(); + /* This can't ever deadlock, because nothing should be written to the pipe anyway. */ + EXPECT_THROW(copyFdRange(source.get(), 0, 512 * 1024, sink, tryCoW()), EndOfFile); +} + +TEST_P(CopyFdRangeTest, toPipe) +{ + auto input = makeData(); + auto source = createAnonymousTempFile(); + writeFull(source.get(), input); + + Pipe pipe; + pipe.create(); + + std::string output; + std::thread reader([&] { output = drainFD(pipe.readSide.get()); }); + + FdSink sink{pipe.writeSide.get()}; + /* This is only a hint and isn't load-bearing. */ + sink.isRegularFile = isRegularFile(); + copyFdRange(source.get(), 0, input.size(), sink, tryCoW()); + sink.flush(); + pipe.writeSide.close(); + reader.join(); + ASSERT_EQ(sink.written, input.size()); + ASSERT_EQ(output, input); +} + +TEST_P(CopyFdRangeTest, nonSeekableInput) +{ + Pipe in; + in.create(); + auto dest = createAnonymousTempFile(); + FdSink sink{dest.get()}; + sink.isRegularFile = isRegularFile(); + // TODO: Test the error code accurately. + EXPECT_THROW(copyFdRange(in.readSide.get(), 0, 128 * 1024, sink, tryCoW()), SystemError); +} + +INSTANTIATE_TEST_SUITE_P( + CopyFdRange, + CopyFdRangeTest, + ::testing::Combine(::testing::Values(false, true), ::testing::Values(false, true)), + [](const ::testing::TestParamInfo> & info) { + std::string res = ""; + res += std::get<0>(info.param) ? "fast" : "slow"; + res += "_"; + res += std::get<1>(info.param) ? "regular" : "nonregular"; + return res; + }); + } // namespace nix diff --git a/src/libutil/file-descriptor-private.hh b/src/libutil/file-descriptor-private.hh new file mode 100644 index 0000000000..d934fe5cb2 --- /dev/null +++ b/src/libutil/file-descriptor-private.hh @@ -0,0 +1,23 @@ +#pragma once + +#include "nix/util/file-descriptor.hh" + +namespace nix { + +struct FdSink; + +/** + * Try to copy @p nbytes bytes from @p from fd to @p to fd, starting at @p offset. + * The intent is to try to use optimized file copies like copy_file_range (on Linux) first + * and only fall back to the slower copying when the fast path fails or is unsupported. + * + * @throws SystemError in case the copy was started, but failed halfway. + * + * @return true if the copy succeeded, false if it's unsupported. Partial copies + * should be reported as errors. + * + * @pre written should be set to 0 before calling this function. + */ +bool tryCopyFdRangeFast(Descriptor from, Descriptor to, off_t offset, size_t nbytes, size_t & written); + +} // namespace nix diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index f5f5ee05a6..d0d8fa1ffb 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -3,6 +3,8 @@ #include "nix/util/util.hh" #include "nix/util/signals.hh" +#include "file-descriptor-private.hh" + #include #include #include @@ -202,10 +204,30 @@ std::string drainFD(Descriptor fd, DrainFdOpts opts) return std::move(sink.s); } -void copyFdRange(Descriptor fd, off_t offset, size_t nbytes, Sink & sink) +void copyFdRange(Descriptor fd, off_t offset, size_t nbytes, Sink & sink, bool tryCoW) { + /* Nothing to do. */ + if (!nbytes) + return; + + static constexpr size_t copyBufferSize = 64 * 1024; + + if (tryCoW) { + /* dynamic_cast here is slightly annoying, but it's much cheaper than + doing I/O so it doesn't really matter. */ + FdSink * fdSink = dynamic_cast(&sink); + /* Destination is a regular file which hasn't been written to and we've + been hinted that we should try block cloning. Try it and in case we + can't do block cloning then fall back to the loop below. Also, don't + bother with files smaller than the copy buffer. */ + if (fdSink && !fdSink->hasData() && fdSink->written == 0 && fdSink->isRegularFile && nbytes > copyBufferSize + && tryCopyFdRangeFast(fd, fdSink->fd, offset, nbytes, fdSink->written)) { + return; + } + } + auto left = nbytes; - std::array buf; + std::array buf; while (left) { auto limit = std::min(left, buf.size()); diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index c95265e637..bf71770311 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -210,6 +210,7 @@ struct RestoreRegularFile : CreateRegularFileSink, FdSink , fd(std::move(fd_)) , startFsync(startFSync_) { + isRegularFile = true; /* Hint for copying contents via CoW copy_file_range. */ } void anchor() override; diff --git a/src/libutil/include/nix/util/file-descriptor.hh b/src/libutil/include/nix/util/file-descriptor.hh index a6796f2269..a0df4222b4 100644 --- a/src/libutil/include/nix/util/file-descriptor.hh +++ b/src/libutil/include/nix/util/file-descriptor.hh @@ -115,11 +115,13 @@ size_t readOffset(Descriptor fd, off_t offset, std::span buffer); /** * Read \ref nbytes starting at \ref offset from a seekable file into a sink. * - * @throws SystemError if fd is not seekable or any operation fails + * @param tryCoW Used as a hint to use optimised file copying like copy_file_range. + * + * @throws SystemError if @p fd is not seekable or any operation fails * @throws Interrupted if the operation was interrupted * @throws EndOfFile if an EOF was reached before reading \ref nbytes */ -void copyFdRange(Descriptor fd, off_t offset, size_t nbytes, Sink & sink); +void copyFdRange(Descriptor fd, off_t offset, size_t nbytes, Sink & sink, bool tryCoW = false); /** * Wrappers around read()/write() that read/write exactly the diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index d933a996a0..b0bb0fd49f 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -79,6 +79,14 @@ public: void flush(); + /** + * Check whether the buffer has unflushed data. + */ + bool hasData() const noexcept + { + return bufPos; + } + protected: virtual void writeUnbuffered(std::string_view data) = 0; @@ -194,6 +202,12 @@ private: public: Descriptor fd; size_t written = 0; + /** + * A hint to the sink as to whether the destination is a regular file. + * Specifying a true for a non regular file doesn't affect correctness - only + * efficiency. + */ + bool isRegularFile:1 = false; FdSink() : fd(INVALID_DESCRIPTOR) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index a067653d1b..5fe44d4e54 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -150,7 +150,7 @@ void PosixFileSourceAccessor::readFile(const CanonPath & path, Sink & sink, fun< /* The most important invariant we care about here is writing exactly size bytes to the sink. copyFdRange should throw an EndOfFile if we fail to read `size` bytes. */ - copyFdRange(fd.get(), /*offset=*/0, size, sink); + copyFdRange(fd.get(), /*offset=*/0, size, sink, /*tryCoW=*/true); } bool PosixFileSourceAccessor::pathExists(const CanonPath & path) diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index 545f4f2b70..5f6dbed39f 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -5,8 +5,10 @@ #include #include #include +#include #include "util-unix-config-private.hh" +#include "../file-descriptor-private.hh" namespace nix { @@ -186,4 +188,72 @@ void unix::SelfPipe::drain() } } +bool tryCopyFdRangeFast(Descriptor from, Descriptor to, off_t offset, size_t nbytes, size_t & written) +{ +#if HAVE_COPY_FILE_RANGE + + /* Otherwise trying block cloning is pretty pointless. Also used to error + out on partial copies (unless we happen to hit block alignment boundary, + but that seems unlikely to be useful). */ + assert(written == 0); + size_t left = nbytes; + static std::atomic_flag copyFileRangeUnsupported = {}; + + if (copyFileRangeUnsupported.test(std::memory_order_relaxed)) + return false; + + while (left) { + checkInterrupt(); + + ssize_t n = ::copy_file_range( + from, + &offset, + to, + nullptr, + left, + 0 +# ifdef COPY_FILE_RANGE_CLONE /* FreeBSD */ + | COPY_FILE_RANGE_CLONE +# endif + ); + + /* Reached EOF too early. */ + if (n == 0) + throw EndOfFile( + "unexpected end-of-file copying from %1% to %2%", + PathFmt(descriptorToPath(from)), + PathFmt(descriptorToPath(to))); + + if (n == -1) { + if (errno == EINTR) + continue; /* Loop over to hit checkInterrupt. */ + + if (written == 0 && (errno == EINVAL || errno == EOPNOTSUPP || errno == EXDEV || errno == EBADF)) { + /* Retry with fallback. */ + } else if (written == 0 && errno == ENOSYS) { + /* Cache ENOSYS. */ + copyFileRangeUnsupported.test_and_set(); + } else { + throw SysError([&] { + return HintFmt( + "copying contents of %1% to %2% via copy_file_range", + PathFmt(descriptorToPath(from)), + PathFmt(descriptorToPath(to))); + }); + } + + return false; + } + + assert(static_cast(n) <= left); + left -= n; + written += n; + } + + return true; +#else + return false; +#endif +} + } // namespace nix diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build index dbe142d3a8..c78896b713 100644 --- a/src/libutil/unix/meson.build +++ b/src/libutil/unix/meson.build @@ -48,6 +48,23 @@ foreach funcspec : check_funcs_unix configdata_unix.set(define_name, define_value, description : funcspec[1]) endforeach +# Detects the Linux signature of the copy_file_range. +have_copy_file_range = cxx.links( + '''#include +int main() { + static_assert(sizeof(off_t) == 8); + copy_file_range(0, nullptr, 0, nullptr, 0, 0); +} + ''', + name : 'have copy_file_range', +) + +configdata_unix.set( + 'HAVE_COPY_FILE_RANGE', + have_copy_file_range.to_int(), + description : 'Optionally used for optimised file copying', +) + config_unix_priv_h = configure_file( configuration : configdata_unix, output : 'util-unix-config-private.hh', diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index c571d0a3e9..00932de2a7 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -3,6 +3,8 @@ #include "nix/util/finally.hh" #include "nix/util/serialise.hh" +#include "../file-descriptor-private.hh" + #include #include @@ -45,12 +47,19 @@ size_t readOffset(Descriptor fd, off_t offset, std::span buffer) ov.Offset = static_cast(offset); if constexpr (sizeof(offset) > 4) /* We don't build with 32 bit off_t, but let's be safe. */ ov.OffsetHigh = static_cast(offset >> 32); + // TODO: Don't do this on each call maybe? + if (::GetFileType(fd) != FILE_TYPE_DISK) + // Not the most accurate error code, but it will do. + throw windows::WinError( + DWORD(ERROR_SEEK_ON_DEVICE), "reading at offset %1% from a non-seekable handle", offset); DWORD n; if (!ReadFile(fd, buffer.data(), static_cast(buffer.size()), &n, &ov)) { - throw windows::WinError([&] { - return HintFmt( - "reading %1% bytes at offset %2% from %3%", buffer.size(), offset, PathFmt(descriptorToPath(fd))); - }); + auto err = ::GetLastError(); + // We report EOF as 0 return code, not an actual error. + if (err == ERROR_HANDLE_EOF) + return 0; + throw windows::WinError( + err, "reading %1% bytes at offset %2% from %3%", buffer.size(), offset, PathFmt(descriptorToPath(fd))); } return static_cast(n); } @@ -133,4 +142,11 @@ void syncDescriptor(Descriptor fd) } } +bool tryCopyFdRangeFast(Descriptor from, Descriptor to, off_t offset, size_t nbytes, size_t & written) +{ + /* TODO: Implement if this becomes relevant. Seems like windows maybe has some reflinking capabilities: + https://devblogs.microsoft.com/engineering-at-microsoft/dev-drive-and-copy-on-write-for-developer-performance/ */ + return false; +} + } // namespace nix From 22765abd4cda8990098a007da1fa8360db13924c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 17 Sep 2026 21:32:35 +0200 Subject: [PATCH 767/927] Use enum values less likely to collide with upstream --- src/libutil/include/nix/util/logging.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index 609a2774eb..1e069e72cf 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -36,7 +36,7 @@ typedef enum { * paths, cf. the name-based `Logger::startActivity()`. Do not use * directly. */ - actStringly = 113, + actStringly = 10113, } ActivityType; typedef enum { @@ -58,7 +58,7 @@ typedef enum { * (the number of body bytes received). More fields may be added * in the future. */ - resHttpStatus = 111, + resHttpStatus = 10111, } ResultType; typedef uint64_t ActivityId; From 50ee1adc6b401c19460ee03c715e609f5634dc34 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 20:44:47 +0000 Subject: [PATCH 768/927] Prepare release v3.22.5 From 15012f4c2fbc5120ea6c9765e38e1e32ed9fbc5a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 20:44:50 +0000 Subject: [PATCH 769/927] packaging/secure-packages/flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'path:../..' → 'path:../..' • Updated input 'nix/nixpkgs': 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013583%2Brev-f041631d1ed8316e18423533f65925c81c3abce7/01a0a705-a838-70e1-90f7-6126cdd43de6/source.tar.gz' (2026-09-15) → 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013924%2Brev-9f03b01a2c5e850ba180f9ca750979300e118389/01a0b003-ce4b-7973-91ca-5f497160ec59/source.tar.gz' (2026-09-17) --- packaging/secure-packages/flake.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index 50a26d9036..a87a4b0be7 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -77,12 +77,12 @@ }, "nixpkgs": { "locked": { - "lastModified": 1789504077, - "narHash": "sha256-IL+ZRmtgTJIMd2lpYVN/W3MmkfMQU3n1CRtDuHADzNI=", - "rev": "f041631d1ed8316e18423533f65925c81c3abce7", - "revCount": 1013583, + "lastModified": 1789657395, + "narHash": "sha256-y1mqtcQPO2ujSk3yi6QaimZEfGQJixotxAjTlS32oXE=", + "rev": "9f03b01a2c5e850ba180f9ca750979300e118389", + "revCount": 1013924, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013583%2Brev-f041631d1ed8316e18423533f65925c81c3abce7/01a0a705-a838-70e1-90f7-6126cdd43de6/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013924%2Brev-9f03b01a2c5e850ba180f9ca750979300e118389/01a0b003-ce4b-7973-91ca-5f497160ec59/source.tar.gz" }, "original": { "type": "tarball", From f0188d3c16cf49fbd145a915808108cd54b24600 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 20:44:53 +0000 Subject: [PATCH 770/927] Set .version-determinate to 3.22.5 --- .version-determinate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version-determinate b/.version-determinate index 00f104afa6..5956f98f30 100644 --- a/.version-determinate +++ b/.version-determinate @@ -1 +1 @@ -3.22.4 +3.22.5 From 77eefcb90a8400470bb736a69cc71ddb224bb909 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 20:44:58 +0000 Subject: [PATCH 771/927] Generate release notes for 3.22.5 --- doc/manual/source/SUMMARY.md.in | 1 + doc/manual/source/release-notes-determinate/changes.md | 6 +++++- doc/manual/source/release-notes-determinate/v3.22.5.md | 9 +++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 doc/manual/source/release-notes-determinate/v3.22.5.md diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index c6ae4e2906..1cdbdd4eed 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -154,6 +154,7 @@ - [Contributing](development/contributing.md) - [Determinate Nix Release Notes](release-notes-determinate/index.md) - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.22.5 (2026-09-17)](release-notes-determinate/v3.22.5.md) - [Release 3.22.4 (2026-09-16)](release-notes-determinate/v3.22.4.md) - [Release 3.22.3 (2026-09-02)](release-notes-determinate/v3.22.3.md) - [Release 3.22.2 (2026-08-21)](release-notes-determinate/v3.22.2.md) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index d577990594..89a036ff0f 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -1,6 +1,6 @@ # Changes between Nix and Determinate Nix -This section lists the differences between upstream Nix 2.35 and Determinate Nix 3.22.4. +This section lists the differences between upstream Nix 2.35 and Determinate Nix 3.22.5. * In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. @@ -236,3 +236,7 @@ This section lists the differences between upstream Nix 2.35 and Determinate Nix + + + +* Backport libutil/serialise: Use zero-copy copy_file_range for copying PosixFileSourceAccessor -> FdSink by @cole-h in [DeterminateSystems/nix-src#630](https://github.com/DeterminateSystems/nix-src/pull/630) diff --git a/doc/manual/source/release-notes-determinate/v3.22.5.md b/doc/manual/source/release-notes-determinate/v3.22.5.md new file mode 100644 index 0000000000..2fbcdce8ca --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.22.5.md @@ -0,0 +1,9 @@ +# Release 3.22.5 (2026-09-17) + +* Based on [upstream Nix 2.35.2](../release-notes/rl-2.35.md). + +## What's Changed +* Backport libutil/serialise: Use zero-copy copy_file_range for copying PosixFileSourceAccessor -> FdSink by @cole-h in [DeterminateSystems/nix-src#630](https://github.com/DeterminateSystems/nix-src/pull/630) + + +**Full Changelog**: [v3.22.4...v3.22.5](https://github.com/DeterminateSystems/nix-src/compare/v3.22.4...v3.22.5) From 9b10bc67bd1bcd65c74905330ee0a5d6c96701f6 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 17 Sep 2026 13:57:37 -0700 Subject: [PATCH 772/927] Update release notes --- doc/manual/source/release-notes-determinate/changes.md | 2 -- doc/manual/source/release-notes-determinate/v3.22.5.md | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md index 89a036ff0f..bb73f7a1a8 100644 --- a/doc/manual/source/release-notes-determinate/changes.md +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -238,5 +238,3 @@ This section lists the differences between upstream Nix 2.35 and Determinate Nix - -* Backport libutil/serialise: Use zero-copy copy_file_range for copying PosixFileSourceAccessor -> FdSink by @cole-h in [DeterminateSystems/nix-src#630](https://github.com/DeterminateSystems/nix-src/pull/630) diff --git a/doc/manual/source/release-notes-determinate/v3.22.5.md b/doc/manual/source/release-notes-determinate/v3.22.5.md index 2fbcdce8ca..c4b932d419 100644 --- a/doc/manual/source/release-notes-determinate/v3.22.5.md +++ b/doc/manual/source/release-notes-determinate/v3.22.5.md @@ -2,8 +2,9 @@ * Based on [upstream Nix 2.35.2](../release-notes/rl-2.35.md). -## What's Changed -* Backport libutil/serialise: Use zero-copy copy_file_range for copying PosixFileSourceAccessor -> FdSink by @cole-h in [DeterminateSystems/nix-src#630](https://github.com/DeterminateSystems/nix-src/pull/630) +## Performance improvements + +* libutil/serialise: Use zero-copy copy_file_range for copying PosixFileSourceAccessor -> FdSink by @xokdvium in [DeterminateSystems/nix-src#630](https://github.com/DeterminateSystems/nix-src/pull/630) **Full Changelog**: [v3.22.4...v3.22.5](https://github.com/DeterminateSystems/nix-src/compare/v3.22.4...v3.22.5) From f2c04f91afe0cfefdf8de7ddfc9c85827d4900e1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 11:20:20 +0200 Subject: [PATCH 773/927] nix flake bake: Warn about derivations that cannot be baked Derivations whose output paths are not known in advance (e.g. content-addressed or impure derivations) can't be baked. Previously they were emitted with null output paths, causing the baked flake to fail at evaluation time. Now they produce a warning and are omitted. Assisted-by: Claude Fable 5.1 --- src/libcmd/flake-schemas.cc | 17 +++++++++++++++-- tests/functional/flakes/bake.sh | 14 +++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index 361d47266d..9fe3e90176 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -428,6 +428,11 @@ nlohmann::json getFlakeInventory( drvObj.emplace("path", state.store->printStorePath(drvPath)); } + /* Derivations whose output paths are not known in advance (e.g. content-addressed + derivations) cannot be baked, since baking requires the output paths to be known at + evaluation time. */ + bool bakeable = true; + if (options.showOutputPaths) { auto outputs = nlohmann::json::object(); auto drvPath = drv->forceDerivation(); @@ -435,13 +440,21 @@ nlohmann::json getFlakeInventory( for (auto & i : drv.outputsAndOptPaths(*state.store)) { if (auto outPath = i.second.second) outputs.emplace(i.first, state.store->printStorePath(*outPath)); - else + else { outputs.emplace(i.first, nullptr); + bakeable = false; + } } drvObj.emplace("outputs", std::move(outputs)); } - obj.emplace("derivation", std::move(drvObj)); + if (options.bake && !bakeable) + warn( + "cannot bake '%s' because its output paths are not known in advance (e.g. it is a " + "content-addressed derivation)", + leaf.node->getAttrPathStr()); + else + obj.emplace("derivation", std::move(drvObj)); } if (auto forSystems = leaf.forSystems()) diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index dc2e6702d7..5d5c223a43 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -102,6 +102,7 @@ nix flake show --json --legacy "path:$bakedDir" > "$TEST_ROOT/show-baked-legacy. # `meta.mainProgram` is preserved, so `nix run` works on baked packages # whose binary isn't named after the package. +enableFeatures "ca-derivations" greeterDir=$TEST_ROOT/greeter mkdir -p "$greeterDir" cp "${config_nix}" "$greeterDir/" @@ -124,11 +125,22 @@ cat > "$greeterDir/flake.nix" <&1 | grepQuiet "warning: cannot bake 'packages.$system.ca'" +[[ $(nix eval "path:$bakedDir-greeter#packages.$system" --apply 'x: x ? ca') = false ]] +[[ $(nix eval "path:$bakedDir-greeter#packages.$system" --apply 'x: x ? greeter') = true ]] [[ $(jq -r ".packages.output.children.\"$system\".children.greeter.derivation.mainProgram" < "$bakedDir-greeter/outputs.json") = hi ]] [[ $(nix eval --raw "path:$bakedDir-greeter#greeter.meta.mainProgram") = hi ]] From 52b2dcdffc6d91e89a43733c4e87cc3493899096 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 11:34:39 +0200 Subject: [PATCH 774/927] nix flake bake: Document Assisted-by: Claude Fable 5.1 --- src/nix/flake-bake.md | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/nix/flake-bake.md b/src/nix/flake-bake.md index e0a6d5872e..bdff1e41bf 100644 --- a/src/nix/flake-bake.md +++ b/src/nix/flake-bake.md @@ -1,7 +1,46 @@ R""( +# Examples + +* Bake the `patchelf` flake into the directory `./patchelf-baked`: + + ```console + # nix flake bake github:NixOS/patchelf --dest-dir ./patchelf-baked + ``` + + The baked flake can be used like any other flake: + + ```console + # nix flake show path:./patchelf-baked + # nix build path:./patchelf-baked#patchelf + ``` + +* Bake the outputs for all systems, not just the current one: + + ```console + # nix flake bake github:NixOS/patchelf --dest-dir ./patchelf-baked --all-systems + ``` + # Description -Bake a flake. +This command evaluates the outputs of the flake specified by flake reference *flake-url* and writes a new, *baked* flake to the directory *path* (specified by `--dest-dir`). The baked flake provides the same outputs as the original flake, but they are pre-evaluated: every derivation is replaced by a [baked derivation](@docroot@/language/builtins.md#builtins-bakedDerivation), that is, a derivation with the builder `builtin:substitute` that is never built but whose outputs are obtained by substitution from a binary cache. As a result, evaluating a baked flake is much faster than evaluating the original, since it does (almost) no evaluation. + +The baked flake has no inputs, so it can be used without access to the inputs of the original flake. + +Derivation attributes in the baked flake have the same `name`, `system`, `drvPath`, `outPath`, `outputName`, `outputs` and `meta.mainProgram` as in the original flake, so commands such as `nix build` and `nix run` behave the same. The outputs of a baked flake can also be used as inputs of other derivations, e.g. by another flake that has the baked flake as an input. + +Building the outputs of a baked flake requires that their store paths can be [substituted](@docroot@/command-ref/conf-file.md#conf-substituters) (or are already present in the Nix store). If a store path cannot be substituted, the build fails; Nix will not fall back to building the original derivation. + +By default, only the outputs for the current system are baked. Use `--all-systems` to bake the outputs for all systems. + +# Limitations + +* Only outputs that are recognised by the [flake schemas](@docroot@/protocols/flake-schemas.md) are baked. Non-derivation outputs, such as `overlays`, `nixosModules` and `templates`, are omitted from the baked flake. + +* Derivations whose output paths are not known at evaluation time cannot be baked. This includes [content-addressed derivations](@docroot@/store/derivation/outputs/content-address.md) and impure derivations. `nix flake bake` prints a warning for each such derivation and omits it from the baked flake. + +* Outputs whose derivation is nested inside the output attribute (such as `nixosConfigurations..config.system.build.toplevel`) are baked at that nested attribute. Other attributes of the output are not included. Consequently, `nix flake show` may not work on a baked flake for such outputs if the flake schema needs those other attributes. + +This command requires the [`baked-derivation`](@docroot@/development/experimental-features.md#xp-feature-baked-derivation) experimental feature, both in the Nix CLI and in the Nix daemon. )"" From 33c6c48a6f0ab8ba2bbfaad1df660a28c51856fe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 11:57:54 +0200 Subject: [PATCH 775/927] builtins.bakedDerivation: Drop the narHash attribute It produced a fixed content-addressed output, which the builtin:substitute derivation type rejects, so it couldn't work as written. Leave a TODO for adding it back to assert the known contents of a pre-built output. Assisted-by: Claude Fable 5.1 --- src/libexpr/primops/bakedDerivation.cc | 38 +++++--------------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/src/libexpr/primops/bakedDerivation.cc b/src/libexpr/primops/bakedDerivation.cc index f472f1ed34..fc8a9f8c86 100644 --- a/src/libexpr/primops/bakedDerivation.cc +++ b/src/libexpr/primops/bakedDerivation.cc @@ -3,7 +3,6 @@ #include "nix/store/derivations.hh" #include "nix/store/derived-path.hh" #include "nix/store/store-api.hh" -#include "nix/util/hash.hh" #include @@ -15,13 +14,8 @@ static void prim_bakedDerivation(EvalState & state, const PosIdx pos, Value ** a std::optional name; - struct BakedDerivationOutput - { - StorePath path; - std::optional narHash; - }; - - std::map outputs; + /* TODO: add a `narHash` attribute to assert the known contents of a pre-built output. */ + std::map outputs; for (auto & attr : *args[0]->attrs()) { std::string_view attrName = state.symbols[attr.name]; @@ -42,7 +36,6 @@ static void prim_bakedDerivation(EvalState & state, const PosIdx pos, Value ** a fmt("while evaluating the output '%s' passed to builtins.bakedDerivation", outName)); std::optional path; - std::optional narHash; for (auto & outField : *outAttr.value->attrs()) { std::string_view fieldName = state.symbols[outField.name]; @@ -56,11 +49,6 @@ static void prim_bakedDerivation(EvalState & state, const PosIdx pos, Value ** a path = state.store->parseStorePath(s); } - else if (fieldName == "narHash") { - auto s = state.forceStringNoCtx(*outField.value, outField.pos, fieldHint); - narHash = Hash::parseAny(s, std::nullopt); - } - else state .error( @@ -77,7 +65,7 @@ static void prim_bakedDerivation(EvalState & state, const PosIdx pos, Value ** a .atPos(outAttr.pos) .debugThrow(); - outputs.emplace(std::string(outName), BakedDerivationOutput{std::move(*path), std::move(narHash)}); + outputs.emplace(std::string(outName), std::move(*path)); } } @@ -102,20 +90,8 @@ static void prim_bakedDerivation(EvalState & state, const PosIdx pos, Value ** a drv.platform = "builtin"; drv.builder = "builtin:substitute"; - for (auto & [outName, out] : outputs) { - if (out.narHash) - drv.outputs.insert_or_assign( - outName, - DerivationOutput::CAFixed{ - .ca = - ContentAddress{ - .method = ContentAddressMethod::Raw::NixArchive, - .hash = *out.narHash, - }, - }); - else - drv.outputs.insert_or_assign(outName, DerivationOutput::InputAddressed{.path = out.path}); - } + for (auto & [outName, outPath] : outputs) + drv.outputs.insert_or_assign(outName, DerivationOutput::InputAddressed{.path = outPath}); auto drvPath = state.store->writeDerivation(*state.asyncPathWriter, drv, state.repair); @@ -133,14 +109,14 @@ static void prim_bakedDerivation(EvalState & state, const PosIdx pos, Value ** a NixStringContextElem::DrvDeep{.drvPath = drvPath}, }, state.mem); - for (auto & [outName, out] : outputs) + for (auto & [outName, outPath] : outputs) state.mkOutputString( result.alloc(outName), SingleDerivedPath::Built{ .drvPath = makeConstantStorePathRef(drvPath), .output = outName, }, - out.path); + outPath); v.mkAttrs(result); } From bfd439d34a9f2bfa89691cfd4e6b53022a4faeda Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 Jul 2026 00:41:11 -0400 Subject: [PATCH 776/927] Check `experimentalFeature()` of nested subcommands Previously only the top-level subcommand's `experimentalFeature()` was checked, so an override on a nested command (e.g. `nix store roots-daemon`'s `Xp::LocalOverlayStore`) was only reflected in the generated documentation, never actually enforced at runtime. Walk down the whole command chain, requiring each level's feature, the same way the `--help` code walks it just above. Assisted-by: Claude:fable-5 (cherry picked from commit eb74c1691a7106f33debf4f3606ced4fd040cb64) --- src/nix/main.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 5acf75257d..cb697d3bf1 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -683,7 +683,13 @@ void mainWrapped(int argc, char ** argv) if (!args.command) throw UsageError("no subcommand specified"); - experimentalFeatureSettings.require(args.command->second->experimentalFeature()); + { + MultiCommand * command = &args; + while (command && command->command) { + experimentalFeatureSettings.require(command->command->second->experimentalFeature()); + command = dynamic_cast(&*command->command->second); + } + } if (args.useNet && !haveInternet()) { warn("you don't have Internet access; disabling some network-dependent features"); From 19d2b1283b2d60319a589808e6eccbaa5d62bc43 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 12:13:09 +0200 Subject: [PATCH 777/927] nix flake bake: Remove redundant experimental feature check The experimentalFeature() override is now enforced for nested subcommands. Assisted-by: Claude Fable 5.1 --- src/nix/flake-bake.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/nix/flake-bake.cc b/src/nix/flake-bake.cc index 83c1988a6d..60f004c1e5 100644 --- a/src/nix/flake-bake.cc +++ b/src/nix/flake-bake.cc @@ -54,8 +54,6 @@ struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas void run(nix::ref store) override { - experimentalFeatureSettings.require(Xp::BakedDerivation); - auto state = getEvalState(); auto evalStore = getEvalStore(); auto flake = make_ref(lockFlake()); From 5ad67ee52cecd1794028c971c5c069eb3ac53217 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 12:22:09 +0200 Subject: [PATCH 778/927] builtin:substitute: Reject deferred outputs An empty output path parses as Deferred, which maps to an input-addressed type with deferred = true, so it passed the check in BasicDerivation::type() and only failed at build time with "has no store path". Assisted-by: Claude Fable 5.1 --- src/libstore/derivations.cc | 3 ++- tests/functional/flakes/bake.sh | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 17b33394dc..efbaf36480 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -879,7 +879,8 @@ DerivationType BasicDerivation::type() const if (builder == "builtin:substitute") { experimentalFeatureSettings.require(Xp::BakedDerivation, "'builtin:substitute' derivation"); - if (!std::holds_alternative(ty.value().raw)) + auto * ia = std::get_if(&ty.value().raw); + if (!ia || ia->deferred) throw Error("'builtin:substitute' derivation must have input-addressed outputs"); return DerivationType::Substituted{}; } diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index 5d5c223a43..8bc1c5ff39 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -43,6 +43,11 @@ nix derivation show "$drvPath" > "$TEST_ROOT/baked-drv.json" [[ $(jq -r '.derivations[].inputs.drvs | length' < "$TEST_ROOT/baked-drv.json") = 0 ]] [[ $(jq -r '.derivations[].outputs.out.path' < "$TEST_ROOT/baked-drv.json") = "$(basename "$fooPath")" ]] +# A `builtin:substitute` derivation must have input-addressed outputs with known paths. +jq '.derivations[]' < "$TEST_ROOT/baked-drv.json" > "$TEST_ROOT/baked-drv-single.json" +[[ $(nix derivation add < "$TEST_ROOT/baked-drv-single.json") = "$drvPath" ]] +jq '.outputs.out = {}' < "$TEST_ROOT/baked-drv-single.json" | expectStderr 1 nix derivation add | grepQuiet "must have input-addressed outputs" + # Building a baked derivation fails if its outputs cannot be substituted. expectStderr 1 nix build --no-link "path:$bakedDir#foo" | grepQuiet "failed to substitute" From 37f616bfed4f470417681b61c25f3fccf9e26852 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 12:22:09 +0200 Subject: [PATCH 779/927] builtin:substitute: Check the output types in checkInvariants() The early return for builtin:substitute derivations skipped the drv.type() call at the end of processDerivationOutputPaths(), so a derivation with e.g. a fixed content-addressed output passed checkInvariants() and was written to the store, only to fail later. Assisted-by: Claude Fable 5.1 --- src/libstore/derivations.cc | 2 ++ tests/functional/flakes/bake.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index efbaf36480..abc1fa7e48 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1300,6 +1300,8 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string throw Error("'builtin:substitute' derivation must have no environment variables"); if (!drv.inputSrcs.empty()) throw Error("'builtin:substitute' derivation must have no inputs"); + /* Check the output types (and the experimental feature). */ + drv.type(); return; } diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index 8bc1c5ff39..3fdcef6444 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -47,6 +47,7 @@ nix derivation show "$drvPath" > "$TEST_ROOT/baked-drv.json" jq '.derivations[]' < "$TEST_ROOT/baked-drv.json" > "$TEST_ROOT/baked-drv-single.json" [[ $(nix derivation add < "$TEST_ROOT/baked-drv-single.json") = "$drvPath" ]] jq '.outputs.out = {}' < "$TEST_ROOT/baked-drv-single.json" | expectStderr 1 nix derivation add | grepQuiet "must have input-addressed outputs" +jq '.outputs.out = {"method": "nar", "hash": "sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="}' < "$TEST_ROOT/baked-drv-single.json" | expectStderr 1 nix derivation add | grepQuiet "must have input-addressed outputs" # Building a baked derivation fails if its outputs cannot be substituted. expectStderr 1 nix build --no-link "path:$bakedDir#foo" | grepQuiet "failed to substitute" From eebe128153943bb3562a0d9ca8edab0e395428e6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 12:22:09 +0200 Subject: [PATCH 780/927] nix flake bake: Record the system of each derivation Previously the baked flake took the system from the schema's forSystems attribute, which flake-provided schemas can omit. Now the inventory records the derivation's own system, with forSystems as a fallback. Assisted-by: Claude Fable 5.1 --- src/libcmd/flake-schemas.cc | 4 ++++ src/nix/baked-flake.nix | 2 +- tests/functional/flakes/bake.sh | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index 9fe3e90176..1afdaa6a6a 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -421,6 +421,10 @@ nlohmann::json getFlakeInventory( /* Record which output this attribute refers to, since it's not necessarily `out`. */ if (auto aOutputName = drv->maybeGetAttr(state.s.outputName)) drvObj.emplace("outputName", aOutputName->getString()); + + /* Record the system of the derivation. */ + if (auto aSystem = drv->maybeGetAttr(state.s.system)) + drvObj.emplace("system", aSystem->getString()); } if (options.showDrvPaths) { diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix index 16f0d0fc50..1b8b3d2961 100644 --- a/src/nix/baked-flake.nix +++ b/src/nix/baked-flake.nix @@ -34,7 +34,7 @@ commonAttrs = { type = "derivation"; name = output.derivation.name; - system = builtins.head output.forSystems; # FIXME + system = output.derivation.system or (builtins.head output.forSystems); meta = (if output ? shortDescription then { description = output.shortDescription; } else { }) // ( diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index 3fdcef6444..364b66e2fa 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -150,6 +150,10 @@ nix flake bake "$greeterDir" --dest-dir "$bakedDir-greeter" 2>&1 | grepQuiet "wa [[ $(jq -r ".packages.output.children.\"$system\".children.greeter.derivation.mainProgram" < "$bakedDir-greeter/outputs.json") = hi ]] [[ $(nix eval --raw "path:$bakedDir-greeter#greeter.meta.mainProgram") = hi ]] +# The system of each derivation is recorded, so the baked flake doesn't depend on the schema's `forSystems`. +[[ $(jq -r ".packages.output.children.\"$system\".children.greeter.derivation.system" < "$bakedDir-greeter/outputs.json") = "$system" ]] +[[ $(nix eval --raw "path:$bakedDir-greeter#greeter.system") = "$system" ]] + # Build the original so that the baked package's output is valid, then run it. nix build --no-link "$greeterDir#greeter" [[ $(nix run "path:$bakedDir-greeter#greeter") = "hello from hi" ]] From a4137388521ef6f58ff2a122b2bd0661063f7252 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 12:22:09 +0200 Subject: [PATCH 781/927] flake-bake.md: Don't claim that drvPath is preserved Assisted-by: Claude Fable 5.1 --- src/nix/flake-bake.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake-bake.md b/src/nix/flake-bake.md index bdff1e41bf..35a841d755 100644 --- a/src/nix/flake-bake.md +++ b/src/nix/flake-bake.md @@ -27,7 +27,7 @@ This command evaluates the outputs of the flake specified by flake reference *fl The baked flake has no inputs, so it can be used without access to the inputs of the original flake. -Derivation attributes in the baked flake have the same `name`, `system`, `drvPath`, `outPath`, `outputName`, `outputs` and `meta.mainProgram` as in the original flake, so commands such as `nix build` and `nix run` behave the same. The outputs of a baked flake can also be used as inputs of other derivations, e.g. by another flake that has the baked flake as an input. +Derivation attributes in the baked flake have the same `name`, `system`, `outPath`, `outputName`, `outputs` and `meta.mainProgram` as in the original flake, so commands such as `nix build` and `nix run` behave the same. Their `drvPath` is different, however: it refers to the baked derivation rather than the original one. The outputs of a baked flake can also be used as inputs of other derivations, e.g. by another flake that has the baked flake as an input. Building the outputs of a baked flake requires that their store paths can be [substituted](@docroot@/command-ref/conf-file.md#conf-substituters) (or are already present in the Nix store). If a store path cannot be substituted, the build fails; Nix will not fall back to building the original derivation. From fad9b621911c0c0e90dbbd912c32e57df8d6a16a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 12:34:47 +0200 Subject: [PATCH 782/927] Rename experimental feature 'baked-derivation' to 'baked-derivations' For consistency with 'ca-derivations' and 'impure-derivations'. Assisted-by: Claude Fable 5.1 --- src/libexpr/primops/bakedDerivation.cc | 2 +- src/libstore/derivations.cc | 2 +- src/libutil/experimental-features.cc | 6 +++--- src/libutil/include/nix/util/experimental-features.hh | 2 +- src/nix/flake-bake.cc | 2 +- src/nix/flake-bake.md | 2 +- tests/functional/flakes/bake.sh | 6 +++--- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libexpr/primops/bakedDerivation.cc b/src/libexpr/primops/bakedDerivation.cc index fc8a9f8c86..a1f4985801 100644 --- a/src/libexpr/primops/bakedDerivation.cc +++ b/src/libexpr/primops/bakedDerivation.cc @@ -172,7 +172,7 @@ static RegisterPrimOp primop_bakedDerivation({ ``` )", .impl = prim_bakedDerivation, - .experimentalFeature = Xp::BakedDerivation, + .experimentalFeature = Xp::BakedDerivations, }); } // namespace nix diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index abc1fa7e48..7cd9457aca 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -878,7 +878,7 @@ DerivationType BasicDerivation::type() const throw Error("must have at least one output"); if (builder == "builtin:substitute") { - experimentalFeatureSettings.require(Xp::BakedDerivation, "'builtin:substitute' derivation"); + experimentalFeatureSettings.require(Xp::BakedDerivations, "'builtin:substitute' derivation"); auto * ia = std::get_if(&ty.value().raw); if (!ia || ia->deferred) throw Error("'builtin:substitute' derivation must have input-addressed outputs"); diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 3f3b31d690..faa0926da1 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -27,7 +27,7 @@ void MissingExperimentalFeature::anchor() {} * feature, we either have no issue at all if few features are not added * at the end of the list, or a proper merge conflict if they are. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::BakedDerivation); +constexpr size_t numXpFeatures = 1 + static_cast(Xp::BakedDerivations); constexpr std::array xpFeatureDetails = {{ { @@ -318,8 +318,8 @@ constexpr std::array xpFeatureDetails .trackingUrl = "", }, { - .tag = Xp::BakedDerivation, - .name = "baked-derivation", + .tag = Xp::BakedDerivations, + .name = "baked-derivations", .description = R"( Enable *baked derivations*: derivations with the builder `builtin:substitute` that are never built, but whose outputs are diff --git a/src/libutil/include/nix/util/experimental-features.hh b/src/libutil/include/nix/util/experimental-features.hh index 7c2f922bac..18374246e3 100644 --- a/src/libutil/include/nix/util/experimental-features.hh +++ b/src/libutil/include/nix/util/experimental-features.hh @@ -42,7 +42,7 @@ enum struct ExperimentalFeature { WasmDerivations, Provenance, CNSA, - BakedDerivation, + BakedDerivations, }; extern std::set stabilizedFeatures; diff --git a/src/nix/flake-bake.cc b/src/nix/flake-bake.cc index 60f004c1e5..2c4fadb2c8 100644 --- a/src/nix/flake-bake.cc +++ b/src/nix/flake-bake.cc @@ -42,7 +42,7 @@ struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas std::optional experimentalFeature() override { - return Xp::BakedDerivation; + return Xp::BakedDerivations; } std::string doc() override diff --git a/src/nix/flake-bake.md b/src/nix/flake-bake.md index 35a841d755..5d461c75fc 100644 --- a/src/nix/flake-bake.md +++ b/src/nix/flake-bake.md @@ -41,6 +41,6 @@ By default, only the outputs for the current system are baked. Use `--all-system * Outputs whose derivation is nested inside the output attribute (such as `nixosConfigurations..config.system.build.toplevel`) are baked at that nested attribute. Other attributes of the output are not included. Consequently, `nix flake show` may not work on a baked flake for such outputs if the flake schema needs those other attributes. -This command requires the [`baked-derivation`](@docroot@/development/experimental-features.md#xp-feature-baked-derivation) experimental feature, both in the Nix CLI and in the Nix daemon. +This command requires the [`baked-derivations`](@docroot@/development/experimental-features.md#xp-feature-baked-derivations) experimental feature, both in the Nix CLI and in the Nix daemon. )"" diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index 364b66e2fa..58603e5532 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -13,9 +13,9 @@ cacheDir=$TEST_ROOT/binary-cache mkdir -p "$flakeDir" writeSimpleFlake "$flakeDir" -# Baking requires the `baked-derivation` experimental feature. -expectStderr 1 nix flake bake "$flakeDir" --dest-dir "$bakedDir" | grepQuiet "experimental Nix feature 'baked-derivation' is disabled" -enableFeatures "baked-derivation" +# Baking requires the `baked-derivations` experimental feature. +expectStderr 1 nix flake bake "$flakeDir" --dest-dir "$bakedDir" | grepQuiet "experimental Nix feature 'baked-derivations' is disabled" +enableFeatures "baked-derivations" # Bake the flake. nix flake bake "$flakeDir" --dest-dir "$bakedDir" From 6e31ab2daf75aa276b2077c6730091e437e1e4cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 12:56:25 +0200 Subject: [PATCH 783/927] nix flake bake: Provide flake schemas in the baked flake The original flake's schemas can't be expected to work on a baked flake, since they may need attributes that the baked flake doesn't have (e.g. pkgs.stdenv.system for nixosConfigurations). So the baked flake now provides its own 'schemas' output that just reproduces the recorded inventory (without evaluation checks), making 'nix flake show' and 'nix flake check' work on any baked flake. To that end, the inventory written by 'nix flake bake' now also records the schema-level attributes (roles, appendSystem, defaultAttrPath) and the node attributes isLegacy, isFlakeCheck and forSystems on non-leaf nodes. Assisted-by: Claude Fable 5.1 --- src/libcmd/flake-schemas.cc | 34 +++++++++- src/nix/baked-flake.nix | 112 ++++++++++++++++++++++++++------ src/nix/flake-bake.md | 4 +- tests/functional/flakes/bake.sh | 24 +++++-- 4 files changed, 145 insertions(+), 29 deletions(-) diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index 1afdaa6a6a..5304b2ce26 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -381,6 +381,16 @@ nlohmann::json getFlakeInventory( std::function node, nlohmann::json & obj)> visit; visit = [&](ref node, nlohmann::json & obj) { + /* Record the schema attributes that apply to both leaf and non-leaf nodes. Non-leaf `forSystems` and + `isLegacy` are only needed by `nix flake bake` to reproduce the schema. */ + auto addNodeInfo = [&](bool isLeaf) { + if (auto forSystems = flake_schemas::Node(node).forSystems(); forSystems && (isLeaf || options.bake)) + obj.emplace("forSystems", *forSystems); + if (options.bake) + if (auto b = node->maybeGetAttr("isLegacy"); b && b->getBool()) + obj.emplace("isLegacy", true); + }; + flake_schemas::visit( options.showAllSystems ? std::optional() : localSystem, options.showLegacy, @@ -461,8 +471,10 @@ nlohmann::json getFlakeInventory( obj.emplace("derivation", std::move(drvObj)); } - if (auto forSystems = leaf.forSystems()) - obj.emplace("forSystems", *forSystems); + if (options.bake && leaf.isFlakeCheck()) + obj.emplace("isFlakeCheck", true); + + addNodeInfo(true); }, [&](std::function forEachChild) { @@ -482,6 +494,8 @@ nlohmann::json getFlakeInventory( }); }); obj.emplace("children", std::move(children)); + + addNodeInfo(false); }, [&](ref node, const std::vector & systems) { @@ -493,6 +507,8 @@ nlohmann::json getFlakeInventory( auto inv = nlohmann::json::object(); + auto schemas = getSchemas(inventory); + flake_schemas::forEachOutput( inventory, [&](Symbol outputName, std::shared_ptr output, const std::string & doc, bool isLast) { @@ -500,6 +516,20 @@ nlohmann::json getFlakeInventory( if (output) { j.emplace("doc", doc); + + /* Record the schema-level attributes, so that `nix flake bake` can reproduce the schema. */ + if (auto schema = get(schemas, std::string(state.symbols[outputName])); schema && options.bake) { + if (!schema->roles.empty()) + j.emplace("roles", schema->roles); + if (schema->appendSystem) + j.emplace("appendSystem", true); + if (schema->defaultAttrPath) { + auto attrPath = nlohmann::json::array(); + for (auto & attr : *schema->defaultAttrPath) + attrPath.push_back(std::string(state.symbols[attr])); + j.emplace("defaultAttrPath", std::move(attrPath)); + } + } auto & j2 = j.emplace("output", nlohmann::json::object()).first.value(); state.spawn(futures, 1, [&visit, output, &j2]() { visit(ref(output), j2); }); } else diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix index 1b8b3d2961..972c85d0bb 100644 --- a/src/nix/baked-flake.nix +++ b/src/nix/baked-flake.nix @@ -4,7 +4,7 @@ let data = builtins.fromJSON (builtins.readFile ./outputs.json); - cleanup = builtins.filterAttrs (name: value: value != { }); + optionalAttrs = cond: attrs: if cond then attrs else { }; # `setAttrByPath [ "a" "b" ] x` returns `{ a.b = x; }`. setAttrByPath = @@ -14,32 +14,52 @@ else { ${builtins.head path} = setAttrByPath (builtins.tail path) value; }; + # Prune the inventory to the nodes that can be baked: derivation + # leaves, and non-leaf nodes that contain at least one of them. + prune = + node: + if node ? children then + let + children = builtins.filterAttrs (name: child: child != null) ( + builtins.mapAttrs (name: child: prune child) node.children + ); + in + if children == { } then null else node // { inherit children; } + else if node ? derivation then + node + else + null; + + inventories = builtins.filterAttrs (name: node: node != null) ( + builtins.mapAttrs (name: output: prune (output.output or null)) data + ); + + # Convert an inventory node into the corresponding flake output. convert = - output: - if output ? children then - cleanup (builtins.mapAttrs (childName: child: convert child) output.children) - else if output ? "derivation" then + node: + if node ? children then + builtins.mapAttrs (name: child: convert child) node.children + else let baked = builtins.bakedDerivation { - name = output.derivation.name; - outputs = builtins.mapAttrs (outputName: path: { inherit path; }) output.derivation.outputs; + name = node.derivation.name; + outputs = builtins.mapAttrs (outputName: path: { inherit path; }) node.derivation.outputs; }; - outputNames = builtins.attrNames output.derivation.outputs; + + outputNames = builtins.attrNames node.derivation.outputs; # The output that this attribute refers to. - outputName = output.derivation.outputName or (builtins.head outputNames); + outputName = node.derivation.outputName or (builtins.head outputNames); # Mirror the attribute set produced by `derivation`: the common # attributes, plus one attribute per output. commonAttrs = { type = "derivation"; - name = output.derivation.name; - system = output.derivation.system or (builtins.head output.forSystems); + name = node.derivation.name; + system = node.derivation.system or (builtins.head node.forSystems); meta = - (if output ? shortDescription then { description = output.shortDescription; } else { }) - // ( - if output.derivation ? mainProgram then { mainProgram = output.derivation.mainProgram; } else { } - ); + optionalAttrs (node ? shortDescription) { description = node.shortDescription; } + // optionalAttrs (node.derivation ? mainProgram) { mainProgram = node.derivation.mainProgram; }; drvPath = baked.drvPath; outputs = outputNames; all = map (outputName: outputAttrs.${outputName}) outputNames; @@ -55,14 +75,64 @@ }; }) outputNames ); - - drv = outputAttrs.${outputName}; in # The derivation may live at a sub-path of the output attribute # (e.g. `config.system.build.toplevel` for `nixosConfigurations`). - setAttrByPath (output.derivationAttrPath or [ ]) drv - else - { }; + setAttrByPath (node.derivationAttrPath or [ ]) outputAttrs.${outputName}; + + # Convert an inventory node into a flake schema inventory node. The + # original flake's schemas may need attributes that aren't present + # in the baked flake (e.g. `pkgs.stdenv.system` for NixOS + # configurations), so the baked flake provides its own schemas that + # just reproduce the recorded inventory. + mkInventory = + node: + optionalAttrs (node ? forSystems) { inherit (node) forSystems; } + // optionalAttrs (node ? isLegacy) { inherit (node) isLegacy; } + // ( + if node ? children then + { children = builtins.mapAttrs (name: child: mkInventory child) node.children; } + else + { + derivationAttrPath = node.derivationAttrPath or [ ]; + } + // optionalAttrs (node ? what) { inherit (node) what; } + // optionalAttrs (node ? shortDescription) { inherit (node) shortDescription; } + // optionalAttrs (node ? isFlakeCheck) { inherit (node) isFlakeCheck; } + ); + + schemas = builtins.mapAttrs ( + name: node: + let + schema = data.${name}; + in + { + version = 1; + inherit (schema) doc; + inventory = output: mkInventory node; + } + // optionalAttrs (schema ? roles) { + roles = builtins.listToAttrs ( + map (role: { + name = role; + value = { }; + }) schema.roles + ); + } + // optionalAttrs (schema ? appendSystem) { inherit (schema) appendSystem; } + // optionalAttrs (schema ? defaultAttrPath) { inherit (schema) defaultAttrPath; } + ) inventories; in - cleanup (builtins.mapAttrs (outputName: output: convert (output.output or { })) data); + builtins.mapAttrs (name: node: convert node) inventories + // { + schemas = schemas // { + schemas = { + version = 1; + doc = "The `schemas` flake output defines the flake schemas of this flake."; + inventory = output: { + children = builtins.mapAttrs (name: schema: { what = "flake schema"; }) output; + }; + }; + }; + }; } diff --git a/src/nix/flake-bake.md b/src/nix/flake-bake.md index 5d461c75fc..a381090292 100644 --- a/src/nix/flake-bake.md +++ b/src/nix/flake-bake.md @@ -39,7 +39,9 @@ By default, only the outputs for the current system are baked. Use `--all-system * Derivations whose output paths are not known at evaluation time cannot be baked. This includes [content-addressed derivations](@docroot@/store/derivation/outputs/content-address.md) and impure derivations. `nix flake bake` prints a warning for each such derivation and omits it from the baked flake. -* Outputs whose derivation is nested inside the output attribute (such as `nixosConfigurations..config.system.build.toplevel`) are baked at that nested attribute. Other attributes of the output are not included. Consequently, `nix flake show` may not work on a baked flake for such outputs if the flake schema needs those other attributes. +* Outputs whose derivation is nested inside the output attribute (such as `nixosConfigurations..config.system.build.toplevel`) are baked at that nested attribute. Other attributes of the output are not included. + +* The baked flake provides its own [flake schemas](@docroot@/protocols/flake-schemas.md) (in the `schemas` output) that reproduce the structure of the original flake's outputs, so `nix flake show` and `nix flake check` work on it. However, any evaluation checks defined by the original flake's schemas are not performed on the baked flake. This command requires the [`baked-derivations`](@docroot@/development/experimental-features.md#xp-feature-baked-derivations) experimental feature, both in the Nix CLI and in the Nix daemon. diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index 58603e5532..de8b7a619a 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -223,11 +223,25 @@ activationPath=$(nix eval --raw "$configsDir#homeConfigurations.bar.activationPa [[ $(nix eval "path:$bakedDir-configs#nixosConfigurations.foo" --apply 'x: x ? drvPath') = false ]] [[ $(nix eval "path:$bakedDir-configs#homeConfigurations.bar" --apply 'x: x ? drvPath') = false ]] -# `nix flake show` on a baked flake evaluates the flake schemas against -# the baked values. This works for `homeConfigurations`, whose schema -# only needs `activationPackage.system`. It does not currently work for -# `nixosConfigurations`, whose schema reads `pkgs.stdenv.system`, which -# the baked flake does not provide, so we don't test that here. +# The baked flake provides its own flake schemas, so `nix flake show` +# and `nix flake check` work even for outputs whose original schema +# needs attributes that the baked flake doesn't have (e.g. +# `pkgs.stdenv.system` for `nixosConfigurations`). +nix flake show --json "path:$bakedDir-configs" > "$TEST_ROOT/show-baked-configs.json" +[[ $(jq -r '.inventory.nixosConfigurations.output.children.foo.what' < "$TEST_ROOT/show-baked-configs.json") = "NixOS configuration" ]] +[[ $(jq -c '.inventory.nixosConfigurations.output.children.foo.forSystems' < "$TEST_ROOT/show-baked-configs.json") = "[\"$system\"]" ]] +[[ $(jq -r '.inventory.nixosConfigurations.output.children.foo.derivation.name' < "$TEST_ROOT/show-baked-configs.json") = simple ]] +nix flake show "path:$bakedDir-configs" | grepQuiet "NixOS configuration" +nix flake check --no-build "path:$bakedDir-configs" +nix flake check "path:$bakedDir-greeter" + +# Schema-level attributes are preserved, so e.g. `foo` still resolves to `packages..foo`... +[[ $(jq -c '.packages.defaultAttrPath' < "$bakedDir/outputs.json") = '["default"]' ]] +[[ $(jq -c '.packages.roles | sort' < "$bakedDir/outputs.json") = '["nix-build","nix-develop","nix-run","nix-search"]' ]] +[[ $(nix eval --raw "path:$bakedDir#default.outPath") = "$fooPath" ]] +# ... and `legacyPackages` is still hidden without `--legacy`. +[[ $(jq -r ".inventory.legacyPackages.output.children.\"$system\".isLegacy" < "$TEST_ROOT/show-baked.json") = true ]] + homeDir=$TEST_ROOT/home mkdir -p "$homeDir" cp ../simple.nix ../simple.builder.sh "${config_nix}" "$homeDir/" From 9996b3236c30d24ccfd0ebcb354274e78cf23a4b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 13:02:28 +0200 Subject: [PATCH 784/927] nix flake bake: Prune the inventory at bake time Rather than having the baked flake filter out nodes that can't be baked (non-derivation leaves, filtered/failed nodes, unbakeable derivations, unknown outputs), do so when writing the inventory. This makes the baked flake.nix a plain projection of outputs.json. Assisted-by: Claude Fable 5.1 --- src/libcmd/flake-schemas.cc | 26 ++++++++++++++++++++++++++ src/nix/baked-flake.nix | 23 ++++------------------- tests/functional/flakes/bake.sh | 6 ++++++ 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index 5304b2ce26..5b8301e8e3 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -538,6 +538,32 @@ nlohmann::json getFlakeInventory( futures.finishAll(); + if (options.bake) { + /* Prune the inventory to the nodes that can be baked: derivation leaves, and non-leaf nodes that contain at + least one of them. This drops non-derivation leaves (e.g. overlays), filtered/failed nodes, derivations + that cannot be baked, and unknown outputs. */ + auto prune = [](this const auto & prune, nlohmann::json & node) -> bool { + if (auto children = node.find("children"); children != node.end()) { + for (auto i = children->begin(); i != children->end();) { + if (prune(i.value())) + ++i; + else + i = children->erase(i); + } + return !children->empty(); + } + return node.contains("derivation"); + }; + + for (auto i = inv.begin(); i != inv.end();) { + auto output = i.value().find("output"); + if (output != i.value().end() && prune(*output)) + ++i; + else + i = inv.erase(i); + } + } + return inv; } diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix index 972c85d0bb..6337821e56 100644 --- a/src/nix/baked-flake.nix +++ b/src/nix/baked-flake.nix @@ -14,25 +14,10 @@ else { ${builtins.head path} = setAttrByPath (builtins.tail path) value; }; - # Prune the inventory to the nodes that can be baked: derivation - # leaves, and non-leaf nodes that contain at least one of them. - prune = - node: - if node ? children then - let - children = builtins.filterAttrs (name: child: child != null) ( - builtins.mapAttrs (name: child: prune child) node.children - ); - in - if children == { } then null else node // { inherit children; } - else if node ? derivation then - node - else - null; - - inventories = builtins.filterAttrs (name: node: node != null) ( - builtins.mapAttrs (name: output: prune (output.output or null)) data - ); + # The inventory of each output. `nix flake bake` only records + # nodes that can be baked: derivation leaves, and non-leaf nodes + # that contain at least one of them. + inventories = builtins.mapAttrs (name: output: output.output) data; # Convert an inventory node into the corresponding flake output. convert = diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index de8b7a619a..c1ed1e6ab4 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -22,6 +22,11 @@ nix flake bake "$flakeDir" --dest-dir "$bakedDir" [[ -e $bakedDir/flake.nix ]] [[ -e $bakedDir/outputs.json ]] +# Only outputs that can be baked are recorded: unknown outputs (like +# `number`) and outputs for other systems are omitted. +[[ $(jq -c 'keys' < "$bakedDir/outputs.json") = '["legacyPackages","packages"]' ]] +[[ $(jq -c '.packages.output.children | keys' < "$bakedDir/outputs.json") = "[\"$system\"]" ]] + # The baked flake has the same outputs as the original for the current # system. Outputs for other systems are not included by default. nix flake show --json "path:$bakedDir" > "$TEST_ROOT/show-baked.json" @@ -145,6 +150,7 @@ EOF # Derivations that cannot be baked produce a warning and are omitted. nix flake bake "$greeterDir" --dest-dir "$bakedDir-greeter" 2>&1 | grepQuiet "warning: cannot bake 'packages.$system.ca'" +[[ $(jq -r ".packages.output.children.\"$system\".children | has(\"ca\")" < "$bakedDir-greeter/outputs.json") = false ]] [[ $(nix eval "path:$bakedDir-greeter#packages.$system" --apply 'x: x ? ca') = false ]] [[ $(nix eval "path:$bakedDir-greeter#packages.$system" --apply 'x: x ? greeter') = true ]] [[ $(jq -r ".packages.output.children.\"$system\".children.greeter.derivation.mainProgram" < "$bakedDir-greeter/outputs.json") = hi ]] From 7028d5da9e960ccbe9bd223d00de75c2c73e3b02 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 13:03:21 +0200 Subject: [PATCH 785/927] CLAUDE.md: Note that C++23 features can be used Assisted-by: Claude Fable 5.1 --- CLAUDE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index abd454bed0..b3cbf31eab 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,3 +5,5 @@ Observe the following rules when contributing to this repository: * Use "Assisted-by:" instead of "Co-Authored-By:" for the Claude trailer in commits. * Do not create PRs unless prompted. Create PRs using `gh pr create --repo DeterminateSystems/nix-src --base main`, observing `.github/PULL_REQUEST_TEMPLATE.md`. + +* The code base uses C++23, so C++23 features (e.g. deducing-this lambdas) can be used freely. From cd0eb2e6b60efbb5347d5f03ba3cb483ccf2f44e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 13:07:33 +0200 Subject: [PATCH 786/927] builtins.bakedDerivation: Respect read-only mode As with derivationStrict, don't write the derivation to the store in read-only mode, but just compute its path. Assisted-by: Claude Fable 5.1 --- src/libexpr/primops/bakedDerivation.cc | 5 ++++- tests/functional/flakes/bake.sh | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/bakedDerivation.cc b/src/libexpr/primops/bakedDerivation.cc index a1f4985801..d7cf2064cd 100644 --- a/src/libexpr/primops/bakedDerivation.cc +++ b/src/libexpr/primops/bakedDerivation.cc @@ -3,6 +3,7 @@ #include "nix/store/derivations.hh" #include "nix/store/derived-path.hh" #include "nix/store/store-api.hh" +#include "nix/store/globals.hh" #include @@ -93,7 +94,9 @@ static void prim_bakedDerivation(EvalState & state, const PosIdx pos, Value ** a for (auto & [outName, outPath] : outputs) drv.outputs.insert_or_assign(outName, DerivationOutput::InputAddressed{.path = outPath}); - auto drvPath = state.store->writeDerivation(*state.asyncPathWriter, drv, state.repair); + /* As in `derivationStrict`, don't write the derivation in read-only mode. */ + auto drvPath = settings.readOnlyMode ? computeStorePath(*state.store, drv) + : state.store->writeDerivation(*state.asyncPathWriter, drv, state.repair); /* As in `derivationStrict`, cache the derivation hash so that derivations depending on this one don't need to read it back from the store. */ diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index c1ed1e6ab4..f01f22c93c 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -27,6 +27,9 @@ nix flake bake "$flakeDir" --dest-dir "$bakedDir" [[ $(jq -c 'keys' < "$bakedDir/outputs.json") = '["legacyPackages","packages"]' ]] [[ $(jq -c '.packages.output.children | keys' < "$bakedDir/outputs.json") = "[\"$system\"]" ]] +# In read-only mode, evaluating the baked flake doesn't write the baked derivations to the store. +[[ ! -e $(nix eval --read-only --no-eval-cache --raw "path:$bakedDir#foo.drvPath") ]] + # The baked flake has the same outputs as the original for the current # system. Outputs for other systems are not included by default. nix flake show --json "path:$bakedDir" > "$TEST_ROOT/show-baked.json" From 764f09d68649f46f4ffb7790547c6fa763a691b9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 13:18:38 +0200 Subject: [PATCH 787/927] flake-bake.md: Mention that baked flakes provide their own schemas Assisted-by: Claude Fable 5.1 --- src/nix/flake-bake.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/nix/flake-bake.md b/src/nix/flake-bake.md index a381090292..c4bc62cd2c 100644 --- a/src/nix/flake-bake.md +++ b/src/nix/flake-bake.md @@ -25,7 +25,7 @@ R""( This command evaluates the outputs of the flake specified by flake reference *flake-url* and writes a new, *baked* flake to the directory *path* (specified by `--dest-dir`). The baked flake provides the same outputs as the original flake, but they are pre-evaluated: every derivation is replaced by a [baked derivation](@docroot@/language/builtins.md#builtins-bakedDerivation), that is, a derivation with the builder `builtin:substitute` that is never built but whose outputs are obtained by substitution from a binary cache. As a result, evaluating a baked flake is much faster than evaluating the original, since it does (almost) no evaluation. -The baked flake has no inputs, so it can be used without access to the inputs of the original flake. +The baked flake has no inputs, so it can be used without access to the inputs of the original flake. It provides its own [flake schemas](@docroot@/protocols/flake-schemas.md) (in the `schemas` output) that reproduce the structure of the original flake's outputs, so `nix flake show` and `nix flake check` work on it without needing the original flake's schemas. Derivation attributes in the baked flake have the same `name`, `system`, `outPath`, `outputName`, `outputs` and `meta.mainProgram` as in the original flake, so commands such as `nix build` and `nix run` behave the same. Their `drvPath` is different, however: it refers to the baked derivation rather than the original one. The outputs of a baked flake can also be used as inputs of other derivations, e.g. by another flake that has the baked flake as an input. @@ -41,8 +41,6 @@ By default, only the outputs for the current system are baked. Use `--all-system * Outputs whose derivation is nested inside the output attribute (such as `nixosConfigurations..config.system.build.toplevel`) are baked at that nested attribute. Other attributes of the output are not included. -* The baked flake provides its own [flake schemas](@docroot@/protocols/flake-schemas.md) (in the `schemas` output) that reproduce the structure of the original flake's outputs, so `nix flake show` and `nix flake check` work on it. However, any evaluation checks defined by the original flake's schemas are not performed on the baked flake. - This command requires the [`baked-derivations`](@docroot@/development/experimental-features.md#xp-feature-baked-derivations) experimental feature, both in the Nix CLI and in the Nix daemon. )"" From 275459c23a96b298de7272e48e74d10dcd5917cd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 15:40:23 +0200 Subject: [PATCH 788/927] builtins.appendContext: Wait for the async path writer before ensurePath() The context keys may refer to derivations that have been queued for writing but haven't landed in the store yet. Then ensurePath() would try to substitute them and fail with "path ... is required, but there is no substituter that can build it". This was easy to trigger with nixpkgs' writeStringReferencesToFile during a parallel evaluation. Assisted-by: Claude Fable 5.1 --- src/libexpr/primops/context.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 569e8a924d..3410a4b4a4 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -292,8 +292,11 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value ** arg if (!state.store->isStorePath(name)) state.error("context key '%s' is not a store path", name).atPos(i.pos).debugThrow(); auto namePath = state.store->parseStorePath(name); - if (!settings.readOnlyMode) + if (!settings.readOnlyMode) { + /* The path may be a derivation that is still being written asynchronously. */ + state.waitForPath(namePath); state.store->ensurePath(namePath); + } state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); if (auto attr = i.value->attrs()->get(sPath)) { From 411bf871ffd87252ff0bfb47a38279169273a52d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 15:42:26 +0200 Subject: [PATCH 789/927] nix flake bake: Don't use the eval cache Baking evaluates everything exactly once, so caching every attribute in SQLite is pure overhead. Worse, it serialised parallel evaluation on the database writer: baking Nixpkgs went from ~100% CPU to ~1200% on a 24-core machine (27 seconds with --read-only). Assisted-by: Claude Fable 5.1 --- src/nix/flake-bake.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/flake-bake.cc b/src/nix/flake-bake.cc index 2c4fadb2c8..c5ae897097 100644 --- a/src/nix/flake-bake.cc +++ b/src/nix/flake-bake.cc @@ -58,7 +58,9 @@ struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas auto evalStore = getEvalStore(); auto flake = make_ref(lockFlake()); - auto cache = flake_schemas::call(*state, flake, getDefaultFlakeSchemas()); + /* Don't use the eval cache: baking evaluates everything exactly once, so caching every attribute in + SQLite is pure overhead (and it serialises parallel evaluation on the database writer). */ + auto cache = flake_schemas::call(*state, flake, getDefaultFlakeSchemas(), /*allowEvalCache=*/false); auto inv = flake_schemas::getFlakeInventory(*state, *getEvalStore(), *flake, cache, options); From a849cc36d1479a114dc5a0bad762ecb8a37b7c62 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 15:45:39 +0200 Subject: [PATCH 790/927] nix flake bake: Add --read-only To support this, get the output paths of each derivation from its attribute set (via the eval cache cursor) rather than from the .drv file in the store. This also avoids waiting for the derivation to be written, which serialised parallel evaluation on the async path writer. Outputs whose path is a placeholder (e.g. content-addressed outputs) are detected as unbakeable. Assisted-by: Claude Fable 5.1 --- src/libcmd/flake-schemas.cc | 40 +++++++++++++++++++++++++++++---- src/nix/flake-bake.cc | 3 +-- tests/functional/flakes/bake.sh | 23 +++++++++++++++++++ 3 files changed, 60 insertions(+), 6 deletions(-) diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index 5b8301e8e3..c683de899a 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -447,17 +447,49 @@ nlohmann::json getFlakeInventory( evaluation time. */ bool bakeable = true; - if (options.showOutputPaths) { + if (options.bake) { + // FIXME: remove this once we have on-demand writing of .drvs. + /* Get the output paths from the derivation attribute set rather than from the `.drv` file + in the store. This avoids waiting for the derivation to be written to the store (which + would serialise parallel evaluation on the store writer) and works in read-only mode. + Outputs whose path is not known at evaluation time (e.g. content-addressed outputs) + have a placeholder rather than a store path. */ + auto outputs = nlohmann::json::object(); + auto getOutPath = [&](ref out) { + /* `derivation` produces an attribute set per output, but `import`ing a `.drv` + produces plain strings. */ + return out->forceValue().type() == nAttrs ? out->getAttr(state.s.outPath)->getString() + : out->getString(); + }; + auto addOutput = [&](const std::string & outputName, const std::string & outPath) { + if (state.store->isStorePath(outPath)) + outputs.emplace(outputName, outPath); + else { + outputs.emplace(outputName, nullptr); + bakeable = false; + } + }; + if (auto aOutputs = drv->maybeGetAttr(state.s.outputs)) + for (auto & outputName : aOutputs->getListOfStrings()) + addOutput(outputName, getOutPath(drv->getAttr(outputName))); + else { + auto aOutputName = drv->maybeGetAttr(state.s.outputName); + addOutput( + aOutputName ? aOutputName->getString() : "out", + drv->getAttr(state.s.outPath)->getString()); + } + drvObj.emplace("outputs", std::move(outputs)); + } + + else if (options.showOutputPaths) { auto outputs = nlohmann::json::object(); auto drvPath = drv->forceDerivation(); auto drv = evalStore.derivationFromPath(drvPath); for (auto & i : drv.outputsAndOptPaths(*state.store)) { if (auto outPath = i.second.second) outputs.emplace(i.first, state.store->printStorePath(*outPath)); - else { + else outputs.emplace(i.first, nullptr); - bakeable = false; - } } drvObj.emplace("outputs", std::move(outputs)); } diff --git a/src/nix/flake-bake.cc b/src/nix/flake-bake.cc index c5ae897097..1c47c10879 100644 --- a/src/nix/flake-bake.cc +++ b/src/nix/flake-bake.cc @@ -7,13 +7,12 @@ using namespace nix; using namespace nix::flake; -struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas +struct CmdFlakeBake : FlakeCommand, MixFlakeSchemas, MixReadOnlyOption { std::filesystem::path destDir; flake_schemas::FlakeInventoryOptions options{ .showLegacy = true, - .showOutputPaths = true, .showDrvNames = true, .bake = true, }; diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index f01f22c93c..b42cd962a7 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -190,6 +190,29 @@ nix build --no-link "$greeterDir#multi^*" [[ $(nix build --no-link --json "path:$bakedDir-greeter#multi^dev" | jq -r '.[0].outputs.dev') = "$multiDev" ]] [[ $(cat "$multiDev/what") = dev ]] +# With --read-only, baking doesn't write any derivations to the store, +# and produces the same result. +roDir=$TEST_ROOT/ro +mkdir -p "$roDir" +cp "${config_nix}" "$roDir/" +cat > "$roDir/flake.nix" <.config.system.build.toplevel`) are baked From 0c31e6f7bff2b20fdac689b7798ec9dea3eb3110 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 16:25:20 +0200 Subject: [PATCH 791/927] nix flake bake: Keep derivations that cannot be baked as stubs Derivations whose evaluation fails (e.g. packages marked as broken) or whose output paths aren't known were dropped from the baked flake. That made e.g. 'nix search' on a baked Nixpkgs miss about 60% of haskellPackages, whereas on the original it lists them (since it only evaluates 'name' and 'meta'). Now such derivations are recorded with 'failed = true', and the baked flake provides a stub for them that can be listed but fails when built, like the original. Also mark all attribute sets in the baked flake with 'recurseForDerivations = true', so that 'nix search' looks into nested package sets like 'haskellPackages'. Assisted-by: Claude Fable 5.1 --- src/libcmd/flake-schemas.cc | 111 +++++++++++++++++--------------- src/nix/baked-flake.nix | 27 ++++++++ tests/functional/flakes/bake.sh | 25 ++++++- 3 files changed, 108 insertions(+), 55 deletions(-) diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index c683de899a..e0d01fe1e9 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -422,21 +422,6 @@ nlohmann::json getFlakeInventory( if (options.showDrvNames) drvObj.emplace("name", drv->getAttr(state.s.name)->getString()); - if (options.bake) { - /* Record `meta.mainProgram` so that the baked flake can preserve it for `nix run`. */ - if (auto aMeta = drv->maybeGetAttr(state.s.meta)) - if (auto aMainProgram = aMeta->maybeGetAttr("mainProgram")) - drvObj.emplace("mainProgram", aMainProgram->getString()); - - /* Record which output this attribute refers to, since it's not necessarily `out`. */ - if (auto aOutputName = drv->maybeGetAttr(state.s.outputName)) - drvObj.emplace("outputName", aOutputName->getString()); - - /* Record the system of the derivation. */ - if (auto aSystem = drv->maybeGetAttr(state.s.system)) - drvObj.emplace("system", aSystem->getString()); - } - if (options.showDrvPaths) { auto drvPath = drv->forceDerivation(); drvObj.emplace("path", state.store->printStorePath(drvPath)); @@ -447,39 +432,67 @@ nlohmann::json getFlakeInventory( evaluation time. */ bool bakeable = true; - if (options.bake) { - // FIXME: remove this once we have on-demand writing of .drvs. - /* Get the output paths from the derivation attribute set rather than from the `.drv` file - in the store. This avoids waiting for the derivation to be written to the store (which - would serialise parallel evaluation on the store writer) and works in read-only mode. - Outputs whose path is not known at evaluation time (e.g. content-addressed outputs) - have a placeholder rather than a store path. */ - auto outputs = nlohmann::json::object(); - auto getOutPath = [&](ref out) { - /* `derivation` produces an attribute set per output, but `import`ing a `.drv` - produces plain strings. */ - return out->forceValue().type() == nAttrs ? out->getAttr(state.s.outPath)->getString() - : out->getString(); - }; - auto addOutput = [&](const std::string & outputName, const std::string & outPath) { - if (state.store->isStorePath(outPath)) - outputs.emplace(outputName, outPath); + if (options.bake) + try { + /* Record `meta.mainProgram` so that the baked flake can preserve it for `nix run`. */ + if (auto aMeta = drv->maybeGetAttr(state.s.meta)) + if (auto aMainProgram = aMeta->maybeGetAttr("mainProgram")) + drvObj.emplace("mainProgram", aMainProgram->getString()); + + /* Record which output this attribute refers to, since it's not necessarily `out`. */ + if (auto aOutputName = drv->maybeGetAttr(state.s.outputName)) + drvObj.emplace("outputName", aOutputName->getString()); + + /* Record the system of the derivation. */ + if (auto aSystem = drv->maybeGetAttr(state.s.system)) + drvObj.emplace("system", aSystem->getString()); + + // FIXME: remove this once we have on-demand writing of .drvs. + /* Get the output paths from the derivation attribute set rather than from the `.drv` file + in the store. This avoids waiting for the derivation to be written to the store (which + would serialise parallel evaluation on the store writer) and works in read-only mode. + Outputs whose path is not known at evaluation time (e.g. content-addressed outputs) + have a placeholder rather than a store path. */ + auto outputs = nlohmann::json::object(); + auto getOutPath = [&](ref out) { + /* `derivation` produces an attribute set per output, but `import`ing a `.drv` + produces plain strings. */ + return out->forceValue().type() == nAttrs ? out->getAttr(state.s.outPath)->getString() + : out->getString(); + }; + auto addOutput = [&](const std::string & outputName, const std::string & outPath) { + if (state.store->isStorePath(outPath)) + outputs.emplace(outputName, outPath); + else { + outputs.emplace(outputName, nullptr); + bakeable = false; + } + }; + if (auto aOutputs = drv->maybeGetAttr(state.s.outputs)) + for (auto & outputName : aOutputs->getListOfStrings()) + addOutput(outputName, getOutPath(drv->getAttr(outputName))); else { - outputs.emplace(outputName, nullptr); - bakeable = false; + auto aOutputName = drv->maybeGetAttr(state.s.outputName); + addOutput( + aOutputName ? aOutputName->getString() : "out", + drv->getAttr(state.s.outPath)->getString()); + } + drvObj.emplace("outputs", std::move(outputs)); + + if (!bakeable) { + warn( + "cannot bake '%s' because its output paths are not known in advance (e.g. it is a " + "content-addressed derivation)", + leaf.node->getAttrPathStr()); + drvObj.emplace("failed", true); } - }; - if (auto aOutputs = drv->maybeGetAttr(state.s.outputs)) - for (auto & outputName : aOutputs->getListOfStrings()) - addOutput(outputName, getOutPath(drv->getAttr(outputName))); - else { - auto aOutputName = drv->maybeGetAttr(state.s.outputName); - addOutput( - aOutputName ? aOutputName->getString() : "out", - drv->getAttr(state.s.outPath)->getString()); + } catch (EvalError & e) { + /* The derivation cannot be evaluated (e.g. a package marked as broken). Record it anyway, + so that the baked flake can still list it (like the original flake would), but fail + when it is built. */ + drvObj.erase("outputs"); + drvObj.emplace("failed", true); } - drvObj.emplace("outputs", std::move(outputs)); - } else if (options.showOutputPaths) { auto outputs = nlohmann::json::object(); @@ -494,13 +507,7 @@ nlohmann::json getFlakeInventory( drvObj.emplace("outputs", std::move(outputs)); } - if (options.bake && !bakeable) - warn( - "cannot bake '%s' because its output paths are not known in advance (e.g. it is a " - "content-addressed derivation)", - leaf.node->getAttrPathStr()); - else - obj.emplace("derivation", std::move(drvObj)); + obj.emplace("derivation", std::move(drvObj)); } if (options.bake && leaf.isFlakeCheck()) diff --git a/src/nix/baked-flake.nix b/src/nix/baked-flake.nix index 6337821e56..c4505bca7e 100644 --- a/src/nix/baked-flake.nix +++ b/src/nix/baked-flake.nix @@ -24,6 +24,33 @@ node: if node ? children then builtins.mapAttrs (name: child: convert child) node.children + # Unlike in Nixpkgs, every attribute set in a baked flake only + # contains things worth recursing into, so mark them all with the + # Nixpkgs `recurseForDerivations` convention for `nix search`. + // { + recurseForDerivations = true; + } + else if node.derivation.failed or false then + # The derivation could not be baked (e.g. because it is marked as + # broken, or is a content-addressed derivation). Provide a stub + # that can be listed (e.g. by `nix search`) but fails when built, + # like the original. + let + error = throw "package '${node.derivation.name}' cannot be built because it could not be baked (e.g. because it is marked as broken)"; + in + setAttrByPath (node.derivationAttrPath or [ ]) ( + { + type = "derivation"; + name = node.derivation.name; + meta = + optionalAttrs (node ? shortDescription) { description = node.shortDescription; } + // optionalAttrs (node.derivation ? mainProgram) { mainProgram = node.derivation.mainProgram; }; + drvPath = error; + outPath = error; + outputName = node.derivation.outputName or "out"; + } + // optionalAttrs (node.derivation ? system) { inherit (node.derivation) system; } + ) else let baked = builtins.bakedDerivation { diff --git a/tests/functional/flakes/bake.sh b/tests/functional/flakes/bake.sh index b42cd962a7..477f4ac5a3 100755 --- a/tests/functional/flakes/bake.sh +++ b/tests/functional/flakes/bake.sh @@ -139,6 +139,16 @@ cat > "$greeterDir/flake.nix" <&1 | grepQuiet "warning: cannot bake 'packages.$system.ca'" -[[ $(jq -r ".packages.output.children.\"$system\".children | has(\"ca\")" < "$bakedDir-greeter/outputs.json") = false ]] -[[ $(nix eval "path:$bakedDir-greeter#packages.$system" --apply 'x: x ? ca') = false ]] -[[ $(nix eval "path:$bakedDir-greeter#packages.$system" --apply 'x: x ? greeter') = true ]] +# Unbakeable derivations are recorded as failed. The baked flake still +# lists them (so `nix search` and `nix flake show` behave like the original)... +[[ $(jq -r ".packages.output.children.\"$system\".children.ca.derivation.failed" < "$bakedDir-greeter/outputs.json") = true ]] +[[ $(jq -r ".packages.output.children.\"$system\".children.broken.derivation.failed" < "$bakedDir-greeter/outputs.json") = true ]] +[[ $(jq -r ".packages.output.children.\"$system\".children.greeter.derivation | has(\"failed\")" < "$bakedDir-greeter/outputs.json") = false ]] +[[ $(nix eval --raw "path:$bakedDir-greeter#ca.name") = ca ]] +[[ $(nix eval --raw "path:$bakedDir-greeter#broken.meta.description") = "a broken package" ]] +nix search --json "path:$bakedDir-greeter" broken | jq -e ".\"packages.$system.broken\"" +nix flake show --json "path:$bakedDir-greeter" | jq -e ".inventory.packages.output.children.\"$system\".children.broken.derivation.name == \"broken\"" +# ... but building them fails. +expectStderr 1 nix build --no-link "path:$bakedDir-greeter#ca" | grepQuiet "package 'ca' cannot be built because it could not be baked" +expectStderr 1 nix build --no-link "path:$bakedDir-greeter#broken" | grepQuiet "package 'broken' cannot be built because it could not be baked" [[ $(jq -r ".packages.output.children.\"$system\".children.greeter.derivation.mainProgram" < "$bakedDir-greeter/outputs.json") = hi ]] [[ $(nix eval --raw "path:$bakedDir-greeter#greeter.meta.mainProgram") = hi ]] From 01163e689aca36fb8b39923564f98a0949e4f456 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Sep 2026 16:58:24 +0200 Subject: [PATCH 792/927] Include builtins.bakedDerivation in the manual 'nix __dump-language' only registers builtins whose experimental feature is in its hard-coded list, so the builtin was missing from the generated builtins page and the links to it were broken. Assisted-by: Claude Fable 5.1 --- src/nix/main.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index cb697d3bf1..0d6e6be8ed 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -578,6 +578,7 @@ void mainWrapped(int argc, char ** argv) Xp::FetchClosure, Xp::DynamicDerivations, Xp::FetchTree, + Xp::BakedDerivations, }; evalSettings.pureEval = false; auto statePtr = std::make_shared( From 528e705b875db2aab295d5ca28a22f569d6668d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 18 Sep 2026 21:31:50 +0200 Subject: [PATCH 793/927] nix flake prefetch-inputs: skip relative path inputs Relative path inputs have no narHash and PathInputScheme refuses to fetch them ("cannot fetch input 'path:..' because it uses a relative path"), so prefetch-inputs failed on any flake using them. Their content is part of the parent input, so there is nothing to fetch. Their own inputs are still visited. (cherry picked from commit db9b1a345c55e04c041bb12ab2eb58b0a368b3eb) --- src/nix/flake-prefetch-inputs.cc | 4 +++- tests/functional/flakes/relative-paths.sh | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nix/flake-prefetch-inputs.cc b/src/nix/flake-prefetch-inputs.cc index 32f8387714..f1eb7d7738 100644 --- a/src/nix/flake-prefetch-inputs.cc +++ b/src/nix/flake-prefetch-inputs.cc @@ -42,7 +42,9 @@ struct CmdFlakePrefetchInputs : FlakeCommand if (!state_.lock()->done.insert(&node).second) return; - if (auto lockedNode = dynamic_cast(&node)) { + // Relative path inputs are part of their parent's source tree. + if (auto lockedNode = dynamic_cast(&node); + lockedNode && !lockedNode->lockedRef.input.isRelative()) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("fetching '%s'", lockedNode->lockedRef)); auto accessor = lockedNode->lockedRef.input.getAccessor(fetchSettings, *store).first; diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index cd6442e5c6..6fb203c275 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -68,6 +68,9 @@ git -C "$rootFlake" add flake.nix sub2/flake.nix git -C "$rootFlake" add sub2/flake.lock [[ $(nix eval "$subflake2#y") = 15 ]] +# Relative path inputs must not make prefetch-inputs fail. +nix flake prefetch-inputs "$subflake2" + [[ $(jq --indent 0 --compact-output . < "$subflake2/flake.lock") =~ ^'{"nodes":{"root":{"inputs":{"root":"root_2","sub1":"sub1"}},"root_2":{"inputs":{"sub0":"sub0"},"locked":{"path":"..","type":"path"},"original":{"path":"..","type":"path"},"parent":[]},"root_3":{"inputs":{"sub0":"sub0_2"},"locked":{"path":"../","type":"path"},"original":{"path":"../","type":"path"},"parent":["sub1"]},"sub0":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["root"]},"sub0_2":{"locked":{"path":"sub0","type":"path"},"original":{"path":"sub0","type":"path"},"parent":["sub1","root"]},"sub1":{"inputs":{"root":"root_3"},"locked":{"path":"../sub1","type":"path"},"original":{"path":"../sub1","type":"path"},"parent":[]}},"root":"root","version":7}'$ ]] # Make sure there are no content locks for relative path flakes. From 0030a24d38e32c4df40ae69d77a0152fa9d28cb5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 15:50:23 +0200 Subject: [PATCH 794/927] builtins.wasm: Fix crash when copying layered attrsets into Wasm `copy_attrname` used `Bindings::operator[]` to look up the n-th attribute, but positional indexing is not supported for layered bindings (e.g. the result of `//`), so passing such an attrset to a Wasm function that reads it as an attrset aborted the evaluator: Unexpected condition in const nix::Attr& nix::Bindings::operator[](size_type) const Iterate to the n-th attribute instead, which matches the iteration order used by `copy_attrset`. This makes copying an attrset O(n^2), which is fine for now but noted as a TODO for a more efficient interface. Assisted-by: Claude Fable 5.1 (cherry picked from commit de7b046895875374d43f66725965298a4dd62776) --- src/libexpr/primops/wasm.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/wasm.cc b/src/libexpr/primops/wasm.cc index b59cbec93e..f7e650ba28 100644 --- a/src/libexpr/primops/wasm.cc +++ b/src/libexpr/primops/wasm.cc @@ -410,7 +410,15 @@ struct NixWasmInstance if ((size_t) attrIdx >= attrs.size()) throw Error("copy_attrname: attribute index out of bounds"); - std::string_view name = state.symbols[attrs[attrIdx].name]; + /* Note: `Bindings::operator[]` is not supported for layered + bindings (e.g. the result of `//`), so iterate instead. This + has to match the iteration order used by `copy_attrset`. + + TODO: Since this function is called once per attribute, copying + an attrset is O(n^2) (and n host calls). Come up with a more + efficient interface, e.g. a `copy_attrnames` function that copies + all names in one go. */ + std::string_view name = state.symbols[std::next(attrs.begin(), attrIdx)->name]; if ((size_t) len != name.size()) throw Error("copy_attrname: buffer length does not match attribute name length"); From ee053f9057530c63f00ede600578c5bfbbcc3328 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 09:30:35 +0200 Subject: [PATCH 795/927] builtins.wasm: Enable the wasmtime compilation cache The cache lives in ~/.cache/nix/wasmtime. wasmtime is configured via a TOML file, which we write there as well. Failure to enable the cache only produces a warning. This requires wasmtime's `cache` feature, so add it to our wasmtime build. Example speedup: `nix eval --impure --expr 'builtins.wasm { path = ./nix_wasm_plugin_quickjs.wasm; } "({ sum: 1 + 1 })"'` went from ~0.65s to ~0.04s on cached invocations. Assisted-by: Claude Fable 5.1 (cherry picked from commit 79a3c2396000981854296681f79fafad393486ec) --- packaging/wasmtime.nix | 2 +- src/libexpr/primops/wasm.cc | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packaging/wasmtime.nix b/packaging/wasmtime.nix index d2c2b95f60..3c479d91ed 100644 --- a/packaging/wasmtime.nix +++ b/packaging/wasmtime.nix @@ -29,7 +29,7 @@ rust.packages.stable.rustPlatform.buildRustPackage (finalAttrs: { "--package" "wasmtime-c-api" "--no-default-features" - "--features cranelift,wasi,pooling-allocator,wat,demangle,gc-null" + "--features cache,cranelift,wasi,pooling-allocator,wat,demangle,gc-null" ]; outputs = [ diff --git a/src/libexpr/primops/wasm.cc b/src/libexpr/primops/wasm.cc index f7e650ba28..1342a02920 100644 --- a/src/libexpr/primops/wasm.cc +++ b/src/libexpr/primops/wasm.cc @@ -1,5 +1,7 @@ #include "nix/expr/primops.hh" #include "nix/expr/eval-inline.hh" +#include "nix/util/users.hh" +#include "nix/util/file-system.hh" #include #include @@ -18,12 +20,41 @@ T unwrap(Result && res) throw Error(res.err().message()); } +/** + * Enable wasmtime's compilation cache, so that a module is only compiled + * once per machine rather than once per process. The cache lives in Nix's + * cache directory; wasmtime is configured through a TOML file, which we + * write there as well. + */ +static void enableCompilationCache(wasmtime::Config & config) +{ + try { + auto cacheDir = getCacheDir() / "wasmtime"; + createDirs(cacheDir); + + // TOML literal strings cannot contain single quotes. + auto dir = cacheDir.string(); + if (dir.find('\'') != std::string::npos) + throw Error("cache directory '%s' contains a single quote", dir); + + auto configFile = cacheDir / "config.toml"; + auto contents = fmt("[cache]\ndirectory = '%s'\n", dir); + writeFile(configFile, contents); + + unwrap(config.cache_load(configFile.string())); + } catch (Error & e) { + // Not being able to cache is not fatal. + warn("unable to enable the Wasm compilation cache: %s", e.msg()); + } +} + static Engine & getEngine() { static Engine engine = []() { wasmtime::Config config; config.pooling_allocation_strategy(PoolAllocationConfig()); config.memory_init_cow(true); + enableCompilationCache(config); return Engine(std::move(config)); }(); return engine; From 0cd357d50a6f069946e4a759305349363b06b7a8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 09:44:12 +0200 Subject: [PATCH 796/927] Update wasmtime to 48.0.2 Two changes were needed: * The C API's C++ headers now use GC types that the C header only declares when the C API crate's own `gc` feature is enabled, so enable it alongside `gc-null` (which implies `gc` in the runtime crate but not in the C API crate). * `wasi_config_preopen_dir` takes a single `bool fs_mutable` instead of directory and file permission bitmasks. The closure of the nix executable grows by 0.7 MiB (142.6 -> 143.3). Assisted-by: Claude Fable 5.1 (cherry picked from commit 0daf15f058e8676afe25150c8a9ca188c5427285) --- packaging/wasmtime.nix | 8 ++++---- src/libstore/unix/build/wasi-derivation-builder.cc | 6 ++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packaging/wasmtime.nix b/packaging/wasmtime.nix index 3c479d91ed..ca2c758ae0 100644 --- a/packaging/wasmtime.nix +++ b/packaging/wasmtime.nix @@ -11,25 +11,25 @@ }: rust.packages.stable.rustPlatform.buildRustPackage (finalAttrs: { pname = "wasmtime"; - version = "40.0.2"; + version = "48.0.2"; src = fetchFromGitHub { owner = "bytecodealliance"; repo = "wasmtime"; tag = "v${finalAttrs.version}"; - hash = "sha256-4y9WpCdyuF/Tp2k/1d5rZxwYunWNdeibEsFgHcBC52Q="; + hash = "sha256-XMCHmvPVmiWpMsICNOGhV5GN+yyPTVFfnUgOC68qyo0="; fetchSubmodules = true; }; # Disable cargo-auditable until https://github.com/rust-secure-code/cargo-auditable/issues/124 is solved. auditable = false; - cargoHash = "sha256-aTPgnuBvOIqg1+Sa2ZLdMTLujm8dKGK5xpZ3qHpr3f8="; + cargoHash = "sha256-KNCqgHMP/bCUAmDwTY+90g7bI69ud0rckj1iKlxadfM="; cargoBuildFlags = [ "--package" "wasmtime-c-api" "--no-default-features" - "--features cache,cranelift,wasi,pooling-allocator,wat,demangle,gc-null" + "--features cache,cranelift,wasi,pooling-allocator,wat,demangle,gc,gc-null" ]; outputs = [ diff --git a/src/libstore/unix/build/wasi-derivation-builder.cc b/src/libstore/unix/build/wasi-derivation-builder.cc index 6fb7c41c22..d3968cc2c2 100644 --- a/src/libstore/unix/build/wasi-derivation-builder.cc +++ b/src/libstore/unix/build/wasi-derivation-builder.cc @@ -54,14 +54,12 @@ struct WasiDerivationBuilder : DerivationBuilderImpl if (!wasiConfig.preopen_dir( store.config->realStoreDir.get().string(), store.storeDir, - WASMTIME_WASI_DIR_PERMS_READ | WASMTIME_WASI_DIR_PERMS_WRITE, - WASMTIME_WASI_FILE_PERMS_READ | WASMTIME_WASI_FILE_PERMS_WRITE)) + /* fs_mutable = */ true)) throw Error("cannot add store directory to WASI config"); if (!wasiConfig.preopen_dir( tmpDir, tmpDirInSandbox(), - WASMTIME_WASI_DIR_PERMS_READ | WASMTIME_WASI_DIR_PERMS_WRITE, - WASMTIME_WASI_FILE_PERMS_READ | WASMTIME_WASI_FILE_PERMS_WRITE)) + /* fs_mutable = */ true)) throw Error("cannot add temporary directory to WASI config"); auto module = unwrap(Module::compile(engine, string2span(readFile(realPathInHost(drv.builder))))); From 6f77b6149207702b5015f97a23f24caa18576976 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Aug 2026 17:22:18 +0200 Subject: [PATCH 797/927] Make the parallel evaluator fiber-based Previously, when an evaluation worker thread hit a thunk that was being evaluated by another thread, it blocked on a condition variable until the thunk was finished. To utilize all cores, this requires oversubscribing threads, and even then most threads can end up blocked on a single popular thunk. Instead, run every work item on its own boost::context fiber. When a fiber hits a pending/awaited thunk, it suspends and the worker thread switches to other work; `finish()` re-enqueues exactly the fibers waiting on that value (the wait list is keyed on the value, so unlike the old hashed-domain condition variable, fiber wakeups are never spurious). Thus we always make progress as long as there is runnable work: e.g. 200 work items can be simultaneously suspended on a shared thunk while 8 worker threads keep evaluating. Notes: * The suspension handshake keeps the waiter-domain mutex locked across the context switch: the fiber's continuation only materializes on the scheduler side of the switch, so the scheduler registers it in the wait list and then releases the lock, preventing another thread from resuming a half-suspended fiber. * Non-fiber contexts (e.g. the main thread) still use the old condition variable path in `waitOnThunk()`. * `myEvalThreadId` is now a per-fiber id (set on every fiber switch-in), since with two fibers on one thread, a per-thread id would produce false "infinite recursion" errors from the self-wait check. * On shutdown/interrupt, all suspended fibers are flushed from the wait lists and resumed so that they observe `quit` (or the interrupt) and unwind their stacks via a normal `Interrupted` exception; a suspended fiber is never destroyed. Work items that haven't started get an `Interrupted` exception on their promise. * Fiber stacks are not yet registered as GC roots, so for now, parallel evaluation requires the GC to be disabled (e.g. via GC_DONT_GC=1). Also not yet done: throttling the number of live fibers, fiber stack pooling, and cross-fiber deadlock detection. Assisted-by: Claude Fable 5 --- src/libexpr/eval.cc | 3 + src/libexpr/include/nix/expr/parallel-eval.hh | 55 ++- src/libexpr/parallel-eval.cc | 385 ++++++++++++++++-- 3 files changed, 411 insertions(+), 32 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f0effb0add..087e8eaeb6 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3100,6 +3100,9 @@ void EvalState::printStatistics() topObj["nrSpuriousWakeups"] = nrSpuriousWakeups.load(); topObj["maxWaiting"] = maxWaiting.load(); topObj["waitingTime"] = microsecondsWaiting / (double) 1000000; + topObj["nrFibersSpawned"] = executor->nrFibersSpawned.load(); + topObj["nrFiberWakeups"] = executor->nrFiberWakeups.load(); + topObj["maxSuspendedFibers"] = executor->maxSuspendedFibers.load(); topObj["nrAvoided"] = nrAvoided.load(); topObj["nrLookups"] = nrLookups.load(); topObj["nrPrimOpCalls"] = nrPrimOpCalls.load(); diff --git a/src/libexpr/include/nix/expr/parallel-eval.hh b/src/libexpr/include/nix/expr/parallel-eval.hh index 4f611328f4..05ff1d191b 100644 --- a/src/libexpr/include/nix/expr/parallel-eval.hh +++ b/src/libexpr/include/nix/expr/parallel-eval.hh @@ -1,9 +1,11 @@ #pragma once #include +#include #include #include #include +#include #include @@ -30,9 +32,24 @@ struct Executor work_t work; }; + /** + * A work item running on its own stack. Defined in + * `parallel-eval.cc`; opaque here to keep the Boost.Context + * dependency out of this header. + */ + struct Fiber; + + using FiberPtr = std::unique_ptr; + + /** + * Queue entries are either fresh work items (which get a new fiber + * when they're picked up) or suspended fibers to be resumed. + */ + using QueueEntry = std::variant; + struct State { - std::multimap queue; + std::multimap queue; std::vector threads; }; @@ -48,6 +65,11 @@ struct Executor std::condition_variable wakeup; + std::atomic nrFibersSpawned{0}; + std::atomic nrFiberWakeups{0}; + std::atomic currentSuspendedFibers{0}; + std::atomic maxSuspendedFibers{0}; + static unsigned int getEvalCores(const EvalSettings & evalSettings); Executor(const EvalSettings & evalSettings); @@ -58,6 +80,37 @@ struct Executor void worker(); + /** + * Create a fiber for a fresh work item. If fiber creation fails + * (e.g. stack allocation failure), the item's promise receives the + * exception and a null pointer is returned. + */ + FiberPtr makeFiber(Item && item); + + /** + * Start or resume a fiber on the current thread. On return, the + * fiber has either finished (its promise is fulfilled and the + * fiber is destroyed) or suspended itself waiting on a thunk (in + * which case it has been registered with the thunk's waiter + * domain). + */ + void runFiber(FiberPtr fiber); + + /** + * Put a previously suspended fiber back onto the ready queue, at + * the highest priority. Called when the thunk it was waiting on + * has been finished. + */ + void enqueueFiber(FiberPtr fiber); + + /** + * Drain the queue on shutdown/interrupt: work items that haven't + * started get an `Interrupted` exception on their promise; + * suspended fibers are resumed so they can observe `quit` and + * unwind their stacks. + */ + void drainQueue(); + using WorkItems = std::vector>; std::vector> spawn(WorkItems && items); diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index 1c89b9c4db..bec50daa3a 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -3,17 +3,128 @@ #include "nix/store/globals.hh" #include "nix/expr/primops.hh" +#include +#include + +#include + namespace nix { +struct WaiterDomain; + +[[gnu::tls_model("initial-exec")]] thread_local bool Executor::amWorkerThread{false}; + +static std::atomic nextEvalThreadId{1}; +[[gnu::tls_model("initial-exec")]] thread_local uint32_t myEvalThreadId(nextEvalThreadId++); + +/** + * The fiber currently executing on this thread, or null if we're not + * inside a fiber (e.g. on the main thread or in a worker's scheduler + * loop). + */ +[[gnu::tls_model("initial-exec")]] static thread_local Executor::Fiber * currentFiber{nullptr}; + +struct Executor::Fiber +{ + Executor & executor; + + /** + * The value of `myEvalThreadId` while this fiber is running. Each + * fiber needs its own id (rather than a per-thread id) since the + * self-wait check in `waitOnThunk()` would otherwise produce false + * "infinite recursion" errors when two fibers running on the same + * thread touch the same thunk. + */ + const uint32_t evalThreadId; + + std::promise promise; + + work_t work; + + /** + * The fiber's continuation. Valid while the fiber is queued or + * suspended; invalid while it's running or after it has finished. + */ + boost::context::fiber ctx; + + /** + * The scheduler's continuation. Valid while the fiber is running. + */ + boost::context::fiber schedCtx; + + /** + * Side channel for the suspension handshake, filled in by + * `suspendFiber()` immediately before switching back to the + * scheduler, and consumed by `runFiber()`. + */ + detail::ValueBase * waitingOn = nullptr; + WaiterDomain * suspendDomain = nullptr; + std::unique_lock * suspendLock = nullptr; + + Fiber(Executor & executor, Item && item) + : executor(executor) + , evalThreadId(nextEvalThreadId++) + , promise(std::move(item.promise)) + , work(std::move(item.work)) + { + } + + ~Fiber() + { + /* A fiber must never be destroyed while suspended, since that + would forcibly unwind its stack. Suspended fibers are always + resumed (with `quit` or an interrupt flag set) so that they + exit normally. */ + assert(!ctx); + } +}; + // cache line alignment to prevent false sharing struct alignas(64) WaiterDomain { + /* Note: not using `Sync` because the suspension handshake requires + locking the mutex on the fiber's stack and unlocking it from the + scheduler (see `Executor::runFiber()`), which `Sync::Lock` can't + express. */ + std::mutex mutex; + + /** + * Wakes up non-fiber threads (e.g. the main thread) blocked on a + * thunk in this domain. + */ std::condition_variable cv; + + /** + * Fibers suspended waiting for a specific value to be finished. + * Unlike the condition variable, this is keyed on the exact value, + * so fiber wakeups are never spurious. + */ + std::unordered_multimap waiters; }; -static std::array, 128> waiterDomains; +static std::array waiterDomains; -[[gnu::tls_model("initial-exec")]] thread_local bool Executor::amWorkerThread{false}; +/** + * Move all suspended fibers out of the wait lists and back onto their + * executor's ready queue, and wake up all non-fiber waiters. Called on + * interrupt and shutdown so that waiting fibers/threads can observe + * the interrupt/`quit` flag and unwind. + */ +static void flushWaiters() +{ + std::vector woken; + for (auto & domain : waiterDomains) { + std::unique_lock lk(domain.mutex); + for (auto & i : domain.waiters) + woken.push_back(std::move(i.second)); + domain.waiters.clear(); + domain.cv.notify_all(); + } + for (auto & fiber : woken) { + auto & executor = fiber->executor; + executor.enqueueFiber(std::move(fiber)); + } +} unsigned int Executor::getEvalCores(const EvalSettings & evalSettings) { @@ -28,8 +139,11 @@ Executor::Executor(const EvalSettings & evalSettings) : evalCores(getEvalCores(evalSettings)) , enabled(evalCores > 1) , interruptCallback(createInterruptCallback([&]() { - for (auto & domain : waiterDomains) - domain.lock()->cv.notify_all(); + /* Wake up all waiting fibers and threads so they can observe + the interrupt and unwind. Note: `_isInterrupted` has already + been set at this point. */ + flushWaiters(); + wakeup.notify_all(); })) { debug("executor using %d threads", evalCores); @@ -56,10 +170,19 @@ Executor::~Executor() debug("executor shutting down with %d items left", state->queue.size()); } + /* Wake up suspended fibers and idle workers so they can wind + down. */ + flushWaiters(); wakeup.notify_all(); for (auto & thr : threads) thr.join(); + + /* Handle any stragglers, e.g. fibers that were re-enqueued by + `notifyWaiters()` on the main thread after the workers already + exited. No fiber stack may outlive this destructor. */ + flushWaiters(); + drainQueue(); } void Executor::createWorker(State & state) @@ -79,6 +202,93 @@ void Executor::createWorker(State & state) })); } +Executor::FiberPtr Executor::makeFiber(Item && item) +{ + auto fiber = std::make_unique(*this, std::move(item)); + auto fib = fiber.get(); + try { + fiber->ctx = boost::context::fiber( + std::allocator_arg, + boost::context::protected_fixedsize_stack(evalStackSize), + [fib](boost::context::fiber && sched) -> boost::context::fiber { + fib->schedCtx = std::move(sched); + try { + fib->work(); + fib->promise.set_value(); + } catch (const Interrupted &) { + fib->executor.quit = true; + fib->promise.set_exception(std::current_exception()); + } catch (...) { + fib->promise.set_exception(std::current_exception()); + } + // Terminate the fiber, freeing its stack, and return + // to the scheduler. + return std::move(fib->schedCtx); + }); + } catch (...) { + // Stack allocation failure. Report it to whoever is waiting + // on this work item. + fiber->promise.set_exception(std::current_exception()); + return nullptr; + } + nrFibersSpawned++; + return fiber; +} + +void Executor::runFiber(FiberPtr fiber) +{ + auto fib = fiber.get(); + assert(fib->ctx); + assert(!currentFiber); + + auto savedThreadId = myEvalThreadId; + currentFiber = fib; + myEvalThreadId = fib->evalThreadId; + + fib->ctx = std::move(fib->ctx).resume(); + + currentFiber = nullptr; + myEvalThreadId = savedThreadId; + + if (fib->ctx) { + /* The fiber suspended itself in `waitOnThunk()`. We are still + holding the waiter domain lock, which the fiber acquired on + this thread before switching back to us (so `std::mutex` + thread ownership is respected). Register the fiber in the + wait list, then release the lock. Only after the unlock can + `notifyWaiters()` extract and resume the fiber — at which + point `fib->ctx` is fully formed. Note that `suspendLock` + points into the fiber's (currently quiescent) stack. */ + auto domain = fib->suspendDomain; + auto lock = fib->suspendLock; + assert(domain && lock && fib->waitingOn); + auto n = ++currentSuspendedFibers; + if (n > maxSuspendedFibers) + maxSuspendedFibers = n; + domain->waiters.emplace(fib->waitingOn, std::move(fiber)); + lock->unlock(); + /* `fib` may be resumed by another thread from this point on, + so don't touch it anymore. */ + } else { + /* The fiber has finished; its promise has been fulfilled + inside the fiber. Destroying `fiber` frees the record (the + stack was already freed on fiber termination). */ + } +} + +void Executor::enqueueFiber(FiberPtr fiber) +{ + nrFiberWakeups++; + currentSuspendedFibers--; + { + auto state(state_.lock()); + /* Note: key 0 means that resumed fibers run before any fresh + work items, so existing work is drained first. */ + state->queue.emplace(0, std::move(fiber)); + } + wakeup.notify_one(); +} + void Executor::worker() { ReceiveInterrupts receiveInterrupts; @@ -88,37 +298,64 @@ void Executor::worker() amWorkerThread = true; while (true) { - Item item; + QueueEntry entry; + bool gotEntry = false; while (true) { auto state(state_.lock()); - if (quit) { - // Set an `Interrupted` exception on all promises so - // we get a nicer error than "std::future_error: - // Broken promise". - auto ex = std::make_exception_ptr(Interrupted("interrupted by the user")); - for (auto & item : state->queue) - item.second.promise.set_exception(ex); - state->queue.clear(); - return; - } + if (quit) + break; if (!state->queue.empty()) { - item = std::move(state->queue.begin()->second); + entry = std::move(state->queue.begin()->second); state->queue.erase(state->queue.begin()); + gotEntry = true; break; } state.wait(wakeup); } - try { - item.work(); - item.promise.set_value(); - } catch (const Interrupted &) { - quit = true; - item.promise.set_exception(std::current_exception()); - } catch (...) { - item.promise.set_exception(std::current_exception()); + if (!gotEntry) { + drainQueue(); + return; } + + if (auto * item = std::get_if(&entry)) { + if (auto fiber = makeFiber(std::move(*item))) + runFiber(std::move(fiber)); + } else + runFiber(std::move(std::get(entry))); + } +} + +void Executor::drainQueue() +{ + while (true) { + /* Keep flushing the wait lists: a fiber resumed below can + finish thunks, which normally re-enqueues their waiters, but + late waiters may still be parked. */ + flushWaiters(); + + QueueEntry entry; + { + auto state(state_.lock()); + if (state->queue.empty()) + return; + entry = std::move(state->queue.begin()->second); + state->queue.erase(state->queue.begin()); + } + + if (auto * item = std::get_if(&entry)) + // Set an `Interrupted` exception on work items that + // haven't started, so we get a nicer error than + // "std::future_error: Broken promise". Note: a fresh + // exception per item, not a shared one. + item->promise.set_exception(std::make_exception_ptr(Interrupted("interrupted by the user"))); + else + /* Resume the fiber so it can observe `quit` (or the + interrupt) and unwind; it cannot suspend again thanks to + the guard in `waitOnThunk()`. Never destroy a suspended + fiber. */ + runFiber(std::move(std::get(entry))); } } @@ -199,14 +436,46 @@ void FutureVector::finishAll() std::rethrow_exception(ex); } -static Sync & getWaiterDomain(detail::ValueBase & v) +static WaiterDomain & getWaiterDomain(detail::ValueBase & v) { auto domain = (((size_t) &v) >> 5) % waiterDomains.size(); return waiterDomains[domain]; } -static std::atomic nextEvalThreadId{1}; -[[gnu::tls_model("initial-exec")]] thread_local uint32_t myEvalThreadId(nextEvalThreadId++); +/** + * Suspend the current fiber until `v` is finished. Must be called with + * `lk` holding the domain's mutex and `v` in the "awaited" state. On + * return, the fiber has been resumed (possibly on a different thread) + * and the lock has been released. + * + * Note: `noinline` so that the compiler doesn't move TLS accesses of + * the caller across the context switch (the fiber may wake up on + * another thread). + */ +[[gnu::noinline]] static void +suspendFiber(WaiterDomain & domain, std::unique_lock & lk, detail::ValueBase & v) +{ + auto fib = currentFiber; + assert(fib); + /* A fiber must not suspend while an exception is being handled or + unwound: the C++ exception state lives in thread-local storage, + so it wouldn't survive being resumed on another thread. */ + assert(!std::current_exception() && !std::uncaught_exceptions()); + fib->waitingOn = &v; + fib->suspendDomain = &domain; + fib->suspendLock = &lk; + /* Switch back to the scheduler (`Executor::runFiber()`), which + will register us in the domain's wait list and then release the + lock. We can't do that here: the fiber's continuation only + materializes on the scheduler side, and publishing it before the + switch-out completes would allow another thread to resume a + half-suspended fiber. */ + fib->schedCtx = std::move(fib->schedCtx).resume(); + /* We've been resumed because the value was finished (or because + we're shutting down); the scheduler released the lock long + ago. */ + assert(!lk.owns_lock()); +} template<> ValueStorage::PackedPointer @@ -214,7 +483,8 @@ ValueStorage::waitOnThunk(EvalState & state, PackedPointer expec { state.nrThunksAwaited++; - auto domain = getWaiterDomain(*this).lock(); + auto & domain = getWaiterDomain(*this); + std::unique_lock lk(domain.mutex); auto threadId = expectedP0 >> discriminatorBits; @@ -262,8 +532,44 @@ ValueStorage::waitOnThunk(EvalState & state, PackedPointer expec auto now1 = std::chrono::steady_clock::now(); + if (auto fib = currentFiber) { + /* Shutdown guard: `quit`/`_isInterrupted` are always set + *before* the wait lists are flushed, and flushing takes the + domain lock that we're currently holding. So either we see + the flag here and throw, or our registration completes + before the flush and we get woken by it — we can't be + stranded in the wait list. */ + if (fib->executor.quit) + throw Interrupted("interrupted by the user"); + checkInterrupt(); + + /* We're running on a fiber, so suspend it and let this thread + run other work. `notifyWaiters()` will re-enqueue the fiber + when the value is finished. */ + suspendFiber(domain, lk, *this); + + /* Note: use the `fib` captured before the suspension rather + than TLS variables, since we may have been resumed on a + different thread. */ + if (fib->executor.quit) + throw Interrupted("interrupted by the user"); + checkInterrupt(); + + auto p0_ = p0.load(std::memory_order_acquire); + auto pd = static_cast(p0_ & discriminatorMask); + /* Unlike the condition variable path below, fiber wakeups + cannot be spurious: the wait list is keyed on the exact + value and we're only woken after it has been finished (or on + shutdown/interrupt, handled above). */ + assert(pd != pdThunk && pd != pdPending && pd != pdAwaited); + auto now2 = std::chrono::steady_clock::now(); + state.microsecondsWaiting += std::chrono::duration_cast(now2 - now1).count(); + state.currentlyWaiting--; + return p0_; + } + while (true) { - domain.wait(domain->cv); + domain.cv.wait(lk); auto p0_ = p0.load(std::memory_order_acquire); auto pd = static_cast(p0_ & discriminatorMask); if (pd != pdAwaited) { @@ -281,9 +587,26 @@ ValueStorage::waitOnThunk(EvalState & state, PackedPointer expec template<> void ValueStorage::notifyWaiters() { - auto domain = getWaiterDomain(*this).lock(); + auto & domain = getWaiterDomain(*this); - domain->cv.notify_all(); + /* Extract the fibers waiting on this value, then re-enqueue them + after releasing the domain lock (to keep a trivial lock order + between domain mutexes and the executor's state lock). */ + std::vector woken; + { + std::unique_lock lk(domain.mutex); + auto [b, e] = domain.waiters.equal_range(this); + for (auto i = b; i != e; ++i) + woken.push_back(std::move(i->second)); + domain.waiters.erase(b, e); + /* Wake up any non-fiber waiters (e.g. the main thread). */ + domain.cv.notify_all(); + } + + for (auto & fiber : woken) { + auto & executor = fiber->executor; + executor.enqueueFiber(std::move(fiber)); + } } static void prim_parallel(EvalState & state, const PosIdx pos, Value ** args, Value & v) From f8032270290a1b1231e2f9f3c20d92794ee2c9b3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Aug 2026 18:28:15 +0200 Subject: [PATCH 798/927] Make call-depth accounting fiber-aware `EvalState::callDepth` is a thread-local counter, and the `CallDepth` RAII guard holds a reference to it. A fiber can suspend mid-call-chain (with guards alive on its stack) and be resumed on a different thread; the guards would then decrement the original thread's counter through the stale reference, while the new thread's counter never comes back down, corrupting the depth accounting on both threads and producing spurious "max-call-depth exceeded" errors. Give each fiber its own call-depth counter in the `Fiber` record, and make the thread-local `EvalState::callDepthPtr` point to the counter of the current execution context: the fiber's counter while a fiber is running (switched in `Executor::runFiber()`), or the thread's own `callDepth` otherwise. Since the fiber's counter lives in the heap-allocated `Fiber` record, the references held by `CallDepth` guards remain valid no matter which thread runs or unwinds them. Assisted-by: Claude Fable 5 --- src/libexpr/eval.cc | 1 + src/libexpr/include/nix/expr/eval-inline.hh | 5 +++-- src/libexpr/include/nix/expr/eval.hh | 11 +++++++++++ src/libexpr/parallel-eval.cc | 12 ++++++++++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 087e8eaeb6..24cdb85539 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1582,6 +1582,7 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) } [[gnu::tls_model("initial-exec")]] thread_local size_t EvalState::callDepth = 0; +[[gnu::tls_model("initial-exec")]] thread_local size_t * EvalState::callDepthPtr = &EvalState::callDepth; void EvalState::callFunction(Value & fun, std::span args, Value & vRes, const PosIdx pos) { diff --git a/src/libexpr/include/nix/expr/eval-inline.hh b/src/libexpr/include/nix/expr/eval-inline.hh index 6d439111cf..35e599ee9a 100644 --- a/src/libexpr/include/nix/expr/eval-inline.hh +++ b/src/libexpr/include/nix/expr/eval-inline.hh @@ -189,10 +189,11 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e [[gnu::always_inline]] inline CallDepth EvalState::addCallDepth(const PosIdx pos) { - if (callDepth > settings.maxCallDepth) + auto & depth = *callDepthPtr; + if (depth > settings.maxCallDepth) error().atPos(pos).debugThrow(); - return CallDepth(callDepth); + return CallDepth(depth); }; } // namespace nix diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index b0ae6c6ecb..e079a94592 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -909,6 +909,17 @@ private: public: + /** + * Points to the call-depth counter of the current execution + * context: `callDepth` (this thread's counter) when not running + * on a fiber, or the fiber's own counter while a fiber is running + * (see `Executor::runFiber()`). Fibers need their own counter + * since they can be suspended mid-call-chain and resumed on a + * different thread, and the `CallDepth` guards on their stack + * hold a reference to the counter. + */ + [[gnu::tls_model("initial-exec")]] thread_local static size_t * callDepthPtr; + /** * Check that the call depth is within limits, and increment it, until the returned object is destroyed. */ diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index bec50daa3a..d4cfba240e 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -37,6 +37,15 @@ struct Executor::Fiber */ const uint32_t evalThreadId; + /** + * This fiber's Nix call stack depth (see + * `EvalState::callDepthPtr`). `EvalState::addCallDepth()` uses + * this counter instead of the thread-local one while the fiber is + * running, so that the depth accounting survives the fiber being + * suspended mid-call-chain and resumed on a different thread. + */ + size_t callDepth = 0; + std::promise promise; work_t work; @@ -242,13 +251,16 @@ void Executor::runFiber(FiberPtr fiber) assert(!currentFiber); auto savedThreadId = myEvalThreadId; + auto savedCallDepthPtr = EvalState::callDepthPtr; currentFiber = fib; myEvalThreadId = fib->evalThreadId; + EvalState::callDepthPtr = &fib->callDepth; fib->ctx = std::move(fib->ctx).resume(); currentFiber = nullptr; myEvalThreadId = savedThreadId; + EvalState::callDepthPtr = savedCallDepthPtr; if (fib->ctx) { /* The fiber suspended itself in `waitOnThunk()`. We are still From bb19862179da5a3db680cf00b215c6e43f70016b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Aug 2026 19:40:43 +0200 Subject: [PATCH 799/927] Fix a race on the domain lock in the fiber suspension handshake The scheduler released the waiter-domain mutex through a `std::unique_lock` living on the suspending fiber's stack. However, `unique_lock::unlock()` releases the mutex *before* clearing its owns-flag, and the moment the mutex is released, another thread can extract and resume the fiber, which then reads the owns-flag on its stack concurrently with the scheduler's write. Occasionally the fiber would read a stale `true`, tripping the `!lk.owns_lock()` assertion (and in builds without assertions, the `unique_lock` destructor would unlock a mutex it doesn't hold, which is undefined behavior). This showed up as a flaky abort in `nix search`. Instead, the fiber now calls `lk.release()` (a fiber-local write) before switching out, handing mutex ownership to the scheduler, which unlocks `domain.mutex` directly. That's legal since the mutex was locked by the fiber on the scheduler's own thread. Assisted-by: Claude Fable 5 --- src/libexpr/parallel-eval.cc | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index d4cfba240e..7266ab07c0 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -68,7 +68,6 @@ struct Executor::Fiber */ detail::ValueBase * waitingOn = nullptr; WaiterDomain * suspendDomain = nullptr; - std::unique_lock * suspendLock = nullptr; Fiber(Executor & executor, Item && item) : executor(executor) @@ -264,21 +263,19 @@ void Executor::runFiber(FiberPtr fiber) if (fib->ctx) { /* The fiber suspended itself in `waitOnThunk()`. We are still - holding the waiter domain lock, which the fiber acquired on + holding the waiter domain mutex, which the fiber locked on this thread before switching back to us (so `std::mutex` thread ownership is respected). Register the fiber in the - wait list, then release the lock. Only after the unlock can + wait list, then release the mutex. Only after the unlock can `notifyWaiters()` extract and resume the fiber — at which - point `fib->ctx` is fully formed. Note that `suspendLock` - points into the fiber's (currently quiescent) stack. */ + point `fib->ctx` is fully formed. */ auto domain = fib->suspendDomain; - auto lock = fib->suspendLock; - assert(domain && lock && fib->waitingOn); + assert(domain && fib->waitingOn); auto n = ++currentSuspendedFibers; if (n > maxSuspendedFibers) maxSuspendedFibers = n; domain->waiters.emplace(fib->waitingOn, std::move(fiber)); - lock->unlock(); + domain->mutex.unlock(); /* `fib` may be resumed by another thread from this point on, so don't touch it anymore. */ } else { @@ -475,7 +472,14 @@ suspendFiber(WaiterDomain & domain, std::unique_lock & lk, detail::V assert(!std::current_exception() && !std::uncaught_exceptions()); fib->waitingOn = &v; fib->suspendDomain = &domain; - fib->suspendLock = &lk; + /* Hand ownership of the domain mutex over to the scheduler, which + will unlock it (via `domain.mutex`) after registering us in the + wait list. We must not let the scheduler unlock through `lk`: + `unique_lock::unlock()` releases the mutex *before* clearing its + owns-flag, and the moment the mutex is released, this fiber can + be resumed on another thread, which would then read the flag on + this stack concurrently with the scheduler's write. */ + lk.release(); /* Switch back to the scheduler (`Executor::runFiber()`), which will register us in the domain's wait list and then release the lock. We can't do that here: the fiber's continuation only From d6a9925bc926472aca1ba4e1d198b970b900ec2e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Aug 2026 19:42:36 +0200 Subject: [PATCH 800/927] Pool fiber stacks Allocating a fresh 60 MiB stack (mmap + guard-page mprotect + munmap, plus first-touch page faults) for every work item is expensive: e.g. `nix search nixpkgs fizzbuzz --no-eval-cache` spawns ~117k fibers, of which only ~45 are ever simultaneously alive, making it ~30% slower than the thread-based executor at the default settings. Keep finished fibers' stacks in a pool and reuse them for new fibers. Reused stacks also come with their previously faulted-in pages. This makes the fiber-based evaluator match the thread-based baseline on the `nix search` benchmark (~3.4s), with only 124 real stack allocations for the ~117k fibers (shown by the new `nrFiberStacksAllocated` statistic). Assisted-by: Claude Fable 5 --- src/libexpr/eval.cc | 1 + src/libexpr/include/nix/expr/parallel-eval.hh | 10 ++++ src/libexpr/parallel-eval.cc | 46 ++++++++++++++++++- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 24cdb85539..b13cf5a02f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3104,6 +3104,7 @@ void EvalState::printStatistics() topObj["nrFibersSpawned"] = executor->nrFibersSpawned.load(); topObj["nrFiberWakeups"] = executor->nrFiberWakeups.load(); topObj["maxSuspendedFibers"] = executor->maxSuspendedFibers.load(); + topObj["nrFiberStacksAllocated"] = executor->nrFiberStacksAllocated.load(); topObj["nrAvoided"] = nrAvoided.load(); topObj["nrLookups"] = nrLookups.load(); topObj["nrPrimOpCalls"] = nrPrimOpCalls.load(); diff --git a/src/libexpr/include/nix/expr/parallel-eval.hh b/src/libexpr/include/nix/expr/parallel-eval.hh index 05ff1d191b..a197e08a0b 100644 --- a/src/libexpr/include/nix/expr/parallel-eval.hh +++ b/src/libexpr/include/nix/expr/parallel-eval.hh @@ -41,6 +41,15 @@ struct Executor using FiberPtr = std::unique_ptr; + /** + * A pool of reusable fiber stacks, to avoid the cost of + * allocating and faulting in a fresh stack for every work + * item. Defined in `parallel-eval.cc`. + */ + struct StackPool; + + const std::unique_ptr stackPool; + /** * Queue entries are either fresh work items (which get a new fiber * when they're picked up) or suspended fibers to be resumed. @@ -69,6 +78,7 @@ struct Executor std::atomic nrFiberWakeups{0}; std::atomic currentSuspendedFibers{0}; std::atomic maxSuspendedFibers{0}; + std::atomic nrFiberStacksAllocated{0}; static unsigned int getEvalCores(const EvalSettings & evalSettings); diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index 7266ab07c0..cd338c736f 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -87,6 +87,47 @@ struct Executor::Fiber } }; +struct Executor::StackPool +{ + Sync> stacks; + + ~StackPool() + { + for (auto & sctx : *stacks.lock()) + boost::context::protected_fixedsize_stack(evalStackSize).deallocate(sctx); + } +}; + +/** + * A Boost.Context stack allocator that reuses stacks from the + * executor's pool. Reused stacks also come with their previously + * faulted-in pages, avoiding both the mmap/munmap system calls and + * the page faults of a fresh stack for every work item. + */ +struct PooledStackAllocator +{ + Executor & executor; + + boost::context::stack_context allocate() + { + { + auto stacks(executor.stackPool->stacks.lock()); + if (!stacks->empty()) { + auto sctx = stacks->back(); + stacks->pop_back(); + return sctx; + } + } + executor.nrFiberStacksAllocated++; + return boost::context::protected_fixedsize_stack(evalStackSize).allocate(); + } + + void deallocate(boost::context::stack_context & sctx) + { + executor.stackPool->stacks.lock()->push_back(sctx); + } +}; + // cache line alignment to prevent false sharing struct alignas(64) WaiterDomain { @@ -144,7 +185,8 @@ unsigned int Executor::getEvalCores(const EvalSettings & evalSettings) } Executor::Executor(const EvalSettings & evalSettings) - : evalCores(getEvalCores(evalSettings)) + : stackPool(std::make_unique()) + , evalCores(getEvalCores(evalSettings)) , enabled(evalCores > 1) , interruptCallback(createInterruptCallback([&]() { /* Wake up all waiting fibers and threads so they can observe @@ -217,7 +259,7 @@ Executor::FiberPtr Executor::makeFiber(Item && item) try { fiber->ctx = boost::context::fiber( std::allocator_arg, - boost::context::protected_fixedsize_stack(evalStackSize), + PooledStackAllocator{*this}, [fib](boost::context::fiber && sched) -> boost::context::fiber { fib->schedCtx = std::move(sched); try { From 2cc80ae5ce13fbe71b9a1e5dde4ee4114ad6dfcd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Aug 2026 13:34:20 +0200 Subject: [PATCH 801/927] Use std::thread with default-sized stacks for the worker threads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since the worker threads only run the scheduler loop — the actual evaluation happens on fibers, which have their own stacks — they no longer need a 60 MiB stack, so we don't need boost::thread's stack-size attribute anymore and can use plain std::thread with the default stack size. This also lets us drop the boost::thread dependency from libexpr. Note: the workers must still register themselves with the Boehm GC. That's not for the sake of the (now root-free) worker stacks, but because fibers running on a worker allocate from it: without registration, Boehm has no thread-local allocation freelists for the thread and every allocation takes the global allocation lock, making e.g. `nix search nixpkgs fizzbuzz --no-eval-cache` ~3.5x slower (~12s instead of ~3.4s). Assisted-by: Claude Fable 5 --- src/libexpr/include/nix/expr/parallel-eval.hh | 9 ++------ src/libexpr/meson.build | 1 - src/libexpr/parallel-eval.cc | 22 ++++++++++++++----- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/libexpr/include/nix/expr/parallel-eval.hh b/src/libexpr/include/nix/expr/parallel-eval.hh index a197e08a0b..23ae342a2d 100644 --- a/src/libexpr/include/nix/expr/parallel-eval.hh +++ b/src/libexpr/include/nix/expr/parallel-eval.hh @@ -5,10 +5,9 @@ #include #include #include +#include #include -#include - #include "nix/util/move-only-function.hh" #include "nix/util/sync.hh" #include "nix/util/logging.hh" @@ -16,10 +15,6 @@ #include "nix/util/util.hh" #include "nix/util/signals.hh" -#if NIX_USE_BOEHMGC -# include -#endif - namespace nix { struct Executor @@ -59,7 +54,7 @@ struct Executor struct State { std::multimap queue; - std::vector threads; + std::vector threads; }; std::atomic_bool quit{false}; diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 129f32e155..ac89b000b3 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -43,7 +43,6 @@ boost = dependency( modules : [ 'container', 'context', - 'thread', ], include_type : 'system', ) diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index cd338c736f..e0a1340b5e 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -8,6 +8,10 @@ #include +#if NIX_USE_BOEHMGC +# include +#endif + namespace nix { struct WaiterDomain; @@ -202,7 +206,7 @@ Executor::Executor(const EvalSettings & evalSettings) for (size_t n = 0; n < evalCores; ++n) try { createWorker(*state); - } catch (boost::thread_resource_error & e) { + } catch (std::system_error & e) { if (n == 0) throw Error("could not create any evaluator worker threads: %s", e.what()); warn("could only create %d evaluator worker threads: %s", n, e.what()); @@ -212,7 +216,7 @@ Executor::Executor(const EvalSettings & evalSettings) Executor::~Executor() { - std::vector threads; + std::vector threads; { auto state(state_.lock()); quit = true; @@ -237,10 +241,18 @@ Executor::~Executor() void Executor::createWorker(State & state) { - boost::thread::attributes attrs; - attrs.set_stack_size(evalStackSize); - state.threads.push_back(boost::thread(attrs, [&]() { + /* Note: worker threads can have a small (default-sized) stack, + since they only run the scheduler loop; the actual evaluation + happens on fibers, which have their own stacks. */ + state.threads.push_back(std::thread([&]() { #if NIX_USE_BOEHMGC + /* Register the worker thread with the garbage collector. This + is not for the sake of the worker stack (which holds no GC + roots), but because fibers running on this thread allocate + from it: without registration, Boehm has no thread-local + allocation freelists for this thread and every allocation + takes the global allocation lock, which is several times + slower. */ GC_stack_base sb; GC_get_stack_base(&sb); GC_register_my_thread(&sb); From b38e66ade2286bf89043641e48c3e4726917c77e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Aug 2026 14:26:37 +0200 Subject: [PATCH 802/927] Use a vector of waiting fibers per value in the waiter domains Replace the unordered_multimap wait list with an unordered_map>. This expresses the intent (a list of waiters per value) more directly, and lets notifyWaiters() extract all waiters for a value in one splice via node extraction instead of walking equal_range() and erasing node-by-node while holding the domain lock. It's also cheaper when many fibers wait on the same thunk, since the waiters are stored contiguously instead of in one map node per fiber. Assisted-by: Claude Fable 5 --- src/libexpr/parallel-eval.cc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index e0a1340b5e..64d49be5dc 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -152,7 +152,7 @@ struct alignas(64) WaiterDomain * Unlike the condition variable, this is keyed on the exact value, * so fiber wakeups are never spurious. */ - std::unordered_multimap waiters; + std::unordered_map> waiters; }; static std::array waiterDomains; @@ -168,8 +168,9 @@ static void flushWaiters() std::vector woken; for (auto & domain : waiterDomains) { std::unique_lock lk(domain.mutex); - for (auto & i : domain.waiters) - woken.push_back(std::move(i.second)); + for (auto & [_, fibers] : domain.waiters) + for (auto & fiber : fibers) + woken.push_back(std::move(fiber)); domain.waiters.clear(); domain.cv.notify_all(); } @@ -328,7 +329,7 @@ void Executor::runFiber(FiberPtr fiber) auto n = ++currentSuspendedFibers; if (n > maxSuspendedFibers) maxSuspendedFibers = n; - domain->waiters.emplace(fib->waitingOn, std::move(fiber)); + domain->waiters[fib->waitingOn].push_back(std::move(fiber)); domain->mutex.unlock(); /* `fib` may be resumed by another thread from this point on, so don't touch it anymore. */ @@ -665,10 +666,8 @@ void ValueStorage::notifyWaiters() std::vector woken; { std::unique_lock lk(domain.mutex); - auto [b, e] = domain.waiters.equal_range(this); - for (auto i = b; i != e; ++i) - woken.push_back(std::move(i->second)); - domain.waiters.erase(b, e); + if (auto nh = domain.waiters.extract(this)) + woken = std::move(nh.mapped()); /* Wake up any non-fiber waiters (e.g. the main thread). */ domain.cv.notify_all(); } From a2ee8a2c489aa388ed02325a071971c3b1c9ddda Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Aug 2026 15:52:38 +0200 Subject: [PATCH 803/927] Make garbage collection work with fiber stacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, fiber stacks were invisible to the Boehm GC, so parallel evaluation required disabling the GC (GC_DONT_GC=1): values reachable only from a fiber stack could be collected while alive. * Running fibers: when a collection happens, the thread's captured sp is inside the fiber stack, and the sp corrector (`fixupBoehmStackPointer()`) used to clamp it back to the OS thread stack, so the fiber stack was never scanned. The corrector now detects this case and pushes the fiber stack's used range `[sp, base)` directly onto the mark stack (which is legal there: the corrector runs during root pushing with the GC lock held, same as `GC_push_other_roots`). The worker's own scheduler stack is then excluded from scanning, since it holds no GC roots and scanning it in full would fault in otherwise untouched pages. * Suspended fibers (parked in the wait lists or the ready queue): scanned via a `GC_set_push_other_roots` callback. `suspendFiber()` publishes the used portion of its stack (from just below the current frame — with slack for the register block that the context switch pushes, i.e. bytes that get touched anyway — up to the stack base) right before switching out, and clears it right after being resumed. Thus at every instant, a fiber stack is covered by the thread scan, by the published range, or (briefly, harmlessly) both. The stacks are tracked in an append-only registry with one entry per allocated stack (thanks to the stack pool, only a modest number), updated only through atomic stores. This is required because the GC callbacks run with the world stopped: they cannot take any lock that a frozen thread might hold, and must tolerate threads frozen in the middle of an update. Free (pooled) and unstarted stacks have no published range and are never scanned. Known limitation: if a fiber enters a boost::coroutine2 coroutine, the fiber frames below the coroutine are not scanned (the sp is then inside the coroutine stack, so we can't tell how much of the fiber stack is in use). This extends the existing assumption that coroutine stacks hold no GC roots. Assisted-by: Claude Fable 5 --- src/libexpr/eval-gc.cc | 71 +++++-- src/libexpr/include/nix/expr/parallel-eval.hh | 21 ++ src/libexpr/parallel-eval.cc | 200 +++++++++++++++++- 3 files changed, 272 insertions(+), 20 deletions(-) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 4c6d6457c1..7f99825d7e 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -14,8 +14,11 @@ # endif # include +# include // For GC_push_all and GC_set_push_other_roots # include // For GC_GRANULE_BYTES +# include "nix/expr/parallel-eval.hh" + # include # include # include @@ -78,19 +81,35 @@ static size_t getFreeMem() } /** - * When a thread goes into a coroutine, we lose its original sp until - * control flow returns to the thread. This causes Boehm GC to crash - * since it will scan memory between the coroutine's sp and the - * original stack base of the thread. Therefore, we detect when the - * current sp is outside of the original thread stack and push the - * entire thread stack instead, as an approximation. + * When a thread goes into a fiber (of the parallel evaluator) or a + * coroutine, we lose its original sp until control flow returns to + * the thread. This causes Boehm GC to crash since it will scan memory + * between the fiber's or coroutine's sp and the original stack base + * of the thread. Therefore, we detect when the current sp is outside + * of the original thread stack, and: + * + * - If the sp is inside a fiber stack, we push the used portion of + * the fiber stack (`[sp, base)` — never the untouched pages below) + * directly onto the mark stack. This is a slightly off-label use of + * the sp corrector, but it runs in the same phase as + * `GC_push_other_roots` (during root pushing, with the GC lock + * held), so pushing here is mechanically equivalent while sparing + * us any assumptions about the order in which bdwgc pushes thread + * stacks vs. other roots. * - * This is not optimal, because it causes the stack below sp to be - * scanned. However, we usually we don't have active coroutines during - * evaluation, so this is acceptable. + * - We then point the sp back into the original thread stack: for + * evaluator worker threads, at the hi end (their scheduler stacks + * hold no GC roots, and scanning them in full would fault in + * otherwise untouched pages); for other threads (e.g. the main + * thread running a coroutine), at the lo end, so that the entire + * thread stack is scanned as an approximation, since the frames + * below the coroutine may hold GC roots. * * Note that we don't scan coroutine stacks. It's currently assumed - * that we don't have GC roots in coroutines. + * that we don't have GC roots in coroutines. Also, if a *fiber* + * enters a coroutine, the fiber frames below the coroutine are + * currently not scanned (the sp is then in the coroutine stack, so we + * can't tell how much of the fiber stack is in use). */ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) { @@ -121,8 +140,30 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) osStackHi = osStackLo + osStackSize; # endif - if (sp >= osStackHi || sp < osStackLo) // sp is outside the os stack - sp = osStackLo; + if (sp >= osStackHi || sp < osStackLo) { // sp is outside the os stack + if (auto base = fiberStackContaining(sp)) { + /* Running fiber: push its used range directly, and don't + scan the worker's (root-free) scheduler stack. */ + GC_push_all(sp, base); + sp = isEvalWorkerThread(_pthread_id) ? osStackHi : osStackLo; + } else + /* Coroutine: scan the entire thread stack, since the + frames below the coroutine may hold GC roots. */ + sp = osStackLo; + } +} + +static GC_push_other_roots_proc prevPushOtherRoots; + +/** + * Push the stacks of suspended fibers of the parallel evaluator. + * Called by the GC during root pushing, with the world stopped. + */ +static void GC_CALLBACK pushOtherRoots() +{ + if (prevPushOtherRoots) + prevPushOtherRoots(); + pushSuspendedFiberStacks(); } static inline void initGCReal() @@ -158,6 +199,12 @@ static inline void initGCReal() GC_set_sp_corrector(&fixupBoehmStackPointer); assert(GC_get_sp_corrector()); + /* Also scan the stacks of suspended fibers of the parallel + evaluator. (Running fibers are handled by the sp corrector + above.) */ + prevPushOtherRoots = GC_get_push_other_roots(); + GC_set_push_other_roots(&pushOtherRoots); + /* Funnel boehm warnings into debug logs. */ GC_set_warn_proc([](char * msg, GC_word word) noexcept { std::array buffer{}; diff --git a/src/libexpr/include/nix/expr/parallel-eval.hh b/src/libexpr/include/nix/expr/parallel-eval.hh index 23ae342a2d..c21029eb5c 100644 --- a/src/libexpr/include/nix/expr/parallel-eval.hh +++ b/src/libexpr/include/nix/expr/parallel-eval.hh @@ -17,6 +17,27 @@ namespace nix { +/** + * If `sp` points into an in-use fiber stack, return the hi end + * ("base") of that stack, else null. Called by the garbage + * collector's sp corrector (with the world stopped); lock-free. + */ +void * fiberStackContaining(void * sp); + +/** + * Push the used portions of all suspended fibers' stacks onto the + * garbage collector's mark stack. Called from the GC's + * "push other roots" callback (with the world stopped); lock-free. + */ +void pushSuspendedFiberStacks(); + +/** + * Whether the given pthread id belongs to an evaluator worker thread. + * Called by the garbage collector's sp corrector (with the world + * stopped); lock-free. + */ +bool isEvalWorkerThread(void * pthreadId); + struct Executor { using work_t = MoveOnlyFunction; diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index 64d49be5dc..3fc46ca6d4 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -10,10 +10,132 @@ #if NIX_USE_BOEHMGC # include +# include #endif namespace nix { +#if NIX_USE_BOEHMGC + +/** + * Registry of fiber stacks for the garbage collector. This is + * append-only and grows in fixed-size blocks that never move, so the + * GC callbacks (which run with the world stopped and therefore must + * not take any locks that a frozen thread might hold) can safely + * iterate it at any moment. Entries are keyed by stack, not by fiber: + * thanks to the stack pool, only a modest number of stacks is ever + * allocated, and pooled stacks keep their (inert) entry. + */ +struct FiberStackInfo +{ + /** + * The hi end of the stack (i.e. `stack_context::sp`). Immutable + * once published. + */ + char * base = nullptr; + + /** + * The size of the stack. Immutable once published. + */ + size_t size = 0; + + /** + * If non-null, the GC must scan `[scanSp, base)`, i.e. the used + * portion of the stack of a suspended fiber. If null, the stack + * must not be scanned at all: it's either free (in the stack + * pool), unstarted (no GC roots yet), or running (in which case + * it's covered by the thread stack scan via the sp corrector, see + * `fixupBoehmStackPointer()`). + */ + std::atomic scanSp{nullptr}; +}; + +constexpr static size_t fiberStackBlockSize = 1024; +using FiberStackBlock = std::array; +static std::array, 1024> fiberStackBlocks; +static std::atomic nrFiberStacks{0}; +static std::mutex fiberStackRegistryMutex; + +static FiberStackInfo & getFiberStackInfo(size_t i) +{ + return (*fiberStackBlocks[i / fiberStackBlockSize].load(std::memory_order_acquire))[i % fiberStackBlockSize]; +} + +static FiberStackInfo * registerFiberStack(char * base, size_t size) +{ + std::unique_lock lk(fiberStackRegistryMutex); + auto i = nrFiberStacks.load(std::memory_order_relaxed); + if (i / fiberStackBlockSize >= fiberStackBlocks.size()) + throw Error("too many fiber stacks"); + auto & blockPtr = fiberStackBlocks[i / fiberStackBlockSize]; + if (!blockPtr.load(std::memory_order_relaxed)) + blockPtr.store(new FiberStackBlock(), std::memory_order_release); + auto & info = getFiberStackInfo(i); + info.base = base; + info.size = size; + /* Publish the entry only after its fields are initialized. */ + nrFiberStacks.store(i + 1, std::memory_order_release); + return &info; +} + +void * fiberStackContaining(void * sp) +{ + auto n = nrFiberStacks.load(std::memory_order_acquire); + for (size_t i = 0; i < n; ++i) { + auto & info = getFiberStackInfo(i); + if ((char *) sp < info.base && (char *) sp >= info.base - info.size) + return info.base; + } + return nullptr; +} + +void pushSuspendedFiberStacks() +{ + auto n = nrFiberStacks.load(std::memory_order_acquire); + for (size_t i = 0; i < n; ++i) { + auto & info = getFiberStackInfo(i); + if (auto sp = info.scanSp.load(std::memory_order_acquire)) + GC_push_all(sp, info.base); + } +} + +/** + * Registry of the worker threads' pthread ids, so that + * `isEvalWorkerThread()` can be called from the GC's sp corrector + * (with the world stopped, hence the lock-free fixed-size array). + */ +static std::array, 256> workerPthreads; + +static void registerWorkerPthread(pthread_t p) +{ + for (auto & slot : workerPthreads) { + std::uintptr_t expected = 0; + if (slot.compare_exchange_strong(expected, (std::uintptr_t) p)) + return; + } + /* Table full; harmless (the thread will merely be scanned in full + by the GC). */ +} + +static void unregisterWorkerPthread(pthread_t p) +{ + for (auto & slot : workerPthreads) + if (slot.load(std::memory_order_relaxed) == (std::uintptr_t) p) { + slot.store(0); + return; + } +} + +bool isEvalWorkerThread(void * pthreadId) +{ + for (auto & slot : workerPthreads) + if (slot.load(std::memory_order_relaxed) == (std::uintptr_t) (pthread_t) pthreadId) + return true; + return false; +} + +#endif + struct WaiterDomain; [[gnu::tls_model("initial-exec")]] thread_local bool Executor::amWorkerThread{false}; @@ -73,6 +195,13 @@ struct Executor::Fiber detail::ValueBase * waitingOn = nullptr; WaiterDomain * suspendDomain = nullptr; +#if NIX_USE_BOEHMGC + /** + * The GC registry entry for this fiber's stack. + */ + FiberStackInfo * stackInfo = nullptr; +#endif + Fiber(Executor & executor, Item && item) : executor(executor) , evalThreadId(nextEvalThreadId++) @@ -93,15 +222,36 @@ struct Executor::Fiber struct Executor::StackPool { - Sync> stacks; + struct State + { + std::vector stacks; +#if NIX_USE_BOEHMGC + /** + * The GC registry entry for each stack ever allocated, keyed + * by the stack's hi end (`stack_context::sp`). + */ + std::unordered_map gcInfo; +#endif + }; + + Sync state_; ~StackPool() { - for (auto & sctx : *stacks.lock()) + for (auto & sctx : state_.lock()->stacks) boost::context::protected_fixedsize_stack(evalStackSize).deallocate(sctx); } }; +#if NIX_USE_BOEHMGC +/** + * Side channel from `PooledStackAllocator::allocate()` (called from + * inside the `boost::context::fiber` constructor) to `makeFiber()`: + * the GC registry entry of the most recently allocated stack. + */ +[[gnu::tls_model("initial-exec")]] static thread_local FiberStackInfo * lastFiberStackInfo{nullptr}; +#endif + /** * A Boost.Context stack allocator that reuses stacks from the * executor's pool. Reused stacks also come with their previously @@ -115,20 +265,29 @@ struct PooledStackAllocator boost::context::stack_context allocate() { { - auto stacks(executor.stackPool->stacks.lock()); - if (!stacks->empty()) { - auto sctx = stacks->back(); - stacks->pop_back(); + auto state(executor.stackPool->state_.lock()); + if (!state->stacks.empty()) { + auto sctx = state->stacks.back(); + state->stacks.pop_back(); +#if NIX_USE_BOEHMGC + lastFiberStackInfo = state->gcInfo.at(sctx.sp); +#endif return sctx; } } executor.nrFiberStacksAllocated++; - return boost::context::protected_fixedsize_stack(evalStackSize).allocate(); + auto sctx = boost::context::protected_fixedsize_stack(evalStackSize).allocate(); +#if NIX_USE_BOEHMGC + auto info = registerFiberStack((char *) sctx.sp, sctx.size); + executor.stackPool->state_.lock()->gcInfo.emplace(sctx.sp, info); + lastFiberStackInfo = info; +#endif + return sctx; } void deallocate(boost::context::stack_context & sctx) { - executor.stackPool->stacks.lock()->push_back(sctx); + executor.stackPool->state_.lock()->stacks.push_back(sctx); } }; @@ -257,9 +416,13 @@ void Executor::createWorker(State & state) GC_stack_base sb; GC_get_stack_base(&sb); GC_register_my_thread(&sb); + /* Let the GC's sp corrector know that this thread's own stack + holds no GC roots (see `fixupBoehmStackPointer()`). */ + registerWorkerPthread(pthread_self()); #endif worker(); #if NIX_USE_BOEHMGC + unregisterWorkerPthread(pthread_self()); GC_unregister_my_thread(); #endif })); @@ -294,6 +457,10 @@ Executor::FiberPtr Executor::makeFiber(Item && item) fiber->promise.set_exception(std::current_exception()); return nullptr; } +#if NIX_USE_BOEHMGC + fiber->stackInfo = lastFiberStackInfo; + assert(fiber->stackInfo); +#endif nrFibersSpawned++; return fiber; } @@ -535,6 +702,18 @@ suspendFiber(WaiterDomain & domain, std::unique_lock & lk, detail::V be resumed on another thread, which would then read the flag on this stack concurrently with the scheduler's write. */ lk.release(); +#if NIX_USE_BOEHMGC + /* Publish the used portion of our stack, so that the garbage + collector will scan it while we're suspended. The slack below + the current frame covers the register block that the context + switch is about to push; those bytes are touched by the switch + anyway, so this doesn't fault in any untouched stack pages. + Note: from this point until the resumed fiber clears `scanSp`, + the stack may be scanned both through this entry and through + the thread stack scan (see `fixupBoehmStackPointer()`); the + overlap is harmless. */ + fib->stackInfo->scanSp.store((char *) __builtin_frame_address(0) - 512, std::memory_order_release); +#endif /* Switch back to the scheduler (`Executor::runFiber()`), which will register us in the domain's wait list and then release the lock. We can't do that here: the fiber's continuation only @@ -542,6 +721,11 @@ suspendFiber(WaiterDomain & domain, std::unique_lock & lk, detail::V switch-out completes would allow another thread to resume a half-suspended fiber. */ fib->schedCtx = std::move(fib->schedCtx).resume(); +#if NIX_USE_BOEHMGC + /* We're running again (possibly on another thread), so our stack + is covered by the thread stack scan from here on. */ + fib->stackInfo->scanSp.store(nullptr, std::memory_order_release); +#endif /* We've been resumed because the value was finished (or because we're shutting down); the scheduler released the lock long ago. */ From c42ab63cfbe9b90ac532d047ae7c21440d82ebc8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Aug 2026 18:01:11 +0200 Subject: [PATCH 804/927] Simplify fiber call-depth accounting using the new CallDepth class Now that `CallDepth` guards increment/decrement the thread-local counter of whatever thread they run on (instead of holding a reference to a specific counter), the fiber scheduler no longer needs to redirect the accounting through a pointer. Instead, `runFiber()` simply swaps the thread-local counter with the fiber's saved depth on every switch-in/out: a fiber suspended mid-call-chain carries its depth to whatever thread resumes it, and its guards unwind against the correct value there. Assisted-by: Claude Fable 5 --- src/libexpr/eval.cc | 3 +- src/libexpr/include/nix/expr/eval-inline.hh | 5 ++- src/libexpr/include/nix/expr/eval.hh | 36 +++++++-------------- src/libexpr/parallel-eval.cc | 17 +++++----- 4 files changed, 24 insertions(+), 37 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b13cf5a02f..31088fbf5c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1581,8 +1581,7 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) v.mkLambda(&env, this); } -[[gnu::tls_model("initial-exec")]] thread_local size_t EvalState::callDepth = 0; -[[gnu::tls_model("initial-exec")]] thread_local size_t * EvalState::callDepthPtr = &EvalState::callDepth; +[[gnu::tls_model("initial-exec")]] thread_local size_t CallDepth::callDepth = 0; void EvalState::callFunction(Value & fun, std::span args, Value & vRes, const PosIdx pos) { diff --git a/src/libexpr/include/nix/expr/eval-inline.hh b/src/libexpr/include/nix/expr/eval-inline.hh index 35e599ee9a..b0aed3f1d5 100644 --- a/src/libexpr/include/nix/expr/eval-inline.hh +++ b/src/libexpr/include/nix/expr/eval-inline.hh @@ -189,11 +189,10 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e [[gnu::always_inline]] inline CallDepth EvalState::addCallDepth(const PosIdx pos) { - auto & depth = *callDepthPtr; - if (depth > settings.maxCallDepth) + if (CallDepth::callDepth > settings.maxCallDepth) error().atPos(pos).debugThrow(); - return CallDepth(depth); + return CallDepth(); }; } // namespace nix diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index e079a94592..0f3ce9f990 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -66,18 +66,24 @@ class EvalCache; */ class CallDepth { - size_t & count; - public: - CallDepth(size_t & count) - : count(count) + /** + * Current Nix call stack depth, used with `max-call-depth` + * setting to throw stack overflow hopefully before we run out of + * system stack. The fiber scheduler saves/restores this on fiber + * switches, since a fiber suspended mid-call-chain carries its + * depth to whatever thread resumes it. + */ + [[gnu::tls_model("initial-exec")]] thread_local static size_t callDepth; + + CallDepth() { - ++count; + ++callDepth; } ~CallDepth() { - --count; + --callDepth; } }; @@ -900,26 +906,8 @@ private: const SourcePath & basePath, const std::shared_ptr & staticEnv); - /** - * Current Nix call stack depth, used with `max-call-depth` - * setting to throw stack overflow hopefully before we run out of - * system stack. - */ - [[gnu::tls_model("initial-exec")]] thread_local static size_t callDepth; - public: - /** - * Points to the call-depth counter of the current execution - * context: `callDepth` (this thread's counter) when not running - * on a fiber, or the fiber's own counter while a fiber is running - * (see `Executor::runFiber()`). Fibers need their own counter - * since they can be suspended mid-call-chain and resumed on a - * different thread, and the `CallDepth` guards on their stack - * hold a reference to the counter. - */ - [[gnu::tls_model("initial-exec")]] thread_local static size_t * callDepthPtr; - /** * Check that the call depth is within limits, and increment it, until the returned object is destroyed. */ diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index 3fc46ca6d4..689e119364 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -164,11 +164,11 @@ struct Executor::Fiber const uint32_t evalThreadId; /** - * This fiber's Nix call stack depth (see - * `EvalState::callDepthPtr`). `EvalState::addCallDepth()` uses - * this counter instead of the thread-local one while the fiber is - * running, so that the depth accounting survives the fiber being - * suspended mid-call-chain and resumed on a different thread. + * This fiber's Nix call stack depth while it's not running. + * `runFiber()` swaps it with the thread-local counter (see + * `CallDepth`) on every switch-in/out, so that the depth + * accounting survives the fiber being suspended mid-call-chain + * and resumed on a different thread. */ size_t callDepth = 0; @@ -472,16 +472,17 @@ void Executor::runFiber(FiberPtr fiber) assert(!currentFiber); auto savedThreadId = myEvalThreadId; - auto savedCallDepthPtr = EvalState::callDepthPtr; + auto savedCallDepth = CallDepth::callDepth; currentFiber = fib; myEvalThreadId = fib->evalThreadId; - EvalState::callDepthPtr = &fib->callDepth; + CallDepth::callDepth = fib->callDepth; fib->ctx = std::move(fib->ctx).resume(); currentFiber = nullptr; myEvalThreadId = savedThreadId; - EvalState::callDepthPtr = savedCallDepthPtr; + fib->callDepth = CallDepth::callDepth; + CallDepth::callDepth = savedCallDepth; if (fib->ctx) { /* The fiber suspended itself in `waitOnThunk()`. We are still From 8e2765715601c5b61f9560060b8adb0d65c3730c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Aug 2026 18:53:23 +0200 Subject: [PATCH 805/927] Make the evaluation context fiber-aware `EvalState::evalContext` was a thread-local value, which is broken with fibers: a fiber that is suspended and resumed on another thread loses its context, and two fibers on different threads could race non-atomic shared_ptr assignments against the same thread's slot. Make the thread-local a plain *pointer* to the active `EvalContext`: each fiber owns its own context (living in the heap-allocated `Fiber` record, so it travels with the fiber), and `runFiber()` points the thread-local at it on switch-in and restores it on switch-out. Using a pointer means fiber switches don't touch the `provenance` shared_ptr's atomic reference count. Non-fiber contexts (i.e. the main thread) share a global default context, preserving the old behavior. As a side effect, a work item's context can no longer leak into subsequent work items executed on the same worker thread, since every fiber starts with a fresh context. Assisted-by: Claude Fable 5 --- src/libexpr/eval.cc | 8 +++++++- src/libexpr/include/nix/expr/eval.hh | 21 ++++++++++++++------- src/libexpr/parallel-eval.cc | 11 +++++++++++ src/libexpr/primops.cc | 6 +++--- 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 31088fbf5c..0556d6b7d7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -252,7 +252,13 @@ EvalMemory::EvalMemory() assertGCInitialized(); } -[[gnu::tls_model("initial-exec")]] thread_local EvalState::EvalContext EvalState::evalContext; +/** + * The evaluation context of non-fiber execution contexts (i.e. the + * main thread). + */ +static EvalState::EvalContext globalEvalContext; + +[[gnu::tls_model("initial-exec")]] thread_local EvalState::EvalContext * EvalState::evalContext = &globalEvalContext; EvalState::EvalState( const LookupPath & lookupPathFromArguments, diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 0f3ce9f990..51eb35c101 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -1132,7 +1132,7 @@ private: public: /** - * Per-thread evaluation context. This context is propagated to worker threads when a value is evaluated + * Evaluation context. This context is propagated to worker threads when a value is evaluated * asynchronously. */ struct EvalContext @@ -1140,7 +1140,14 @@ public: std::shared_ptr provenance; }; - [[gnu::tls_model("initial-exec")]] thread_local static EvalContext evalContext; + /** + * The evaluation context of the current execution context: a + * fiber's own context while a fiber is running (switched by + * `Executor::runFiber()`), or a global default context otherwise. + * A plain pointer so that fiber switches don't need to touch the + * `provenance` shared_ptr's (atomic) reference count. + */ + [[gnu::tls_model("initial-exec")]] thread_local static EvalContext * evalContext; /** * Create a work item that propagates the current evaluation context. @@ -1148,8 +1155,8 @@ public: template auto makeWork(T && t) { - return [this, t{std::move(t)}, evalContext(evalContext)]() { - this->evalContext = evalContext; + return [this, t{std::move(t)}, evalContext(*evalContext)]() { + *this->evalContext = evalContext; t(); }; } @@ -1222,13 +1229,13 @@ struct PushProvenance PushProvenance(EvalState & state, std::shared_ptr prov) : state(state) { - state.evalContext.provenance.swap(prev); - state.evalContext.provenance.swap(prov); + state.evalContext->provenance.swap(prev); + state.evalContext->provenance.swap(prov); } ~PushProvenance() { - state.evalContext.provenance.swap(prev); + state.evalContext->provenance.swap(prev); } }; diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index 689e119364..c5b53e2345 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -172,6 +172,14 @@ struct Executor::Fiber */ size_t callDepth = 0; + /** + * This fiber's evaluation context. `runFiber()` points the + * thread-local `EvalState::evalContext` at it while the fiber is + * running, so that the context travels with the fiber across + * threads. + */ + EvalState::EvalContext evalContext; + std::promise promise; work_t work; @@ -473,9 +481,11 @@ void Executor::runFiber(FiberPtr fiber) auto savedThreadId = myEvalThreadId; auto savedCallDepth = CallDepth::callDepth; + auto savedEvalContext = EvalState::evalContext; currentFiber = fib; myEvalThreadId = fib->evalThreadId; CallDepth::callDepth = fib->callDepth; + EvalState::evalContext = &fib->evalContext; fib->ctx = std::move(fib->ctx).resume(); @@ -483,6 +493,7 @@ void Executor::runFiber(FiberPtr fiber) myEvalThreadId = savedThreadId; fib->callDepth = CallDepth::callDepth; CallDepth::callDepth = savedCallDepth; + EvalState::evalContext = savedEvalContext; if (fib->ctx) { /* The fiber suspended itself in `waitOnThunk()`. We are still diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cf009515c5..e99cf419c0 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1434,7 +1434,7 @@ static void prim_derivationStrictGeneric(EvalState & state, const PosIdx pos, Va } try { - derivationStrictInternal(state, drvName, attrs, v, state.evalContext.provenance, acceptMeta); + derivationStrictInternal(state, drvName, attrs, v, state.evalContext->provenance, acceptMeta); } catch (Error & e) { Pos pos = state.positions[nameAttr->pos]; /* @@ -2823,7 +2823,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value ** args, Valu HashAlgorithm::SHA256, refs, state.repair, - state.evalContext.provenance); + state.evalContext->provenance); }); /* Note: we don't need to add `context' to the context of the @@ -2975,7 +2975,7 @@ static void addPath( auto path3 = filter ? SourcePath{ make_ref( - path2.accessor, state.evalContext.provenance), + path2.accessor, state.evalContext->provenance), path2.path } : path2; From b6db70de57a91b0fe040e49cc0ff01877f768e0a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Aug 2026 20:31:27 +0200 Subject: [PATCH 806/927] Avoid needless worker wakeups in the executor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Producers used to wake workers unconditionally: `spawn()` did a `notify_all()` (waking every worker for possibly a handful of items) and `enqueueFiber()` an unconditional `notify_one()`. Instead, track the number of sleeping workers (under the state lock, so there are no lost wakeups) and wake exactly `min(nrItems, nrSleeping)` workers — zero futex calls when all workers are busy. Note: on the `nix search nixpkgs --no-eval-cache` benchmark this turns out not to reduce the overall context-switch count measurably: tracing shows that ~65% of the ~750k futex waits in that workload come from the Boehm GC (the global allocation lock `GC_allocate_ml` and the parallel-marker coordination lock), and most of the remainder are genuine executor sleeps between work bursts. Still, exact wakeups are strictly better than the previous thundering herd as the worker count grows. Assisted-by: Claude Fable 5 --- src/libexpr/include/nix/expr/parallel-eval.hh | 8 +++++++ src/libexpr/parallel-eval.cc | 21 +++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/libexpr/include/nix/expr/parallel-eval.hh b/src/libexpr/include/nix/expr/parallel-eval.hh index c21029eb5c..89a846ffb6 100644 --- a/src/libexpr/include/nix/expr/parallel-eval.hh +++ b/src/libexpr/include/nix/expr/parallel-eval.hh @@ -76,6 +76,14 @@ struct Executor { std::multimap queue; std::vector threads; + + /** + * The number of workers currently blocked on `wakeup`. + * Producers wake up at most this many workers (and none when + * all workers are busy), to avoid pointless futex traffic and + * thundering herds. + */ + size_t nrSleeping = 0; }; std::atomic_bool quit{false}; diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index c5b53e2345..d05a3c371a 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -523,13 +523,16 @@ void Executor::enqueueFiber(FiberPtr fiber) { nrFiberWakeups++; currentSuspendedFibers--; + bool doWake; { auto state(state_.lock()); /* Note: key 0 means that resumed fibers run before any fresh work items, so existing work is drained first. */ state->queue.emplace(0, std::move(fiber)); + doWake = state->nrSleeping > 0; } - wakeup.notify_one(); + if (doWake) + wakeup.notify_one(); } void Executor::worker() @@ -554,7 +557,14 @@ void Executor::worker() gotEntry = true; break; } + /* Nothing to do; sleep until a producer wakes us up. + Note: `nrSleeping` is maintained under the state lock, + so a producer either sees our increment (and wakes us), + or its work insertion happened before our queue check + above — no lost wakeups. */ + state->nrSleeping++; state.wait(wakeup); + state->nrSleeping--; } if (!gotEntry) { @@ -609,6 +619,7 @@ std::vector> Executor::spawn(WorkItems && items) std::vector> futures; + size_t toWake; { auto state(state_.lock()); for (auto & item : items) { @@ -625,12 +636,14 @@ std::vector> Executor::spawn(WorkItems && items) auto key = (uint64_t(item.second) << 48) | dist(rng); state->queue.emplace(key, Item{.promise = std::move(promise), .work = std::move(item.first)}); } + /* Wake up one worker per item, but only workers that are + actually asleep. In the steady state (all workers busy), + this does no futex calls at all. */ + toWake = std::min(items.size(), state->nrSleeping); } - if (items.size() == 1) + for (size_t n = 0; n < toWake; ++n) wakeup.notify_one(); - else - wakeup.notify_all(); return futures; } From f23ea3d0e2f051e01c58c707db4b80cd119cce88 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Aug 2026 12:47:56 +0200 Subject: [PATCH 807/927] Drop randomization of the WorkItem key It's no longer useful. --- src/libexpr/parallel-eval.cc | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index d05a3c371a..23be09d990 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -625,15 +625,8 @@ std::vector> Executor::spawn(WorkItems && items) for (auto & item : items) { std::promise promise; futures.push_back(promise.get_future()); - /* Note: this uses a cheap PRNG rather than std::random_device, - since the latter costs hundreds of cycles per call (RDRAND or - /dev/urandom), which adds up when spawning many work items. The - key only needs to spread items of the same priority around the - queue, not be cryptographically random. */ - [[gnu::tls_model("initial-exec")]] static thread_local std::mt19937_64 rng{std::random_device{}()}; - [[gnu::tls_model("initial-exec")]] static thread_local std::uniform_int_distribution dist( - 0, 1ULL << 48); - auto key = (uint64_t(item.second) << 48) | dist(rng); + static thread_local uint32_t local = 0; + auto key = (uint64_t(item.second) << 48) | local++; state->queue.emplace(key, Item{.promise = std::move(promise), .work = std::move(item.first)}); } /* Wake up one worker per item, but only workers that are From 2ddbf4bc8f628519e12cefe8d8d49b3fcc9142fd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Aug 2026 13:31:46 +0200 Subject: [PATCH 808/927] Drop the unused hugepages stuff --- .../patches/boehmgc-batch-malloc-many.patch | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/packaging/patches/boehmgc-batch-malloc-many.patch b/packaging/patches/boehmgc-batch-malloc-many.patch index 2e40f85796..81742641ef 100644 --- a/packaging/patches/boehmgc-batch-malloc-many.patch +++ b/packaging/patches/boehmgc-batch-malloc-many.patch @@ -124,31 +124,3 @@ diff -ur a/misc.c b/misc.c char * space_divisor_string = GETENV("GC_FREE_SPACE_DIVISOR"); if (space_divisor_string != NULL) { int space_divisor = atoi(space_divisor_string); -diff -ur a/os_dep.c b/os_dep.c ---- a/os_dep.c 2026-07-03 12:56:51.390341791 +0200 -+++ b/os_dep.c 2026-07-04 01:00:23.191858912 +0200 -@@ -2286,6 +2286,24 @@ - if (((word)result % HBLKSIZE) != 0) - ABORT( - "GC_unix_get_mem: Memory returned by mmap is not aligned to HBLKSIZE."); -+# if defined(LINUX) && defined(MADV_HUGEPAGE) -+ /* Optionally ask for transparent huge pages. The heap is large */ -+ /* and densely touched, so 2 MiB pages greatly reduce first-touch */ -+ /* page-fault and TLB overhead. However, under the "madvise" THP */ -+ /* policy, faults in a MADV_HUGEPAGE region perform synchronous */ -+ /* memory compaction, which can stall for a long time when */ -+ /* physical memory is fragmented. Hence this is opt-in via the */ -+ /* GC_MADVISE_HUGEPAGES environment variable; setting the system */ -+ /* THP policy to "always" (which takes huge pages opportunistically */ -+ /* without direct compaction) is generally preferable. */ -+ { -+ static int use_hugepages = -1; -+ if (use_hugepages < 0) -+ use_hugepages = GETENV("GC_MADVISE_HUGEPAGES") != NULL; -+ if (use_hugepages) -+ (void)madvise(result, bytes, MADV_HUGEPAGE); -+ } -+# endif - return((ptr_t)result); - } - # endif /* !MSWIN_XBOX1 */ From 0b1accd38d84d85fb3a39787478cd05cf0b24d4b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Aug 2026 18:19:44 +0200 Subject: [PATCH 809/927] Fix compatibility with upstream bdwgc --- src/libexpr/eval-gc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 7f99825d7e..af45cba69f 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -206,7 +206,7 @@ static inline void initGCReal() GC_set_push_other_roots(&pushOtherRoots); /* Funnel boehm warnings into debug logs. */ - GC_set_warn_proc([](char * msg, GC_word word) noexcept { + GC_set_warn_proc([](const char * msg, GC_word word) noexcept { std::array buffer{}; auto res = snprintf(buffer.data(), buffer.size(), msg, word); /* Ignore garbage. */ From 1f7bb829d4a3d2b81aa6644e2840c6a34dfdff4b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Aug 2026 18:32:14 +0200 Subject: [PATCH 810/927] Build bdwgc from a flake input instead of patching the nixpkgs package The boehmgc dependency is now built from the nix-patches-master branch of github:edolstra/bdwgc (bdwgc master plus our patches, which were previously applied as patch files to nixpkgs' 8.2.12 package). This makes it easier to develop GC changes: point the input at a local tree with `--override-input bdwgc ~/Dev/bdwgc`. Also adjust to a bdwgc API change: GC_warn_proc now takes a `const char *` message. Assisted-by: Claude Fable 5 --- flake.lock | 19 +++ flake.nix | 5 + packaging/dependencies.nix | 7 +- .../patches/boehmgc-batch-malloc-many.patch | 126 ------------------ ...gc-gctest-tiny-freelists-heap-growth.patch | 38 ------ 5 files changed, 28 insertions(+), 167 deletions(-) delete mode 100644 packaging/patches/boehmgc-batch-malloc-many.patch delete mode 100644 packaging/patches/boehmgc-gctest-tiny-freelists-heap-growth.patch diff --git a/flake.lock b/flake.lock index f11f48a2aa..5160a9e259 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,23 @@ { "nodes": { + "bdwgc": { + "flake": false, + "locked": { + "lastModified": 1787927442, + "narHash": "sha256-Z5fTPKJJOxlfawRm4VGG4TmLbqfl/WuGCx6bLiMVTeo=", + "ref": "refs/heads/nix-patches-master", + "rev": "4ae90901832091c29d45a6d80be18ccbc6b51e2f", + "revCount": 6621, + "type": "git", + "url": "file:///home/eelco/Dev/bdwgc" + }, + "original": { + "owner": "edolstra", + "ref": "nix-patches-master", + "repo": "bdwgc", + "type": "github" + } + }, "flake-compat": { "flake": false, "locked": { @@ -103,6 +121,7 @@ }, "root": { "inputs": { + "bdwgc": "bdwgc", "flake-parts": "flake-parts", "git-hooks-nix": "git-hooks-nix", "nixpkgs": "nixpkgs", diff --git a/flake.nix b/flake.nix index d46b468314..1d77f81928 100644 --- a/flake.nix +++ b/flake.nix @@ -6,6 +6,11 @@ inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; + inputs.bdwgc = { + url = "github:edolstra/bdwgc/nix-patches-master"; + flake = false; + }; + # dev tooling inputs.flake-parts.url = "https://flakehub.com/f/hercules-ci/flake-parts/0.1"; inputs.git-hooks-nix.url = "https://flakehub.com/f/cachix/git-hooks.nix/0.1.941"; diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index c78a37b9c7..035075a983 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -36,9 +36,10 @@ scope: { inherit stdenv; }).overrideAttrs (attrs: { - patches = (attrs.patches or [ ]) ++ [ - ./patches/boehmgc-batch-malloc-many.patch - ./patches/boehmgc-gctest-tiny-freelists-heap-growth.patch + src = inputs.bdwgc; + + nativeBuildInputs = (attrs.nativeBuildInputs or [ ]) ++ [ + pkgs.buildPackages.autoreconfHook ]; env = (attrs.env or { }) // { diff --git a/packaging/patches/boehmgc-batch-malloc-many.patch b/packaging/patches/boehmgc-batch-malloc-many.patch deleted file mode 100644 index 81742641ef..0000000000 --- a/packaging/patches/boehmgc-batch-malloc-many.patch +++ /dev/null @@ -1,126 +0,0 @@ -Reduce contention on the GC allocation lock: make GC_generic_malloc_many() -hand out up to GC_many_blocks heap blocks worth of objects per acquisition -of the allocation lock, instead of exactly one. This matters for -parallel evaluation in Nix, where all evaluator threads replenish their -thread-local free lists (Values, Envs, Bindings) through this function -and otherwise serialize on the allocation lock. - -The batch size is settable via the GC_MALLOC_MANY_BLOCKS environment -variable (1..64); the compile-time default is GC_MANY_BLOCKS_DEFAULT -(1 = upstream behavior, overridden in packaging/dependencies.nix). - -diff -ur a/include/private/gc_priv.h b/include/private/gc_priv.h ---- a/include/private/gc_priv.h 2026-07-03 12:56:51.388341746 +0200 -+++ b/include/private/gc_priv.h 2026-07-09 20:00:13.787169270 +0200 -@@ -1767,6 +1767,17 @@ - - GC_EXTERN unsigned GC_n_kinds; - -+/* Number of heap blocks worth of objects to hand out per allocation */ -+/* lock acquisition in GC_generic_malloc_many(). Larger values reduce */ -+/* contention on the allocation lock when many threads allocate small */ -+/* objects heavily. Settable at runtime via the GC_MALLOC_MANY_BLOCKS */ -+/* environment variable (clamped to [1, GC_MANY_BLOCKS_MAX]). */ -+#ifndef GC_MANY_BLOCKS_DEFAULT -+# define GC_MANY_BLOCKS_DEFAULT 1 -+#endif -+#define GC_MANY_BLOCKS_MAX 64 -+GC_EXTERN int GC_many_blocks; -+ - GC_EXTERN size_t GC_page_size; - - /* Round up allocation size to a multiple of a page size. */ -diff -ur a/mallocx.c b/mallocx.c ---- a/mallocx.c 2026-07-03 12:56:51.389341768 +0200 -+++ b/mallocx.c 2026-07-09 20:00:13.787169270 +0200 -@@ -300,6 +300,9 @@ - /* since the collector would not retain the entire list if it were */ - /* invoked just as we were returning. */ - /* Note that the client should usually clear the link field. */ -+/* See the description in gc_priv.h. */ -+GC_INNER int GC_many_blocks = GC_MANY_BLOCKS_DEFAULT; -+ - GC_API void GC_CALL GC_generic_malloc_many(size_t lb, int k, void **result) - { - void *op; -@@ -433,7 +436,7 @@ - my_bytes_allocd = 0; - for (p = op; p != 0; p = obj_link(p)) { - my_bytes_allocd += lb; -- if ((word)my_bytes_allocd >= HBLKSIZE) { -+ if ((word)my_bytes_allocd >= (word)GC_many_blocks * HBLKSIZE) { - *opp = obj_link(p); - obj_link(p) = 0; - break; -@@ -442,12 +445,24 @@ - GC_bytes_allocd += my_bytes_allocd; - goto out; - } -- /* Next try to allocate a new block worth of objects of this size. */ -+ /* Next try to allocate new blocks worth of objects of this size. */ -+ /* Up to GC_many_blocks heap blocks are allocated per acquisition */ -+ /* of the allocation lock, to reduce contention on it. */ - { -- struct hblk *h = GC_allochblk(lb, k, 0); -- if (h /* != NULL */) { /* CPPCHECK */ -- if (IS_UNCOLLECTABLE(k)) GC_set_hdr_marks(HDR(h)); -+ struct hblk *hbs[GC_MANY_BLOCKS_MAX]; -+ int nblocks = GC_many_blocks; -+ int i; -+ -+ if (nblocks < 1) nblocks = 1; -+ if (nblocks > GC_MANY_BLOCKS_MAX) nblocks = GC_MANY_BLOCKS_MAX; -+ for (i = 0; i < nblocks; i++) { -+ hbs[i] = GC_allochblk(lb, k, 0); -+ if (NULL == hbs[i]) break; -+ if (IS_UNCOLLECTABLE(k)) GC_set_hdr_marks(HDR(hbs[i])); - GC_bytes_allocd += HBLKSIZE - HBLKSIZE % lb; -+ } -+ nblocks = i; -+ if (nblocks > 0) { - # ifdef PARALLEL_MARK - if (GC_parallel) { - GC_acquire_mark_lock(); -@@ -455,8 +470,10 @@ - UNLOCK(); - GC_release_mark_lock(); - -- op = GC_build_fl(h, lw, -- (ok -> ok_init || GC_debugging_started), 0); -+ op = 0; -+ for (i = 0; i < nblocks; i++) -+ op = GC_build_fl(hbs[i], lw, -+ (ok -> ok_init || GC_debugging_started), (ptr_t)op); - - *result = op; - GC_acquire_mark_lock(); -@@ -467,7 +484,10 @@ - return; - } - # endif -- op = GC_build_fl(h, lw, (ok -> ok_init || GC_debugging_started), 0); -+ op = 0; -+ for (i = 0; i < nblocks; i++) -+ op = GC_build_fl(hbs[i], lw, -+ (ok -> ok_init || GC_debugging_started), (ptr_t)op); - goto out; - } - } -diff -ur a/misc.c b/misc.c ---- a/misc.c 2026-07-03 12:56:51.390341791 +0200 -+++ b/misc.c 2026-07-09 20:00:13.788169293 +0200 -@@ -1157,6 +1157,14 @@ - } - } - { -+ char * many_blocks_string = GETENV("GC_MALLOC_MANY_BLOCKS"); -+ if (many_blocks_string != NULL) { -+ int many_blocks = atoi(many_blocks_string); -+ if (many_blocks > 0 && many_blocks <= GC_MANY_BLOCKS_MAX) -+ GC_many_blocks = many_blocks; -+ } -+ } -+ { - char * space_divisor_string = GETENV("GC_FREE_SPACE_DIVISOR"); - if (space_divisor_string != NULL) { - int space_divisor = atoi(space_divisor_string); diff --git a/packaging/patches/boehmgc-gctest-tiny-freelists-heap-growth.patch b/packaging/patches/boehmgc-gctest-tiny-freelists-heap-growth.patch deleted file mode 100644 index a7cd51d694..0000000000 --- a/packaging/patches/boehmgc-gctest-tiny-freelists-heap-growth.patch +++ /dev/null @@ -1,38 +0,0 @@ -From 7cd4f090acc0e801502f918060106bba926c7153 Mon Sep 17 00:00:00 2001 -From: Eelco Dolstra -Date: Tue, 25 Aug 2026 14:37:18 +0200 -Subject: [PATCH] Scale gctest's heap growth limit with GC_TINY_FREELISTS - -The 'unexpected heap growth' check in check_heap_stats() uses a fixed -limit that is exceeded when the collector is built with a larger -GC_TINY_FREELISTS: bigger thread-local free lists cause each thread to -retain more nearly-empty heap blocks (up to one per size class and -object kind) and increase block-level fragmentation. Scale the limit -proportionally so that gctest passes with such configurations. - -Assisted-by: Claude Fable 5 ---- - tests/test.c | 7 +++++++ - 1 file changed, 7 insertions(+) - -diff --git a/tests/test.c b/tests/test.c -index 0ae3f95b..b62699fa 100644 ---- a/tests/test.c -+++ b/tests/test.c -@@ -1753,6 +1753,13 @@ void check_heap_stats(void) - # endif - # ifdef MEMORY_SANITIZER - max_heap_sz += max_heap_sz / 4; -+# endif -+# if defined(GC_TINY_FREELISTS) && GC_TINY_FREELISTS > 25 -+ /* Larger thread-local free lists cause each thread to retain */ -+ /* more nearly-empty heap blocks (up to one per size class and */ -+ /* object kind) and increase block-level fragmentation, so */ -+ /* allow the heap to grow proportionally. */ -+ max_heap_sz = max_heap_sz / 25 * GC_TINY_FREELISTS; - # endif - max_heap_sz *= n_tests; - # if defined(USE_MMAP) || defined(MSWIN32) --- -2.54.0 - From 0af5e54468ddaea8c355b6cc09ba1665a0b73766 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 17:34:13 +0200 Subject: [PATCH 811/927] Use bdwgc's registered stacks for fiber and coroutine scanning Replace the custom GC/stack integration (the sp corrector in fixupBoehmStackPointer(), the FiberStackInfo registry and the worker pthread registry) with bdwgc's new first-class support for client-registered stacks (edolstra/bdwgc#1, so the bdwgc flake input now points at the user-defined-stacks branch): fiber stacks are registered with GC_register_stack(), and every stack switch updates GC_current_stack / saved_sp so the collector always knows which part of which stack to scan. This also closes the long-standing hole where the frames of a fiber that had switched onto a sourceToSink()/sinkToSource() coroutine stack (e.g. an evaluation calling Store::addToStore() through a path filter) were invisible to the collector, causing random segfaults and spurious "attribute missing" evaluation errors under GC pressure, reproducible with `nix flake show --all-systems --no-eval-cache` on the nix 2.18.1 flake (~4 out of 6 runs failed; now 0 out of 10, including runs with GC_INITIAL_HEAP_SIZE=32M to force frequent collections). The coroutine stacks themselves are now scanned too, via hooks in libutil (coroutine-gc.hh) that libexpr implements in terms of GC_stack, so libutil still does not depend on bdwgc. Assisted-by: Claude Fable 5 --- flake.lock | 15 +- flake.nix | 2 +- src/libexpr/eval-gc.cc | 152 +++++--------- src/libexpr/include/nix/expr/parallel-eval.hh | 21 -- src/libexpr/parallel-eval.cc | 198 +++++------------- src/libutil/include/nix/util/coroutine-gc.hh | 45 ++++ src/libutil/include/nix/util/meson.build | 1 + src/libutil/serialise.cc | 183 ++++++++++++---- 8 files changed, 302 insertions(+), 315 deletions(-) create mode 100644 src/libutil/include/nix/util/coroutine-gc.hh diff --git a/flake.lock b/flake.lock index 5160a9e259..f8d0ef4c81 100644 --- a/flake.lock +++ b/flake.lock @@ -3,17 +3,16 @@ "bdwgc": { "flake": false, "locked": { - "lastModified": 1787927442, - "narHash": "sha256-Z5fTPKJJOxlfawRm4VGG4TmLbqfl/WuGCx6bLiMVTeo=", - "ref": "refs/heads/nix-patches-master", - "rev": "4ae90901832091c29d45a6d80be18ccbc6b51e2f", - "revCount": 6621, - "type": "git", - "url": "file:///home/eelco/Dev/bdwgc" + "lastModified": 1789388623, + "narHash": "sha256-qk2ONj1UU3zRzyJMtVns5ZDH0uCWpvTtqW2b/lZmotU=", + "owner": "edolstra", + "repo": "bdwgc", + "rev": "c6afef5518ef71d446339a5aa4c1b97fc6392534", + "type": "github" }, "original": { "owner": "edolstra", - "ref": "nix-patches-master", + "ref": "user-defined-stacks", "repo": "bdwgc", "type": "github" } diff --git a/flake.nix b/flake.nix index 1d77f81928..445397e453 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,7 @@ inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.bdwgc = { - url = "github:edolstra/bdwgc/nix-patches-master"; + url = "github:edolstra/bdwgc/user-defined-stacks"; flake = false; }; diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index af45cba69f..15d668722a 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -8,20 +8,10 @@ #if NIX_USE_BOEHMGC -# include -# ifdef __FreeBSD__ -# include -# endif - # include -# include // For GC_push_all and GC_set_push_other_roots # include // For GC_GRANULE_BYTES -# include "nix/expr/parallel-eval.hh" - -# include -# include -# include +# include "nix/util/coroutine-gc.hh" #endif @@ -81,89 +71,60 @@ static size_t getFreeMem() } /** - * When a thread goes into a fiber (of the parallel evaluator) or a - * coroutine, we lose its original sp until control flow returns to - * the thread. This causes Boehm GC to crash since it will scan memory - * between the fiber's or coroutine's sp and the original stack base - * of the thread. Therefore, we detect when the current sp is outside - * of the original thread stack, and: - * - * - If the sp is inside a fiber stack, we push the used portion of - * the fiber stack (`[sp, base)` — never the untouched pages below) - * directly onto the mark stack. This is a slightly off-label use of - * the sp corrector, but it runs in the same phase as - * `GC_push_other_roots` (during root pushing, with the GC lock - * held), so pushing here is mechanically equivalent while sparing - * us any assumptions about the order in which bdwgc pushes thread - * stacks vs. other roots. - * - * - We then point the sp back into the original thread stack: for - * evaluator worker threads, at the hi end (their scheduler stacks - * hold no GC roots, and scanning them in full would fault in - * otherwise untouched pages); for other threads (e.g. the main - * thread running a coroutine), at the lo end, so that the entire - * thread stack is scanned as an approximation, since the frames - * below the coroutine may hold GC roots. - * - * Note that we don't scan coroutine stacks. It's currently assumed - * that we don't have GC roots in coroutines. Also, if a *fiber* - * enters a coroutine, the fiber frames below the coroutine are - * currently not scanned (the sp is then in the coroutine stack, so we - * can't tell how much of the fiber stack is in use). + * Implementations of the libutil coroutine GC hooks (see + * `coroutine-gc.hh`) in terms of bdwgc's registered stacks. Together + * with the fiber stack registration in `parallel-eval.cc`, this makes + * every stack that can hold GC roots — thread stacks (registered + * automatically), fiber stacks and coroutine stacks — scannable by + * the collector, including the frames of a fiber that has switched + * onto a coroutine stack. */ -void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) + +static void * coroStackRegisterImpl(void * base, size_t size) { - void *& sp = *sp_ptr; - auto pthread_id = reinterpret_cast(_pthread_id); - size_t osStackSize; - char * osStackHi; - char * osStackLo; - -# ifdef __APPLE__ - osStackSize = pthread_get_stacksize_np(pthread_id); - osStackHi = (char *) pthread_get_stackaddr_np(pthread_id); - osStackLo = osStackHi - osStackSize; -# else - pthread_attr_t pattr; - if (pthread_attr_init(&pattr)) - throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); -# ifdef HAVE_PTHREAD_GETATTR_NP - if (pthread_getattr_np(pthread_id, &pattr)) - throw Error("fixupBoehmStackPointer: pthread_getattr_np failed"); -# else -# error "Need `pthread_attr_get_np`" -# endif - if (pthread_attr_getstack(&pattr, (void **) &osStackLo, &osStackSize)) - throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); - if (pthread_attr_destroy(&pattr)) - throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); - osStackHi = osStackLo + osStackSize; -# endif + auto stk = new GC_stack{}; + stk->base = base; + stk->limit = (char *) base - size; + GC_register_stack(stk); + return stk; +} - if (sp >= osStackHi || sp < osStackLo) { // sp is outside the os stack - if (auto base = fiberStackContaining(sp)) { - /* Running fiber: push its used range directly, and don't - scan the worker's (root-free) scheduler stack. */ - GC_push_all(sp, base); - sp = isEvalWorkerThread(_pthread_id) ? osStackHi : osStackLo; - } else - /* Coroutine: scan the entire thread stack, since the - frames below the coroutine may hold GC roots. */ - sp = osStackLo; - } +static void coroStackUnregisterImpl(void * cookie) +{ + auto stk = (struct GC_stack *) cookie; + GC_unregister_stack(stk); + delete stk; } -static GC_push_other_roots_proc prevPushOtherRoots; +static void * coroSwitchToImpl(void * cookie, void * callerSp) +{ + auto prev = GC_current_stack; + /* `prev` is null on threads not registered with the GC; such + threads hold no GC roots and need no scanning. */ + if (prev) + prev->saved_sp = callerSp; + GC_current_stack = (struct GC_stack *) cookie; + return prev; +} -/** - * Push the stacks of suspended fibers of the parallel evaluator. - * Called by the GC during root pushing, with the world stopped. - */ -static void GC_CALLBACK pushOtherRoots() +static void coroSwitchBackImpl(void * prevHandle) { - if (prevPushOtherRoots) - prevPushOtherRoots(); - pushSuspendedFiberStacks(); + auto prev = (struct GC_stack *) prevHandle; + GC_current_stack = prev; + if (prev) + prev->saved_sp = nullptr; +} + +static void coroMarkSuspendedImpl(void * cookie, void * sp) +{ + if (cookie) + ((struct GC_stack *) cookie)->saved_sp = sp; +} + +static void coroMarkActiveImpl(void * cookie) +{ + if (cookie) + ((struct GC_stack *) cookie)->saved_sp = nullptr; } static inline void initGCReal() @@ -196,14 +157,15 @@ static inline void initGCReal() GC_set_oom_fn(oomHandler); - GC_set_sp_corrector(&fixupBoehmStackPointer); - assert(GC_get_sp_corrector()); - - /* Also scan the stacks of suspended fibers of the parallel - evaluator. (Running fibers are handled by the sp corrector - above.) */ - prevPushOtherRoots = GC_get_push_other_roots(); - GC_set_push_other_roots(&pushOtherRoots); + /* Make the coroutine stacks of libutil scannable by the GC (fiber + stacks are registered in `parallel-eval.cc`; thread stacks are + registered automatically by bdwgc). */ + coroStackRegister = coroStackRegisterImpl; + coroStackUnregister = coroStackUnregisterImpl; + coroSwitchTo = coroSwitchToImpl; + coroSwitchBack = coroSwitchBackImpl; + coroMarkSuspended = coroMarkSuspendedImpl; + coroMarkActive = coroMarkActiveImpl; /* Funnel boehm warnings into debug logs. */ GC_set_warn_proc([](const char * msg, GC_word word) noexcept { diff --git a/src/libexpr/include/nix/expr/parallel-eval.hh b/src/libexpr/include/nix/expr/parallel-eval.hh index 89a846ffb6..33b31c7258 100644 --- a/src/libexpr/include/nix/expr/parallel-eval.hh +++ b/src/libexpr/include/nix/expr/parallel-eval.hh @@ -17,27 +17,6 @@ namespace nix { -/** - * If `sp` points into an in-use fiber stack, return the hi end - * ("base") of that stack, else null. Called by the garbage - * collector's sp corrector (with the world stopped); lock-free. - */ -void * fiberStackContaining(void * sp); - -/** - * Push the used portions of all suspended fibers' stacks onto the - * garbage collector's mark stack. Called from the GC's - * "push other roots" callback (with the world stopped); lock-free. - */ -void pushSuspendedFiberStacks(); - -/** - * Whether the given pthread id belongs to an evaluator worker thread. - * Called by the garbage collector's sp corrector (with the world - * stopped); lock-free. - */ -bool isEvalWorkerThread(void * pthreadId); - struct Executor { using work_t = MoveOnlyFunction; diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index 23be09d990..279018d97d 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -10,132 +10,10 @@ #if NIX_USE_BOEHMGC # include -# include #endif namespace nix { -#if NIX_USE_BOEHMGC - -/** - * Registry of fiber stacks for the garbage collector. This is - * append-only and grows in fixed-size blocks that never move, so the - * GC callbacks (which run with the world stopped and therefore must - * not take any locks that a frozen thread might hold) can safely - * iterate it at any moment. Entries are keyed by stack, not by fiber: - * thanks to the stack pool, only a modest number of stacks is ever - * allocated, and pooled stacks keep their (inert) entry. - */ -struct FiberStackInfo -{ - /** - * The hi end of the stack (i.e. `stack_context::sp`). Immutable - * once published. - */ - char * base = nullptr; - - /** - * The size of the stack. Immutable once published. - */ - size_t size = 0; - - /** - * If non-null, the GC must scan `[scanSp, base)`, i.e. the used - * portion of the stack of a suspended fiber. If null, the stack - * must not be scanned at all: it's either free (in the stack - * pool), unstarted (no GC roots yet), or running (in which case - * it's covered by the thread stack scan via the sp corrector, see - * `fixupBoehmStackPointer()`). - */ - std::atomic scanSp{nullptr}; -}; - -constexpr static size_t fiberStackBlockSize = 1024; -using FiberStackBlock = std::array; -static std::array, 1024> fiberStackBlocks; -static std::atomic nrFiberStacks{0}; -static std::mutex fiberStackRegistryMutex; - -static FiberStackInfo & getFiberStackInfo(size_t i) -{ - return (*fiberStackBlocks[i / fiberStackBlockSize].load(std::memory_order_acquire))[i % fiberStackBlockSize]; -} - -static FiberStackInfo * registerFiberStack(char * base, size_t size) -{ - std::unique_lock lk(fiberStackRegistryMutex); - auto i = nrFiberStacks.load(std::memory_order_relaxed); - if (i / fiberStackBlockSize >= fiberStackBlocks.size()) - throw Error("too many fiber stacks"); - auto & blockPtr = fiberStackBlocks[i / fiberStackBlockSize]; - if (!blockPtr.load(std::memory_order_relaxed)) - blockPtr.store(new FiberStackBlock(), std::memory_order_release); - auto & info = getFiberStackInfo(i); - info.base = base; - info.size = size; - /* Publish the entry only after its fields are initialized. */ - nrFiberStacks.store(i + 1, std::memory_order_release); - return &info; -} - -void * fiberStackContaining(void * sp) -{ - auto n = nrFiberStacks.load(std::memory_order_acquire); - for (size_t i = 0; i < n; ++i) { - auto & info = getFiberStackInfo(i); - if ((char *) sp < info.base && (char *) sp >= info.base - info.size) - return info.base; - } - return nullptr; -} - -void pushSuspendedFiberStacks() -{ - auto n = nrFiberStacks.load(std::memory_order_acquire); - for (size_t i = 0; i < n; ++i) { - auto & info = getFiberStackInfo(i); - if (auto sp = info.scanSp.load(std::memory_order_acquire)) - GC_push_all(sp, info.base); - } -} - -/** - * Registry of the worker threads' pthread ids, so that - * `isEvalWorkerThread()` can be called from the GC's sp corrector - * (with the world stopped, hence the lock-free fixed-size array). - */ -static std::array, 256> workerPthreads; - -static void registerWorkerPthread(pthread_t p) -{ - for (auto & slot : workerPthreads) { - std::uintptr_t expected = 0; - if (slot.compare_exchange_strong(expected, (std::uintptr_t) p)) - return; - } - /* Table full; harmless (the thread will merely be scanned in full - by the GC). */ -} - -static void unregisterWorkerPthread(pthread_t p) -{ - for (auto & slot : workerPthreads) - if (slot.load(std::memory_order_relaxed) == (std::uintptr_t) p) { - slot.store(0); - return; - } -} - -bool isEvalWorkerThread(void * pthreadId) -{ - for (auto & slot : workerPthreads) - if (slot.load(std::memory_order_relaxed) == (std::uintptr_t) (pthread_t) pthreadId) - return true; - return false; -} - -#endif - struct WaiterDomain; [[gnu::tls_model("initial-exec")]] thread_local bool Executor::amWorkerThread{false}; @@ -205,9 +83,9 @@ struct Executor::Fiber #if NIX_USE_BOEHMGC /** - * The GC registry entry for this fiber's stack. + * The registered GC descriptor of this fiber's stack. */ - FiberStackInfo * stackInfo = nullptr; + struct GC_stack * gcStack = nullptr; #endif Fiber(Executor & executor, Item && item) @@ -235,10 +113,12 @@ struct Executor::StackPool std::vector stacks; #if NIX_USE_BOEHMGC /** - * The GC registry entry for each stack ever allocated, keyed - * by the stack's hi end (`stack_context::sp`). + * The registered GC descriptor of each stack ever allocated, + * keyed by the stack's hi end (`stack_context::sp`). Pooled + * stacks keep their registration; their descriptors have a + * null `saved_sp`, so the GC never scans them. */ - std::unordered_map gcInfo; + std::unordered_map gcInfo; #endif }; @@ -246,7 +126,14 @@ struct Executor::StackPool ~StackPool() { - for (auto & sctx : state_.lock()->stacks) + auto state(state_.lock()); +#if NIX_USE_BOEHMGC + for (auto & [_, gs] : state->gcInfo) { + GC_unregister_stack(gs); + delete gs; + } +#endif + for (auto & sctx : state->stacks) boost::context::protected_fixedsize_stack(evalStackSize).deallocate(sctx); } }; @@ -255,9 +142,9 @@ struct Executor::StackPool /** * Side channel from `PooledStackAllocator::allocate()` (called from * inside the `boost::context::fiber` constructor) to `makeFiber()`: - * the GC registry entry of the most recently allocated stack. + * the GC descriptor of the most recently allocated stack. */ -[[gnu::tls_model("initial-exec")]] static thread_local FiberStackInfo * lastFiberStackInfo{nullptr}; +[[gnu::tls_model("initial-exec")]] static thread_local struct GC_stack * lastFiberGCStack{nullptr}; #endif /** @@ -278,7 +165,7 @@ struct PooledStackAllocator auto sctx = state->stacks.back(); state->stacks.pop_back(); #if NIX_USE_BOEHMGC - lastFiberStackInfo = state->gcInfo.at(sctx.sp); + lastFiberGCStack = state->gcInfo.at(sctx.sp); #endif return sctx; } @@ -286,9 +173,12 @@ struct PooledStackAllocator executor.nrFiberStacksAllocated++; auto sctx = boost::context::protected_fixedsize_stack(evalStackSize).allocate(); #if NIX_USE_BOEHMGC - auto info = registerFiberStack((char *) sctx.sp, sctx.size); - executor.stackPool->state_.lock()->gcInfo.emplace(sctx.sp, info); - lastFiberStackInfo = info; + auto gs = new GC_stack{}; + gs->base = sctx.sp; + gs->limit = (char *) sctx.sp - sctx.size; + GC_register_stack(gs); + executor.stackPool->state_.lock()->gcInfo.emplace(sctx.sp, gs); + lastFiberGCStack = gs; #endif return sctx; } @@ -424,13 +314,9 @@ void Executor::createWorker(State & state) GC_stack_base sb; GC_get_stack_base(&sb); GC_register_my_thread(&sb); - /* Let the GC's sp corrector know that this thread's own stack - holds no GC roots (see `fixupBoehmStackPointer()`). */ - registerWorkerPthread(pthread_self()); #endif worker(); #if NIX_USE_BOEHMGC - unregisterWorkerPthread(pthread_self()); GC_unregister_my_thread(); #endif })); @@ -466,8 +352,8 @@ Executor::FiberPtr Executor::makeFiber(Item && item) return nullptr; } #if NIX_USE_BOEHMGC - fiber->stackInfo = lastFiberStackInfo; - assert(fiber->stackInfo); + fiber->gcStack = lastFiberGCStack; + assert(fiber->gcStack); #endif nrFibersSpawned++; return fiber; @@ -487,8 +373,26 @@ void Executor::runFiber(FiberPtr fiber) CallDepth::callDepth = fib->callDepth; EvalState::evalContext = &fib->evalContext; +#if NIX_USE_BOEHMGC + /* Make this thread's stack scannable by the GC while the fiber + runs (its frames may reference GC values in non-worker callers, + e.g. when draining the queue from the main thread), and make the + fiber's stack the current one. The fiber clears its own + `saved_sp` after it has been resumed (see `suspendFiber()`). */ + auto prevStack = GC_current_stack; + if (prevStack) + prevStack->saved_sp = (char *) __builtin_frame_address(0) - 512; + GC_current_stack = fib->gcStack; +#endif + fib->ctx = std::move(fib->ctx).resume(); +#if NIX_USE_BOEHMGC + GC_current_stack = prevStack; + if (prevStack) + prevStack->saved_sp = nullptr; +#endif + currentFiber = nullptr; myEvalThreadId = savedThreadId; fib->callDepth = CallDepth::callDepth; @@ -726,11 +630,11 @@ suspendFiber(WaiterDomain & domain, std::unique_lock & lk, detail::V the current frame covers the register block that the context switch is about to push; those bytes are touched by the switch anyway, so this doesn't fault in any untouched stack pages. - Note: from this point until the resumed fiber clears `scanSp`, - the stack may be scanned both through this entry and through - the thread stack scan (see `fixupBoehmStackPointer()`); the - overlap is harmless. */ - fib->stackInfo->scanSp.store((char *) __builtin_frame_address(0) - 512, std::memory_order_release); + Note: from this point until we clear `saved_sp` after being + resumed, the GC may see the stack both as some thread's active + stack and as a suspended one; it scans it only once (from the + lower stack pointer). */ + fib->gcStack->saved_sp = (char *) __builtin_frame_address(0) - 512; #endif /* Switch back to the scheduler (`Executor::runFiber()`), which will register us in the domain's wait list and then release the @@ -741,8 +645,8 @@ suspendFiber(WaiterDomain & domain, std::unique_lock & lk, detail::V fib->schedCtx = std::move(fib->schedCtx).resume(); #if NIX_USE_BOEHMGC /* We're running again (possibly on another thread), so our stack - is covered by the thread stack scan from here on. */ - fib->stackInfo->scanSp.store(nullptr, std::memory_order_release); + is scanned as that thread's active stack from here on. */ + fib->gcStack->saved_sp = nullptr; #endif /* We've been resumed because the value was finished (or because we're shutting down); the scheduler released the lock long diff --git a/src/libutil/include/nix/util/coroutine-gc.hh b/src/libutil/include/nix/util/coroutine-gc.hh new file mode 100644 index 0000000000..5e11901463 --- /dev/null +++ b/src/libutil/include/nix/util/coroutine-gc.hh @@ -0,0 +1,45 @@ +#pragma once +///@file + +#include + +namespace nix { + +/** + * Hooks that make the boost::coroutines2 coroutines in libutil (see + * `sourceToSink`/`sinkToSource` in `serialise.hh`) visible to a + * conservative garbage collector. libutil itself does not depend on + * any GC; these default to null and are installed by the evaluator + * (`libexpr`), which implements them in terms of bdwgc's registered + * stacks (`GC_register_stack` etc.). + * + * The contract: + * + * - `coroStackRegister(base, size)` is called when a coroutine stack + * is allocated, with `base` its cold (hi) end. It returns an opaque + * cookie identifying the stack. `coroStackUnregister(cookie)` is + * called when the stack is deallocated. + * + * - `coroSwitchTo(cookie, callerSp)` is called just before the current + * thread switches onto the coroutine stack identified by `cookie`; + * `callerSp` approximates the caller's stack pointer at the switch + * (everything from `callerSp` up to the caller's stack base must be + * scannable). It returns an opaque handle for the caller's stack, + * to be passed to `coroSwitchBack` when the switch returns. + * + * - `coroMarkSuspended(cookie, sp)` is called on the coroutine stack + * just before yielding back to the caller, with `sp` approximating + * the coroutine's stack pointer; `coroMarkActive(cookie)` is called + * when the coroutine is resumed again. + * + * All hooks are invoked with the corresponding stack switches + * strictly balanced. + */ +extern void * (*coroStackRegister)(void * base, size_t size); +extern void (*coroStackUnregister)(void * cookie); +extern void * (*coroSwitchTo)(void * cookie, void * callerSp); +extern void (*coroSwitchBack)(void * prevHandle); +extern void (*coroMarkSuspended)(void * cookie, void * sp); +extern void (*coroMarkActive)(void * cookie); + +} // namespace nix diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 3af780eb29..1379fd8d2f 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -32,6 +32,7 @@ headers = [ config_pub_h ] + files( 'config-global.hh', 'config-impl.hh', 'configuration.hh', + 'coroutine-gc.hh', 'current-process.hh', 'deleter.hh', 'demangle.hh', diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index eb12098431..a781529258 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,4 +1,5 @@ #include "nix/util/serialise.hh" +#include "nix/util/coroutine-gc.hh" #include "nix/util/file-descriptor.hh" #include "nix/util/signals.hh" #include "nix/util/socket.hh" @@ -372,10 +373,76 @@ void StringSource::skip(size_t len) /* 512KiB is a conservative estimate for deeply nested NARs, which are limited to 64 levels. We also tend to allocate rather large buffers on the stack, so we should leave plenty of headroom. Note that no evaluation is supposed to - happen on sourceToSink/sinkToSource coroutine stacks (for Boehm GC reasons), - which requires much more stack space. */ + happen on sourceToSink/sinkToSource coroutine stacks (they are too small for + that), though the GC hooks below do make them scannable. */ static constexpr size_t defaultCoroutineStackSize = 512 * 1024; +void * (*coroStackRegister)(void * base, size_t size) = nullptr; +void (*coroStackUnregister)(void * cookie) = nullptr; +void * (*coroSwitchTo)(void * cookie, void * callerSp) = nullptr; +void (*coroSwitchBack)(void * prevHandle) = nullptr; +void (*coroMarkSuspended)(void * cookie, void * sp) = nullptr; +void (*coroMarkActive)(void * cookie) = nullptr; + +namespace { + +/** + * A stack allocator that reports the coroutine stacks to the GC hooks + * (see `coroutine-gc.hh`), storing the resulting cookie in the owning + * object (which must outlive the coroutine). + */ +struct GCTrackedStackAllocator +{ + void ** cookie; + + boost::context::stack_context allocate() + { + auto sctx = boost::coroutines2::protected_fixedsize_stack(defaultCoroutineStackSize).allocate(); + *cookie = coroStackRegister ? coroStackRegister(sctx.sp, sctx.size) : nullptr; + return sctx; + } + + void deallocate(boost::context::stack_context & sctx) + { + if (*cookie) { + coroStackUnregister(*cookie); + *cookie = nullptr; + } + boost::coroutines2::protected_fixedsize_stack(defaultCoroutineStackSize).deallocate(sctx); + } +}; + +/** + * Notify the GC hooks that the current thread is about to switch onto + * (and, on destruction, back off) the coroutine stack identified by + * `cookie`. `callerSp` must be the frame address of the function + * performing the switch. Construct just before resuming a coroutine + * (this includes destroying a suspended one, which unwinds on its own + * stack); the switch back happens when the coroutine yields or + * finishes. + */ +struct CoroutineGuard +{ + void * prev = nullptr; + bool active = false; + + CoroutineGuard(void * cookie, void * callerSp) + { + if (coroSwitchTo) { + prev = coroSwitchTo(cookie, callerSp); + active = true; + } + } + + ~CoroutineGuard() + { + if (active) + coroSwitchBack(prev); + } +}; + +} // namespace + std::unique_ptr sourceToSink(fun reader) { struct SourceToSink : FinishSink @@ -384,12 +451,22 @@ std::unique_ptr sourceToSink(fun reader) fun reader; std::optional coro; + void * stackCookie = nullptr; SourceToSink(fun reader) : reader(reader) { } + ~SourceToSink() + { + /* Destroying a suspended coroutine unwinds it on its own + stack, so this too is a switch onto the coroutine + stack. */ + CoroutineGuard guard{stackCookie, __builtin_frame_address(0)}; + coro.reset(); + } + std::string_view cur; void operator()(std::string_view in) override @@ -398,23 +475,27 @@ std::unique_ptr sourceToSink(fun reader) return; cur = in; + CoroutineGuard guard{stackCookie, __builtin_frame_address(0)}; + if (!coro) { - coro = coro_t::push_type( - boost::coroutines2::protected_fixedsize_stack(defaultCoroutineStackSize), - [&](coro_t::pull_type & yield) { - LambdaSource source([&](char * out, size_t out_len) { - if (cur.empty()) { - yield(); - if (yield.get()) - throw EndOfFile("coroutine has finished"); - } - - size_t n = cur.copy(out, out_len); - cur.remove_prefix(n); - return n; - }); - reader(source); + coro = coro_t::push_type(GCTrackedStackAllocator{&stackCookie}, [&](coro_t::pull_type & yield) { + LambdaSource source([&](char * out, size_t out_len) { + if (cur.empty()) { + if (coroMarkSuspended) + coroMarkSuspended(stackCookie, (char *) __builtin_frame_address(0) - 512); + yield(); + if (coroMarkActive) + coroMarkActive(stackCookie); + if (yield.get()) + throw EndOfFile("coroutine has finished"); + } + + size_t n = cur.copy(out, out_len); + cur.remove_prefix(n); + return n; }); + reader(source); + }); } if (!*coro) { @@ -428,8 +509,10 @@ std::unique_ptr sourceToSink(fun reader) void finish() override { - if (coro && *coro) + if (coro && *coro) { + CoroutineGuard guard{stackCookie, __builtin_frame_address(0)}; (*coro)(true); + } } }; @@ -445,6 +528,7 @@ std::unique_ptr sinkToSource(fun writer, fun eof) fun writer; fun eof; std::optional coro; + void * stackCookie = nullptr; SinkToSource(fun writer, fun eof) : writer(writer) @@ -452,38 +536,51 @@ std::unique_ptr sinkToSource(fun writer, fun eof) { } + ~SinkToSource() + { + /* See `SourceToSink::~SourceToSink()`. */ + CoroutineGuard guard{stackCookie, __builtin_frame_address(0)}; + coro.reset(); + } + std::string_view cur; size_t read(char * data, size_t len) override { + CoroutineGuard guard{stackCookie, __builtin_frame_address(0)}; + bool hasCoro = coro.has_value(); if (!hasCoro) { - coro = coro_t::pull_type( - boost::coroutines2::protected_fixedsize_stack(defaultCoroutineStackSize), - [&](coro_t::push_type & yield) { - /* Feed the consumer in chunks, instead of on each write - to avoid excessive context switching. parseDump does - lots of small writes to the sink, which we should - accumulate. */ - struct CoroBufferedSink : BufferedSink + coro = coro_t::pull_type(GCTrackedStackAllocator{&stackCookie}, [&](coro_t::push_type & yield) { + /* Feed the consumer in chunks, instead of on each write + to avoid excessive context switching. parseDump does + lots of small writes to the sink, which we should + accumulate. */ + struct CoroBufferedSink : BufferedSink + { + coro_t::push_type & yield; + void * stackCookie; + + void writeUnbuffered(std::string_view data) override { - coro_t::push_type & yield; - - void writeUnbuffered(std::string_view data) override - { - yield(data); - } - - CoroBufferedSink(coro_t::push_type & yield) - : yield(yield) - { - } - }; - - CoroBufferedSink sink(yield); - writer(sink); - sink.flush(); - }); + if (coroMarkSuspended) + coroMarkSuspended(stackCookie, (char *) __builtin_frame_address(0) - 512); + yield(data); + if (coroMarkActive) + coroMarkActive(stackCookie); + } + + CoroBufferedSink(coro_t::push_type & yield, void * stackCookie) + : yield(yield) + , stackCookie(stackCookie) + { + } + }; + + CoroBufferedSink sink(yield, stackCookie); + writer(sink); + sink.flush(); + }); } if (cur.empty()) { From c57c2ac0da9802eb7e96b0c8754344ec4abe3555 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Sep 2026 19:42:40 +0200 Subject: [PATCH 812/927] packaging/secure-packages/flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nix': 'path:../..' → 'path:../..' • Added input 'nix/bdwgc': 'github:edolstra/bdwgc/c6afef5' (2026-09-14) • Updated input 'nix/nixpkgs': 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013551%2Brev-07b3c48788b0deb9ee48c40ed5ff1dff7f4643e7/01a08dbc-1a6d-7e60-8dd0-bc2bf8ecb4fb/source.tar.gz' (2026-09-10) → 'https://api.flakehub.com/f/pinned/DeterminateSystems/secure-packages-26.05/0.1.1013564%2Brev-ad42e7d403fdb69c4be8150f5be383f3c3c4cf0f/01a09fa0-9a34-7f52-93da-2a98c2d0027a/source.tar.gz' (2026-09-14) --- packaging/secure-packages/flake.lock | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index a87a4b0be7..c348556c4c 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -1,5 +1,22 @@ { "nodes": { + "bdwgc": { + "flake": false, + "locked": { + "lastModified": 1789388623, + "narHash": "sha256-qk2ONj1UU3zRzyJMtVns5ZDH0uCWpvTtqW2b/lZmotU=", + "owner": "edolstra", + "repo": "bdwgc", + "rev": "c6afef5518ef71d446339a5aa4c1b97fc6392534", + "type": "github" + }, + "original": { + "owner": "edolstra", + "ref": "user-defined-stacks", + "repo": "bdwgc", + "type": "github" + } + }, "flake-compat": { "flake": false, "locked": { @@ -59,6 +76,7 @@ }, "nix": { "inputs": { + "bdwgc": "bdwgc", "flake-parts": "flake-parts", "git-hooks-nix": "git-hooks-nix", "nixpkgs": "nixpkgs", From d7d459130d82f7979ebc6207f7da664fe88c7106 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Sep 2026 11:48:25 +0200 Subject: [PATCH 813/927] Update the bdwgc input to include Darwin registered-stacks scanning Assisted-by: Claude Fable 5 --- flake.lock | 6 +++--- packaging/secure-packages/flake.lock | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index f8d0ef4c81..caa2443bbe 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "bdwgc": { "flake": false, "locked": { - "lastModified": 1789388623, - "narHash": "sha256-qk2ONj1UU3zRzyJMtVns5ZDH0uCWpvTtqW2b/lZmotU=", + "lastModified": 1789465360, + "narHash": "sha256-S18DMeECNCKZUhy8PspMyXSPzO7ZFsv/IQsIpYNkIls=", "owner": "edolstra", "repo": "bdwgc", - "rev": "c6afef5518ef71d446339a5aa4c1b97fc6392534", + "rev": "82eb0a94e28c38b42ab43bc32e1f3740ce6e7d6a", "type": "github" }, "original": { diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index c348556c4c..b902daf8b0 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -3,11 +3,11 @@ "bdwgc": { "flake": false, "locked": { - "lastModified": 1789388623, - "narHash": "sha256-qk2ONj1UU3zRzyJMtVns5ZDH0uCWpvTtqW2b/lZmotU=", + "lastModified": 1789465360, + "narHash": "sha256-S18DMeECNCKZUhy8PspMyXSPzO7ZFsv/IQsIpYNkIls=", "owner": "edolstra", "repo": "bdwgc", - "rev": "c6afef5518ef71d446339a5aa4c1b97fc6392534", + "rev": "82eb0a94e28c38b42ab43bc32e1f3740ce6e7d6a", "type": "github" }, "original": { From a049f6be9d397cf81363bf59f361f47c5b6594eb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Sep 2026 12:06:10 +0200 Subject: [PATCH 814/927] Update the bdwgc input to fix stackstest linking on musl Assisted-by: Claude Fable 5 --- flake.lock | 6 +++--- packaging/secure-packages/flake.lock | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index caa2443bbe..ca58a53f5d 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "bdwgc": { "flake": false, "locked": { - "lastModified": 1789465360, - "narHash": "sha256-S18DMeECNCKZUhy8PspMyXSPzO7ZFsv/IQsIpYNkIls=", + "lastModified": 1789466139, + "narHash": "sha256-N+E7uhp1UNUIO708qX5UGfoXK1fvPQKKARsTU1GUb4U=", "owner": "edolstra", "repo": "bdwgc", - "rev": "82eb0a94e28c38b42ab43bc32e1f3740ce6e7d6a", + "rev": "af7ab5ba87bde1ec1630f57852271269438d555d", "type": "github" }, "original": { diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index b902daf8b0..0a09a1213a 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -3,11 +3,11 @@ "bdwgc": { "flake": false, "locked": { - "lastModified": 1789465360, - "narHash": "sha256-S18DMeECNCKZUhy8PspMyXSPzO7ZFsv/IQsIpYNkIls=", + "lastModified": 1789466139, + "narHash": "sha256-N+E7uhp1UNUIO708qX5UGfoXK1fvPQKKARsTU1GUb4U=", "owner": "edolstra", "repo": "bdwgc", - "rev": "82eb0a94e28c38b42ab43bc32e1f3740ce6e7d6a", + "rev": "af7ab5ba87bde1ec1630f57852271269438d555d", "type": "github" }, "original": { From 579d1f76ef44f0589da4f9a90566658b9d11d427 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Sep 2026 19:20:58 +0200 Subject: [PATCH 815/927] Update the bdwgc input to fix the test suite hang on macOS The collector wrote its scan-epoch stamps into the registered stack descriptors, which for thread stacks live in the collector's heap and are write-protected during incremental collections; on Darwin the resulting write fault could not be serviced with the world stopped, hanging gctest and weakmaptest in boehm-gc's check phase. The stamps now live in collector-owned scratch memory. Assisted-by: Claude Fable 5.1 --- flake.lock | 6 +++--- packaging/secure-packages/flake.lock | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index ca58a53f5d..f043a19efb 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "bdwgc": { "flake": false, "locked": { - "lastModified": 1789466139, - "narHash": "sha256-N+E7uhp1UNUIO708qX5UGfoXK1fvPQKKARsTU1GUb4U=", + "lastModified": 1789490831, + "narHash": "sha256-AZwK4PzsERmLJ/F+Xjqj8kERb7zYiW0U+SZ586b5rBM=", "owner": "edolstra", "repo": "bdwgc", - "rev": "af7ab5ba87bde1ec1630f57852271269438d555d", + "rev": "015d683f032dec4c71f2bda61e8f1a88f4a4acd4", "type": "github" }, "original": { diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index 0a09a1213a..ed2f5cf709 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -3,11 +3,11 @@ "bdwgc": { "flake": false, "locked": { - "lastModified": 1789466139, - "narHash": "sha256-N+E7uhp1UNUIO708qX5UGfoXK1fvPQKKARsTU1GUb4U=", + "lastModified": 1789490831, + "narHash": "sha256-AZwK4PzsERmLJ/F+Xjqj8kERb7zYiW0U+SZ586b5rBM=", "owner": "edolstra", "repo": "bdwgc", - "rev": "af7ab5ba87bde1ec1630f57852271269438d555d", + "rev": "015d683f032dec4c71f2bda61e8f1a88f4a4acd4", "type": "github" }, "original": { From 3dc9a67682b9fc02916808ce8c61618748fc44c6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 13:08:05 +0200 Subject: [PATCH 816/927] builtins.wasm: Add read_file_v2 With `read_file`, the guest has to guess the buffer size, so for any file larger than its guess the host reads and copies the file twice. `read_file_v2` instead reads the file once into a buffer that it allocates in the guest by calling the guest's `nix_wasm_alloc` export, and stores the length via an out-pointer. The export is looked up once per instance. `read_file` is kept for now for modules built against the old runtime. Scanning all of Nix's components: identical results, ~1200 fewer file reads; wall time unchanged since the second reads came from the page cache. Assisted-by: Claude Fable 5.1 (cherry picked from commit 150e88227adc5859e186431be26233b33edd135f) --- src/libexpr/primops/wasm.cc | 51 ++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/wasm.cc b/src/libexpr/primops/wasm.cc index 1342a02920..df8a2e6345 100644 --- a/src/libexpr/primops/wasm.cc +++ b/src/libexpr/primops/wasm.cc @@ -163,6 +163,8 @@ struct NixWasmInstance wasmtime::Store::Context wasmCtx; Instance instance; Memory memory_; + // The guest's allocator, if it exports one (see `allocInGuest`). + std::optional allocFn; ValueVector values; std::exception_ptr ex; @@ -184,6 +186,13 @@ struct NixWasmInstance { wasmCtx.set_data(this); + if (auto ext = instance.get(wasmCtx, "nix_wasm_alloc")) { + auto fun = std::get_if(&*ext); + if (!fun) + throw Error("export 'nix_wasm_alloc' of Wasm module '%s' is not a function", pre->name); + allocFn = *fun; + } + /* Reserve value ID 0 so it can be used in functions like get_attr() to denote a missing attribute. */ values.push_back(nullptr); } @@ -507,9 +516,25 @@ struct NixWasmInstance return addValue(res); } + /** + * Allocate `size` bytes in the guest by calling its `nix_wasm_alloc` export. The guest is responsible for freeing + * the buffer (e.g. by reconstructing a `Vec` from it). + */ + uint32_t allocInGuest(uint32_t size) + { + if (!allocFn) + throw Error("Wasm module '%s' does not export 'nix_wasm_alloc'", pre->name); + auto res = unwrap(allocFn->call(wasmCtx, {(int32_t) size})); + if (res.size() != 1 || res[0].kind() != ValKind::I32) + throw Error("'nix_wasm_alloc' of Wasm module '%s' did not return an i32", pre->name); + return (uint32_t) res[0].i32(); + } + /** * Read the contents of a file into Wasm memory. This is like calling `builtins.readFile`, except that it can handle * binary files that cannot be represented as Nix strings. + * + * Deprecated: the guest has to guess the buffer size, so the file may be read twice. Use `read_file_v2`. */ uint32_t read_file(ValueId pathId, uint32_t ptr, uint32_t len) { @@ -521,7 +546,6 @@ struct NixWasmInstance if (contents.size() > std::numeric_limits::max()) throw Error("file '%s' is too large to process in Wasm (size: %d)", path, contents.size()); - // FIXME: this is an inefficient interface since it may cause the file to be read twice. if (contents.size() <= len) { auto buf = memory().subspan(ptr, len); memcpy(buf.data(), contents.data(), contents.size()); @@ -529,6 +553,30 @@ struct NixWasmInstance return contents.size(); } + + /** + * Read the contents of a file into a buffer allocated in the guest via `nix_wasm_alloc`, so that the file is read + * and copied only once. Returns the buffer and stores its length at `lenPtr`. + */ + uint32_t read_file_v2(ValueId pathId, uint32_t lenPtr) + { + auto & pathValue = getValue(pathId); + auto path = state.realisePath(noPos, pathValue); + + auto contents = path.readFile(); + + if (contents.size() > std::numeric_limits::max()) + throw Error("file '%s' is too large to process in Wasm (size: %d)", path, contents.size()); + + auto ptr = allocInGuest(contents.size()); + + // Note: the allocation may have grown the memory, so get the data pointer only now. + auto mem = memory(); + memcpy(mem.subspan(ptr, contents.size()).data(), contents.data(), contents.size()); + subspan(mem.subspan(lenPtr), 1)[0] = contents.size(); // FIXME: endianness + + return ptr; + } }; template @@ -571,6 +619,7 @@ static void regFuns(Linker & linker, bool useWasi) regFun(linker, "call_function", &NixWasmInstance::call_function); regFun(linker, "make_app", &NixWasmInstance::make_app); regFun(linker, "read_file", &NixWasmInstance::read_file); + regFun(linker, "read_file_v2", &NixWasmInstance::read_file_v2); if (useWasi) { unwrap(linker.func_wrap( From 29955328b03a0b742c34c1a49091df36ce9ff007 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 14:12:36 +0200 Subject: [PATCH 817/927] builtins.wasm: Count guest allocations Add `nrWasmGuestAllocs` and `wasmGuestAllocBytes` to the evaluator statistics: the number and total size of the buffers allocated in Wasm guests on behalf of host functions (currently `read_file_v2`). Assisted-by: Claude Fable 5.1 (cherry picked from commit e7efa0a049860e7e79ee77259d8496ac29157c01) --- src/libexpr/eval.cc | 2 ++ src/libexpr/include/nix/expr/eval.hh | 4 ++++ src/libexpr/primops/wasm.cc | 2 ++ 3 files changed, 8 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f0effb0add..b745aa4c74 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3104,6 +3104,8 @@ void EvalState::printStatistics() topObj["nrLookups"] = nrLookups.load(); topObj["nrPrimOpCalls"] = nrPrimOpCalls.load(); topObj["nrFunctionCalls"] = nrFunctionCalls.load(); + topObj["nrWasmGuestAllocs"] = nrWasmGuestAllocs.load(); + topObj["wasmGuestAllocBytes"] = wasmGuestAllocBytes.load(); #if NIX_USE_BOEHMGC topObj["gc"] = { {"heapSize", heapSize}, diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index b0ae6c6ecb..0cd6ee654d 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -1091,6 +1091,10 @@ private: Counter nrFunctionCalls; public: + // Buffers allocated in Wasm guests on behalf of host functions (e.g. `read_file_v2`). + Counter nrWasmGuestAllocs; + Counter wasmGuestAllocBytes; + Counter nrThunksAwaited; Counter nrThunksAwaitedSlow; Counter microsecondsWaiting; diff --git a/src/libexpr/primops/wasm.cc b/src/libexpr/primops/wasm.cc index df8a2e6345..b69af81667 100644 --- a/src/libexpr/primops/wasm.cc +++ b/src/libexpr/primops/wasm.cc @@ -527,6 +527,8 @@ struct NixWasmInstance auto res = unwrap(allocFn->call(wasmCtx, {(int32_t) size})); if (res.size() != 1 || res[0].kind() != ValKind::I32) throw Error("'nix_wasm_alloc' of Wasm module '%s' did not return an i32", pre->name); + state.nrWasmGuestAllocs++; + state.wasmGuestAllocBytes += size; return (uint32_t) res[0].i32(); } From 36f42bc33a14bbe403b2dde96b455817a33c4205 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 17:51:25 +0200 Subject: [PATCH 818/927] builtins.wasm: Bounds-check all guest memory accesses Host functions indexed the guest's linear memory with guest-supplied offsets through `std::span::subspan`, which does no bounds checking, so a module could make the host read or write outside the Wasm memory. (Flagged by a review comment for `read_file_v2`, but the same pattern was in every host function, and the checked `subspan` helper only ran after an unchecked `subspan`.) All accesses now go through `guestSpan`, which checks the offset and length against the current memory size and throws the "Wasm memory access out of bounds" error, which surfaces as a trap. Add a functional test with a module that passes an out-of-range pointer to `make_string`. Assisted-by: Claude Fable 5.1 --- src/libexpr/primops/wasm.cc | 75 ++++++++++++++++++++++--------------- tests/functional/oob.wat | 12 ++++++ tests/functional/wasm.sh | 7 ++++ 3 files changed, 64 insertions(+), 30 deletions(-) create mode 100644 tests/functional/oob.wat diff --git a/src/libexpr/primops/wasm.cc b/src/libexpr/primops/wasm.cc index b69af81667..c4568582ea 100644 --- a/src/libexpr/primops/wasm.cc +++ b/src/libexpr/primops/wasm.cc @@ -70,14 +70,6 @@ static std::string_view span2string(std::span s) return std::string_view((char *) s.data(), s.size()); } -template -static std::span subspan(std::span s, size_t len) -{ - if (s.size() < len * sizeof(T)) - throw Error("Wasm memory access out of bounds"); - return std::span((T *) s.data(), len); -} - // FIXME: move to wasmtime C++ wrapper. class InstancePre { @@ -241,14 +233,40 @@ struct NixWasmInstance return memory_.data(wasmCtx); } + /** + * A view of `len` bytes of guest memory at `ptr`, checked against the current size of the memory. All accesses + * to guest memory with guest-supplied offsets must go through this, since `std::span::subspan` does not check + * bounds. The memory is fetched afresh each time, since it may have grown (e.g. by `allocInGuest`). + */ + std::span guestSpan(uint32_t ptr, size_t len) + { + auto mem = memory(); + if (ptr > mem.size() || len > mem.size() - ptr) + throw Error( + "Wasm memory access out of bounds (offset %d, length %d, memory size %d)", ptr, len, mem.size()); + return mem.subspan(ptr, len); + } + + /** + * Like `guestSpan`, for an array of `count` values of type `T`. + */ + template + std::span guestSpan(uint32_t ptr, size_t count) + { + if (count > std::numeric_limits::max() / sizeof(T)) + throw Error("Wasm memory access out of bounds (offset %d, count %d)", ptr, count); + auto s = guestSpan(ptr, count * sizeof(T)); + return std::span((T *) s.data(), count); + } + std::monostate panic(uint32_t ptr, uint32_t len) { - throw Error("Wasm panic: %s", Uncolored(span2string(memory().subspan(ptr, len)))); + throw Error("Wasm panic: %s", Uncolored(span2string(guestSpan(ptr, len)))); } std::monostate warn(uint32_t ptr, uint32_t len) { - doWarn(span2string(memory().subspan(ptr, len))); + doWarn(span2string(guestSpan(ptr, len))); return {}; } @@ -304,7 +322,7 @@ struct NixWasmInstance ValueId make_string(uint32_t ptr, uint32_t len) { auto [valueId, value] = allocValue(); - value.mkString(span2string(memory().subspan(ptr, len)), state.mem); + value.mkString(span2string(guestSpan(ptr, len)), state.mem); return valueId; } @@ -312,7 +330,7 @@ struct NixWasmInstance { auto s = state.forceString(getValue(valueId), noPos, "while evaluating a value from Wasm"); if (s.size() <= maxLen) { - auto buf = memory().subspan(ptr, maxLen); + auto buf = guestSpan(ptr, maxLen); memcpy(buf.data(), s.data(), s.size()); } return s.size(); @@ -327,7 +345,7 @@ struct NixWasmInstance auto base = baseValue.path(); auto [valueId, value] = allocValue(); - value.mkPath({base.accessor, CanonPath(span2string(memory().subspan(ptr, len)), base.path)}, state.mem); + value.mkPath({base.accessor, CanonPath(span2string(guestSpan(ptr, len)), base.path)}, state.mem); return valueId; } @@ -340,7 +358,7 @@ struct NixWasmInstance auto path = v.path().path; auto s = path.abs(); if (s.size() <= maxLen) { - auto buf = memory().subspan(ptr, maxLen); + auto buf = guestSpan(ptr, maxLen); memcpy(buf.data(), s.data(), s.size()); } return s.size(); @@ -363,7 +381,7 @@ struct NixWasmInstance ValueId make_list(uint32_t ptr, uint32_t len) { - auto vs = subspan(memory().subspan(ptr), len); + auto vs = guestSpan(ptr, len); auto [valueId, value] = allocValue(); @@ -381,7 +399,7 @@ struct NixWasmInstance state.forceList(value, noPos, "while getting a list from Wasm"); if (value.listSize() <= maxLen) { - auto out = subspan(memory().subspan(ptr), value.listSize()); + auto out = guestSpan(ptr, value.listSize()); for (const auto & [n, elem] : enumerate(value.listView())) out[n] = addValue(elem); @@ -392,8 +410,6 @@ struct NixWasmInstance ValueId make_attrset(uint32_t ptr, uint32_t len) { - auto mem = memory(); - struct Attr { // FIXME: endianness @@ -402,13 +418,13 @@ struct NixWasmInstance ValueId value; }; - auto attrs = subspan(mem.subspan(ptr), len); + auto attrs = guestSpan(ptr, len); auto [valueId, value] = allocValue(); auto builder = state.buildBindings(len); for (auto & attr : attrs) builder.insert( - state.symbols.create(span2string(mem.subspan(attr.attrNamePtr, attr.attrNameLen))), + state.symbols.create(span2string(guestSpan(attr.attrNamePtr, attr.attrNameLen))), &getValue(attr.value)); value.mkAttrs(builder); @@ -428,7 +444,7 @@ struct NixWasmInstance uint32_t nameLen; }; - auto buf = subspan(memory().subspan(ptr), maxLen); + auto buf = guestSpan(ptr, maxLen); // FIXME: for determinism, we should return attributes in lexicographically sorted order. for (const auto & [n, attr] : enumerate(*value.attrs())) { @@ -463,14 +479,14 @@ struct NixWasmInstance if ((size_t) len != name.size()) throw Error("copy_attrname: buffer length does not match attribute name length"); - memcpy(memory().subspan(ptr, len).data(), name.data(), name.size()); + memcpy(guestSpan(ptr, len).data(), name.data(), name.size()); return {}; } ValueId get_attr(ValueId valueId, uint32_t ptr, uint32_t len) { - auto attrName = span2string(memory().subspan(ptr, len)); + auto attrName = span2string(guestSpan(ptr, len)); auto & value = getValue(valueId); state.forceAttrs(value, noPos, "while getting an attribute from Wasm"); @@ -486,7 +502,7 @@ struct NixWasmInstance state.forceFunction(fun, noPos, "while calling a function from Wasm"); ValueVector args; - for (auto argId : subspan(memory().subspan(ptr), len)) + for (auto argId : guestSpan(ptr, len)) args.push_back(&getValue(argId)); auto [valueId, value] = allocValue(); @@ -501,7 +517,7 @@ struct NixWasmInstance if (!len) return funId; - auto args = subspan(memory().subspan(ptr), len); + auto args = guestSpan(ptr, len); auto res = &getValue(funId); @@ -549,7 +565,7 @@ struct NixWasmInstance throw Error("file '%s' is too large to process in Wasm (size: %d)", path, contents.size()); if (contents.size() <= len) { - auto buf = memory().subspan(ptr, len); + auto buf = guestSpan(ptr, len); memcpy(buf.data(), contents.data(), contents.size()); } @@ -572,10 +588,9 @@ struct NixWasmInstance auto ptr = allocInGuest(contents.size()); - // Note: the allocation may have grown the memory, so get the data pointer only now. - auto mem = memory(); - memcpy(mem.subspan(ptr, contents.size()).data(), contents.data(), contents.size()); - subspan(mem.subspan(lenPtr), 1)[0] = contents.size(); // FIXME: endianness + // Note: the allocation may have grown the memory; `guestSpan` fetches it afresh. + memcpy(guestSpan(ptr, contents.size()).data(), contents.data(), contents.size()); + guestSpan(lenPtr, 1)[0] = contents.size(); // FIXME: endianness return ptr; } diff --git a/tests/functional/oob.wat b/tests/functional/oob.wat new file mode 100644 index 0000000000..dd5da01d7e --- /dev/null +++ b/tests/functional/oob.wat @@ -0,0 +1,12 @@ +(module + ;; A module that passes an out-of-range pointer to a host function. The + ;; host must reject it rather than access memory outside the Wasm memory. + (import "env" "make_string" (func $make_string (param i32 i32) (result i32))) + + (memory (export "memory") 1) + + (func (export "nix_wasm_init_v1")) + + ;; The memory is one page (64 KiB); this offset is far beyond it. + (func (export "oob") (param i32) (result i32) + (call $make_string (i32.const 0xFFFFFF00) (i32.const 16)))) diff --git a/tests/functional/wasm.sh b/tests/functional/wasm.sh index f0b64fd65f..a813cb0531 100644 --- a/tests/functional/wasm.sh +++ b/tests/functional/wasm.sh @@ -15,3 +15,10 @@ fi [[ $(nix eval --json --impure \ --extra-experimental-features wasm-builtin \ --expr "builtins.wasm { path = ./fib.wasm; function = \"fib\"; } 40") = 165580141 ]] + +# A host function called with an out-of-range pointer must fail with an +# error rather than access memory outside the Wasm memory. +expectStderr 1 nix eval --impure \ + --extra-experimental-features wasm-builtin \ + --expr "builtins.wasm { wat = builtins.readFile ./oob.wat; function = \"oob\"; } 0" \ + | grepQuiet "Wasm memory access out of bounds" From 21bb98c3838f74d0eb8096891a00b0ba171f5d5c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 17:59:48 +0200 Subject: [PATCH 819/927] builtins.wasm: Return the read_file_v2 result as a packed u64 Instead of storing the length via an out-pointer, return the buffer pointer in the low 32 bits and the length in the high 32 bits. Rust's C ABI cannot express Wasm multi-value returns, so this is the usual way to return two values from an imported function. `read_file_v2` is unreleased, so the name stays. Assisted-by: Claude Fable 5.1 --- src/libexpr/primops/wasm.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/wasm.cc b/src/libexpr/primops/wasm.cc index c4568582ea..bb248efd99 100644 --- a/src/libexpr/primops/wasm.cc +++ b/src/libexpr/primops/wasm.cc @@ -574,9 +574,10 @@ struct NixWasmInstance /** * Read the contents of a file into a buffer allocated in the guest via `nix_wasm_alloc`, so that the file is read - * and copied only once. Returns the buffer and stores its length at `lenPtr`. + * and copied only once. Returns the buffer pointer in the low 32 bits and its length in the high 32 bits (Rust's + * C ABI cannot express Wasm multi-value returns, so pack both into one `u64`). */ - uint32_t read_file_v2(ValueId pathId, uint32_t lenPtr) + uint64_t read_file_v2(ValueId pathId) { auto & pathValue = getValue(pathId); auto path = state.realisePath(noPos, pathValue); @@ -590,9 +591,8 @@ struct NixWasmInstance // Note: the allocation may have grown the memory; `guestSpan` fetches it afresh. memcpy(guestSpan(ptr, contents.size()).data(), contents.data(), contents.size()); - guestSpan(lenPtr, 1)[0] = contents.size(); // FIXME: endianness - return ptr; + return ((uint64_t) contents.size() << 32) | ptr; } }; From 763dfc88aab69dc1c2123b3076aa1e1b9f7a41cb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 20:14:59 +0200 Subject: [PATCH 820/927] Don't use full store paths in examples --- src/libexpr/primops/bakedDerivation.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/bakedDerivation.cc b/src/libexpr/primops/bakedDerivation.cc index d7cf2064cd..74f72ad2b2 100644 --- a/src/libexpr/primops/bakedDerivation.cc +++ b/src/libexpr/primops/bakedDerivation.cc @@ -160,7 +160,7 @@ static RegisterPrimOp primop_bakedDerivation({ builtins.bakedDerivation { name = "hello-2.12.1"; outputs = { - out.path = "/nix/store/1q8w6grhdj6pn0cvlw18cw07nvjb7pj5-hello-2.12.1"; + out.path = "/nix/store/1q8w6grh…-hello-2.12.1"; }; } ``` @@ -169,8 +169,8 @@ static RegisterPrimOp primop_bakedDerivation({ ```nix { - drvPath = "/nix/store/1jczli5n8zgxl2vgfsc5vy3z0f5lfn3q-hello-2.12.1.drv"; - out = "/nix/store/1q8w6grhdj6pn0cvlw18cw07nvjb7pj5-hello-2.12.1"; + drvPath = "/nix/store/1jczli5n…-hello-2.12.1.drv"; + out = "/nix/store/1q8w6grh…-hello-2.12.1"; } ``` )", From c5711cbe82386fdb16bb5bb8ab206280bbc32a97 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 20:46:34 +0200 Subject: [PATCH 821/927] Use GC_get_approx_sp() when recording the stack pointer of a stack Previously the stack switch code recorded `__builtin_frame_address(0)` minus a fixed slack of 512 bytes as the saved stack pointer of the stack being switched away from. The frame address is the *top* of the current function's frame, so that slack had to cover the frame's own size (compiler-dependent and unbounded) as well as what the context switch pushes below it, with nothing checking that it did. Instead, use bdwgc's new GC_get_approx_sp() (its own approximation of the stack pointer, taken in a callee, so it lies below the caller's entire frame). The remaining slack now only needs to cover the frames of the Boost.Context resume path and its trampoline's callee-saved register block (which may hold GC pointers of the callers), both of which are small and bounded; it is defined once (`gcStackSwitchSlack`) with that rationale. The libutil coroutine hooks no longer take a stack pointer argument: the hook implementations in libexpr compute it themselves. Assisted-by: Claude Fable 5.1 --- flake.lock | 6 ++--- packaging/secure-packages/flake.lock | 6 ++--- src/libexpr/eval-gc.cc | 8 +++---- src/libexpr/include/nix/expr/eval-gc.hh | 15 ++++++++++++ src/libexpr/parallel-eval.cc | 10 ++++---- src/libutil/include/nix/util/coroutine-gc.hh | 24 +++++++++---------- src/libutil/serialise.cc | 25 ++++++++++---------- 7 files changed, 53 insertions(+), 41 deletions(-) diff --git a/flake.lock b/flake.lock index f043a19efb..86112bbdbd 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "bdwgc": { "flake": false, "locked": { - "lastModified": 1789490831, - "narHash": "sha256-AZwK4PzsERmLJ/F+Xjqj8kERb7zYiW0U+SZ586b5rBM=", + "lastModified": 1790015987, + "narHash": "sha256-bPVosUgucJR0hGSap3tJW84AR6eGYegVROV+O8VYX5A=", "owner": "edolstra", "repo": "bdwgc", - "rev": "015d683f032dec4c71f2bda61e8f1a88f4a4acd4", + "rev": "373957929b9ca3b2ae69a3e16b830c44c15b42ca", "type": "github" }, "original": { diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index ed2f5cf709..2ce548d021 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -3,11 +3,11 @@ "bdwgc": { "flake": false, "locked": { - "lastModified": 1789490831, - "narHash": "sha256-AZwK4PzsERmLJ/F+Xjqj8kERb7zYiW0U+SZ586b5rBM=", + "lastModified": 1790015987, + "narHash": "sha256-bPVosUgucJR0hGSap3tJW84AR6eGYegVROV+O8VYX5A=", "owner": "edolstra", "repo": "bdwgc", - "rev": "015d683f032dec4c71f2bda61e8f1a88f4a4acd4", + "rev": "373957929b9ca3b2ae69a3e16b830c44c15b42ca", "type": "github" }, "original": { diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 15d668722a..6ab320725e 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -96,13 +96,13 @@ static void coroStackUnregisterImpl(void * cookie) delete stk; } -static void * coroSwitchToImpl(void * cookie, void * callerSp) +static void * coroSwitchToImpl(void * cookie) { auto prev = GC_current_stack; /* `prev` is null on threads not registered with the GC; such threads hold no GC roots and need no scanning. */ if (prev) - prev->saved_sp = callerSp; + prev->saved_sp = (char *) GC_get_approx_sp() - gcStackSwitchSlack; GC_current_stack = (struct GC_stack *) cookie; return prev; } @@ -115,10 +115,10 @@ static void coroSwitchBackImpl(void * prevHandle) prev->saved_sp = nullptr; } -static void coroMarkSuspendedImpl(void * cookie, void * sp) +static void coroMarkSuspendedImpl(void * cookie) { if (cookie) - ((struct GC_stack *) cookie)->saved_sp = sp; + ((struct GC_stack *) cookie)->saved_sp = (char *) GC_get_approx_sp() - gcStackSwitchSlack; } static void coroMarkActiveImpl(void * cookie) diff --git a/src/libexpr/include/nix/expr/eval-gc.hh b/src/libexpr/include/nix/expr/eval-gc.hh index 1b1eb7e627..5c48ad5a09 100644 --- a/src/libexpr/include/nix/expr/eval-gc.hh +++ b/src/libexpr/include/nix/expr/eval-gc.hh @@ -55,6 +55,21 @@ void assertGCInitialized(); * The number of GC cycles since initGC(). */ size_t getGCCycles(); + +/** + * Slack to subtract from `GC_get_approx_sp()` when recording the stack + * pointer of a stack that is about to be switched away from (see + * `GC_stack::saved_sp`). The approximation is taken in a callee of the + * function performing the switch, so it already lies below that + * function's own frame; the slack additionally covers what the switch + * itself pushes below the call site: the frames of the Boost.Context + * resume path and the block of callee-saved registers stored by its + * trampoline (72 bytes on x86-64, 176 bytes on aarch64). The latter is + * essential, since those spilled registers may hold GC pointers of the + * callers. Erring on the low side merely makes the GC scan a few bytes + * of (mapped) stack below the true stack pointer. + */ +constexpr size_t gcStackSwitchSlack = 512; #endif } // namespace nix diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index 279018d97d..95b4ee38c2 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -381,7 +381,7 @@ void Executor::runFiber(FiberPtr fiber) `saved_sp` after it has been resumed (see `suspendFiber()`). */ auto prevStack = GC_current_stack; if (prevStack) - prevStack->saved_sp = (char *) __builtin_frame_address(0) - 512; + prevStack->saved_sp = (char *) GC_get_approx_sp() - gcStackSwitchSlack; GC_current_stack = fib->gcStack; #endif @@ -626,15 +626,13 @@ suspendFiber(WaiterDomain & domain, std::unique_lock & lk, detail::V lk.release(); #if NIX_USE_BOEHMGC /* Publish the used portion of our stack, so that the garbage - collector will scan it while we're suspended. The slack below - the current frame covers the register block that the context - switch is about to push; those bytes are touched by the switch - anyway, so this doesn't fault in any untouched stack pages. + collector will scan it while we're suspended (see + `gcStackSwitchSlack` for why the approximation is lowered). Note: from this point until we clear `saved_sp` after being resumed, the GC may see the stack both as some thread's active stack and as a suspended one; it scans it only once (from the lower stack pointer). */ - fib->gcStack->saved_sp = (char *) __builtin_frame_address(0) - 512; + fib->gcStack->saved_sp = (char *) GC_get_approx_sp() - gcStackSwitchSlack; #endif /* Switch back to the scheduler (`Executor::runFiber()`), which will register us in the domain's wait list and then release the diff --git a/src/libutil/include/nix/util/coroutine-gc.hh b/src/libutil/include/nix/util/coroutine-gc.hh index 5e11901463..282eea73eb 100644 --- a/src/libutil/include/nix/util/coroutine-gc.hh +++ b/src/libutil/include/nix/util/coroutine-gc.hh @@ -20,26 +20,26 @@ namespace nix { * cookie identifying the stack. `coroStackUnregister(cookie)` is * called when the stack is deallocated. * - * - `coroSwitchTo(cookie, callerSp)` is called just before the current - * thread switches onto the coroutine stack identified by `cookie`; - * `callerSp` approximates the caller's stack pointer at the switch - * (everything from `callerSp` up to the caller's stack base must be - * scannable). It returns an opaque handle for the caller's stack, - * to be passed to `coroSwitchBack` when the switch returns. + * - `coroSwitchTo(cookie)` is called just before the current thread + * switches onto the coroutine stack identified by `cookie`. It + * records the stack pointer of the stack being left (so that + * everything from there up to that stack's base is scannable) and + * returns an opaque handle for it, to be passed to `coroSwitchBack` + * when the switch returns. * - * - `coroMarkSuspended(cookie, sp)` is called on the coroutine stack - * just before yielding back to the caller, with `sp` approximating - * the coroutine's stack pointer; `coroMarkActive(cookie)` is called - * when the coroutine is resumed again. + * - `coroMarkSuspended(cookie)` is called on the coroutine stack just + * before yielding back to the caller, and records the coroutine's + * stack pointer; `coroMarkActive(cookie)` is called when the + * coroutine is resumed again. * * All hooks are invoked with the corresponding stack switches * strictly balanced. */ extern void * (*coroStackRegister)(void * base, size_t size); extern void (*coroStackUnregister)(void * cookie); -extern void * (*coroSwitchTo)(void * cookie, void * callerSp); +extern void * (*coroSwitchTo)(void * cookie); extern void (*coroSwitchBack)(void * prevHandle); -extern void (*coroMarkSuspended)(void * cookie, void * sp); +extern void (*coroMarkSuspended)(void * cookie); extern void (*coroMarkActive)(void * cookie); } // namespace nix diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index a781529258..2d4f9ac1ad 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -379,9 +379,9 @@ static constexpr size_t defaultCoroutineStackSize = 512 * 1024; void * (*coroStackRegister)(void * base, size_t size) = nullptr; void (*coroStackUnregister)(void * cookie) = nullptr; -void * (*coroSwitchTo)(void * cookie, void * callerSp) = nullptr; +void * (*coroSwitchTo)(void * cookie) = nullptr; void (*coroSwitchBack)(void * prevHandle) = nullptr; -void (*coroMarkSuspended)(void * cookie, void * sp) = nullptr; +void (*coroMarkSuspended)(void * cookie) = nullptr; void (*coroMarkActive)(void * cookie) = nullptr; namespace { @@ -415,8 +415,7 @@ struct GCTrackedStackAllocator /** * Notify the GC hooks that the current thread is about to switch onto * (and, on destruction, back off) the coroutine stack identified by - * `cookie`. `callerSp` must be the frame address of the function - * performing the switch. Construct just before resuming a coroutine + * `cookie`. Construct just before resuming a coroutine * (this includes destroying a suspended one, which unwinds on its own * stack); the switch back happens when the coroutine yields or * finishes. @@ -426,10 +425,10 @@ struct CoroutineGuard void * prev = nullptr; bool active = false; - CoroutineGuard(void * cookie, void * callerSp) + CoroutineGuard(void * cookie) { if (coroSwitchTo) { - prev = coroSwitchTo(cookie, callerSp); + prev = coroSwitchTo(cookie); active = true; } } @@ -463,7 +462,7 @@ std::unique_ptr sourceToSink(fun reader) /* Destroying a suspended coroutine unwinds it on its own stack, so this too is a switch onto the coroutine stack. */ - CoroutineGuard guard{stackCookie, __builtin_frame_address(0)}; + CoroutineGuard guard{stackCookie}; coro.reset(); } @@ -475,14 +474,14 @@ std::unique_ptr sourceToSink(fun reader) return; cur = in; - CoroutineGuard guard{stackCookie, __builtin_frame_address(0)}; + CoroutineGuard guard{stackCookie}; if (!coro) { coro = coro_t::push_type(GCTrackedStackAllocator{&stackCookie}, [&](coro_t::pull_type & yield) { LambdaSource source([&](char * out, size_t out_len) { if (cur.empty()) { if (coroMarkSuspended) - coroMarkSuspended(stackCookie, (char *) __builtin_frame_address(0) - 512); + coroMarkSuspended(stackCookie); yield(); if (coroMarkActive) coroMarkActive(stackCookie); @@ -510,7 +509,7 @@ std::unique_ptr sourceToSink(fun reader) void finish() override { if (coro && *coro) { - CoroutineGuard guard{stackCookie, __builtin_frame_address(0)}; + CoroutineGuard guard{stackCookie}; (*coro)(true); } } @@ -539,7 +538,7 @@ std::unique_ptr sinkToSource(fun writer, fun eof) ~SinkToSource() { /* See `SourceToSink::~SourceToSink()`. */ - CoroutineGuard guard{stackCookie, __builtin_frame_address(0)}; + CoroutineGuard guard{stackCookie}; coro.reset(); } @@ -547,7 +546,7 @@ std::unique_ptr sinkToSource(fun writer, fun eof) size_t read(char * data, size_t len) override { - CoroutineGuard guard{stackCookie, __builtin_frame_address(0)}; + CoroutineGuard guard{stackCookie}; bool hasCoro = coro.has_value(); if (!hasCoro) { @@ -564,7 +563,7 @@ std::unique_ptr sinkToSource(fun writer, fun eof) void writeUnbuffered(std::string_view data) override { if (coroMarkSuspended) - coroMarkSuspended(stackCookie, (char *) __builtin_frame_address(0) - 512); + coroMarkSuspended(stackCookie); yield(data); if (coroMarkActive) coroMarkActive(stackCookie); From 883d6a718ca01cadac52172fafb2652604211878 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 20:49:04 +0200 Subject: [PATCH 822/927] Move bdwgc fork --- flake.lock | 4 ++-- flake.nix | 2 +- packaging/secure-packages/flake.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 86112bbdbd..f61a4a296a 100644 --- a/flake.lock +++ b/flake.lock @@ -5,13 +5,13 @@ "locked": { "lastModified": 1790015987, "narHash": "sha256-bPVosUgucJR0hGSap3tJW84AR6eGYegVROV+O8VYX5A=", - "owner": "edolstra", + "owner": "DeterminateSystems", "repo": "bdwgc", "rev": "373957929b9ca3b2ae69a3e16b830c44c15b42ca", "type": "github" }, "original": { - "owner": "edolstra", + "owner": "DeterminateSystems", "ref": "user-defined-stacks", "repo": "bdwgc", "type": "github" diff --git a/flake.nix b/flake.nix index 445397e453..5d4dbfb17a 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,7 @@ inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.bdwgc = { - url = "github:edolstra/bdwgc/user-defined-stacks"; + url = "github:DeterminateSystems/bdwgc/user-defined-stacks"; flake = false; }; diff --git a/packaging/secure-packages/flake.lock b/packaging/secure-packages/flake.lock index 2ce548d021..283c7bbeda 100644 --- a/packaging/secure-packages/flake.lock +++ b/packaging/secure-packages/flake.lock @@ -5,13 +5,13 @@ "locked": { "lastModified": 1790015987, "narHash": "sha256-bPVosUgucJR0hGSap3tJW84AR6eGYegVROV+O8VYX5A=", - "owner": "edolstra", + "owner": "DeterminateSystems", "repo": "bdwgc", "rev": "373957929b9ca3b2ae69a3e16b830c44c15b42ca", "type": "github" }, "original": { - "owner": "edolstra", + "owner": "DeterminateSystems", "ref": "user-defined-stacks", "repo": "bdwgc", "type": "github" From b745ccbaf9095da46078c35ddb410de72fdd0a45 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 21:10:46 +0200 Subject: [PATCH 823/927] Make the default evaluation context thread-local The evaluation context of non-fiber execution contexts was a single global object, shared by every thread evaluating outside of the executor (e.g. the main thread plus any other thread calling into the evaluator). The context is mutable and unsynchronized: PushProvenance swaps the provenance in and out of it, so concurrent use from several such threads was a data race. Give each thread its own default context instead. Assisted-by: Claude Fable 5.1 --- src/libexpr/eval.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0556d6b7d7..e392067ede 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -253,12 +253,14 @@ EvalMemory::EvalMemory() } /** - * The evaluation context of non-fiber execution contexts (i.e. the - * main thread). + * The evaluation context of non-fiber execution contexts (e.g. the + * main thread, or any other thread that evaluates outside of the + * executor). One per thread, since the context is mutable (see + * `PushProvenance`) and unsynchronized. */ -static EvalState::EvalContext globalEvalContext; +static thread_local EvalState::EvalContext defaultEvalContext; -[[gnu::tls_model("initial-exec")]] thread_local EvalState::EvalContext * EvalState::evalContext = &globalEvalContext; +[[gnu::tls_model("initial-exec")]] thread_local EvalState::EvalContext * EvalState::evalContext = &defaultEvalContext; EvalState::EvalState( const LookupPath & lookupPathFromArguments, From a2dea02cb422c3951f75122dfc1cb4abac60c208 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 21:46:37 +0200 Subject: [PATCH 824/927] Store the evaluation context by value and swap it on fiber switches EvalState::evalContext was a thread-local pointer to the current context (a fiber's own, or a per-thread default) so that fiber switches didn't have to copy the context's provenance shared_ptr, i.e. touch its atomic reference count. But the same can be had without the indirection: make the thread-local context a value and have runFiber() std::swap() it with the context stored in the fiber record on every switch-in/out, which merely exchanges pointers. The fiber record then holds the resuming thread's own context while the fiber runs. This also removes the separate per-thread default context object (the thread-local value is the default). While at it, move (rather than copy) the captured context into the thread-local one when a work item starts: the lambda must be `mutable` for that, since std::move on an init-capture of a non-mutable lambda silently degrades to a copy. This halves the atomic refcount operations per work item. Assisted-by: Claude Fable 5.1 --- src/libexpr/eval.cc | 10 +--------- src/libexpr/include/nix/expr/eval.hh | 21 +++++++++++---------- src/libexpr/parallel-eval.cc | 15 ++++++++------- src/libexpr/primops.cc | 6 +++--- 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e392067ede..31088fbf5c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -252,15 +252,7 @@ EvalMemory::EvalMemory() assertGCInitialized(); } -/** - * The evaluation context of non-fiber execution contexts (e.g. the - * main thread, or any other thread that evaluates outside of the - * executor). One per thread, since the context is mutable (see - * `PushProvenance`) and unsynchronized. - */ -static thread_local EvalState::EvalContext defaultEvalContext; - -[[gnu::tls_model("initial-exec")]] thread_local EvalState::EvalContext * EvalState::evalContext = &defaultEvalContext; +[[gnu::tls_model("initial-exec")]] thread_local EvalState::EvalContext EvalState::evalContext; EvalState::EvalState( const LookupPath & lookupPathFromArguments, diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 51eb35c101..90c5634d0c 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -1142,12 +1142,13 @@ public: /** * The evaluation context of the current execution context: a - * fiber's own context while a fiber is running (switched by - * `Executor::runFiber()`), or a global default context otherwise. - * A plain pointer so that fiber switches don't need to touch the - * `provenance` shared_ptr's (atomic) reference count. + * fiber's own context while a fiber is running (`Executor::runFiber()` + * swaps it with the context stored in the fiber record on every + * switch-in/out, which is just a pointer exchange, i.e. doesn't + * touch the `provenance` shared_ptr's atomic reference count), or + * the thread's own context otherwise. */ - [[gnu::tls_model("initial-exec")]] thread_local static EvalContext * evalContext; + [[gnu::tls_model("initial-exec")]] thread_local static EvalContext evalContext; /** * Create a work item that propagates the current evaluation context. @@ -1155,8 +1156,8 @@ public: template auto makeWork(T && t) { - return [this, t{std::move(t)}, evalContext(*evalContext)]() { - *this->evalContext = evalContext; + return [this, t{std::move(t)}, evalContext(evalContext)]() mutable { + this->evalContext = std::move(evalContext); t(); }; } @@ -1229,13 +1230,13 @@ struct PushProvenance PushProvenance(EvalState & state, std::shared_ptr prov) : state(state) { - state.evalContext->provenance.swap(prev); - state.evalContext->provenance.swap(prov); + state.evalContext.provenance.swap(prev); + state.evalContext.provenance.swap(prov); } ~PushProvenance() { - state.evalContext->provenance.swap(prev); + state.evalContext.provenance.swap(prev); } }; diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc index 95b4ee38c2..74288efc17 100644 --- a/src/libexpr/parallel-eval.cc +++ b/src/libexpr/parallel-eval.cc @@ -51,10 +51,12 @@ struct Executor::Fiber size_t callDepth = 0; /** - * This fiber's evaluation context. `runFiber()` points the - * thread-local `EvalState::evalContext` at it while the fiber is - * running, so that the context travels with the fiber across - * threads. + * This fiber's evaluation context while it's not running. + * `runFiber()` swaps it with the thread-local + * `EvalState::evalContext` on every switch-in/out (a mere pointer + * exchange), so that the context travels with the fiber across + * threads, while this slot holds the resuming thread's own context + * in the meantime. */ EvalState::EvalContext evalContext; @@ -367,11 +369,10 @@ void Executor::runFiber(FiberPtr fiber) auto savedThreadId = myEvalThreadId; auto savedCallDepth = CallDepth::callDepth; - auto savedEvalContext = EvalState::evalContext; currentFiber = fib; myEvalThreadId = fib->evalThreadId; CallDepth::callDepth = fib->callDepth; - EvalState::evalContext = &fib->evalContext; + std::swap(EvalState::evalContext, fib->evalContext); #if NIX_USE_BOEHMGC /* Make this thread's stack scannable by the GC while the fiber @@ -397,7 +398,7 @@ void Executor::runFiber(FiberPtr fiber) myEvalThreadId = savedThreadId; fib->callDepth = CallDepth::callDepth; CallDepth::callDepth = savedCallDepth; - EvalState::evalContext = savedEvalContext; + std::swap(EvalState::evalContext, fib->evalContext); if (fib->ctx) { /* The fiber suspended itself in `waitOnThunk()`. We are still diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e99cf419c0..cf009515c5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1434,7 +1434,7 @@ static void prim_derivationStrictGeneric(EvalState & state, const PosIdx pos, Va } try { - derivationStrictInternal(state, drvName, attrs, v, state.evalContext->provenance, acceptMeta); + derivationStrictInternal(state, drvName, attrs, v, state.evalContext.provenance, acceptMeta); } catch (Error & e) { Pos pos = state.positions[nameAttr->pos]; /* @@ -2823,7 +2823,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value ** args, Valu HashAlgorithm::SHA256, refs, state.repair, - state.evalContext->provenance); + state.evalContext.provenance); }); /* Note: we don't need to add `context' to the context of the @@ -2975,7 +2975,7 @@ static void addPath( auto path3 = filter ? SourcePath{ make_ref( - path2.accessor, state.evalContext->provenance), + path2.accessor, state.evalContext.provenance), path2.path } : path2; From 228852a93a669b4f352001489d91223f7e83cc3a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 15:18:40 +0200 Subject: [PATCH 825/927] packaging/nix-make: Build libnixutil with Nix as the build system Proof of concept of using Nix as the C++ build system for Nix itself, with one derivation per compilation unit. The #include closure of each unit is computed at evaluation time by a scanner (from nix-wasm-rust, checked in as scanner.wasm) running under `builtins.wasm`, so only the headers a unit actually needs are inputs to its derivation. Editing a source file rebuilds just that unit and the link step; editing a header rebuilds only its includers. * lib.nix: generic machinery: config header generation, per-unit compile derivations, shared library linking and mkComponent. * libutil.nix: libnixutil transcribed from src/libutil/meson.build, including the nix-meson-build-support sources and the generated config headers. * flake.nix: `nix build ./packaging/nix-make#nix-util` builds lib/libnixutil.so, using the same dependency overrides as the Meson build via `nix.overlays.internal`. The build is currently unoptimized (-O0) for faster iteration. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/flake.lock | 136 +++++++++++++++++++ packaging/nix-make/flake.nix | 35 +++++ packaging/nix-make/lib.nix | 219 +++++++++++++++++++++++++++++++ packaging/nix-make/libutil.nix | 223 ++++++++++++++++++++++++++++++++ packaging/nix-make/scanner.wasm | Bin 0 -> 57566 bytes 5 files changed, 613 insertions(+) create mode 100644 packaging/nix-make/flake.lock create mode 100644 packaging/nix-make/flake.nix create mode 100644 packaging/nix-make/lib.nix create mode 100644 packaging/nix-make/libutil.nix create mode 100644 packaging/nix-make/scanner.wasm diff --git a/packaging/nix-make/flake.lock b/packaging/nix-make/flake.lock new file mode 100644 index 0000000000..8bf1e31120 --- /dev/null +++ b/packaging/nix-make/flake.lock @@ -0,0 +1,136 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1782949081, + "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=", + "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e", + "revCount": 480, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/hercules-ci/flake-parts/0.1.480%2Brev-17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e/019f2195-dee5-7233-9747-eca0c27f7406/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/hercules-ci/flake-parts/0.1" + } + }, + "git-hooks-nix": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": [ + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1784288435, + "narHash": "sha256-ReRHaLgr/uVqdD8afFSn+myXIfpHeOhP0yYe0TJqAA8=", + "rev": "43b3c1ab9d40fb1dbb008f451988a91e375825e9", + "revCount": 1231, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/cachix/git-hooks.nix/0.1.1231%2Brev-43b3c1ab9d40fb1dbb008f451988a91e375825e9/019f7135-8fdf-76f0-b1a1-d2c67e91af8d/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/cachix/git-hooks.nix/0.1.941" + } + }, + "nix": { + "inputs": { + "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", + "nixpkgs": "nixpkgs", + "nixpkgs-23-11": "nixpkgs-23-11", + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "path": "../..", + "type": "path" + }, + "original": { + "path": "../..", + "type": "path" + }, + "parent": [] + }, + "nixpkgs": { + "locked": { + "lastModified": 1787962033, + "narHash": "sha256-u6z9VTZA4Kf3RkHQo9sQI7NI4Ei/uiU9vrMqOiwWP1Y=", + "rev": "c5c4a43b0e8056328ec4529f735cabdb8f1942bb", + "revCount": 1012930, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2605.1012930%2Brev-c5c4a43b0e8056328ec4529f735cabdb8f1942bb/01a053c0-7061-7382-b001-102e9148a5b5/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/NixOS/nixpkgs/0.2605" + } + }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "root": { + "inputs": { + "nix": "nix", + "nixpkgs": [ + "nix", + "nixpkgs" + ] + } + } + }, + "root": "root", + "version": 7 +} diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix new file mode 100644 index 0000000000..ab09a42298 --- /dev/null +++ b/packaging/nix-make/flake.nix @@ -0,0 +1,35 @@ +# Proof of concept: building Nix with Nix as the C++ build system, with one +# derivation per compilation unit. The `#include` closure of each unit is +# computed at evaluation time by a scanner running under `builtins.wasm`. +{ + inputs.nix.url = "../.."; + inputs.nixpkgs.follows = "nix/nixpkgs"; + + outputs = + { + self, + nix, + nixpkgs, + }: + let + system = "x86_64-linux"; + + pkgs = import nixpkgs { + inherit system; + # Provides `nixDependencies2`, the same dependency overrides used by the Meson build. + overlays = [ nix.overlays.internal ]; + }; + + nixMake = import ./lib.nix { inherit pkgs; }; + + components = { + nix-util = import ./libutil.nix { inherit pkgs nixMake; }; + }; + in + { + packages.${system} = components; + + # The raw scanner output for each component, for debugging. + lib.${system}.scan = pkgs.lib.mapAttrs (_: c: c.units) components; + }; +} diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix new file mode 100644 index 0000000000..461a2a92dc --- /dev/null +++ b/packaging/nix-make/lib.nix @@ -0,0 +1,219 @@ +# Generic machinery for building C++ components with one derivation per +# compilation unit. +{ pkgs }: + +let + inherit (pkgs) lib; + + getDeps = builtins.wasm { + path = ./scanner.wasm; + function = "getDeps"; + }; + + # Non-optimized for now, for faster iteration. + optimizationFlags = [ "-O0" ]; + + # From nix-meson-build-support/common/meson.build (GCC-supported subset). + warningFlags = [ + "-Wall" + "-Wdeprecated-copy" + "-Werror=suggest-override" + "-Werror=switch" + "-Werror=switch-enum" + "-Werror=undef" + "-Werror=unused-result" + "-Werror=sign-compare" + "-Werror=return-type" + "-Werror=non-virtual-dtor" + "-Wignored-qualifiers" + "-Wimplicit-fallthrough" + "-Wno-deprecated-declarations" + "-Wno-interference-size" + "-Wno-subobject-linkage" + ]; + + commonCxxFlags = [ + "-std=c++23" + "-fPIC" + "-pthread" + "-D_FILE_OFFSET_BITS=64" + "-D_GLIBCXX_USE_TBB_PAR_BACKEND=0" + "-fno-semantic-interposition" + ] + ++ optimizationFlags + ++ warningFlags; + + # Generate a header with `#define`s, like Meson's `configure_file()`. + # Integers and booleans become bare defines, strings become quoted + # defines, and `null` becomes an `#undef`. + mkConfigHeader = + name: attrs: + builtins.toFile name ( + "#pragma once\n" + + lib.concatStrings ( + lib.mapAttrsToList ( + k: v: + if v == null then + "#undef ${k}\n" + else if builtins.isBool v then + "#define ${k} ${if v then "1" else "0"}\n" + else if builtins.isInt v then + "#define ${k} ${toString v}\n" + else + "#define ${k} ${builtins.toJSON v}\n" + ) attrs + ) + ); + + # Map the external (angle-bracket) includes of a unit to the entries of + # the component's `externalDeps` table, by prefix match. Unmatched + # includes (libc, libstdc++) are ignored. + matchExternalDeps = + externalDeps: externalIncludes: + let + matches = lib.filter (d: d != null) ( + map (inc: lib.findFirst (d: lib.hasPrefix d.prefix inc) null externalDeps) externalIncludes + ); + in + # Deduplicate by prefix. + lib.attrValues (lib.listToAttrs (map (d: lib.nameValuePair d.prefix d) matches)); + + compileUnit = + component: unit: + let + deps = matchExternalDeps component.externalDeps unit.externalIncludes; + in + pkgs.runCommandCC "${baseNameOf unit.path}.o" + { + __structuredAttrs = true; + inherit (unit) src includes; + srcPath = unit.path; + inherit (component) includeDirs; + cxxFlags = commonCxxFlags ++ component.extraCxxFlags; + pkgConfigDeps = lib.concatMap (d: d.pkgconfig or [ ]) deps; + buildInputs = map (d: d.pkg) deps; + nativeBuildInputs = [ pkgs.pkg-config ]; + # `_FORTIFY_SOURCE` warns on every unit when not optimizing. + hardeningDisable = [ "fortify" ]; + } + '' + mkdir tree + cd tree + + # Recreate the source layout: each header and the source file is a + # separate store path. + for name in "''${!includes[@]}"; do + mkdir -p "$(dirname "$name")" + ln -s "''${includes[$name]}" "$name" + done + mkdir -p "$(dirname "$srcPath")" + ln -s "$src" "$srcPath" + + flags=() + for dir in "''${includeDirs[@]}"; do + flags+=("-I''${dir:-.}") + done + if [[ ''${#pkgConfigDeps[@]} -gt 0 ]]; then + flags+=($(pkg-config --cflags "''${pkgConfigDeps[@]}")) + fi + + $CXX "''${cxxFlags[@]}" "''${flags[@]}" -c "$srcPath" -o "$out" + ''; + + linkSharedLibrary = + component: objects: + pkgs.runCommandCC component.name + { + __structuredAttrs = true; + inherit objects; + inherit (component) + libName + linkFlags + linkLibs + linkPkgConfig + ; + buildInputs = component.linkDeps; + nativeBuildInputs = [ pkgs.pkg-config ]; + passthru = { + inherit objects; + inherit (component) units; + }; + } + '' + mkdir -p $out/lib + + libs=() + if [[ ''${#linkPkgConfig[@]} -gt 0 ]]; then + libs+=($(pkg-config --libs "''${linkPkgConfig[@]}")) + fi + + $CXX -shared -fPIC \ + -Wl,-soname,lib$libName.so \ + -Wl,--as-needed -Wl,--no-undefined \ + "''${linkFlags[@]}" \ + -o $out/lib/lib$libName.so \ + "''${objects[@]}" \ + "''${libs[@]}" \ + "''${linkLibs[@]}" + ''; + + /** + Build a shared library component. + + - `name`: derivation name (e.g. `nix-util`). + - `libName`: library name without `lib` prefix (e.g. `nixutil`). + - `roots`: list of `{ root; prefix; }` directories scanned for sources and headers. + - `includeDirs`: include search path, relative to the root namespace. + - `sources`: compilation units, as paths in the root namespace. + - `files`: extra files (e.g. generated headers) by path in the root namespace. + - `externalDeps`: list of `{ prefix; pkg; pkgconfig ? []; }` mapping + external `#include`s to packages. + - `extraCxxFlags`, `linkFlags`, `linkLibs`, `linkPkgConfig`, `linkDeps`: what they say. + */ + mkComponent = + { + name, + libName, + roots, + includeDirs, + sources, + files ? { }, + externalDeps ? [ ], + extraCxxFlags ? [ ], + linkFlags ? [ ], + linkLibs ? [ ], + linkPkgConfig ? [ ], + linkDeps ? [ ], + }@args: + let + units = getDeps { + inherit builtins; + inherit + roots + includeDirs + files + sources + ; + }; + component = args // { + inherit + units + externalDeps + extraCxxFlags + linkFlags + linkLibs + linkPkgConfig + linkDeps + ; + }; + in + linkSharedLibrary component (map (compileUnit component) units); + +in +{ + inherit + getDeps + mkConfigHeader + mkComponent + ; +} diff --git a/packaging/nix-make/libutil.nix b/packaging/nix-make/libutil.nix new file mode 100644 index 0000000000..2064c41914 --- /dev/null +++ b/packaging/nix-make/libutil.nix @@ -0,0 +1,223 @@ +# libnixutil, transcribed from src/libutil/meson.build and its subdirectories. +{ pkgs, nixMake }: + +let + # The overridden dependencies used by the Meson build, layered over Nixpkgs. + deps = pkgs // pkgs.nixDependencies2; +in + +nixMake.mkComponent { + name = "nix-util"; + libName = "nixutil"; + + roots = [ + { + root = ../../src/libutil; + prefix = ""; + } + { + root = ../../nix-meson-build-support; + prefix = "nix-meson-build-support"; + } + ]; + + includeDirs = [ + "" + "include" + "unix" + "unix/include" + "linux/include" + "widecharwidth" + ]; + + files = { + "include/nix/util/config.hh" = nixMake.mkConfigHeader "config.hh" { + NIX_UBSAN_ENABLED = 0; + NIX_ASAN_ENABLED = 0; + }; + "util-config-private.hh" = nixMake.mkConfigHeader "util-config-private.hh" { + HAVE_LIBCPUID = 1; + HAVE_POSIX_FALLOCATE = 1; + }; + "unix/util-unix-config-private.hh" = nixMake.mkConfigHeader "util-unix-config-private.hh" { + HAVE_CLOSE_RANGE = 1; + HAVE_COPY_FILE_RANGE = 1; + HAVE_DECL_AT_SYMLINK_NOFOLLOW = 1; + HAVE_F_GETPATH = 0; + HAVE_LUTIMES = 1; + HAVE_PIPE2 = 1; + HAVE_STRSIGNAL = 1; + HAVE_SYSCONF = 1; + HAVE_UTIMENSAT = 1; + }; + }; + + sources = [ + "archive.cc" + "args.cc" + "base-n.cc" + "base-nix-32.cc" + "bump-memory-resource.cc" + "caching-source-accessor.cc" + "canon-path.cc" + "checked-arithmetic.cc" + "compression-algo.cc" + "compression-settings.cc" + "compression.cc" + "compute-levels.cc" + "config-global.cc" + "configuration.cc" + "current-process.cc" + "english.cc" + "environment-variables.cc" + "error.cc" + "executable-path.cc" + "exit.cc" + "experimental-features.cc" + "file-content-address.cc" + "file-descriptor.cc" + "file-system.cc" + "forwarding-source-accessor.cc" + "fs-sink.cc" + "git.cc" + "hash.cc" + "hilite.cc" + "json-utils.cc" + "logging.cc" + "memory-source-accessor.cc" + "memory-source-accessor/json.cc" + "mounted-source-accessor.cc" + "nar-accessor.cc" + "nar-cache.cc" + "nar-listing.cc" + "pos-table.cc" + "position.cc" + "posix-source-accessor.cc" + "processes.cc" + "provenance.cc" + "serialise.cc" + "signature/local-keys.cc" + "signature/signer.cc" + "source-accessor.cc" + "source-path.cc" + "strings.cc" + "suggestions.cc" + "table.cc" + "tarfile.cc" + "tee-logger.cc" + "terminal.cc" + "thread-pool.cc" + "union-source-accessor.cc" + "unix-domain-socket.cc" + "url.cc" + "users.cc" + "util.cc" + "xml-writer.cc" + + # linux/meson.build + "linux/cgroup.cc" + "linux/linux-namespaces.cc" + + # unix/meson.build + "unix/current-process.cc" + "unix/environment-variables.cc" + "unix/file-descriptor.cc" + "unix/file-path.cc" + "unix/file-system-at.cc" + "unix/file-system.cc" + "unix/muxable-pipe.cc" + "unix/processes.cc" + "unix/signals.cc" + "unix/users.cc" + "unix/xdg-dirs.cc" + + # nix-meson-build-support/common + "nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc" + "nix-meson-build-support/common/cxa-throw/interpose-cxa-throw.cc" + ]; + + externalDeps = [ + { + prefix = "boost/"; + pkg = deps.boost; + } + { + prefix = "nlohmann/"; + pkg = deps.nlohmann_json; + pkgconfig = [ "nlohmann_json" ]; + } + { + prefix = "archive"; + pkg = deps.libarchive; + pkgconfig = [ "libarchive" ]; + } + { + prefix = "openssl/"; + pkg = deps.openssl; + pkgconfig = [ "libcrypto" ]; + } + { + prefix = "sodium.h"; + pkg = deps.libsodium; + pkgconfig = [ "libsodium" ]; + } + { + prefix = "blake3.h"; + pkg = deps.libblake3; + pkgconfig = [ "libblake3" ]; + } + { + prefix = "brotli/"; + pkg = deps.brotli; + pkgconfig = [ + "libbrotlicommon" + "libbrotlidec" + "libbrotlienc" + ]; + } + { + prefix = "zstd.h"; + pkg = deps.zstd; + pkgconfig = [ "libzstd" ]; + } + { + prefix = "libcpuid/"; + pkg = deps.libcpuid; + pkgconfig = [ "libcpuid" ]; + } + ]; + + linkFlags = [ "-Wl,--wrap=__assert_fail" ]; + + linkPkgConfig = [ + "libarchive" + "libblake3" + "libcrypto" + "libsodium" + "libbrotlicommon" + "libbrotlidec" + "libbrotlienc" + "libzstd" + "libcpuid" + ]; + + linkLibs = [ + "-lboost_context" + "-lboost_coroutine" + "-lboost_iostreams" + "-lboost_url" + "-pthread" + "-ldl" + ]; + + linkDeps = with deps; [ + boost + libarchive + libblake3 + openssl + libsodium + brotli + zstd + libcpuid + ]; +} diff --git a/packaging/nix-make/scanner.wasm b/packaging/nix-make/scanner.wasm new file mode 100644 index 0000000000000000000000000000000000000000..f09bb6c17353f7ae9763e8391c0886f5de715785 GIT binary patch literal 57566 zcmdqKeViTDdFNTDs_skQx~(p~fl$k)ZXpvXMwW1}Bm?X8kpUqChRJv{p1ceQNr1YA zq;540P6%p)@W>ew;{A{~BX;-KsjT&-?S7=bVam9=g>pruhho-0Y?!P%a>DrT>)6-Lj_SA)T9XR-r>#IW6L)V_vh3wIA|Iz-P zx9*8jwcfsY&-C>wRgKGvSm$A!Q+syqzW%1Y`}TzQje|S)@7={fzv>=yNs=^On#NJu zY&ggNleE=JoEvH+jmGejW}^S%W;>;1BW36Sq_y2hNp}T&^eZd`fj{voAxF_5<-Q#Z8?wfY*J#cVx*Te4j z-Phf>T-S9ve{gs5n0tFv#NJJ&V?|Ur{hY~{jAY>^AIU!A(!=LPKA$L)wNa7ybeuAD z=3V~&B3T!=d#Ns|ciU-#U)^4#hNkrtzcR}HDt0Y(T}+fIU8j@@BRy=RY{X-Q+Ym)%$1Z2l7ta>H{MHl z*O>FsmFmvF?k4@LyMo`%BH5@K<2`;Ajd34Swj0wg#ga+C>h`kj8`EHFb~c?X+V&%^ zezYfjYkF$Rzinom(r(7blll?ck6uzVFE3hCy_SZe=IB)&@v~^s9}jJ?!%>zMP4A}5 z#7F$duN|d$3k-~A`3*(7E?!~7=SLo3nhKcRBGtQ90xJL)SW|G}PdzYaXv=TBeAxM; zpPGyOzZ0y^9rv&D%Aev%ehZ*h<=#+{2(rLET_2}gKuYA>;}q|XuZ!tLH48A2o#WhL z(3D(}9;R?yxcsDJ&e`SJld&7A2%@<*0?v+smMvY{kV}io*4c?Ld1W-m+3W%Zqd${6;1UHoNEqjM z2{CrLNp~;{E)!64T-coQ%ZFpy$;OK%5a&q;76q4iBB%R~2+0*K8liiT3xfPBp8)>L zt#n851>u74L$PZd766!O?CJf^NzJ#x+?*(<2SN0j2AxzBELv9#rwjl#8!yxC>RwZA zPn6B8hGQc+ZhB3?)L;-s@*lnXNPAx|O~(8zznDp5yljEyB0W^LCyMq|c;14Hs;8|9 zC>UO$<{i~#I{<0`#1bmLMzqU-A?YiC5SIzSsdjrY9jDdM9m5t(ZQv3LVbDcFTYv~s zR1F^Xng%EF27?Ncq6v_)PsWg6pYtm)0d8nPEnmf$qQYGRN)RrABV5HIQ1XdwV&DA8 zmSmNfhQZH2$^F;vo)cz}B}x9%fU)0#Az$TxPz%3y&*B#7TwHnP7K9u?^JYah(}9TsDprnw5^ULY_CJHAx1&@7F^l6 ze7ND~T+yNNId|=7nb}~NbLQjea>&2jnk-ZF0`pA+0wB4T?utg;D@0$W{_r8+ngROF znJF)hL8eQ|y~)e=o4qtadd6m@Cn38MVJ{G73_hD+Vhv5KCH&9$*FXANYm%RK9rqud zP51eHrf3WPO;9U=7Ec3pM3H|DwH8kUL`@68pgU1*)r1=J6M_+~Z!QQwp9aAGzz(~q zq*gIC4o9nb5=*t*fHc)jqML)&O*)>H zz+*y<#tf>)Hd8?wAZsRim<3o68(p_-T#K}66v(#*?={__K;uyO+h(M<-Ncd@Lynp# z0r;)q|Mqn~o{^mIdZ;Zzq#FOAh2S8%5yW_tOri*M*6E}1kHMGv+I2Qf;G!O~LtcNLxFN~Vmm|~)4Ptdk|1@O4 z>EiMzKc{F&7s8SH(WPlFY1yRxwLC+U6Yp8&rU8Or8Z7M4Z&qo*~uN$-tb$o_5ebw=;Wi^ zJ{gS{&76U(i?1xf@VfYF`1~{XW8<`P)U)#+Hk`sHqVgAbF^;{{^r!6CFu!1bfqQDTiIC&BIFUziI(0D#YN>KJ%7SRb{<9FU< zdb4Ke#Qx!Uyc9H({I?J~f)wJ`<)3qfWXvmH1C*oweu46sp)NMmCHdzatpTc|vHspi zqu5?Zw+i@kHSj}|Y8sS>W?kLvGoe|*>oLR7C#wnEkIlJo0_}kbwDU|8Xa~9qLl-#c z0z-UYXxI}Bjp7T2_W18rbO8;lkB_wb9!l)*Gqqcx@cy=l=G#M>!LreVHr#|&#)|C*@ zJ`GV?8VZ~UT!>r=bk${AJ&V~Nj{&A;5tm@XNzFl0*nlMl+LdUtmjB5~0Oq`S zWrLp2;R%j?5YV;!xs!rVyDpNPp~y(T2yI~Qs14tm^v&t1tp5k~mgVn|NyXrpi^Jv^ zm-CeP^8&<*YHD6QDCL-qmP}VD%Fhq9RFn<{LX}k_G8$T&{}^{Uez_?oJzAwdYx47T z$?NhkgOy*Q@)BwQJ_HXYyY@DiTMfTVA&r-#Nfk``1*bB4DRJ?Agyi;@BF6pudNFQ3sH*}xjK$#rTs;hN zKo>$4#9_FH*+dI{Vu4C3V!4mU;+^~T8uTK@jDqILrNpq(#2?@-*072JWF9=|{HuU2 zUl$0}s0lf$SCfPhC1{iN@f{L4$S)@oA9_WlT^w~oPfSna#0VHKhmbSCV?%VC><788 zRKBb)trGuqq!z4(Kjwldzzju7{zc7ieU$yj;dtb5e3w+Xpt?4agyKIapSoom?4uH( zI9xF~ehAF8G`ZQ%KSo0m9-fqx)9uvmL`Y*8U|-IocAcdmT8BIbBdT2_m7?(q4m( z)JoBWD;jtLu>;Lb-y$ADr5CEoFRHR{T5(ME(_b~j{WQI<*UdlWkT~G)F$eTxjfIGo zd@DjImvBeBN1=les1#cV@)@e8okYxQNB~G?agBZVS|nWTAEz*;vRN*?6~bhHVxa0N zFNgJt;L`8}9ko@yXyxyr`C&EBi#GT7&{WZ=`j^xnE=JZTbfr$y_4K$UImpl$_->wb zrC1E~mvO-^zl96d{8lb_FqFE8Umg3kiE5&?R4OGUf`_vHsDs*Jq1ijG7#`x`bM(jv z2YWoYCAkjSNXWGVOdE)G=4 z7BwYu#0q!_49E|>Oj&14UJRwUmz&r=jNNH?Nkk>izl7=F3;+Drg9YKAmnktOgHPqL z?n8gv$a-vv1fKvw1+4tIz=~ZA8b!TfrkbTOJaD~EzvQN3W(tl>0Q3D4_k^wt>M_?v z2r!%{V%W#X`${e}eVL*k6GdbDFcMbow+(s3XNta5gbkeHnwDj!$ATPBYD$hp$&$UI zU=)}c^FVw4QKJ`Uf-d^gX35n^HxOwqRlAr-Th&mO@=8=)QPn)@LaWyT2QNBW3y-op2iWXrKd@ROxdNbXjOcDiX4H7+K z6*Tqf8p2s0BP|0pR`N9VCygoB)apdu9JJb(v26vKTR!uTe*=0a(Cl9l2SE7#A%6Hx z{^^<{o~49;kH7DUibLeC4Aj$Y*FRY|^O|a}^AGD@o?0y))((gV)I*RD;!7tcN3mcc z61quIYO>KcIP{qU9i`U4Ds`-Q2MFTQ&Uc8faToJ^dzM{A>Ar$Tn4oxwR^SMl04e2& zp_D|E&i`)Y>0mM(_@&uOOnXN7$+vVnB$J&cw0mRC=6&xKNr0k>bB4PxTVNl zTE}sRo89>ba_ViugggkKEs4TXX87x5I`|tV{jtNWJSYH|`p+Hy0Ba5WY-)K)t7=_| zv8KOk((m7{XuKc0a@f|>?3b-eRK3@o0bhb6bk}fE6v-iP`jOBWaecAQ*azz-h@n_7 zn*g~P6aEv70INdM8fQ4g%QTQ^$zx!Wf|a7r!cY-k%fd^1MTYJy5JnvQ6GsijljZ*BgN2>Ezn`)v2C=3iIGR3)j}WPR~RsMdPO#hc1Ksr zEC@&p5p|hjMZ)@-nUg3LEuYe)T1HGn*)A4hfgTq8J0NFN=Ec!F!u{IdR>BtrE+m&V zrXuvKU_N?K#&}wc4T*}h;|aeQr6y4=(V}9bY*tIIdY#MoRcz3zcL~VhoiP4bL3ETJ z>`Q|<9`F?^0GGf5C1RUo&f>rJO7PT7@WAA}&~<|Ig6);~KX-}|$YCiOQ^1SSsWIWK zs$m|D`8ihaWrw+k_HW9!nXjN|mMS#oog8{IC&%hi)Tl$cq-9HdITi*W)Z8mDqY1!T zCJ5Z`5J7;INF9!&Kbt{%*K_`Jv$KF~h9yJbh`4Z+n00Wtm2J-i+2l7VIu`qam^hcS zwnNdB+i*os5oG;573nv~jzroUqK6+FZ4C)tsOi*;!GxyC3E1$48jG$O@6Bi#FKJb{ ztQalhDh3@&%G9^e)^-gL>n)5^%SWma7W-(mzlbCP6pGbnDhDWxQiIh?I>HL1neI^K1b=IH7W$a7K-GB@=yaR!YV1~4S_6hR5TBXK4!NAtJN5nM+!Z2 zxrPhYVt-8B{SYH%A`2@CR`3?lRi@#>x}5@#E2{O(xrimst&9%6J@{yoJxT{wJ=gg! z&CW@`uS0p;s^@9#h_e>86Pg}eJv~#)N@`iyGJqyfDr^<8kb_*0^g~)0*b8o45Ff?{ z6`YXFvV0AE23V$TLVq46SBrpXPQ2BSRYBDdD?vgkR$?q#1<=w7R5>u zijsC%OKtQmkQXX_rgO zHTgSxy@EuLww8)oM8kS3Xl>I($+}}a7XH)wZn;wQ(H8cJKFDTYRxHz_+$S?cEL!`x zZSNQ@SEH{)mKlaGK13VllRWHG$;$JCaVOh$pF9<}1+|eGfTVshO`VJ+ zE&6C>dKoYf1Q_rR)=Sq2Km_e@5w{(_cbj~HqxT=1n>~C_kA{)iy&M6*(o!pm>lnf? zW1(xRGo&-s0v=hh!kR6LUpP*8Orkt1|Fr9s%$)22bTb}zng36&TsB^m$Nf1o<#GU^ zH!I2xo;MSom>d^8?YST@y$=*kfJu$FIIF%tYkd@Lp{hW{Q+%o@+RQy9Fwt9C6x-Y{ z6$Nv#Q2`Z^2e1&(=xq&)^9F+=_i6s?E{qMz;g`o_qzJGCd(7932b`{5ttWt+VBUoO zuFJ2N*dRQhV12Y*Ne>3pPH!0432(#Z)R%=o+&a3g2C}<{0lxmXfAiP>?f?DrKX^Sl za;G9$*lfM#F7UVW%MbrxPZu9$7t(APFYUn-EUu0Hfa9>|yULXoSQQ)h|eK zlZB^QFgjL}$IDTSUQ9vxKhUG#*;~qRhl^!kj}joPERL7wOqOR)aKEf5gs|ns*%OC~ zWzagl9gmlfmuJ&ZadvTz(0Go}cvi6-{84DUu=}@hujZFSy;?^aN{S1VUNpc-+P zc_c0`mR7S(8&Jh^pMWD~9`PSRlhwsau-Df5^=RmR>B43U-P@wG)yRku;@S&2DLmdg zJ_&F2ww-gdEVsQL-7}AtyL#st277Hu72&Yxj5Ct5SDbgRVG+Fbk@!pZEMyzX7u%wr zVxp@G2&`s;&Ui6ADL9by)1D9!BR6!-FTn0vNT|JqEi#J3#S%*kDvImmzf_E6ZT4{9 z3g+QHb`RYmQLq7*5Q~CAyNsJ&N%3^bpz>l* zl2FR7AlnjDfCc8g03YrZ`;w)KtSUyBS>RhMPXjfdN?su2FRkN2ZAI)ZB0d?9*%Cmx zQpBvtNV$r<%atZ}as}m}*W7|^3N`n-{%~)J;iM}yFx*=rYpgAo&n_%wDhxNl;@uJv zyMoeP+MpYFrGT&XZ^tO^2a<-L(6-XBq7O)#UP(@jn+!QQ@E1)exJv7y7j>fh-YC8@ zLpiIE$=*AMOERuR$Y?_0asNEw&NV?DRpc{Vj9xJeOG#%ftKKNUGrS=JdSVGOJKBtu zOWGR|Z?pO+e=aoO9g}lKcL#*h<-g?23zvuYSB>^^=F7xKBm}Bq zh>9iJkOC+KkPDvuMl;j(s#roID}6)1%)D&NZ3S(#9uh7i^0}ZLqup}Ef920k{$+e< zq8O?2F*|{pDN(=aEXv80D=(^IqzRoMX34rK&XLk&Ksf9D@z`vMJlGuiPk}9Ko7f0o z_>oVdBG<`4D1Z%@wQ^J;d)}RVkar3xGGcBY?cZk5(vZT@PyysBw2m0)%O&LM3~BXC zP3;3XTn^?Nbh$;tv;3KrFf^1-CDc-ZmR~1)4CPmtLM5)7!rb)5ErG>u>pjE*+8;h!5diyEQerqY5pY= z<2chOwiQQ@m&5r+>!gd&Cy(iJLfAx>wZPKNF9f2)*z)q!aWPrU`Edhkf3Bio8<5BN zp2nJE-TolG;70A^ZpdGNbhw2p!1Z?@nX5*S4~HoX!ZWTXWH9t&6Iw))z%ZRCNx3m5 z^umuaGhx`XbTl5MZ~!e`iR9pNnF$Z1t7J#E(fzWsWh$0>4#QPrkX|Rqj;_fLBvWrG zAfh;mFMv`)4H43p9XZHap^+@rFn(fCO3+wRh%0aXg#g_O60EWNzb6jx#KP>RA*YYeWTw1=lnvODCpR!4X#h4*~pNY94{}9&^ z<#JGr=6AEe(Xc=d_g&bszcAC)!VI+Lk5n~T5~OC+)CBV4^j8h31KPZyw=};9V6t5E z%MPqyeuA6;V?d_`nYO(GaM!}!5RpZ^R{&#H26{!YB48Bpv;t`;F605`@O${#&Nr)u zH9+FLEz=JbOEuF%&}U0Ee0zCKhe0VA7Pkmu)!{k)%_#^gha5}h|$#1tXikzJJp}k1q@H3cCaQg&qd=#nMgHJ zz0FOOnQv8>Ih5Gcj};{dRyrp5lz0NHHrC;yo4;2g4Aim^4(l#x8f~_nc2RPYT^N$*MDETE& zQNT7=4H=WdIzOBKgzu$>uBE~^#y!X^-!8Vn%*nN0jMWGzLARdbTAiy`Sc~UQz#CwG zDVSf%-^~m-kWM!=FQhCj&Q`mcOxqUDgwMBOvr`wUr`n}VfE*U~U*!J-Aj1z5_66uw zGV?_+3M&IXMllGHc>;)qbLhY5F!y$E*zCm$D{HtSw-!?mFe+9D`L+VY2XWON&)=us z+CsHp)yHzcFpD)+Yd@*gHfTct!HmK~qgSK@1=@aKWvgBa_NT9aEgdX}R2p0XJE9bx zv;2e7klUiH7?I95x*jo;#w60oxR={ZeO3HuU7RUgRICc}OvGSmAld)M7PZ5P-1k$2 zH&fhqTdi!8tBKD-w0%pmjz_DsMD4cy#UK8gKRSNj_rCf{bg#XJ9mw=@+y2X+eey3p z{q*zSc%gb;P`jX-e90o(_8(sO)L;C&m%exMrRudB*Rp4{?fZZDoqPV|(SQGqzo;vs zk@A1WVuyJSEJ^cgjIaWF+MX&emS|hO@ado8J~4YYj7KQ}N;;wW=6m93+t!aADLaZe z)a~<%&SC!?;iCoA9Rq7jgH3pg+(x4rJ}y%W_cL$k}T4g(+l_`PLx zC#p8uwp2fPzVq6ta^{ju2@3dwas#?BbyCj%R4NRA;=a7_y9wFr@&oO=U`{H16 z^IriB@9_^jR*4Pa19_zRR$C?wSBO)PBNz+$R|n**)qt-I$i6JN&O@ef$Svr>dVH-&t zov!cZDG4zHo-P=aZLt;}*6Y+`|5W`2^6OtT~>5h#gZ4Ob3UecIIF$OC7U;l#JZ@2gs zqcUVaAS-sE+s|l}nj*5_uGo@AY-JXbFca*Mb);a~CO`}ZGw7FV3dV&CW(0E*OXp42 z&mw_wT0F!lml0rAjp*a3t)(W)?vk*CybCTc7&IpX6e(BI46}clH3+1Iex9hmz z=T?$Ts4_X`pCuBI$ZFs(9mn~j4yUei8888aD5*M}feVN|LvfhLCWOoh7!i>gl_Pvv zoq|B~6Ogh{{RGLm6`X8yQbcW~S1SPv8!7*?sjo=dE%6iCq@P@<9uNjx*= z$dqx$lCN4JPYvknMqs3!_3;JD@o?}06S+vRR4XbWkYOPoRr%QNfC^(Ho#PTmE*D`^ zigwuvf-gK#uy+>(ZRAavDiI!Vj@Ss61ScdB$dHGSQgA|2G3W@GNQeN)(aFE!x+B32 z8KAdR^rk|9)9uK&$p?LwzHjq}X@83oV^4R2I)e?$xq9O?z&g$LzOQ55cYUyR3@hyl~<^+vY+RVd2cO8DU2(qw@1Pi|43Q>GZ`fvdffdBBCg@9gagG2*Zf+ zjZ)f@FF6}_M4g0Z47ajBlv1M>5hmcAgseJQFd*ii{*AWykcBIFGt0)@&s?G;j995`}XjFjWtOu0|=CNLs@nFMQtKDKZE4+CMG{2>TKodBsfBaC}oumy|Y zpE&_lHlU9#j$|m9*I+tt1yfU8Fik4_C4R+tJ`324>UC-g^$;@11!g3QV)xA@4!{B% z34U3$5J3II;5d|eylSQ^9@?ZZ9eDlu2IC`lOBj=NN$NnO%o?qNIVxe=OL%YmbMwRZ8 z6O~&_QcK(a&XGAy5Kqsl9&@n^rYk0JgW4Fu(s@uwdZKVA?kby#oI@YY<5*8-7a$Wz z7%EaDa7cj}8BFlb2hpNTa*Ym=(-8c4grH??-7dEb=#uIx^8POCO9p-+Swa^?Z;$z1NA{tD7jhf4s9 z0mNIe39a#|i^O79H``AWX<~25$V!@F??a!FfTmWeDXFe77@!^wxzLlroRQrB<|pR# zkbP+NP=3CTMDs-OLl|z<12i-yaGEQv){}kqv^~!70aOgCMBkPKBu7D!0RvI~v(ySu zq(-z*TKkg^0E+YcGh76U4N)>yh%aT$F*LJBgKr?|As0~N?HR*l|Lh&{iV!7xhCP4| z7?3@V?vMO)@-N%@RSXL`JWEyaAe+W8F?nzVcKtOjsa-*5Gu0wHkUvf(p}`pAVA*tLw2iiVbQXa|T$vnb z_@{oIcGJ-AEwpRtfeM?$LsFEG+bb&43`K6RB8sF`eeY(wdj~?aP6@l&ICzaf%2$OU zz-xejJ?Z%Cwf~&T+^65|6eT{#1fLz4;D@aj#Btrr&v6Vum=-;RK0NT&1j`1QEpw)} zZPN?vfNiT0LK%u5Rp?QBxEXkbLJ#Yn_s3$P-#-Wrtz-88kvy#Je*MK8 z07!Vh+hS<+EqToKAtzw_S=0#l1NazVY|?jr4y9@kIT90~&Ee-iYfyRz#gOD?4$>vj% znV4V06M+2`m?_-)#ZNj`(Dm#oMZu2=;Aau8+ybFWJQtOU6}t0l z)g2X<>iU%HEz@;Q+XA@)IjkWgoaI3jYX>0CE}lYf8UL2EoUGNv$iJZZz+*v{hreoK ztezQw*ZpN-X0JMo1E-@Bc=npZ-U}0B#7KQpBurp^vUOl~QGa$TnGpiqX121P8CYf< zG9rwgG&4_xU9yUu8xrtpk-2S2Slv=!kOK7d7>{**L|4c$zfM|(VL@p+#KJ;y4Zo5+ zAq4RK!9rkOUwUpGHcUfTQUwSAs_YE%8HpK_;Ks3AmJw{aFU&7dKMFXQeE<`z2*2AU z&p9rFHe{qU{J)BJNApaN+ve?cZko5jzI;vjFBe-9NLkMmX%RzGY#8xDTDkZkd zkM4N>4+_XeqQEHmKMPT-{n3hYHBmNX%c|L=WtP4W*rXUv{!{8%<^tXjC6RO(F(&DT zf(Gm{Ct`BeudvRTK8n)5L;8#KzmWA#@4!TONCFv|;mU{rG>_Xfp!^gVJhF%%jhNzB zT7xx8OA@Cx58Lk=0bVNtK^E;EDjhmFdK{6y@_4TmUYX`Vi_tjiW#!7_gqf`r2Z~%k zYv}VzEVSskqsXExRubwNmbJ<8Y4Bs99Whh&Q?z$PT!j*z)fKXujkhFoj`LLlV31){*z!b5sX*}ib@N&IVE5ZldV7mUcsYb++BWLP%g13m?FA5q9iohArA%H z(P1s-?*r@%U-Chr{s!I)K^QGOg#IKBh|K2iogW3rr0hHj8@ytLPo-?7L5BfRU=6_F zvdSXInU2e;R`GyLQ9+7{u%5~W1K-_mN8D&frw|N^2K9I%JuZP`^ruQIv^WW~Z!eYr z>|v`M3mIn8Vp3Qf4Mrw!r6X=Gv9IoTWpFW2V_1To^!sqcPKnW{9DtJe30_e9Dt7h| z`y=HW)&_6S?Bv}~lTQ&YP8IC3gs*F1f*Dqn9JoDY=QGP5k{lDI!RH>t~z zw2cvpVo}xkM^HgBdx?x-PV?8s3MWg_bRfAc$uU$7&jAcjU=@r}R5rI|R7U`V%*)^g zQb<+7!FuZ1{!+;vnMUSsc(P0ZNZFtQgSHqx-G>-{74S4QCd(}&mJ+eT(8Nyy{9qwa zw$YD5B9eNFu@QYE-5^N6b+N>Wp~u6EFr&>4t`$*Oi8z z<0>)ttli5P;O3dC5X#mHNqMQzle*_Rr+d`V2}`YHt-IGUMlrRPf2hQi3zkqOSjwq|QM|3XlM{<&YB12yW$KOyladt8SJkw~QWnAW_R z^azUJA1Plp|1hav#H>WKCh92383iJf56Ir|xl3W;=OjC>S3 zNE>NiAFS!;STk&DM8F+)myNiZ79Ey}*1e*PTa`6;(*Vp-r*@J;ZKLv1d4Q4{rTGC> zbb?KwCQkGW*s~Q>lkxoATqZ{!%i zw6=Vz}Gi&e~yNk)_n|T!trP#;+IoWd=0B4B}u&wDfRCEO)UbX?h73~3t! z=08{FydZ-`OKn_I1f=D&h{98m>etyzhfOEa68nREDJWh%?x+$B1yVXHI`>ajKj^{e zRkVsZ5QpR6YKmp=wCyJA0NX}QxkDmKQ7)zq@}8~v+H#pI=Y1|nB`io&{T3*r?a1gZM;cL*LuCKTwuhrYQgUs-a+x2_nt}sZfPsX($5)3sF`{W5XMT3~v-#fH!7)j5Vad;U;ea z!h#S+i-0TrY3Eo$MmdV}8MJk-Vm(b0$|jG6*{)(Fq-hejYxJ0=+Ry3vY0aOg`ct|d zM<$~ZGy|@4dPKxtGx49ZSDK6_CqGj;k(!5p$X+d&3Imhm_Yc}DW)oz7O5glOWCCJc zA6J4fHhox(We&oGCovlIn&44Bg2j9+$}tf0;hubi5b~)49%}Fj4qSjx<%2Lx%~Moc zLPzLNmR`-1;&2(We<+Wqsyz(+nvHvrcmtgh+sQws&={_y%i`@LE1~{GY{H5@mzCN) zk`O*HQ|pZ7B~^pygveJ~GP>oA0C zEz0qqXt2j(pr9X*#1pza9Ou_V0rIl*e}v}X5rQj5e*{@(@>Wk@LY!4p3ZxcLDbEd? zJRc-X6LWwwGx`DhA`lUZQ8U*X&qNHQA_mq8{1McJ|1tllv<6zqKlf$s&+)Ht(LFG< z$;ngb>%)e!U2IAKPAZi$&*H$8Xobmt=594pF$I-rl=rdbI^B%OGM8A|?0zF3eo3QG=lD3`6X$rPl$ z+765?uqiN$C+h~)#6$9r?YxTk0e<)g@N81V2;Dpw>kKL6Qe4`CZx1k=bj0weP7kC3 zx<*SgFbT=v9Pq?mQ1<}~Rfw;(r`qKmt!pxR4O^pUeOPN^;0;K|9vjS7HL3&X9B5H)dI`(2h@>;;N|_Ym)Zb?3j`_;QeLzKpG972o7;ids+{MwQFfjJ=Hg}(TI^UZ zY}dHtwb;9xUzTNGbPJ;YOh>*b7sIo8p7aesnMcz7XZ1%d=6 z9hy;%(a!Id7mJM@){+Dm?f2&ysUgfkK%q4E)-$a$^rC4D4+muGL^C&M8gqlyE1Nxo z3Pgc><#e7vSbBl{8C@(vBsMphDtlWy;~RmFpej+Hs0tcGV`m?PdW2T~BLD-XIz(|n za=iSfSZ%Z6)NENsDu~#s3=M>lC|pVv$|_|9s<5LEfNsGWl_VIufiyu`GDSRQa5q8> zc{kOWl%mPRP@}oaE$KPkhRBQon&Ayh&PHcFl1|7n-8`>*%}Z9a#?EZyXI=E2JB&s@FMQ!T482gSJnk6L3impCo1A?YnBhNnzOEGT4$=u9L! zb&i;rJk%lX$3kW(5>Z6tRvJ9Sv?Wc-Z4&2#OSRx;0*RPDj{FiIPM#MGiMnlwn9YW0 zDE~bNwMHRsBx<1LFyY)=WYnm8afMGMLDX0F15;NfGMA`~N9FpU5(4tEf#NY^E**m< z8nV!TLt_PLiBymzIpZ}YZlqqhbx;e^HmyBF)up7WCOz;%bzF8@S=a>gtGLBSJ%|V; zw%9X7BH@PGsEo7<(?#eEcpriLPl2^y%K9uKPXr9!C4$Wb6Q|i!pr;5*2?Q0HTPT8( zeS+MRTPW@W$@Esnm0JiQP^)qap_d?}f`s(0hs+PX^c_7~vw=ap{ehW=wh_6Ypesx4 zhgw)tNYc1`s{|_0;)pU~0L1Iiu0ZCD8HywkD++3tGsQT53FaSVrR!IFHNX2=s`q#ilOs zK22$KvTYCy{K4OiIHOdw0k=6ODK%`5)f%UuXu4|M)eTE0nax7vhnX-=dRPP0>iLuq z6hk$guMFEC!}_9Jc{Nn^=d<=dzD#bvtd{U zLCpo{-ClH6vkyb2EA~isXaa~zX*3Aa8)m!^}Fj96TG64UW$j7>U<(4s>0BuF)&@nzo5+?Ip~V zUpgKoP7^~8#{C^viOo7=vRj$^V^1^)Amt@5KpV(AndyUK zFkUPMF-2-I7zZ&ZP}O2E2B(bHkV`Kd3)aVQ5QWw0?q?8%5a56)G^VRXp)B7Z3TYE; z!8>4Bwx3M`QD_q-<}85nVny{rR3ue1z=v_ohj5#5p=c~{A-_b?Nng|?feUFf$-fr( zXE6`d(Va>_io-bnO>=ST_$=Z*=V!dtRMdCy(m!v8s%&{R)o^zIj|XdZK|nHh6PV zH+rHsPYm8XsT)1fo4LW8r*)$zdh^ua&9l1E6TNw6@aB2l=!xDuH+b`cZuCTNP7U6? zq#Hfan^$!6Vs)cy|IKTE{0GjjsJ-59{W^>W!}bVvp+PSoKENezC`O^GNkZ*M6~+x_PX6qies|le&4LdZTN< z*web1tKR6^FZQf%o~qvH+AsFJZl0;$=-My#f^MFx-ssvdc53jA*w=WWzbbJLb@F(ZS5g+Yq1iK(G^b-EfRHft2?A!lOvU_d^&v1L`z= zF(FHam2>4ANk!qJB%bjtoS;dzAZ{dK11uCA^UuM$c>Y=jS40hvX@ud4y|l7t6kL6- zr{)^mJjqILdps0jD=`~@qyUTSL%3dUYRZA7vY$0w+|&v`n!6zCi1edPI48A2qv3&l zS#l16L#>YN+|n>@p#j~*2jgg?mfu&%{-}*YeZ5VPHIFUCGwDgjk<0gMAaa^{!hk8& zGmYx1VdAIt4NoPjYmhwCAhBZuBxS>hhOF~?1WcY$NxU;YA_IvQ3m5^cq`S_Ts(&;)hkqLpf53s}io7@#zPh8k($I&95Qz;vJh*l(KS zM{Sm}p=zrcn^UubOF&aiEy0ww25ia|Y?K~PU%{vjHbUJtplXl_9}s3x{yQQ}MxaF{ znRT{|`Gn9eF_L|d?5)`?Y-#|38YtXnpH&bsNsdsX2VKMd7G1;~i@1+rd5~g3S6dXt zhQhQxLs*jdsJheQaWeXYg6ye+?01J~pd=D`-eig2RW_K`U|6kdw+8?(Po!gg>1 zht0K?0567?JE(JUiJ8gr*8rH2-prVvu#^+GKsc!C+E-;jl2K# zOh}R%>4d zP?(FX-I3v&eSE4_zu7}VP@MPaq?|83B`c7pEZ}8$J1l4FMXDG2j-Fnmp+>IW^vw?T zjBj+Q(%bZLltSe>#-MskDqwbrTf(cMOI()=8w9kA($F7Z=%p(H=qzN!%2xp0Fl(Y= zuFNN3({f?Zinbt?L=kOk%>Y{1BNk+}5B$ZKm*Pz0PFg~;h+0B=m(c-&koa=;sw3)b zJZ*LB6|E2xPZSeZJp`gfA#H8U7VgYNCFn<}OhSMxNrJ5GKvw(!i8(F7&4*6;q7Aq0 zYq%Occ~%R&pwH^i;p%gZ03yHHXfgkpEN}>mf|AqbbAbi@X5oUCd$GrgVZXV_ckm$k zGnLz4!{b|k03nQ2(2y}hYcP#9RLCG;B`^E0^~~Fv14GwHXWb4(4OSE2;wDFUtoF=T zH?wwNWGes}uB2_~7EoJ)h>oB3vxhkinzGFqjaBu9YHsGqa5-ZAb?|1NkP!{%!S}%#m z%Z7wnY-C8dDIbS8my7^Kvx42A&SfdmZeQIE@$jS`!D)@a4JrSOfHh(juM4A_!*GMaR1G&!@__kRMbkvO!Rk>7(T+hQ z!Ga>(@dj-t6)0@C|2z~Dpnwe}i$vHDVAjE^i<$}6i;YvA29v_NHAWJ#+SuVlg}cmZ zh|zBSh0WOC>F=l$`I1R}U-6Df5*I|W$OIV%PW)KOqx{ywWj~AlMC`D**FfyBG$Y-I zm=Wt*eyb(4OL8hVzTS^?O6MQz2<^s+`F&!!|NN0TYQgDXp2SRcYRV?@FMn!I`MyFn zdJhGZ{kIs75P3u3&JT&gBLAJ&e^Jph9^{f-6Ixc`H*W%?ghdj}74qSw(|L_tn&{Ds ztPU8|7)AJp=(Ijvl30w#WLyT^F+|*ic{GwUlE=}6`Vw74n4&WDS!rQe&p522&WeHn ze?Z*;g&SlS5;m2odzG6#0XSoPH@2(X(a;yWW_%;l<#MP)@#*9Kv-OR z24SOtJPZaPM@uaV00<*Zo{81%a*Jvek@m;7U@*Y=OvC9{lYn;kS{U=@*^Azd;reWf z(-zwA&*ik9f)l0j+ArD)-jUjpEb9xjGqhEYUl|W(!3I>5&4Mioa9KyAh%tr{I-~e8 zXn^INCusOx{=sjOeS~`UUkdlnanXJD#dc6O_*P&qI1``?Fmi-I!1*F{i$_5>uLWe# z-B%U1w08z{n{=3mulfm66^>%^0H1dH*bX>{wEyl{r)6QiZr;#mo%e^>$9mQAnMT#G z2G)ZXW$~7w$`Rq9-b4d)P3K|Z zTR8?EX>6e$O**o3vSH!MZ4F^=&LvT}?69C%g|lfL!|dE^6bmyGRnDta16{8{veyf} z!9-%RY^mV0TVeL51dv|Tr3M0a)q>TCuog+HB1_`3YK&Wt0Idunlzm}pb70Wo zA**K>eGm#r8hrGJK}+X}^jO3ufJtEO%|Ltxz-HNzgp@gf0YJ#SG-;!it)zktO?LU3 zr)g)cY;mt8d0XHQD_fSh!egy$Rigz*wz4(9lCMR4V8?-7mX}s9cp-OID9J{KjinFN z4MTGzTQW72CGZuR&a|o(+0r&xC{Pw)S_5D?9)3ps)2I;un~^H(O)FO}7|J#;<2|rr z@zg-nJTjuTW$SuPTg%AslC59`$tL{KL|79uL~2cp{#X;^IXMtvFEx#ka9}S+GPHvT zw5C#8tro>zj9s_-JXTAdpF%fF)Q%2o=AzT|xU_rScnl~G4Wq#U3G9rY!~U>@uPJL|Xu6jx)ELRpAhJNUTS zqbR<=@l?!a5{qYG!76FKqVe(=Z)-SCPl&v{#K#whkDht zb9pf_Wg#>09w*J{a3laboLy_Sj{s}9XN@{G(t-$RUN}qYfxx&L2+uq=0u4KQS4U8p zm|P!_4E4N=F>2B4Z>^r{Tx@AF4# z*^^~^)Dw*<+M`&b7{)jp*29Ax;i8#|g7Hb$hbX@NVh@8^{zP%57VrVYS(x|0-~cdeR0FXa+$?hk7(zNG zQ7)U<`Vm-y>)h`OOXk8pB{G(m5MWnpm5aBZ+BbA*8YShmnSp1lB(+G-y zDxk$ZpKyhfgt~A)Q=`&6l5nWufJcHZx55AeqM{4}?T!1IZ}ztNpEKFllx!MeI4St= zRAg2*jnJC@#TKl|W5C#pk}>9}&lvlTT=~?q{)>mIXTShe_}yrK$>!kiM*Ajk)Cp5e z`WJWvg-maZo_=ubl1Q+QCs@F5^JZVzO*#0MmElMK&-AK7+rE;XmXuL?(D;y?X&wk4 zC{xB@-LnY6%YH}f2R}ojFs4L=)}f^z=j(#L=?&3$9vrjq2e7B0O^QXaP7zP!AK_Mv zh#wNR{0Wf{tG(& zkwu_)QsYVF%4n}Lcs4pd@liBGN(8O7^#Md4@Hq|Y)1dyNQLG#&j!*g}k&6)f#Op*a zZU48TM80H3!W%*5*ZKTG&{M==4H~@&(hdZRa1l8Xk3$$L&VEZ;{x{UADQ%zaaGI<*3&rEl1()&JN=db@%#t}MkhE|BOCIY zq@C({Yv~p}N$)%~=}yN*mw3KSr;%L%A!(_D)_}(ak$nmc9}emVj*(dk)$W2g+Zx4U zNFm$y8&d@-7W$$*@KM0D%70g!lajB20r^sp%}z)#NL5gADCp?oXjqfghqD2Z zB)`53WaCGk(v%%_u2Nt>oUC(YMj_H~d~A%@ifda$o-NSaqMerZwzN-_3FWe+d~ zMi`{jy?Ws6K~^#f3;|@!B!jTH^1s%O9gQ0)q~qrB?fw=rZO@1{bsoK}C)rOxi9}Hy zg+PwuhKRKe9f#S`Z{q5*He8`S`p|o;peG$xCr3uLsmmnUybk#A8zaK~P%0}3_%j-% zAdvb8zkp!l`wE~9H;HDgQK8J43?qTs*6od3i3bgIgvlapYpceA*ooov94(7hP_qK@f5Lh(hI46sGkDE( z(_%ys-v_zEykY-(<0_>o`(g*D?t#N-$FRPKTEsYm4vpfUKgAhaZ7p!kKYEev2yHEB z%|Cj<9w{Q&ulT$@0`~Kdp0G#Ax%o%mwdV0*iK{Sx-~DXL86g5G(*Q#t{7kCV3+F1n(?t z4&e0{kJArSt_g6k1H%wm5j+bXu$Yv-V*hr+uR)EkRnef@`^pzk5Q2G_T)T!(jItH+LP|Ulb$`v2c8r(>ZWe??DT?Rb=Mzm|GxcHWL2k9 zjYEN9oPWdh9aKRRbFu{^q<#^vcWXEj&DKnjwkhk`R8$;^Sw{Xp$QXH-|MqWkdDi() zewOR2N&ZW6Llpu>d+K{Z&VTu{oU0-*oYDh->N|K}g7$B{Y(Iz{{z*3B59;H&H2*Tc z|8!q-I><$wB6xO@4Ytt=O5StSDVTIr>skAHqH^5^?M z*5#qt|9}6SEBiee9{5Ll+{5XnRT?Mi44jME$yM@INPS%&iSIfy63;jI_mQ~XZU>RL z!JZ5v@dA4?h{Si=lR+f@IAb3`;`x6!ByKzd63+*TneSvJNM1>Qs1)}1Q4pP`kQyrm zwe?}1gXDV?IVgY#RGYM#sl^kx#BmSBry)ddUtVWlj5Y}v+h~sr4YDVYYjr4Mbqh%* zm2X814Laiw5?7&*>q6cK3ZzXlaIuBtC53f-3Y?kBnQ%yYi1^~jJbSwni`rrWecJK@ zebo(yc2ez+W@X)d%f=1E%C=~%fBgHDz`_oRmj4#t;N}ocHbk|1Pa2xD@b9BxjB(vqJ;_SNy zp8nJcqIS-Iv@lNQmqgsTheS0qz{+Xub{djt%#c=}D`3CrE=s$qSbwzbG+nsw8RcsO#@*8~WkV83$G(ZlGVn|DQM3(Ge0t;D@ zI1_FdrEt#PV#S`I2ayHTQJJKJ4KZ}$P-;x+sBhYs4DmCxHl%Mva9lMLVMcuF1qVwf z0F?S9oWKloHvO|YcH}plPPllMvqp4%CZ_99NK(70lNMm#rhj3+>@(J-eRz|AZK#Y( z2V^1li88PDWj5Jc!07mJ7=Bcd%ORv``&c7Dgm^og!vq}k5Db#Aq=$00l{u05BVS{5 z1WItOhWsn)UN;P0XSqr(fj9lzG0G#ICxMsM@r4yHuviJYBk90v{uy&1R4wDED>pNK z;x5iD!#kasI8h9rxD)EU_an!99BRIzPbMRf4AgQ~0!h^WNg650j6W=@>H1%d$7KQ* zOIWhkiRj$(?TliH>?r$=Q4n92zsr{loobrfMc4X<>u}K_K#3vGDI>@YkzP1;mvS>q zKtn9qhRJz=_*heL#^s7yK+HwX@n!o&L=@G+y{s&X*KtA3J^xJ`)TwY>L6HL{MAA_B zNCMQ?FH_euK?S}ME}?d;xf(Z+F(al(cn1R(*;Qr6XK?kXszl2Ykai}SgfSQ-4bEMUMV64joWG< zQAWSlL;hm!6kl*`Xt4i;_RS*Dz!}$f!0@Mj#54>O{HY&dkyOzpb2^P!OQK(arQ|vX0xp}VB&b!E33C-q zQ$g;4excuh!Scg)TN1`6vq>xolE4a`l5dke!Ar`8IBm_-=I-HQj z4P}r@IuFZUNK^o*N{HxK62MB5?imXZLBpQmH3v_fv4B?6VKT;Gr%dPrF;1VlXK6T%W~dhDf_qCg2_zt)OUzJXH!VmY zA8ZD3E19n&DB>}C<+li*c#NRI-1wJjhkDr<9KQxojo(UK+wWcgBnuk>onAq0X_PAn z>iQ?XFcDO6taw1x za|Y#}#7q`G5b%=H9F?{{cp??4hOhQcrW3~H$H*_;t!-}!=mhsO773~{4 z+UEBxn@tAS3*#~UCttiM!?7IVT9g-*6l@#w(`kYtsiMgGp$K|ni)Ie+g53!out5C; zv4C)-WPO~Z0iZtj#&LLGg9YOSrDfPJEb-@63bYEeK(RO49%#$BQ*jmThg000a zO-9wtBNhau=yZ%a>YzGeQc+O*tP7;+>{GvT23XDvhxK8zq5N5w^e7TBBi`76ho`WG z#H}6@a+KSL#lk~^S-w6T^R(kdVBEp6+NhahuzbK2^&#M}Y_Rx@|B$yD1|q42x`luw z3+4~_7ZWg|?ElK$NVRaXjD-_~EAE$2m>41hGJQ|FEPV9w$EqBX((PCV2rf&lRELbegw&?l?)s&#J(~>S z+sOzwc_>QVHYtd|PzgxWxLAWU9X^k)FUO%&vqPiysa=D6klMu7@=rJ(R4@TL==ap0 z;^U*AK*dlV^MbV!MW~n60<`24)+UCR=_Fq}-;;6UFYMQ1P+{a>iRKC_$lV~SRg3Kn-3ypqS>{hB9Pt)FPg=O8Gk0BF@2$s1*L=cfd}Z7(B~C;SfS~PHsl8xSG|{7>y)!k1q^?B# zo3yr8tpz)yiPC+O)>i!xt+9wi2)VLjnyBP6w3ji_r4(9Qx3_fRTr<&Q6Kkq+hXrW? zS8HoHR?sGR)Dj&;YA#HDv?rug&38yGR+?ALTm=anM~V{v5hU&SM)7#K<^Tg!^wf~i`Im=8^&BFkGzHa zY@#9hNX0@?^?94YjznHen&pxJyJ|27xbT5z5eeU_7vLC)#Syal6DG$LOZaKIDMrn8Y6#XWxEvD!7=8 ziNQ|g0-Ydw2Z%aE1qck#y}m+6AxOCBXryYG`D05e?UoClt&?qmN<)+P_@p#g>8+5W zMxxk3QA!_7h(X<06Z<}C>z=&`JwEAYW zvM~Pv=md&}r%FDEIW-G0VG-dcO1B0n5}q#$M#}t2rGQo;Vj_RKS&^5<4QJBR8yl#R( zSY3{TPUIE0Si)|x-M3S1_w5{~h@#tf5LuyJ>^~itZnPjRVlqZnC1jSUc z=CI;QW%LJFaX613C+My{n&i6V8}bCf~cuX(~AN|#4dF?jjlUb4sr|K|3c`#+W9342(=}A8v3Y& z4AYAFLUKBNsc(EKj8Xe#h@mve#!vp<9015~fu5~z&Uj=LVT|_m1dXZ@K>PDywT`XR z^D`G{0q*-X>+0y-YVtggxJqnusoe<3)sjpFazk7-vH1gNQ|xl{U+|)ggx+6yEau4s z(3&U1e5wOzbsMl~wtwX(d?<=Rhb_gKlS6U*RDpbc175htBA?h0)%&-sigo^)7@a}P zx1DWUh<`(r$tXXE{Pe^AJGvmv{E+|O!%v(HvMKXBYaIS*95Qx5q994wQ)J17pq*EY z)6n|xr)dYd%wuLpN>vX`SYBBRwXd_{c}>O#KQ&L%mHErUb3aKr#MZ3^(iuYX?tE)l z$&y@p05hmZGlN15e#jpQy;ofiQxY~(hB4w(&_F+lG~#}~G2+mOoiq4&8nTS}6;2%$ zj`J@%BD-n`?yi%G`CTUyTYwTzigG=m4kQ#lkrqQ*C>h;$z-yrXpPmL9r4t#I7R_w? zVhmA*_<_z0k$3bbLduXorHhy^^Z(hN&;V%a&Z9{I5CP`zlT`fM+A10HNkK6vN@J#3P7BPc6b}&l6utbneJOmuhT;YtN4-+4cg`;hg_0z*H~nyKnu}4;aFG@fk+Ss?GPevyF7f2b>>RCuSB2|$wkloB6WiUv95fFfk z`T??L7YgIL?phIf~6%i1G_* zoGre_y^4V-x)HJygpB^7+fhsGNj6GJr39XWX(3S=c1{l* zGEX7fD&n$#2tgU;2^lo5Pg%f4S}#yVMyFgEf0XqOtP6D0R~fhRW%$Z6PNty3I4VU9 zC}5j2kB0;TGj)Ezcy3T;iE<6hg4bxOB|Pmd0Xk5KhEcDZVk6mSs$pu4fdbb;?6U0| zJLVUd29?n>LNN2!LW_Ow7;!m}x?(d8=_|}spfCc}cqEU0MCmI4?@Z|{`rwCyZ7>2C zrLR0Ls2k2t&tCcCYBvUMTcsO`Jj2Nm<28ZV@DDu!;UURItI|(pQ{oO?WCmUtEG^kM zILQOw<}B>V!aRP-$ok2gcE@W=OzOXI_Z-bZRq(=%w2Ub&S#1`i_1IqO0J`+10g}W0 z^KJj-TPME$oqM1C{(p?_CEqYvwe8irf8*Pq`hWhH|M{DT?>(X*G#&?RrK)+4-Kb`#A-+Z^O@Rxse{NH@>cfWt^E1?2XPoHQp7$G+SnpmZ0qfj`$;U5ts@C$;% zP`nze+HT}uA$?-Ycq8V%1Ng;b{_$!PBFUn!x5)7HPkn1nT8Y(N|7dteuaEJ|iv-Cs zKJ`t0|KE{+fP(o?V-8(`RunF1dvW%hGFWA8iI<5WbB*mfkUyr!vih>$OQ2kHWvV@l zwz-$zO3F0Tgy9bEj8(XE!TGeHJDg}SuroHU&R4gcvDNXq2EzW%Sk#u$p6CM->}s&# z0$*uotn=sdPdj6iurn6wt9HgBvbWimRwlM+OKXGC0ulyTZxys$2CTXuP4H9}ZD_T@ zqXq&DEQo)G4Xy62ZD_61c@gXVvz%?($T=wimMk?B0YQd2vp1Ka8Z&s(*2o%6JZ)z~ z+^S}jeLD_Wv#Fo5OgPO?zbX<8Rcb0MV^xgp z)+mkUj4mNotR0+2ln0|0Gm*n_ar8DW>?vF$d8&!AfM*}>$?ICHgEeMK#e7ba_MVR@ zt8PcikUDJ`D`HDx0Jw%tnC}mHv+}%98D{7q`5>6G5jHZT zOhC6Z7!~9b#3m%f%{n9*lPAmm*R36S!*fygM@xrq+;j8Z{l)u=J^ObT>*f-E&E@Nh z+i%&kzc{pS@2T^zWn zxbeVk`*(A9Q&oIK4gRo&&(QYyYJ2~I>0)QG>z18Uq5YjxAGxI1JFQoH_aEB3drvXf zF4n1+L&aQk{hRdu)|#7h7wiAgoZC1Toh7)`FfgDTDt1m6bHl}rfP!g<+o*!l0{C29 zOy9Dn*avW#!rU^ep(g>LdKN94+PVMcJwN+r4LZans&?dkpF~37r?y>Rhb98>enNG`(x%d)~Ws)0WK_Zr*d@ zO*`NHo{KhZdG|#(-gM*6T^C-od&_&bY~H%hK!dv;ClJ+S}K#v7-n_Uzes>&}B4_w1jZ`pCPc4*k75*mYoPkF{?|*?8!{ z)bz$f(953P8xP$|?|SefZ-08{jr;a&oV;!Cu8FtW{yczgy7es$biScO(^Egf8Q;3+ z)?K&!h#H)p+IVp1^z@#o{Xdcc0YI+5Uixis+`Z@Ep^dlfJao&AQwMI} zzhTq6F5LXC3k`fzJ8$23>w(?Fz+@&&wzZ)On35&dK!d#ckNGQ#ZNz2A5G z*11)6s_N9KQ|Fw%kKALX+&fwov~6S;2sXPrIzZvut;Vnp+i*HyHADxT&&@?h_-0Q9i>I z5*;&}VVHLj!`R{6;fGxd^so|9`tTc2R(X2@-W6SK?tsS^0JZH0E#2I#Z*A4L>VwPm z?VbWr$FNYhIC?g~83S&L;@0Rny0b7ALB~vh{I3C00b=;KkWL?kKMU!kQTS#^J4WGy zkf!BJ^zZ`*_0Z^X4BD&s5WZll+R85P^0mSEgYCXxklHV~u?xA1JFudw9R@qnKBXlC z1iWs~bYEcPdInrG((80`MhHc)QB>}XV|i12I$CE zP)boML2ozJnC`ZkVXDEAKGr~au#zKv{D&?A^-w6_19QY16Of{=xs-cYm5KB6e%1F(%N|EAHl(lk^oR=rdkK`{^9)V0a zVxvrkbP52K(#3kc0tfnkKA4KXJO?~8Fgi^JlhI@{nN1dx)nqf- zO%9XOY%m+mCbQXWFO*XU5Vzb(8HoMJXbJ`7dqupdT+bwph-DbDj9d4dtM1_L<;@;%OEYMF5T>Ttom4@( zTwNUx2i%=``ArI_OSjTg>+kS3DXZOWU0#Mq2m&ye69g=Z5|%j?)({w|notUt2Ga`QZ~! zZzFM<_%ZoLi>tD#=Kk8*9`BP+Z5%lGo41Y+{`TCGgRiA$2%@Y?NVYj$FYWs9Bgy{E zrk4b{w7kVP@S}KtfIA`R#NrJ;y<2awS9-Bw?zOOVpB$b#Z9|Q44i7 zc5|Fr(2Fvk!)A&&eC9zNFG}O|VnI?`PDVnqR11wwk4qI~zFN$cx>Wa-7x1N=%+KKw zr(-$QLJ?;!&#D&X@Xlr9D^)TdH`c|=d_qPMNe&;H?3t^smgK6+^lEW#T#X>l7Z;4z z$b6NY$AGvQ0z>sT9cqoP&Y`a;UD(SU?00ro4CF%o9Dx4 z3%T-fsI!<&Be-{JcD##2y=Mx-=L!i5AF~0oE{DI#pG9PX`{Sq2DhvO|WFC=vF5QBA zHATduo-c>@Ix^#mh$LVQAKudc5gX6OvEd86njo#3IAHL3ViCa_DYtilm#D$Maa=ML~*}GgPVS)Ht0+t%YwWj~O#oNKNv zWFSUtLc8#8vPb+5-{fwyL(;Fhdp2zRg`r`=hNqs%_*kQzIqT-%^(9l5EN%Q`{l=#^ z{b<*L!*84zJoV1StDg=rB(fyzuF|ra`ZjM^;wx)B-thWY-aK;j?D^|o_CL1a zg`G!^o;Z2>!tbj$|N7m*_fFT;&Td$^tnrDb2M!#3?dWd?Po9rYN?yF=#y7Wz!tD=T zyrjwQ@MmN;KJw_^eHSl1@Wz`-$ywP|GiqnkMf2#oe?RfT**{$W^6NlwAk>wkFL`m_ zYe!F>K7VNoGxfR6hJoz&hH7UoS}cfKU2e&>FFO48vMKjf&Rx;<_Ng=PfB4a7w}%*| zaeUt;(l=d9CwzSG%bM^WxDBZ{gH06?DIpdjz)jb@5HHth#|ZTTOEP4M61C*ZZ= zr%2zHBvs<$KjuofQjo6{DSTvnNF6?x)~n;fSERpgW9?G!l7#T$u4lrz0n84Hg9)Ku%5*XUQw&xF1cMQ( zRv8j?$SYtTV?-vPtn|eymn5MSf+W~(xI(s6Do8;JJgbX{2#gkFoQYhtIuW6WjMEUt zPS}?ShKWU4n1%Q_;2ab{2{;LMEhr7NMI^8?&VsjpAc)6GC=bX3RhEM(7qBWq75adH zNCeNoPQYTVWTI+#p$QO2BASD+pca=SEUAPVoDTF5vTG2u$f?jo3AGT!L#J>mCaezb z2*;R5S_Ccv$zU^a7DVrnAR?@i5QrP&go;=Lj14*od-XKnJ%(opM0jx83=F`SgOL|t5nZE| zhJ|sGJeHteLK_tvgCF3lR7SEE*|hmER+%*dT>+J`6FDT7!yO|Q3AjcuYGg41#-Oc; z6%!-e9i9n|z;e?NFo_BTOJHysTrbi=CR1JyUk7RV6ZDveb2`n1!Yd|I7UMF zYa|nbo`LoWf|o7vk=f%htA|M#1lkCl5`bCg!6fJ)P+G&oC72D6L)sjI*I!`^$Fa#U zW?l;)wPP7rqYRot=73ZI2lP0Z;aFZ2v5-Z6%rbUj5|IWaaY)MuUN@t?0=S#suc_fmzqw(L4u*H{g43ndaPQSS0x za=>vZfVDaafI2?zASj54_uS$Je-7fHG-6w>7$EN*&@GGM><9ma+R=BH^|yH%+q^9y z_<>4gpkKPc;f&S8{IJuO0CcF{SO9v0(%lqwR}Mg$I>0ne*4`NcRimu*L&N|gJ$@h+ z3!*9y`yl8$+^w#lZ;jXGdZ1##C^}mpP4yoY^XLKpuj~h30M-y(ha9G&{{Y|!61yS^ z#8e!QCFm07o>_^xM7+b1OJKtLJdItQjUj&| zgHA}TubX;`5r?tW-_-{HQFxX5c~!*@J(CVG3hMd&3xE^g?x_U=CZ((_+P_%&9w-|R z5X09%nu<{j|2CRW+a7Vul@511MEU<&AFh6n#;GWc7|k3IKiVeU(x^{P0B9_OIxlnm z?cQAAT@&Ejp&SjB+&vbbr(2niclQS(9(v$!UOuJn35JG|7@c{-@OB!*&%l5u0Ie84 z7WP^ye|OWpSHC-GTr)Zr==>-;EYPLn9}Wo9m_tZ$$#+xjq-&0@?#Q`NY4>$B(oje% z#F4H?)yaV}+@_eZ!$= z|EJFP$9KQ0?YWh|WzpG+eKfvEV}gEH{{b8*hw&7 zW&!PI04N{mAp$^Gm`s4teaT|_k)VgDaPewG_s=$NJfL|?QBNN_VV>^(v|+a<>7fY; z`#1F-eDL_u=R$w^-Iv?0UEEq{$asA&x309;IJW!pw+BmJ{^+M4eDn5=cT%;Oo&CGC z$5!~3{^6C{2Y Date: Sun, 20 Sep 2026 15:50:56 +0200 Subject: [PATCH 826/927] packaging/nix-make: Discover compilation units automatically Instead of transcribing Meson's source list, every .cc file under the component root is a compilation unit, minus `excludeSources` (the windows/ and freebsd/ directories for libutil). Adding a source file no longer requires a build system change. The nix-meson-build-support sources that Meson links into every component are passed explicitly via `files`, with `commonSupportFiles` for reuse by later components. The scanner (updated scanner.wasm) now takes `sourceExtensions`, with the default living in lib.nix. Also work around a crash in `builtins.wasm` in Nix <= 3.22.5 when copying layered attribute sets (the result of `//`) into Wasm, fixed in the previous commit, by passing `files` through `mapAttrs`. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/lib.nix | 48 +++++++++++---- packaging/nix-make/libutil.nix | 102 ++++---------------------------- packaging/nix-make/scanner.wasm | Bin 57566 -> 59746 bytes 3 files changed, 50 insertions(+), 100 deletions(-) diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 461a2a92dc..0e19617af0 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -164,7 +164,12 @@ let - `libName`: library name without `lib` prefix (e.g. `nixutil`). - `roots`: list of `{ root; prefix; }` directories scanned for sources and headers. - `includeDirs`: include search path, relative to the root namespace. - - `sources`: compilation units, as paths in the root namespace. + - `sourceExtensions`: suffixes identifying compilation units. + - `sources`: explicit list of compilation units, as paths in the root + namespace. By default, every file in the roots or `files` with a + source extension is a compilation unit. + - `excludeSources`: paths (files, or directories with everything below + them) to leave out of the compilation units. - `files`: extra files (e.g. generated headers) by path in the root namespace. - `externalDeps`: list of `{ prefix; pkg; pkgconfig ? []; }` mapping external `#include`s to packages. @@ -176,7 +181,14 @@ let libName, roots, includeDirs, - sources, + sourceExtensions ? [ + ".cc" + ".cpp" + ".cxx" + ".c" + ], + sources ? null, + excludeSources ? [ ], files ? { }, externalDeps ? [ ], extraCxxFlags ? [ ], @@ -186,15 +198,22 @@ let linkDeps ? [ ], }@args: let - units = getDeps { - inherit builtins; - inherit - roots - includeDirs - files - sources - ; - }; + units = getDeps ( + { + inherit builtins; + inherit + roots + includeDirs + sourceExtensions + excludeSources + ; + # Work around a crash in `builtins.wasm` (Nix <= 3.22.5) when + # copying "layered" attribute sets (the result of `//`) into Wasm: + # `mapAttrs` produces a fresh, non-layered attribute set. + files = lib.mapAttrs (_: v: v) files; + } + // lib.optionalAttrs (sources != null) { inherit sources; } + ); component = args // { inherit units @@ -209,11 +228,18 @@ let in linkSharedLibrary component (map (compileUnit component) units); + # Sources from nix-meson-build-support that Meson links into every component. + commonSupportFiles = { + "nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc" = + ../../nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc; + }; + in { inherit getDeps mkConfigHeader mkComponent + commonSupportFiles ; } diff --git a/packaging/nix-make/libutil.nix b/packaging/nix-make/libutil.nix index 2064c41914..c2e0ea7365 100644 --- a/packaging/nix-make/libutil.nix +++ b/packaging/nix-make/libutil.nix @@ -15,10 +15,12 @@ nixMake.mkComponent { root = ../../src/libutil; prefix = ""; } - { - root = ../../nix-meson-build-support; - prefix = "nix-meson-build-support"; - } + ]; + + # Every .cc file under the root is a compilation unit, except these. + excludeSources = [ + "windows" + "freebsd" ]; includeDirs = [ @@ -30,7 +32,13 @@ nixMake.mkComponent { "widecharwidth" ]; - files = { + files = nixMake.commonSupportFiles // { + # Interposes __cxa_throw; linked into libnixutil only. + "nix-meson-build-support/common/cxa-throw/interpose-cxa-throw.cc" = + ../../nix-meson-build-support/common/cxa-throw/interpose-cxa-throw.cc; + "nix-meson-build-support/common/cxa-throw/is-logic-error.hh" = + ../../nix-meson-build-support/common/cxa-throw/is-logic-error.hh; + "include/nix/util/config.hh" = nixMake.mkConfigHeader "config.hh" { NIX_UBSAN_ENABLED = 0; NIX_ASAN_ENABLED = 0; @@ -52,90 +60,6 @@ nixMake.mkComponent { }; }; - sources = [ - "archive.cc" - "args.cc" - "base-n.cc" - "base-nix-32.cc" - "bump-memory-resource.cc" - "caching-source-accessor.cc" - "canon-path.cc" - "checked-arithmetic.cc" - "compression-algo.cc" - "compression-settings.cc" - "compression.cc" - "compute-levels.cc" - "config-global.cc" - "configuration.cc" - "current-process.cc" - "english.cc" - "environment-variables.cc" - "error.cc" - "executable-path.cc" - "exit.cc" - "experimental-features.cc" - "file-content-address.cc" - "file-descriptor.cc" - "file-system.cc" - "forwarding-source-accessor.cc" - "fs-sink.cc" - "git.cc" - "hash.cc" - "hilite.cc" - "json-utils.cc" - "logging.cc" - "memory-source-accessor.cc" - "memory-source-accessor/json.cc" - "mounted-source-accessor.cc" - "nar-accessor.cc" - "nar-cache.cc" - "nar-listing.cc" - "pos-table.cc" - "position.cc" - "posix-source-accessor.cc" - "processes.cc" - "provenance.cc" - "serialise.cc" - "signature/local-keys.cc" - "signature/signer.cc" - "source-accessor.cc" - "source-path.cc" - "strings.cc" - "suggestions.cc" - "table.cc" - "tarfile.cc" - "tee-logger.cc" - "terminal.cc" - "thread-pool.cc" - "union-source-accessor.cc" - "unix-domain-socket.cc" - "url.cc" - "users.cc" - "util.cc" - "xml-writer.cc" - - # linux/meson.build - "linux/cgroup.cc" - "linux/linux-namespaces.cc" - - # unix/meson.build - "unix/current-process.cc" - "unix/environment-variables.cc" - "unix/file-descriptor.cc" - "unix/file-path.cc" - "unix/file-system-at.cc" - "unix/file-system.cc" - "unix/muxable-pipe.cc" - "unix/processes.cc" - "unix/signals.cc" - "unix/users.cc" - "unix/xdg-dirs.cc" - - # nix-meson-build-support/common - "nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc" - "nix-meson-build-support/common/cxa-throw/interpose-cxa-throw.cc" - ]; - externalDeps = [ { prefix = "boost/"; diff --git a/packaging/nix-make/scanner.wasm b/packaging/nix-make/scanner.wasm index f09bb6c17353f7ae9763e8391c0886f5de715785..43290f9c212ae6c44f8985205749d3f51b495efb 100644 GIT binary patch delta 17054 zcma)k3w#t+x@VoL>PONYsDuCs;Za>g=*TM|B)rK1!W+@+n^6H3g${25q9Ppy#tcS9 z4`Ni*;2@wPMuQL)F*;rccXUJt9rg}8%glORXP4Q_+{=vdciHj&|EH>(z&Ll8{H*i- z&Ue1|`KsFAzGwaZb1QBvSiMvjh9RC1g^$LE4-jdQo^Rm4P@4Zx3F}JQm~XMYVSdDK zr;XPZ6pHH)iUY#3tgx_c)3C#NLWD$ESau|0+rq-1h=#0Cs8eSvj6XA+gI+6S=h#-n zj-VE@ZOj!>OPJvZhN4y^YDGmbc-i9Gd25FnZ>YP>USoy{ zLyB1Mb(MvcSKV{x{o+OOl6YC{!=c_5hs8UhRot;CwQ%X`+BJ9V74M3-#1Rn}xw-Yq zI-ri3W5hJ|>9`+>*=mvXXK|al*&cdX!{cp+D_qmLy*f|G18B)F4cC@^QgRm?4cprc zXNeJW?W9?h$Z;)awtC;L5N$2r*)bv7R49C@Xm2SG?+{|H`XnOp>>OPv=C-Vft~bSo zmVeECLZtf$k;X3896RlrE;qDlQh?PaLO4%#>|a2ibYDZBF>1U>so_RkQDLHxb_%lr z)G?f5Hi`bIJR4=z8Z5H}79DhdK zpAkhXx5BioDzuX+JnWRy!>lsVPMS%tldQwPbyIEK9-fE-Zww6dfJ4$O_i`PPAgGZk z9LjQY++3#w%@2%}8+Cw1(wyeyu}IJIV)EIoAZNsJs(o-{0WQyt$rsq2gYG;MChh{M zBoMIOkQ;W2;%1C+*>rkgrMx~71-jAmK;m``$2<)vsRV1^=vrsgqA6^_#hueh5Hp5{ z)4e*+h{b^9F(c(g-MG^m^;4+la@(st3?>W^*bC5St{dG`5W|Nm$^M<=Nminl%ZFyF?Qq7Bp~dJ1!Dvl zV?$uox6Y`}&u4IrQcqq`6Y{&~1qVOnANKLGQaCF*N}WMyIVmkAGQzGRK6*L>SvZ;b*i_FibOyCtS(q=vb&rA? zgVmz0qkHZYuE0q=Q{I=+4z%MvU_)lBBV9Lz&p-m5A!=H;@nVa5pxb0ItL5WvJ%o5h z{m<^3^G8Eh@}ymxaywDhod?vtMa#ryb-HMdcvg)qPKtW9u=uiWjR(OvSJof$*&yrD zcB<9>;{T0lPxW{RfWmHE1ImYhh5%?}$s#P?Su#WXM4c{~68VXU=fs?6)To~G(x(Wg z3#6x!m1fUQqYz_quQtddi)Q@@qif7jgyl(OspUka4FU+AxCUP$av|idLFQ zbil?5r#}~zLmma-2sdr%Rp38_Hihm`!i417ayYb-CZi`t2w9Fzp-HoB0i-#+5;dHL zLX4$c37xO;1Pn~V!qGV?$w>y6TU>~ZduqM59iCc~rxPqJ#z?`cno>jk>82aSZgu(1 zSK#^F&Ar3|_0G-djOZWpMSuFdUcIJKV512wrnVGQa2RyN6E==GTWy>-)|?(wdwU%c zziye|`*sV8@$*Fel*Z<&gw`xWelJ!IGSHUhN-5SRfDZ6q8SAglN1{7d13Hj)%^Elf z)Ns(0g!3v`XTpIi$FW>c?r_{Df9}!Qh(gAMq zE$lAs!Dk^K18ew9I=8Bu27H0Fp1a2@Ug>qO0e}M6#oFqD_haqRfxi-t+A-+o;m$Hz z$NUYy0%&SflSCA5>{BJc$ovuNm~@?c@m^|uS@7P&UN=r=l424Z68%E4l+8eiE@Oi~N+oU9rud(Y0A9^dr+poz-A&_v~T zG*R`8EV{`iatBVO((4)YY}GvwLXc{>TNKcD#K>p6KLW=K#TaeC0BD8u;IN%X)c+be zM{HH2M;%jrM<0Q|`Tgii3f}`13_5f$Cc`=irYadTLhMvmj_H9QbIF)DdhdN5lSar> zD4eO-(W2c#RHpZwrY4NNQcPEmj;$RygW8*f8ADMDB}M_m1BsGt{ysAykbY1!Lk%6* zOUzVr#!X7j0z`x|I`ZI3?S+h3XILHR9_{7`c+}Wx^(V87>D(5CzXQgX!Zlqn{t)8V z=!8LVP!$sf2=S`ArD{#r(GZDQVZNZJp&tS-KX3&6E%52)hRV?d5+)qSEjdj>O`^Wv8NKM-?Svub^E1PnKl|GeZcg^Vvl-!`gM@&KTa?45P`x( zYiB)1yM*bWMNuG6BQWO%!>wT8jTzUb=h4&Ym21**Go(k%OrPv4Fr2Zdfis5pPiY?q|H?@g zB96E-D>lf$QlYjlEKJX$tE(YMV3IHxfW-bd*vwB`8xVWcS@7ZrK7JjhR{O(o-6d8h_wm27QqpUCKFo8 zxFBinFi9X>hZ;#on0?;ug!~3@=-fdrFDlO}z_WdLi!?kSpEQ@?t|xd`g8gbDpTho> z`FaX&tUMnl-iD9~p{c(gE?F7{J{y2@(%j0cu-qn6P7#}pX4VgaQwXDUILI&&K@ujO z&}VESPQe+k&)}5V@J8H?0SSkrs-T7wgMEbUnYbRe;>>Qi!aTsztT5BfsF0ZZbO-Oq zc7?e&n6)dI)f7zH!8XP<$-o=UA300UUOh!+nAc!2r zRW$4*;2I$Qj1xdU1Ap=xN^ObU1mU3Pkvmvn+<*A7G)DT$Q<;%JQ+|r%R*+cFY$vVJ zrV?`HZZf-vYzx}hnv6E&3!E2o`oj4E=ZqZt5>-c02;|t)u`d|oOL9B;pES=g43&ls zo56}YPe43>RF+_-@))#jtZFRdrTzt{8J#2Mc3n~;>K zb2)^D-HlqQSlfa9SZ))Z0bM`MjlqU*SOSr+P!ge2x=mG&ARTrxm<3hRvxqKd9b-EJ z8n6@$Vk*ot9P_oF3Trt3AngQX(8sSSAjy7gH?n;B71h9MD%_0(RhTtF5sQLi6D)_yP{V-e_I& zVil$~m`IQ{5o;Jq=vrVfQ9?CI#J#4hP2{<@7f$3@=_+Vdj6xLV9zY5F79Qf)C=*9G zfp+=y1`!G^vqmUvr1L1kiLhA(N|@TW@?za2OgJ;lb3;73&LH9*iR>zqN<`GZox^sX z14h(;@UwzuR$C7FGy@f15s)}U+zW(`nO-i-c@TWc5M{=>21LQmzAymj37(cRuKh2(igP)3%mW6bo{U3+ zQT)4@4H9B!@E%0YV_KH#do7W#0rG)Z(8nNF+UnMLMyX*|$(K$fAz>!VIVFf$Sr=<- zyby1eh#SN5OLGoDBa}+R6w%tD4t$DSC_iqa9!ON^*P|YY=Wt&L3NnWBJ!87_u)ur= zSknY>NJo}ma36n|`IN0Lvq(5-dCHlD<3atj0)dg`UY<0Z>%hx2Lj~WAmgH)-v#Pn3zW5l`c@~aKCL^bo%WI_v`fP=f z4=zUbOjuk9SB?Dsdut^G4#N!y#q@A=W*&1IuorlrG6qNhPADktR2pf12;dUxAd;F( z8V1sDyDqGKMo@6l$;*z5(wZ^jgSG85Yu1TgdQ1 zKk+_FKrv+za4>7V5by!kfX4Vel0iI5a&abD2@P^2)-hbap0*J8p0ouaGvicYs|r-k zrNrbicO;m5%R?kJjwUbZ-#=!-bVjFloAp z2yL_*qZasVM?3|Jg`Dw_zl=0!eWa7u>IkQw8xO<$GC{-Yy0?B6H7f3v9XEC7T2eg|of&D~_ab3cq zFp*L4@vU5q$=WDu;|1}uw9NAE;jhFYc;$xC*PPRc?7$rmW(7UFaQUT7+K4|ekuDrT z!(jxLae=&>k9$0#Ws93)iodURr#B)g1f!yp5D){}YTd$_{cvQ6cngY1@99f7pgT8l z-;aR--6*PvVu8xdI;{0Cx(2ABk3m6Wx`%d}=7B2QW+K1<$}|XYpoiJiczG7yGDri# zR}{Pj&6u9oAqJMJ#C1hTaS#YzGkIDH_$EV30cBtg(Kqzl4KNgMV~AR-rb7%DN2(7~ z^qLuGDJ$6EbWrN>0`0RnwYfnaJJMzhG9ZBRC~9SvktmRDsHIWmNw9XbLAL0075o6~ zxie2sc8yoOf=bKKFmyWKEsQ0)K!9?v3_+ahX(R&X$m_AcuB_L4gh-`k!Qfo22X@Pc=Co~P< zFJJn?NOyo}3DJwD%U;0b{6Y_bo_2$1I9?hgYQkw<3+XY;gvWznK!O7+r*r;Z!DH&| z09`AUUn@wyABS|*4=wT-?-*xi^u+h$R0VAVGBRsI#Qi96zL$#;Sa=#Q6gH%u_Fe}u zkoqCwpqLKfgeDmJ?BEp*E(Q(=ZSS<4%DNU`D&*FnVnT_iew10}s}nKu8?Vo_|QPsgPoL5(ctSI8QCd@D=oi2%WVv1<`zvcu|cT62x71 zz|WSMc&kUZ36PqU83`s?5n=??)y4o9AlzQ)TQPPH>}Gk^#EsDdPcG!8ep|@pWEymM zK>$cAFxUnK&Ke{je%`EM1x}!?nvte~huPA+(=jAgQ6>UQrA;r!VYvcj528v&?!HCW z)&+VE);WNnrANP?#Fz+Pht&m#X(4{1Y82a>k2IVhEc zi$pAm@+4nT;>O_U3xl>rSauR(pucBDC&+{ody(^R*U`q?^@zjFGh==o;3v&f z0^gz7lr~15;3yh^287_1)0qTfSgV}2aLBTSA+W%uT_g<1FKU}iFT_s-=pbp&VQwa- zH&3j)@BU5oTbtVI(i@RXf`NbpW1c`!a0OtP$b)MC@q_d6zBrvtWV!`+?~&G8S&PUJ z*V`J}C48I0G6+tfK{cRN!9;PdTY&vC@dfD^mxy6;Eaw(t+i3QH{G-xj*09)NI=9ls z@-hc+>?h)A!>ZG;$k1`=8)42pgZ#|jk=7sd!7?y_X9e_;0z#qJ;_Qp9Z(cmg;RPxD zGq2N7OlD{a%$S$E$PsGAM8t=W+59*PSr#2b)LCGyYsVUeu=x#uqkb{37~h0}Gq@H& zD0qoN(9R2~aE%&Xc=6GXjg6qhkOFCI>V`mCXBc<{b%&)+p-K4w8jQqIV<1kDEP@+` zp)FCMW?ICWOc70+RcRh0i7!clbOX#Iq5_xFN4>FXejtu3+<>cF1*RSWzx0+ z_hx7T=8~0|gb^4ke6rx@SANjAhJE9j_6??gCk#x5jtlS-Q5)DCAQ;7_NQ*K);5SM% ziC&IxTrKK%6m$R$V5MnlNeW(jcvFEr)@wlwSZr3oX}m{dG6vIONTc}f3dDijF%2*r zUpN{5l@fqrT@7-A%kT(nRo7VYMPIv}|IY17$|&`j58D&uVGxvQG6Nhq33%B|OCKGl zUHDE1PJmN%dPFtMA0QrAt@Cffv)h9A&EE+1PYe2_orfUr)Kqdue|vWR6YdSe%bgFyVhtfi`0{yupFP8w*}*xcb6|%c*~X zjQC6e{mB&5X9_4zrWgPKrr`Xl=+=GWP4$nrmZ#qoM(j&VgsBhZ{%a|XZwUMA8#mse z5K5un!rKHNrsj%GjqiyO6q@3ogghToyZ(sd7Y5=v{E;?19E8BY+p?$j=BBmzCK1sG zkO+cNL9P>1HHEXGbOXrNF_aEn7)oP5a>Zue4wv<40wM*70E_}klL@1@2-on#Xnag$ z;sYJ$LE#H!CX614rNAMnFQmAtpO2xz4seB$2o?K0KLU1y0(O85$e2N_gBaTOsfPFK zzIq^+3-zFky2jP8Bfu|^NNnr++hzl0>6^U3;%S^OKv^SCTmySo4l}2oTsr~oSN+r4 zGIeBOrI@GwvasCTc0?s_pJ;A9qL$n~I$nPm-!yu_KyE#(UcbGxZzBS0U^1H*oIF!* z^P9Yv;3dT(oL080?{5E8w5yNq*h46*7R|{*!Mnx7>XStt`~S44@|FM?l|(lAkP^bI zl3N3)&;|&owYJu?Rze_jKkdgVO(D5%rS$Q`?NE~&J+LRbkBY|Kwk+=7o1kfe49N}k z9VH%$Y25KppDrGMYX%c$YqO|894|lxK?qQZoJR#3rBM;*QL%kZWDl&Nj|!s#pt2%0 zJFZdTB?+jgk5Wm{>0eXt;d!8Dp^2`NC9P;2U$QGYOhZ*$O~cb<1zgLyz(H+u9%6001kvB?egc+-pF>fd|6rbfuP5b zGy&%95)zx8)LIB=E2{GQJ_LR4ZS>NB%XZc{F1JBQWWAcVe2m()yoY;$&AoVVjHPS} z8i(0vR-Y{|yz&E9TZ4Xpc%tl+#V40n^i46_)IBQ3PIV@$H8nxmY_ZwDE1^s8) zU=Q&)qZ?}Hio&Vwtg_AiP4A$=9yaPaHX7Jq4@c@dHny=*-;%R3Ceo)_KNZYkDNh8A zvuu15G+1V*onzxnFv2oBZ4(LI9*nTePHW&u9UdCb?6mR3t>X}xtGNQ>gO3dn=;Bif zxLt?~2DsYAcuFSO*R1BO>K7)X1GYKu8xjD%Z$(L^KdN0 z!Lkmd(>X-C4kUPdx$t}Rpx*Nb#U{&6tn7(bFtFu#&!;h@1KPj35`zEr>am>zft|wC z2b5C{txbqk>Z;nxo`(b#*+&U!STBn!5LVZ%?W;bjjg@9B3#8wLzhZ*3GiM}lpl~%b@7^^%C;Vpw3cRW96B?i5`4rTOLv#t_ z#F}0i3yAL?vW(IuLQmBA1S~^r$r^kLsN5MoBS9G27DK;7vyq6_WY7n^hz#RGD_p}_ z348w_zVy1}&PQSD58Zi*xLN(TJ7?q`!#j1*1;0LM8M-cJip6T&-3##P*mHMZ7pf-PO{trpAG4fhVDIeXyVO}@#~ z7@?#4hN+R|0Lt?(+~DSGaRtu$&ZMcbeM2b_dwauZu~PkI!;N{!0OETmxIBz*SM$MjhQF(uU;>ByjYZsd?0uJ-t77WrjorK~j(kZIhyo=Ep}2Mz z&MMV(U(dW?WGc)*)_-*0pl*i*$gJ_xI+x*E{Qe%pE|?5v>Iv-BH7sM@7ku)N5aah3 ztCjbU>b31nv@)NKeF$)ndYzFXG>|yZ zvXk{^!P(y3I58kQ6+3gRrcK>PX2|36nyh{0MJHq2O{v!b0K=TJ>3J4>$bwCS^YxJf zHseS*a?_@MvofCCl)&#|PYxDn_X(RJGOipk7-dd(;V|_uFa8Ay4zytNOtbE=nzy-k zOzU2n2WVCGo6Cl`zr}f$_HU5oTKHZ%P#{WgjhI0zq`p5EWgY)NZ+^c-yEZ;3UqmDgvsrdXq86W)cVtnD(`0yyPL4E%4+aj$V ze&h~3|MJLmk#4ElDnx9pi}xOQ4Up@sRkI$QTyQ{B%N6;CPA!=7HTC?XFNzP;tjAu< zIldccA$9y(OIVE(;<&2X*1hETD^)U9%OD89OTis!m0#2%8?G9*6=N2jqdLw)B#S}4 z%tH=o3iK;96L)#OjDvU~d-Vz>PJcReT>W<2jp7S6=JB!fK7WezgQ+pbh@ScaP`GxG zp?mx2roC2n#z*hEr?2jbcIfeRPrO4?%l)Frhe!qR(1^D&TptJau>1Id#2I-!T_zl!O9q}8W zdouX7JpIHpA+A(kHTI2fMBK|4@5o>NSQR|^>_s1kzVoF1IJCO#Q|4M+gdnt7%vXTK z%*=x1{Nxm-j8y<>S^AVC@Co>aryqm|%iX~%3+^zg>XwN5?`uodq#fVl0zT%M3OrXm zGYijmpBW$C*8+ifPFc_P3m^8YL!Pb1*Ysg#8} z6;0~5&-E>7f)3&xp0*==J%U@A%d4H`G^w6Vk7AeoP3!RNzw;h-aOWuTs5-OrGJL6B z`utURu6w>edYYd9XLZ?2qr}5%=}Qx%3bKTET1p*yX_?rj`n`M^md|~;XZ7O(HX6dt zU!dgyr?B(5aAT`W@J5n05u4(kuCIl*vz>(DZQau^Uh%Mz5KKAS)!Q%M0C-*Zt-HWxVhJq2GAyLosxkGIsmVhNmn-Tq1}Yu;a; z7kpLQ&KELs)gSjS7f-00UY&vG-dAVi`IlGch$mX6G)IM5->j~9Z8-Wizjku;wWL`M za=*wo>WmPJuz&qzBF}Id^=nQ)b?)~UX!D|O+SMPMZ{Bq&QP?#T&vUyj!P9$TD4tip zP_H)sbU^=YBEhf)GKPdYUp`w7x*#(6Ll@E;x=a1~r-RZsJKk9_dE{qyp#7$QqxNi& zc#8f-B$)zP1!F+;$m$aKw^*zT+&I9K+aqNiyi{autPlH42V6ux1UvCN4>SM~7P(Qx z95M=_Dq4@jjINHUlP}x|%t!5>gy+4x+X}wg4V`=vZ!Gtas@Nu`KHpP@ADW!obCl1*Rb|2_rh z(1WM{&fvwLQ0pJQ#9m6O9j_Ogj|p}3^}m^#&tL=_p5?8OErFF@mG*w$zrwS&7$6~`zztwNiwY0NgzKq zt~;one{(IIc=21wo)J5jy{;jf%!-g42K^rh`aQrQdzyCMm zdD-#qZ%g@tpw5w%_=T^-r1(I-LkaVznYMIkyd!c4WI*9(fuefOxmO>SPcJevf|^W$u*`S)MOJ9?vI^NXWknjPN;tG&lbehwT>uMJ+TfKPsveio$uU@@)*&S~GiwF9<3s&8+X6eFZwb?!T z0Pb)#@{<9BEhwMOrKmyt4FeS_=gjuqf5h+?H99l3MnnrP zuvC#^MFvHMCMqgutklvL-=(cu?DO4n-=+Q0*8Xbmv%kml^1HtOxAr+9(B69q*=zpR zx4!jF>su@T`L-PA2*hW&@&oIM7($;PA(dWMB=K5 zTII!QDsm+$Qbp$1s;aiccOw9e(2u?`FP@6g&+!t8RH=$4BeA5%RLpAs0X5-8-OgsP z+pqTE+igxV=jb*`<5Qxs#q$^6vFvWwiGO?kimCJOT<%=zFIl+yrn^}2O$(PSTyfK? z5zYbgu-j|g2<2#%?7iZmvWu?1Yx&ye)h@ML?O~0ttApwd)v9j0{ad#zyyMQLx9wDK zsn^t-Dy52x-&yDFH@|hqt0|_!yFtw~cX@wPH=D(g5f`7_+~x!-aQ$1SrIl{p(&p%o zogkvC>h&3#^hs*`1x_-ERJ-MwqQLWKnqNd})uEO|G^x}fQx+SjzG#^g+oset=F_<5 z+dDB|UDMK(SmUaDT2%78D%V@79J6?HBe}p0WJ0f@9#~JPsCJUc-+HPj*T>PHI}O=r zUnhtLs@5gZk$O5uwelUmM-Z)c6PY-TWKx>?YB!RNRn)qDsf*XT0ZFXZtz>{x>y}GN zJ2SG0S~ph~_VB|Vsjx?a5{(hBj&gN9pGdvm)2$1C1blTir5pLTdPamk)$Yhl4AjXC z&Vom^J3L$Ls|;}Tn0nUb1@WNRub_CusSjd4m{hwnv&kSwdReweKeZ8x#(iH%Jueeu z;@Gsb2b!m7sE4{SebVOAt=?$l>vLBj7v{v`Qgndqu) zktFl{-jZA%6ln$FX2>FQweF%I7DPfsi~TB)3sMmQ#ga25pF8z^of0al-3d&aWc0Iw zY>`ElO-L}F#=0Qlm#0*rqfN(vtewi1rUuFU#Hj#o5EN48`A%1NeF0WX2A*zmp%o1g zo$#lal9CaFUQiY_shE*+=~L})aKR$t&vcS{M=N;r(kDrLZ5m1fwY6nKNs-EJhi&yn zw?69y%KyFsNl$;#Lgxy-Q9AFi6S{Un&wp02b#ArW?8&F6*Z7qOrv*4j_2vR_U zXb4snz#b35N~is<2gS?{rDHGNUYI8#EZsg&!p@WMSx<*~T3@xx_(|#2DrytyBg5)YFVj@oneps zYICgXxXNc(9|Dqf_5B5orJU-CXwNj+ZcVY%(znL!>voBH$o#3>R5hbzc=w)4Z85XU zA1oaO|E6_hMSakTy^i}2n13l>qV6}B_n50Tn;ku>Rg?K?kBhr~2L3wGpS~u`V0S*1 z_0*ZpJ^w_v|LFM`fMP+)0xE@jqX61gv4FwR%1hOQ=JLwv@ds6^DCuu8&s5%&Yki&J zU4ZkM0IVakjf@_yj}<2PkRE!J-f%EKA#$&ix_z7!Q~F2&%vl*-t#?Wp^iX7ts-ACO z`IO22+z%5vp;Xtp=H6aole2^|U`OAdVczN0Lv1ym^cs+w0kKCT!&VjSbeD~ zxrUXACK7-VzozBMnio|!sf-8-^GyKvYaFv}z+f|AP^a`Kjcr0J^jyTh#%v!nO${=q z2JKVdG@AxjW$QDEWRO5D?ZqZ(HO_I^ihVDWAFGd&Fyxnm|EbH9Dw9Pc6$eT9FUEPOdT5Z_@-3%?J3_f`3!E5Z?%7uxo_x4jLinSvptq(yEg!!a}Y3g{;+k7 zJw5DKYOpyp{3o$=P@)qJ=8+M*F)$-8dRqM5o7nTnBPM-Gur3xY&m33Uz1rQ-5zN|LMmp^{vdNlZhd%OIRL6!92?a` z4Ku$V^^m&QEUxWN&4ab4)JRi5`s(;7dMEr^^Zw}T)Myioxm%4fPmQTjW6hB<^VB%g zYwS2R-pm_&3q4DNQhK(HtxZipahKOKb%{UGxZ?)XaoD(_0Qly({^}C*Am2%5-?-j$ z+65>SV2Jqlf>wIIpp|wMv?j3*-Sk%3p2?>B_-tjqrrFlOh0tnbP@X67)#JB!e-t&3 zw2X5wBIN2Cu!zck)D%yct2UVHCmb&?&$|iJaTA2MRo+1`H51=KQT}0Kzp^&~&nQ2Q zsJP-xDE++2UNTDbA1+EF8$YN2D5L-4=)dKcyxRNIR~R`;Z{9B$gP*4>>tPe10dI<# zKIsZI)oh)#LQOO2x*nYmh7(PPSM3J+)6AHeq3YN+u6M^-i2TNU0ai!YM1ej)G6(+C}xkJh~Zu z<@*!N|Cw^7+Gl1=)#@el$kYMqWpiNa3H6FuKW#iU?@zmnaYaGr{J2s(u9$IOpFWqq zKbSs{z8_2<%^cAgYjXQks#xU2@3^JyTmPRb-wU<<-}6XuM}W8~9eL!OGVT%@i#)gu zR=1ljGsi=YxihO0*JmB1OYb$SW)2y6Zj>F=!_rhN=mibQspf;3S(P(?ojD9i$jln3 zCYsA;Wn9mK^!clNa`w#1*Pep)1_gFKrFK19tN)fM%dWiyM#j-fWSw)2OgV)=pvydF*Qyxfx>1 z8ZuZKMr9l^iOEO%xn((TE$;epw;`)=*OfK0jy@qGlkjJ|b>fONGIS|AQ|U8!!&EdW zz2OOZ#QoV)T%B*OcGqNz>4nd}86QT{4U*!L%I55pA<=Niodt+H@F4-Q!oN(CJuXs! z8%}no%L##WhntDGb#9$k2bB_V4**Yth8x7|@ub=5qXxK<5bgrtiX+azMHpozPBnD? zA}4)fVV|3AAJI4Y$#2KMFGjuFS;YqxL}=ci>izN%2egE>FXCR*XI$%1M3D6e6ANii zFpZ->?6AUlb{5V*c9tA4Z{*}-rx3)#Nswb|cDUInho3g)47^(RkfhOC_n_ba4O?wl z4nTMo`}3Xl9kYIwC`{sl+s(#J$B}->-S`ioV!(J6QQ#-9U8iRCKU%cCRdq= zq}!CxTQ}l`vxm?hA>vq~bQ_XnY0~6onMC|<4ko0H_}_V|;8)X&Ex{!j5@LT$K~qaZ zw#v(KQYrQWEMeg!p^^?i`b*iEo_e#eD!N=BlXAH}MPkc)g{0;B2#GCkmW+6pCxt~T z^jQ*X1eB=pz|o(`fMkJnFVFK?P8KFeUllT1lWV!-C`0UuvaU|~bHEo;9%XX^RpZSZ z?Y5B-0zj{?QRN;{nQqdkkE(j1HTiCngGWXS`8{z4trE9{_vZ>^29!#gJE}6BCDW!d zT_lBgUY>ip#&lSp)4sm0G)D4k>snYFI}IXY*xJ{ocQ7k<83`;v zWO1Xtt?;fmSeBIgg;vbCCnp)0(!X4sJ`jW55rHx-z;v$#sK{+=U7fAi!u?N%7}E~sZUS44C1%mzaoHj zL?u{u{*&xgPu-knUs>r^l_?8yN)NC1FA2#fwZpS%?f74psexUOM@%I;51VCmUQuW> zGOQ{S4eZ3S_aJ2tg;s`xq~Kwvhbpoa-Z=?(KnYS)od7_NG=iW@EekI6gXJt#H%N+R zwV>>2*MY)eyoCKp$hihyXx@!_@47n^51XtNu`ZQcL;R-eZbB)x7qOJVvNu?$P-(_9 zxkyk98bAUCPY|;}iBln#Ap;42=Ci_pO(I-q!$wIFy+(Jm*QTIAkr;TL#9rp9Nw9?+ z^6gJzZJtmGkJZT*g>1+SI& zMM(}zw3P^Lf;f%PN?HQ;uR*7lWIWi;lMP~B>{3K5U!Jx=ZD3TM31v_j=NfnwyVMDC zr|saJD;r{6BWGF~gUVWf(2y+_sm|h-Y*bmtwqw+0r!mw)yg`5UmA5o&gFzcqHfEa> zv(pg>o(Pf+*~oad0yxop8mT98m9+dUA*0kHeO3TOBXv0#ouJm)Vosxs2E9e~IQTw( zyKqT;HUTtx5OE@K=#LV%@@paA9_&#BWQ*+xz82ZM5-W!9MteK3JJdrG=uj+1xxU7A zM!Vwd!$1z?Naav0qAEmTNfp~r!@$v&oG2mmikN#gyc`_d71<~QAQ>lo{ zJXz$f4g6bWOo?`aG|s_EDFEqHNda_%R21Hawk*y8h2!~7*k0R7ESYpVVq>;s+N1)n z=cPJyK*IpI9|peRD(897{oV&4m0I~I1%bJ~MZxR!HoXMbMMB`6I!7vzwgYTQVc zZxP9Q6%w;%q0pF`jmkz{t&Br?oIXNS!s95oI7%RhQ9!mEkZlY??hpn)B{6D-dq`Kp zx*y1a45q7$jUjna#fQaxbO3bSKoj(jc5>8CaRup=w!_=FYi8%nZ z^hW=3lDtk3$JukIp|;8H!L^kXg&|R;B&T3cbe7BFMJPl)0mZUhn~RnblpsAm?;$Bd z4y45+l1zY%qVyHHa;k-Q9DPu=>k0gSX(cL%r^8;-MHERwI{33fy&`pxpWvA+Ps5{v zls+V<5eY_FR{C7F90oa^PtWGlFHp_yTD^x;$Ph=)BRj~|cuuCf-byY)h+m4G(8uxhfJ2`^1zksb8(j+;IQs;Qyn@tm}{HTh|{|aGvW=yLJ6V&x=w~4RWA>ze9&DcVpJ4#TRcNB1r^YNy2(z zsz~4tQOkkB@^nEGIDc@Zlz^srcCT?0 zqQ*H_)Q2a=HY=xM#>M%P?>3bSt-dH;D1srN_AJaO=@$y+g1-|ve=lo1;L3P=m~#AY zgpfd+F>D*6E1I$mfO7D%yT&e@cWU$Uig`csZ+qFjqt|(R2b3MIwTc)QyCLDGh58ti z!gk3g;o}!s?xB~}5SfUGs2~KQmeH+hmd#B`q`}3c#{Fo^uKHM5Hh=m|vhu8_Wn;>8qI?2aAWaS4Hey1l&Qt)I) zDF6*#!kWe`uo*$F$ch|Cb)pw;HX%|Xhp>zzA|kg)*$7v1s3R`xpg3w~a_+&&jrXr> zYTme`tugmNrYkbwiLh|zN7D6jV0gqAtoQW~`-^@Jm1vOjg0A=M*4`?;0tJoB-yr%Z zsuPmtJvk((0j@%u-4}FaK6|3b(=nxqA+s$1n}Y2Q7dl90kJ}*0PS?K)E|7zue=Syo zDm0jjEkWww!Ia9;EY+W(*z&vzvCiXF>g|qF-;q15!h@p8j3sTJ;puGE5ouwsi71G< z=op7;!gS~XdfzK;)55UfiI=J9p}!!JOC873cVpv*hsT`ZNDP%h~;i9Y3|?IH78DnE9(oCoCL&Lz)?;%!eCdlFXdsX zS&U{R9QF^bg|H<{ravy8RbWc@(!~UAcEN@hqq1by=m`tUIf3<9Js>`F!AnGM;%G(+j;4&2gOH5%GAx7f zc$$DY9|kl924ojclVt+#LQgaFYb{!KD*{J6P1Giw4wpY{FHm3!@iRqhG8+jeZczl{ zX4*YaYyi5)x|uB2x|y;AqU2ywaWjPhHP8u_vcJ*grky{BWoCT~=A2)wq@=xlx zIef=%f%2_8=e9$^D}2kmduLYae|P8TTl2uOI92*g2q~l7I(<416%_(;yj3f*-cs1Z z)*~K~Wuci~y;O(zc_Ev-di7G=Z(AoCMP^&ObU<$bEze$fHX}J+zBs8U7wHuEnD>@m zk}F`smJ)>zrx)9G9x51UP>G*Ig$T7pMKN(6l~6LFFsMZGsK73u0wEz>hPNb8UStcZ zT1n#xk7-;sB4yDOcd05_agf5%6>H)J%xABxQ!Qr0%88}z z_;ZPl+56(K`OV6UxNAwT>ea3BZM$VS00O&-65H8i>Q;42pAA=3cd4|-KdCp8)n8Q!qz_q=s0Xe$Bqz>+{*0VmwG_UPf-GDLlD4;;mBQ!w z7D)@;X6mLxl+NUvByI1vL0TI52slaGyKR<2bG}7V-DrNXYJBul8E)ctRSnrAl{@l1 zTBRU00%(87!XYW_H#gr^HtvXI59T|ybu1i{g4D?Hqa6z$O5vz^@vf^)dUZN>T8bx4 zu)1u<=aQAOjA;_aJCiR+s-;<4Hb_e&A1O%M-fgoKn$0b%%Uag0j;Qe`rF)t zt1{gVwgViorTPSPwILt}#Z0^+!+?azsOx$ypyXPme0GymVS*%SIsvg*4@I(MYHbO#J5rIn1$=jzpcBJs)-NGqxs1P zK2VFy?M({{;?Es{Q2#eggQ0%>!D*_&%z1FIsGNHrYzozq#fXaJLo?zH0F`B4_fSx3 zSp=N15khZj*FV%3#9n!5oLX%D_|T1Mf>T^fizA~y-(s$O_zGH|fB5A@1GHmR*P9!f z?`Fb}nkQA}_fxz^aSF4Z_LgdI2G}w4@J9yTw!}u$-S%czR55zKosT41Uw{Hr$sLfG z1b+$q4~gp8Xe$^}#FksZC=AHYbNG=V-C9&heqSvY<@fc8^*t}j0}uPM+qPXQZH`fr z54m4B0K}hI-@|d_iDXx5`u z+5djQX8og;>GQBPEssvVIoz*stH34CfdMw&Y6&b;UUTG!2I9k8vw6W%**h(AVtoPe zb82R#J#5I;8-|uz8ss^KN!Yy28~R^XFzR?abicr;vs2cRq6SpKl!hS@`cj1ug=U?z zBeufC9-Hl+c+0%@c<-c@R^DxLy=B%sRx|SWn^@Y=tz;%b&5?qaVz~wXO12%3mfNg$ zAK;V6eiXB&ZKHYi@h*ALYNPq+@uO(Q?cbRn7xeK4O!3CK1^e4_^TslzcAJ%(n)&|6 zrs2gKo@Do}ESfP-jEkSy!kE8pF?T;P$^CMRdGm=O?%6Ho4^LF_il@k2RC5O>;;8;o zdbz$7Bw$0_D^ltb9uwK+;=tEmZ85i-D~7GW-PCYqAVf#1SBkIFp}dL%wk=y>d)}Nf z6T7eI(EVI?l^L^nQPr`nb>kgDfJ1Ja)9)!~QFtT4lZ#=_G4tN$tJE=5zNN4Fz)alI zv+Ld17I_f?m#@{udh>3$cC}eV`;%tVmfjVcpOSTm(ADdBy)NQz#lIT~WQ^^#OL}$v zhrLYp$%Q=a*zn|7b&q-f$+y(KX5Dwc&G(Prou%$=8M#%dK0f6zSny}i6itb6J}(Vkr(M^N>ZmM@yISa=IXybM@qEfKJ&Nj6VJo0zXjZmU5zbv#BPYK z>4@E6ThkG{LAIuVUCXBL&s6Gi^P3%gxPOq_AkN_aI#cxQ_6y$qv_3o8zWd3T@*nh7 z|HDlDK_w&b#6=AKbuIIM;42>Xt$OZJEStM?G~efTp5aKn?fF{1zk2>MzQL|bVjoIS z^n2#^UHxOH!tBOf(|8y8@vi<|np%V(31<{>Q}Bhft~MpRN2{mIjNSiNJ#8lK=~MAE zjKXQxDgs_<%Q1_KCOFoA+T6Ei6SI`;UCp<7@4a2RpvOg)X}nlBqLt#SjQhek^_ZFX z!o>*i(ig7e`{Ne|Q1j;(erMizb|3H9H$UEYG0XaY z`zoicS6F29LkbO%h5=$i#pLpeOtGj2XvmMUhw6y`9YuwNGqP-k-4V`dPQCvfbNx%# z0k8F?m3#-jd@0|RFV9J>SJ~2v`fL|sO68{U}X*xdcY)A_e%JA^CIZ2Rv}+sxVhm-3x)U=H6c2j;46Ex$XEQ11Q~^Oqlvq^@tv zr|Jr$S~KOFZ0v#?FaFjcJivFhy7!5vvzpAT2yB7S*T zlb`UmB?`iGk-eNmZPX^Td%$Ar2jz=m-$o7zpriZe`*i^ z(l*#V@v7Ki{ozKYZ1kd3P1Y^o}1@kN;q+*bIrf63bz0l5kf<-M!>z*Be|2|``CfgM865LjG+Kqap<-#(HpLI3M! zT&)C#A$M_tT%;ah}r| zhHpAVl8}*n_~x9`$(6CF3=Tv%oht8i!UW=UHXWVUCG;+%ZXGOwZ|O+@=`Tmi3ofDQ z^wW>#3+6hHK94Zu*4}aJAh$C!O-3IBKQ>uNDp{*s?dl3k42Kj}$-$I?t}Vg{792r} znLit)_M5doo1?ayV?P@-S!falVJuKb922>7kr2Na682Ufg+lp*J0XnrLQ-s<-f71E z{2N{OP#cCPB}OT^h<~ej@#iB{tNC9)?=|2=`zVGSZ!g#v-P#U@FpS6?o1Xm1m|}@0 zv^mM2w=6rhEp9r0xG{IzZ9^9<7`kBS@>_>4zI|ZAcRa^&hUC-Xq%P%)^5s$abm5ZQ z7OlK}zB4lJJAG*X8^1mHEtJoYujo)VP~kz zzxsHRYHaE9>x)(F&)_i^e{J{h?@En-Q)YVQuRQ2OY?~xN%Qcjd6U9FptN5{ zALI3YGT{m_9Z0GlI6zsE*=sY((MJjyG0w?n66Uw3hMS7t)fFvXczbY3uxS1g)9+ND KmK%N-kNkgv{)>kI From 9db6c356e711d0bdd8417bd259651f23844d0a36 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 15:58:02 +0200 Subject: [PATCH 827/927] packaging/nix-make: Update scanner.wasm The scanner now indexes every file under the roots and parses #includes lazily, so it no longer has a hard-coded list of includable file extensions. No change to the resulting derivations. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/scanner.wasm | Bin 59746 -> 59909 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/packaging/nix-make/scanner.wasm b/packaging/nix-make/scanner.wasm index 43290f9c212ae6c44f8985205749d3f51b495efb..9411db96f02326de5f124c17700815821b623d8b 100644 GIT binary patch delta 19160 zcmaic3w%|@x$l~_Uu*BRce3&T5)#N-D*<*Oyc84OW{~$osG#EO0165_5eY^_t8P?O zRJ7or1xuA$6jW3&sHjmxMT<&%s6`ulY>&MMd$FZ0x3#yn$9p{g|E#@3Ku<6HSo5Cw z=6laKv-w}|hweBMN*arNCA?_0;8#%^T2n%7vqJ}Vyh-nHD6^3afO#TUl5@w>% zF~cLVTsRboVC6_GVuzDO5rK&W7mLSj%Ls{RD4G;lMGrKway%3oYfN{RFI&Cfdf;WjvgONK7pxj?yr}Ln`-~ng z3@PlsD~FU0nRQ*u%}FSzRJ<<~5FQgnz{#Tz0i5{dt( zLNBO)H^+!ZRS~*GOj6f}{wn@OT^$~N=Jp3W3|F|Ovv5*M$U`W}4-Gdgt1_~kh3yY@ z7|xZ3?S|84nHP6M&Ls7-aJ|^yW<_iv_N&t9K=a^+w#Ml9gqWv3j!As`T1&*dwzbwx zrnsR^*xwb|zCvWNOlWpE>zXcC3^ipi*&`w%4clHD60vNxA=ii|KT>135m(fkNjD;$ z3_3?-G9!g!IOT36ZI*gbv?Eg{(UdkFKUPt14ntkE-mF6st2fgaW7L~9EJv=8#XDSJyfx#CTGMF2k`No618|TDG+G(o$kzaARwgJ&jj*s= z(g0Baq!a~=rVxbPV)az7GqMS623+gKO>8S|*7=5vOam6fUIY~9#$CgY$VgKGlpwGP zzHRcT8IEEQ2Or~L8P(mT@eNH8FQT`b_H5^2;X@GX&D{dnGlz;-vZkK`hi0z@ACD2Q zn3!;#$oK`2qKMwImD@}RYq}P<6e3&q$U;xQc4Cpo>FMI|b3!(4782f!B)s-#;`8^T&#HC&4&8$v4p8$e%-8oS{RdC>%)kq4dcp`PHhLvct$?*l3EOdOb#QdAP?;oPnn zlw{JGaMs5tG88%+&?}JbrYWhIYk)Q;lz=@A=HT#)kU>{=yhN#m6nCKPj3qO$Kg2ay z5CxJU>J~Sc`#>XV)V-J)^<-WXlnl1YZDKVLyLLfn4zrYW6XXCYu>}b77_pD3-a^ub zGz>+BhY_57DiX}CRaui8nUVtBUML51fwX695j{=@RWJ4)oK1i|O>i^t7y<<3*o2ID zrcJy>bNE84`7&U@!Y;78F<>;ego7QCV{;61k}bi9vxz*-mPI*ySlLtflGVVOt@+Y% z?Izy=JCiRX;7bD}CtsotWw0gCq_LZH3%heiz>rZ*CCwxe48-D=3(?Uyf$;!J{Ybbm zi}~B3w@C;fS#}^F`7&Y9nv*6lG@38uND!$T69OOPa=u_B+txt2YeXW8YT#A?HV9x)Qf>KWTmgqlZ8K zvM!#5gw~p+4FRW}REzCeGtfEQbC-Untb0aqU;dxNXx8f4@! zxjWD9`_Mz4B~P#;*&lM!eH@EPxide_CmfxY>vFw;mL-As*K`wq8xjY>h4IbQ<7+^n zP^WI7p|%?YEskpd^hqf|POOVST}%$jNze(^jx!bWK)R&kOjK|5DTg)vsLuc~QHj3& z%RS5iIyq9bXnCl_`f!^ZUp4maSy7GUfJRN$+K{CUgiOPhI5n!J?^?|9fBX6(p~|a9 z_q6@6>@esK3I_d~u>pW}*#?)!@ zL$IYG8K${1Bz=lTItP{qV+Iu_#bnHnT+#z+;axuBj;)cl{hL8%SvE|2ybbm^fjJU1 zV8f6A{5m(G_75r@2}T_0_a;A%^(&qJp6A9eHQlQiJUOov!-qBb$r^(Ojepsm^n0pQ zUFE>Ij5WK((wbJ`mr1c|7QC+{xCKPDVDwRy@#&vgZd}c+8<)M*O_CxOX@i^tMn;nK zQ&=qpZ*Qq**{%h(c{lueDMlL*G=p;guOK#G+GHgn9Ak#P$p~;_J@A z%yBH?#ntk`L$dLV-^00Ics}5SwSnvG7rDOk9pM*eTz`iwU*(qoT(*?L92F6~rj`sP zN7obH)@C54$uA}X4Yx2<)aZey_oX0rNivJO$n$w!9+1l8w8?ZYiqb zpoj17@GDyU@+OpvU6=GIam$;ob&FfTlzL=I#xKWAZn;}QDpin5J=_wI2c0smF84v1 z^Ot~?C`-MN2K}&E-7i;xVd}?2-8t*%9UAxxw3oR(K>#Zmw%uOA8bzm$by-fbw;qv% zHv?5g{DvVN-i{V8r5c8%MM^CkHX_~I1w~!=#vL-W3TSZSEd&kDnAJqKf$zoC8KByS z^_x)$Ih6*uH6?nVGR7;gUDx#{h&ykeV4Meu61G?9+MtCG!@^m;l;$T`y-b|}LaVyr zeX|7_pvr^oVKw>sE2G7?xt$o{wLm!0!rcbtq*lU7_9mX<0mJM5tjaMw-2gWs$I_rs zSzB(R)*J!f1(c_c@Ao91AP1MKw^!iy)N<(1u!9X9ZK&h0`JOGaUZL3tCa|fW;$9)W zeK@U_2N*COY5}Ia$+O);K!y!l@bNth>wphk2n7F4t8n%U$SAU4n+jmom`<%RwMk~b z5-kNL;ubb_NikI4ns}2R@22}4%uJ6AU$~tBK4)A2P9aKIIW7;;35KFxL#2eZZOp#L za-u#QSBs_~E*l|r(07mOLq=S70zno|Wm)uv>-#YzAZ>nvA+AOCkIRE_590C@=!O7= zki}ZF(`pDc>H!lyha;u}UGRA8%_B%BQ15lXTRg-{z(gr?RskOLoC%u&oFg}fAmfOw zhi2jbu7LDGyEF$gEky!RSd1i5Z!QelE)Ck|2c>yPV1((Kpvz1ge|msI`Kk5Z^!iQ_ zOaiJqp*bjI$U@&^FfyN5egT}LEO^7tfS3hPYFZ(|vBJE7-Z`87M*2yf@R1v9YlyL8 zC-;W~f<_~bKSXR#b{Xc|!7?LsSJ(sHHJ!oqV4I**9hOAjw4bc-tSknDS`}UrUusa8 zlX?Ibq#RAukUSWbg#U3ASt9%x)6YRg4TQ51Bh%)W@J^V1ZuzAZ;A>>Ru)H#+M=gTH zeS1Zkp^Kma?a5gzNxkY@6v-0sB!WF9^6VC1&Xy+XA)bCohu=#!o{PPe$f_2isaL*B z=o`8ec6GrV;h+Pxe6)3?-SvsMCARZGR-_Eg4%m^Mz@Hy=u#k)x^=}gwid|Z zR!Hv&qlFzq?3mR(SK3&{pB&Cg1EDQB7BLz*-UD515D_Ymh6=z4`9^kTDhdR#5-w7X z4Zsd$i!ROq$K*NZIO-`*C)m!j2R#t7Q;v7CPz1j16mi3!W5FquQL-8PV4qYa1~^*b~GVTD_j@r$W}0OVk!1@yxS zMUO!cFD{Rqs=^LIg%LSvFb}-#ETe14o~l@3>TA)9hh6PV0Ag)N;(_kfs)yd*%s^ux zAfN*o0YG}F?=?_A=}s`X!MG_YSnNQz`~Wh9+2RaiB|}}Yw#_+590&)6@oydRKMjb= z50h&dW9uU~F~&yc6h;893=j~oh)B;Dab z6lOddivwcVMBhT-i;RHj<(k{pT9^>L2XfI_fmlBG>-N8!*EYEYD}&}#;Krawz9qG? zMP}FBdfVF7Em9|wQv$35MKmK7!Yc4+A&qVYh)ZdM`tJgWM?xDCuXoW^3-m%^FJ!$5X5U{zv)5=6o4zU=qYUAb^@l!!R#EyX6-mM4iw^ z6>~L%vyrTDYMCr0eB@)dr>l=ubKr5Iex5Us$xKh5t+dvT%ffx7)XsHK!unDgYAfF^dk{Z?v{+K zA(9~m^^aVZdDwP8NfiQ zHH|aKY;>dKL9Pk*P7=|m2ty7q(^RN*BygaB^kf!;LG$ELagXq|%b9}9X=`~%1umzp zT~18y(d3{zjv@gzsn0zyY~JN`K(A7QQbe>2rNzqmmRwQmJFe@A2GewsGOpWTE(FS8dM-tR_|kkLu-2SNeV3rKG7~E$b=Xty zDv0nUc4D>M|Xt7CSZ#Lu{yreca;H(D>!&4g$4zipx zSgmJBa0Ud+5^hmF@Prf?fNj&fw2mpv(PpM}%InQd%ogY@e2ze@5)2syOy-vI?k(+Yl(6Q|5*)n}UgP z4$?$RX$J0-CpI<;pe-3e;2*TM$ifI>=DbBBVjvA0HcdMGfHK1D&=vIX98CXn{*R_M*#AuOQ&Y&v+v2r;4c-<3#tusDtQU|6?|=bwaI3bIjMBp~k2Xf8*$J<7dz(I}^*C6yFs%Z`D1 zA!%U+pvK?^6=PY5F?v|CTg)_n6Cl4HBQ5IQ5Bj5fjI@(ZbWgdD25r0nZil^q5MFoh^Q- zE*^WhtSoo=45!$|Q5{jn;M!RY@83`~4PYQXNua%hENYV+BxVuDIFmKS#h3!MY9X}jZ0P$L`rh;!5hjptNPMD!F9c72nJI4QzkCjH$) zdJKZL%BfTBjpew|4=(r5kTv2Cb)kF-xATP)28)Sp<0cFeVz*j$&UHP8gJol6S(rDQ zS8lH3j8uO)XRg?zrc9LLd3E>10isRqnb;}zs(U7lLCxEfuE#jK$HBNFJuZQ9=S`lC zzK=~Fh`v9W+<-MAQ*O%c5y=EO#_7%pT^#d2hTBiSZTdQECAv>YEBTG#_mUu5pPTN4 zc7s*P)GQK?3ylDf) zC^d7M9}TvnTBlt-=Jfl6(>9b95%3>%3ohAUqV+Xplq7U+QGKUhUUHsY2~L&m!Z`_} znqKFe2Fh*fU#FifvdWn;RSZ)X%@~$Cb*m4}m|FRdhrxe?T#bn$y(cMJrC5oz=Ba!37Pgh^58fIQ4cB)5bUN{~cjb{OtZ=B?4l#{r?H}v6(42OXB`4uFkz9w9*a2@i{MAG<^W!k=cB z&xk<2(Z1D$TaR=bb=D?C&dXTZY@&PM-t$7n3lViL?~%gr`0+Rb^BHnq4koo=PL+tL z?3~{CK0K!e-#6x*mCZdz;aSdfs*)D=mNs_@Ti{*#}ehg=PwMIoyVA;E+RE3WFOxfw=<){Q24y6={5s3MWoNF8WkD9#kgsu^4WgX*)`jrkaP1I$|B36$I3bI5X6f=XnK_QE$kDoT5Na;i;*ZKMiRr7jR=@rtLk$i-BiOI{iG=>6V3vFUMWjq>T93?Vk;I~iY^IrghtFBp*hj5;Vl68uJrIXCFz`j8uR&SeS$Byy5~#|ur^P>i}H zh9w{o44egGg0<;y#(6ryA)20T7)gZwBVL#=+6yZb+-dXO40fvqTv*X>A^->nyJfl+ zQC}C)OK{I}=s?Xr&TMt_g}o=h?SPI;K|^WUg1ezh0PpA4ap$pLseMGlC$h8O0_Q8n_S z)uSSA@eT-0u|CLK&H%`&$!k|3cT!qN7BDK`Ip{3km28JFVDx7f)fJ3MBH$_Hc0320 zjhoZzi-%>SGCU&%73@ZvsR$mL1!Ci%fs%8E`qjm?Da?~0 z%O_w4qN?PQvkUcWjwnWuZm3&yNv}fe8m~k~f&|q3=c&6c=~teFBRGQi#%%zGBZJ@u z4j3u*)+JRD(gaAj#4C(CX8{LLK5#Z|kfr?6ifj}kO`bI%ojjIknJLpRG_(Rf_*{n} z2gnmh^h=1XESJlDxiWYp_lVHdgbS$V;`twi2Od6x$fN>JK}VS4n0Qz}hRdCAO0eiQ zlRQJqh0#y|>g`MWH6-z}^SIU+2v0M`$fHQ9=04G7zV1Zf<6NgsU8(%b>cw1j(Pe`N z)%gyMumnFq2$?&^lg(yRKJlH7oFvLGX)fleJ(u+=r{ErjdOaU$2h}Xn-CEuL=CXgw zUI5<_3_-_)o)H;Tm++t^P*J?@bEA~k0LApqFXXlk;ZYT%V#q?t=5$id0P~U{#Pc{& z5Cs{?@0doPEb(Z^i$Uypp>6sRHEsc3lajgFfs6$x_6l&qK0(IGD>XForjH;7f<*;O z=Mx?tKD@a^F^Dov4O}eNLDKQ01T;&Vcy^`z1G#25u)|kh8ttD8R~=kXnO%oR4ucJP zb8C>pOCPUTe(}1_5q$I=Z2-%GFAEX?GUp=Q*EOE;i}Auvati%U!Y#FNd(HO`T6pCN zI90i~LkRRXgw~mQYPV#9xQS2mgjG! zbdmcnpnMKBN819_$j@^b6GS8>Nky2{kd87Vni$X$Z<9PcgY?2wkU5(AHYo?5=fg+n zhKTBO`G8{GKaHoOG&?W?Sowm>2g1JIdU^GboSolEQG%Vv-NH69_BLq1kaJ0HgPPT_ z;i!85@;Oj*IJ)UA1h!|c56WR$@&`Yb11m*iL$5G^J_<5wM!@8ZKIuq>6ZUYfBRY6a zjXMzl&NFD#fmuF4f*@Md!Spy}NA+G(vi*C zD|{k zYY_A|uSrhTT=Ene;8_vX@||kn)o+N4)bFkyCJw2J<>!clYQgd`c1L>$oC^azrEIg$ZDN`N_iZle!153e*a`&t1kIR2*B6aWfve2k5d2<8j#pYK=9&O10qfW z687{X5Z3UmAW`R$I}S3Z)da+Y5B9)#OrW9>WK(Na4!1RwnxfZ7RZd2t^mO67yOuPCWXW^4v3)8M{pR;f**TPhO zqTJP`*%RIC*S?1OFLIrjN;VU;b{2GQJ^QS|ht*8G2CVDevWW%Ou-k_2g{>^GhAmSz zcQ0*cfi-N}*1fQc1=g@-XZONh7Ivx=t4l|>GrK$2X1Ah zjbhQR?)dgRi0UuDJvJ|Nlj%0gPE~W0FPhbqn;I(i2`Z&vFFalmat)3ASWUmBM!j~E zUEPT-P>X2G%}%B)gmbM`R=~oC)$eag+lPq}N-S3Bd-LcrZVKZ7b76+639bjn9#Jc9 z?lb~V!rxWc6#C%;#AO|_;p@ezcscgu292o zTZG^5soO5JmS{KInXmqQ+jLZoyL~>IAG-Y~BBK_sU6#zD0PhfhlaJO82G(PDOcKk~ ztUKzUU2nYOjz9xzv>#T9JEumMVMc6l{+*tqcMWj0YsFpPeJ5dh?#|JoN&W85OOptI zB32Gt3&p6;yX!nOK7QA;)-vAm835{%bvIzG57wPsk)xrG7s5~QLo(DNLwuHWSPi{< z(B;htcJMEq3AUOvIIv}18^F)J@-~WG1lz66p3@9M0XO({R6qh25t`E!%nt~bX}^0& zkA0#rM@+2~a>V@V?uy|#0CiWAShurM=P*i>R@aj50Q;-Ed#jJ{#@p(9hV?o0GOC8i zFOle@@%=%$U6<9X_e}k#2w{CC~B|j&=-A+bI1riU#`3LRiI+wRnU>J9)wS!Hz~jz$yU>)cOH44hFp; zSX|{auLWe%&H`+T$h$2#CX;D(6$B@67Z^x7s3P~xgu-~?{wiCGCx5yjPpF&jt37k~ zt8^+tWD%@`U$%tvKA2{dP(X3PxL{e{$LYN9&FBdNdP4o+{*qu~aYB7?|3SJGkA7!y zl(1vj_=Y(-@1kwNh8{xvUR}3w9lrmyaaiK;LwVs-BOe$YZAVeQtX4nJC|*}DKQKge zsACVf5T&>}BYia@brJbGyO<6OArx@{4Hua>XmQgBWIFOqb%i=_=t=|vB-qi_Iz%|v z(hKWWZiVBl%UZlXQ%BXfUMstGKbdV+BR4g>8@4o#F}OYsC5WBi7kbS>Fbj7+LyZkv z)PYTNO(f+HR+}5QsPPY$_gD>!!b26f`xcpyojbt2*Q@3SYs4?r#s~XWbn^ZPf(H#J zV;PAjF-^{D2p-1hUcY0n;s2zUVV6RYYJOvam|7C3P#?Q5i9fY%^2$wh_CqsqC$s*c zQQ`*m_Cv3V8`W*!y$avoeRrC;v2Da=A?y|04uWY3XN79qGO_rO#;+@i3|{U)Exe;1 z+VZ6MST#PpCw}}1>;Ykl=i2`9@F-D!{Fz2+x8kQAA%GoTsS~p`Z5~%Mx0Z|JYEAxI z9Q1Sr;-k<$y35mBE0V{bA>Zj+l;i6Ct(S^FsG;8ZH;x9fO}-_l1cD4%x@xeY zYpZlIyh;|f?c1!slVLv7?Q1~d|^A%?Is%k`pCFbpzE(E?t-pX7rR3@Sl4uiu1?oTRoTP45@gF>L`Yq+{>ZKn9x0J3bd#pzMRE>MA5`%Ca zaUrbEt!+ylbA%YGR{ii^*gSK41HOm$jZ@pV{|(2#t&i8^`}40rh62o&How%n}?s&GexSg)q+_?x&-oxQWV;y&;TVw75C zFrtlvE+Viz2i~V{-nkL06h65c-*r!ZJB{DtYRiJgp>hqhS8UbSEn3o06Kc@ZoshFyN9K6x3_y4n}87RzD8_T zU+$ic@02~W@O^O4Y_YlR(><0j545R2JwF0`))`Mpdt)|-ne#Ad-_$1TUBw*G4lbT{0 z8P*>c79NNNjo4x@Q9tcB+T18qynn5VzkIW}Sl#vVFmadK_wxOH^N0Gb03{+U2ml4A z`@7WQR|cptd;4RNj~Ef`W-Cl5n;N z+d(T(f+d}M)blSc!kncqorCZEmpW#C`2<9H3nB(PNEOUx=N<%@Cm#f$NASxstfpTJ z9RKD7F9RMGZD>i%2Cs~fft8j#E}J$_y?$zCWz zr~G&iw?(FoEbmaawog$%daVNgmBg>{ouvNpTK~a=@moKw4YsZ}ygqWw1UwoDo{!@i3>)RU0LB_{)w34}OUXS<=+ngIi$RvL zDw%M;t-@~%!2^La-sRg{()TUe zIB+6toW6PzSRVd?_N(RFYU%zO4P34LETU0)koNB{7w@Wn+g|}!=WqM3r?Fr0qXA+V{u`3vJZioSV5Ai0?fastDZ9U*EY>~fHL*K0Jk+bi*1;2{~-(sM0 z-s}r6;q>-YSNqj(O2ncmO}t7g!<#6EZ0j<(^NRAnVF*i?KNgS4BmBiEF7J`x|6)Kr z*bFg8=q+ck`qP`|SA&1yJK#sKjLLL-Gy?R}d2xk3S6y|$FN7)5Pm%(2^pe_gU~J)O zW7W|Etw7vGZ&jc-&iptj1a{~pwf3!d%{4EpId3Jq~`3(kORf7oa;)%B)r=dTaUp1w-;%gm;gDrY?dv^mN#?n)dDp z6G`^nnc`7(=-ol*5GBDGTu+PZAIL8}SzaJgErPmo7Z8(+DZpvzD%iBRa^0(H^m~`~ zSdVIkzzv@t{TpFtp?dng;pXO7)vw>H9MIl14uke~74(BRsYlb^EL7g1$@bHu5)6xU z820bmRvxO0s(0QWYhAIVxq0X{%dcqL^!_h|T6Sb)cG>bPny*{D#4wE8VvbRbv{V>I zdAD*elqCzb>L!2OoJKXmzZ%bHu4Ex-0^ zHUEPOFTXNqVY0&dmP6QgxYjh+Ur!^_GFUWmxM5{DXou^5q+mPqVDP{PfJOUPtq_ pxl!++tVgxY`RxxyVmzt`qH0fA?fGq)>T#^EIX>2Q*0I6jzX8l5an1k$ delta 18786 zcma)k33ydSp6{u1_boYDAS>jYlK=^85J-?U#lA1%f`|f&3O9fR5Ya|%l&DcrgM}Iu zHQFesps|BW17h5!?UsIx-KglZ`|)P9Kl-(Aq(}Xvr=9Wres%5*f$o{_kJQ89N|BUU+eyZ}tLgg1$ zDzJ9hnw8fW!uOikFMK{J61LycF}7PHsr3xpv>toNqc{MrRS zx#@QCym&#pDE0yehs0s=hBzRuUy;0a_4*ArUcX1YDGrJwA}*rQX64(jJ~Bs(X)4Ee zg_x~Y`@R#))iwU1!?r!tY1qOxt?MQygxrsk{MfMlvMecgv9RsoPQzMd#B6`1S?EM; zpEX}MNlY`9c?8!9sw8ERzMQ8nR-Vo}E} z;mxMl((!%t5h2Rd1Mz}XrwCv-lVe{4W2tgOo;K>;P=#TKY*Aw(k^ZF3L|BGZWQQuv zFaQWK6PKv3H2rS4q{gg3U8Kgek%Vi^QVcL^%tDr=G1ZOMnE74@%j*#LI)v?1$VQ{z zSBHwqBtHJ6RczLoC|8=5ZkBAszfDv9x;)$o15Icc=mLgHv)YYX!r@#aleTI5?1&w; zN>F^)NZMfwP*j@J+ys+ZZcIMe0$heHYqB@n7|fPP*)jPn>m#U7nAmFE4s$>qkQlIo zR$<(Xab6a!UKlDDI$?ktK08a?AHz1k!YoOLxl=lqja)d1H8?oC61a@vV--wJ7_k^& zJZ2_2YxM&Tq1Q7(J2i?&L0p;g9Y9-tFI{VZTDUSqBTXhCncQOm{-9RjCu<{~+5 zBrA+;J$@t>*8_EFlATCTlVwM|T-=Bmu@8h$HCbc(hFnwHvw;| zykdY~hOKGp(JXgV$Tp`Ygzgy79f=p+K8vvUD$Q-C6SV!-Y$GO*?gwIf$!)KKeW$4J zvTU(a_00Yz6?A>F(d2@r++`+RF6y(Mfdt4F6n(a^e#$j`vW+v8$iqmiT^cA;1Ld=x z6@b;OG@DI+x0zPZ_JK_@YktKMQy>c7l$;uy@up3Q%)CE%^RidF3FO5l3SSD###cE$ z(Aekt0V9g_j*qK^9qbt#j}4kKo^;4A0u%|5ssdE?gV|%-1tx(cwS?qGY*e?oJjA#gZc=r%MtOL++66omKU|4Su-Ot(JNsUx&5DQe9P2pNFm;r{v z_-)}NI1SRPcOT}y-iU}OVQLfgt}g|ke3Z{wwr>LEtJA18C!q=~Nu*lfet-#D&XZzF z0-eE^*esHTj^NPIVgsZN3SzA{(JCZjl(7N{qB0=QJF%YdD8Dh ziQ@Fs@(kA25J0Je1eN9{tWE4!nya`dGmXLxPznHJ(xG^YdXG;b{@&Osb{pyXDay!= zi4knVvL7Ao$}OfHEOX)DL2PryG`abFi(DENdW!!kI2U zcCpXvlik$^llaj?Evx963dO7eP*ns5z=O@}lTLt2q1;1}oyMl5^wJi@N?h!ZM6m{X z>|?tE+73es$yb=D!Wx8xOr0= z+@UFd6dHUn$Pn12OVKQAa04>9VI8tdW+#kd8;vF!?I3H>B_QLvb?tPv24wCA*qPHt z13UV>jw$q97sE2{6n{z@5PUfXnoo{%OxQ~h%P$4Z5`Yo%&D5*}_yMdy;(#ACxglXB ze;0p%G1C084b>Ew7B+*RP=T6hG8_VHkTR_tZhZ%urxJ<4Ae00-L^f`RavSV1rDh<% zmBpq>b_5~=F}Lbn0U;OE2R>RuS8GZ5El|~#^|Y`Bv>7&cTgJMg94`jHLv3T2m8?kV ziKjWy7|d5h`ZrKb`>M=MR5Z|Af;t8~D(t{0BknDQ4H8h8n4}A5U=}bk$hgrncxb3H zCv|BlpFC=$wG?J+6Q1RwV&s3PTRN8+)(|81F9|a{Wp1H1C1NWKvrhkk-LbqI1_7W- z!FF&ApqRCGhE=7mw(h}9)EVTGDk$wdRvV;T73kySKbTWk=+$M6B`uiORaNJd^v>Ij zZRw3ynb45b_1J^XLDQ<0IVGhOF29{$YgQt`_A8Kpls1w)^-RtMDcS_!--$vZ0L?%* zCH)B8inZ-oZc=*B`WU1Ty`K005mXs zvTXHLZb?7w)M(62!us$WS0D-2m?M$UU8*sMBNtVJ^NLb;2yhn7)ut(l2sUy8^I+jB zUmALUm2YjWZyt1!Aj&{#s7;W(UARFwYrwST86Slfykb2mAQiqE^DtzCCJop@O5|~4 zzR3xYfih-6n1SDv$L+}ZU5ug?(EJXy4WXZ!fM*1Ro32mBrbAD(z6Q_5md&qweu!*F z*_up~%>GF?=q-o>aT~JpYRv;cJe}zM#FkH!$43d)Tdi7&eQNQ-KIbuKf>})97+Av= z;2Dk<2bcn?b`gg+0(cIX+F)VwY1jtsR(}fJTE?6Vxzg%h+5kN?(!d)Jyq; zQZewH2^Iw&sn*DbF&S{o81V+nqrul+KA6qpDvL(ye69+mm&idA*)YuoXL$|GH5bSW zYF^7lFU@PIbC`|Qz*$@~TaF#CcXPm_WVQg9EeMj6*-(cvcnx4^@DjGwojV#ut#7Rz z(mU2n62L$#9=0J58YdAjwYk9!`0K#6@-V#Lp!(lE3W{(@(OiIY26*Ny%LP?V&mO6e zW?Lvw=oT_9RlYug__aqBw(jo6xiJWf*&V8C@+^n~p0$B;JLH6cc4&fsFikuXql^O? z-PjeD5`IKCj)as~nb1idAEh;xDGvi(ep#B)x~69wJn=V;co?L{repFXSvC*udn+?w zS@}MBxmunf3&W?DjR#O3ArBxyc@PH%x({SAE>CCX*@KL%l*d?}qUIO$n12V@aN5m+eg7$y3vg&kF@?Sh&d z*isvP$R)rxkgh(Pv|b!jb0&W$7O6>7E=O?S$tg2LHp2%&OhoKPC@U&C|0Me2NnUNDh4YEp&aO%q&d#X|#FVh`NC3E&9!q`Wl7&g zn4-`s6vNX?++yHnHjV%>@HjXz3`Rl~%}yA93AL?cZ0c&tYS^0%Im=0~BJM`PFHxMW z*ErzCFf{px2o=#;Gr;*LUPAst`7ILel#@|zu`1x=KoH30F^_r^6OuP<8!i+BAh8MC z$j74AjN7rEkTr`8k5=rV-h~=3v}5Y3(jmn$nV)nlSg#)bI(+SUm^pzZoP^q4IyjX` zx;?E&h2sLwD1k17I&`hagqxeR-QBWiqnihC*^=)@p%wvNeM6G+uImZ!hE+gJJ@AJp zgJJi~lzUbgMlFVNL#V|;(lBaJ2IA|T9=1I}+-j%nt&Xi#lnyS$2|4$O^79Q3^m|D%@_)ybfZoJ z*a!!S4G{6uUR5a%E1a#KYBONgVN{s5ENWXxJL+wR(;>QJfIQe1T7HOvm?+_3-%D3I zJwY#obvWJ*y9ufqf{3M~AhBD5a~1r|so*K_2C?Pj0G7E4JJ-%<7tq7Cce_OmZb3cD zxwcJu+ReIFr^-umvjp-lU-mJkxE6RQcpV%8_bMSk}QEJtBrfFH|Y<=Lh#YyBnOST3c2LrSxdo z@XA6P6tnGHc0+#bXwptJaBB#ah3m-{F`NdGT9*2*tWPQnPLwpjjR{eF&KL(VC);)= zhVFfM|`;Cz$?(+Zi0d$T%G%`Y&tNuJ%{obl`iS%e{+(0JSJP0cNBosFrj8W1-- z2^{g@!Tx0EAB_e#p5jJz;1M)|&LbYYGHrPc7z^Erg@niIarS3bn$;PNgX$TRv4056 zRhh%#NSWG8G~6EK5QcZ$LMPkqp#{*QlL02MhKLk=J9@iLT&A2Xvkv5s10#0A$>O;o z3}xcz1c43u74@Osl7k&oT{R z!U?c21>JPao+1Ic3M-rv*eAf!6J`{wUxdV$l$#rDqaNp+i2NMxTB%u=`2(y8yJSiN zj0&aWEs*5oXHk>Z_9}f$a%9K0V>HW*;dVZ*K}?La-I&n2hOW%a{Lf_^;;rdOLfa*Btd&7Cka8QT26YK?;ER zxcX%HyjEY#2%ir8_OP{4 zoCi@t{l(TWCbJwJe&$NL7pVnqp2xH&*w3d!)KNykB6QvD6c`8RD^M7|?s@C#iM!TK z&uIVMy5w~(omtaUpVWqAZmg_P0?0HZF~>%WcH0<1OldG&=!xCfh&{n>-t8u=;d+%g zwC4yHz=jb#f>wr&uxfzR1UwKp8mL6f+p5-h?beA;g50@HT;+bfMJDY)PY0)VWn(221p zD*Rx2%rp)90?8z;GN3f6F=9i+J;GBMQJG3XlR$ickN!b&lHgx1^gPKyz_O>4J^+=(=s0bO*% z^viKj3m~#aiMQcUA-a)Dhd1I3Z-C)sQ+g<(5f~cO1_D~*@0$WN1ekQGlVe>7eB$7F zj{srdSb;tsnt?nXEyKDAsGAER*XA9zEFV~|G9Mh&1HJ;Tvcfno!tPk3*+KV@a4N*< zB6dy<@P!nJ*vS+u)yxz@idLpsAVCcRa>(*Nr^akSR*P4JM;4FStTBXk1t^MSl~W7~ zc~X#q(#s)rsSV@ch3qt~Fcr9;(#4e;IXzb1%mo=R)bNnQK_;@cMZVU|2xUf2ho6JP zKbX-P%QY=9Uyzx67aKt|g#?6F!cA8i+Jqgq#!+>FmG{JhoUBy_iF@{Nihd6cI&bB51BkIJNxBy@5f@dvwJh|MtjOOn!c$<%+99m)|{KI zos+>@%^WrzOYc>t`?qBJV-v&eLQhFytLoM2$HfC`{?H!yUO#lMXi={ZJ*G}wa70w8 z-NX9izlDLYdlqbl52rT7m*t1*#@m z$Pw}+5)G*jp$?z`vthnz%0FVhn654zu|dpGAB-rf?(`O#35L)n#+tzhdL^!vkW?T* zfTWZk8w3hkGgW@g#Idswmcu2PzGY>%bi_QHPJ^JivPSR*aJ}&JvJFvqpT24XbDGxj z^tJ!#nhJ4`>QVbT?(KhFJ4npx_*?A&AzoJfF1)eFD5$iMj_N^!F;r_?mQ=4?xInb4 zz$htRQBz0t6F*Zoj5;AY)U?r~QPVW~CX9>V%m!M(TIT3+QH-<3%tha&WBQ}-y<=*z z#yev+rw|;BlI&J@vUjn^e;M>SKcIPT5Qq~-yH7|4?Z!uEV?cRc3@JU`0Uc9PAC4Uj zbR@==MXquU+Sm_|s5y0oYWBGN{%42kkwqHWyy`u#AvRgvJI*z?3iZ;sA)-$GVO)PP zK}E*9L2plL==jyn`Jt@ycGW10fD zU)4<$7pgp2oj7-Q^W?PBe|`+?Il$Ii$kAJnVuKX8qp}vM{c;qdFbgKE6uZ?M6E5oo zW)DE5UE?gXqZ~){-ypY7%ulp+V7ffnN^W1IviI0(-NY-bryl6U)UXJdhTl)Tbleh# z)NrF?#2hqg++u7xpmAyk^5yaeBn)WG;{7**YY1^WLPyzD!!^bBEQGY+SWp__?D0Gr zV zF=)qwD1@vE#sP8f+dw6Q>}G5jk#sx>K*Ue(WhcujR~u$bDLu_Xz#FkvkKm0f#%!?q zsE=n1OMR=*!S8igp*vtMT4df;;kgA=LmT%V^tvq7T|(Y+?6@4yM-W(McBcv79BG?g z04@UZunmH@byguut%$pOI*u4gI=*-*QN-Y+ACET#?<#DB!w3_AR;@G3N*PJhlR|Rz z6t0PPG>G6Meyx5tvqA*bnVBO}xZa|^;{gr#qwz5AS1Zju0zrACyM(668HBd z9PdelJPMp?n12xPx6!lF#ETN7XLyj&E+cwK$+aefzOcw z#j-P9#?m0RZOCWYFJ_gSDfP|lGSz=}4K8DZj;8wL$R6F33D zP~v1q2KM53fW1>aHosr$acV0M^>{1=%cVEsA|}F(SQj$|RuJsbCt+-yDhPLzNhgFQ zAYkxq(~Vf0o+vJG7#7lp7TkC7FvC+EM&R7XBx18C#|l+Vpa700v?S#yBpgxYse?^J?kZqo5mE?bXf0XAL= zY3~SXu*I%-4TiPl3nwZedL~oEPjF?0yx3Tew<*9WV%S0XvB^NDe$*4xcN7L$rVu;; z-5YnYw|p-j=Cc@!%U%SO;9^g{WlXp37Z_`St+8FU>5}}2>v+x3;ebcS2LfcnjXauHD9EGTyZnL|FdO@_dwDH$RJJo~wD zK-FHn>B69$yW90sJaO<3!@c9}BjW2q?xEa}O1R{K?0Z=oS8BTiP#;jIE*_XQDIR23 zJQcGpLEzWDq&kJ$>zN5CQ#*io*^fTH`@p)}gzMKne;DB&>=|r{*NpnmfWGMzud%>6 zodB zC0-{$_yiBECrCC(&~c_X#~u@GN(Vx*S{t7SBHd|XH1vmh@3KC%30S_bC~6pZiYW$u zK|uq*pVVcz)h-I3r#rz=p_6hJ)`-j1!i9qdjBqXb3=)E%Wmw2lRKTsKJaNC)G7wq$ z1vD_crS^pdMU?wv(D_#&g{p?`7qXqHI=S$FrWV5q1p`6MsNv`jL9No^aZDY?(Kr2e zfKT^4#MAY^(nGil^#|y*E>5d7O-V=_A29KtPZaR=1nJ9%7(9=5;z?TwDwa1LCTz`^ zMfOrH4D3Kg#Pa(EI0rTiT{1cmjf@Y&)q#_QdUWtYR&zDH`pJuTMM{wH9Dqy(3r>no zR>v0gOC3AXX$&&x7ru`i0`g9-?DTSk4>TBM0F9mx3>;%=Yl-e_8~fZ`yd067LOYap ze$2^(7QuZVrCFYUqmaJqMp(Gk*KF#k?Ys$MGoPHmV25jcEkKt%rnX#hp{P~|u5bzx zwgC}9u*4x(Q2?QRVf9a6F|y}$?gRc0meBqWl?XUt+~Tqbm?mwtu3J1fwbZ-G^GFya z31Lb|LLab<83p3F^j3HZVNIAEQ9t>MQ%@YQpn%9ZeQ|X()-6SN4Z(-f?BQ=&KG~ zS(>vfy-Uqd*mXdCdF8y}dZ6BiXNoS@@M{jm^)7a7!E|Zo9ImJbC#Jrt^zu01MQq3l z02H2A0~U<*yF@>tN}WuPLqzm@61+w6yitkZ$9^^))lKcl(lYL46-Mu2`b+hPtBUjA z(;euC(LloU#nU7eUouQQt?HH(r6Au*F*&mrokgxMIZ6RJ6L`;b@GMQgx54X*G@o^6 zI6h$V2nil?{OY=^ors?gROorBmaBV>*A&m_3Nx)GU~=`l53)GO!q*OjLBoT`?7Vd0 zyD!rkuF9^#?+MhU*Nnhhhi%t5_`Y<_0P&Fe?3yd_9l87qai8jS?X>LkpzGLhZI%!} zS6i>E#&`d9$IxYYb)sgG(8lZ`3pQCdiNavsyx58AO_o}rk-V{qz^~!3oLoHi5 z5s{&1SB{P~AMV66FK{&u5r1CUr=k_A2D_Qf^FRXcEW9GsHGKCK(4_LrK{Y)2KTIV4 z`sXmqi}mxmW)YYrxT+kr{a1}#mY&Nsq5Imri5xhrvL!t!-y)j08QtLe8i0RY&l`z# zWMNZoYLFguKF`x(4W8!cI>?!cwcc4(*^4t%mJG@Dw1X25#5Ck`hYGJAUz~vhk7e;P zJq-xB9uq8ET`-j3fJh;|tT?-@-_dVEScB)~-m(;|7WMqxdU9HF~I7bEVSUm0@JRRT`aVwTbO3LrQ}|g z+R{x-yZY^CVNbe+X;-(yEVQRvn09r0mxTlA7N%X@j&k-1 zn0sFhU6;S?1hXu&#}_QHhJ};e3tzFo8n&G3Uig*;*0AMt_reb>oK~Id@>5N3AbTd= zsj+)u8w;%A@aFD?9W1bhEjw9gNf(%Qwd`V{HQmCrtJ_`{+SpQ#8JOwnx1WVQY(fiD zgiadQ=Z|XdUjHr&2hyFGN;Y%EQ5HJW1*Y$w{n`L;@PxaM4W zc77lH7}*FMryu0;X{YNku~(xTH7x&-$D7m#8!v`{+Bc2KN=wib`lvun@+McTQ}^9e zTk^V~Zt)hy(TLB=J!r7qmYXZ$=#EQA1Lr9W%idI=)dan4;9d>gR2kDcitk#{zI@Xt zu}eL*spn!&2sB{dRK?d;J=EgT$-|A7U(PCbs+^m9_fBgZ z+eegmvuXvV`?adsH@nZCDNJ{uSd@?k%$r^Ivkyb{Lkw z+|ox}r3!vBBYF(cCL5aFx~`-CCoxm3R5#ta4BpA7w_X<3zbApKt43~_fvT-r7NYs{ zEx#6P)eEVW@h-eWPph)VLBRU##>wIar5Xo9r*<~p;pttC_Et6TwrRl|;8|dkPu}KO z9F2to+VSDee{&mQ`tG(-qCr*PzBrBmETY>m(`;H}I~`08meFy%}qj zG>sQ+>WZc!taxM70Ko8cQ-RwB8qfD>BSQXwan*LoTBkm1DvhUG_)7-Zg*yiKcwJE~}n(>`1r z!x_d`Tkabb+=ZfiS{=Es4v$%Xyl=4BqXyq^W7fI%Uy!vP7jOaj9J`p-diB8lMdC&E z{QVaV*@V-r1kc!7AqeYcIxXGGx4^5^WzEJPs{Rj*E!fnp``PR*YRdzw%rmX(s|QM% zL|YaxS<+Ho{nysIQHJ)aar}f!6NHnPl!&d=Teqe!Gr)dZWK?$U2AgeG*7geVw)&){ zcaKiqvp@!*N@OT0BYKRAZtpLC``@&$*xpNBvSSu5VV>D>q1d86-*HH!RO>_6i`0V; zPZz0<84n8)+lUYfgNspXqq_8wNxAzqdTfzn@U{bT_lny4$n)Y|b!qD>k>k$+GMpP< z>98KXPz*c1uTI7`Ag+gBvPu6sotPUCCD8SBkG!gGEo$>)MdCQWx!rrg$;B}B)?+2{ zJsT}bX}M(n#zBUzEz?E+MwyLw#JZ+j*Mz&( zxVk3Zt;W$cR=P%=e0*KeeUAgrgXN!?92fG9Cy<`;S+@z5+<8~gt52iYSKjsvlAWe} zVi(d~CR-yYDEG4bU|i_&T0_Vww>1)xSPj9^VR& z7^i()23U8h(x;w0|L(3y{pu-wcNcA&Bz~jjwv}=S+;k{v4C&a|W(hG;-SzanupxQR z)Z$Ca^7O$0s`;7kaWZQBX$`)w|8y3<{$1mOdwF_$N=@6YMjD zD>j3#`E28;E~syI)rtqzfM>slS}cFAyrd0`gdiIa8~{Rxer!gGd{&!U_S|-?^38Lb z@Lje0C+eRLT`0Cccz75T)bPU#@NGWaPi$B1hrd*{doC3BtA;%j!U`4#@3NITv}X;D zR{ftJhS66&Uz+z2p0n~7Cww@G=r{tC52^c}FDiPN`{zA?8{pXtA3rb5tcTT`&tHzQ zMK9cl?*lK)!1uEkX5(A=;#~1i$EFv30$bUs9XRA~$()FH#?Dp$v40@u z?*D3kd@p%*CcaO<+6UiXygC%$zrNbhZ-;PbV?lM?{1A4Zg(@gm{P7w-U8R2a+8~S& zua{o=5bT70XyCC4Q1uY7vc3eCAr{M{^TN?SyuE6K#t&_bm9fsWfLHt|0#h<{j)IAh zgRojf%%X<~#|)*1)OD{f!#eN3J`vyefzI47o`WPjidf@rA_XhN)S>n&F;sooezThO z%oufB$E{+i`mv*0Y*W?y@9&*CET=!>92> z2&}$J)gP%)qYjnyn#NdV81X7T!S!CGBFEp<@PBu(?oeO#z>yOCXyQeD@xMPfGB~=% zI{;(j>c=A^$4oqTQBWt22yh%{)~_IHG9mxUga$t%|I^FzZbn8e-1@J3b1;4l@${Rs z;qiX`<`6u$DC?|*E&TVl+#Udl_bB^tUL{fhl_g(Psm@}tPd(P@;Bm#_PX3(Zv(Db` z*8NxmFXNcdC5XvKbPcA!QgJR-^wz-t(jqQ?YoKUScfU2b_dePda4>9(zRD0-AEs8j zg7PKx(OVT7x~kYd#6dDJ4Su_**U!9x#5d4~;X3@{d6kW7`P(IcS)Yu7 zhDnLd!pJK7o!&5*sr+G`sItK?O#)Y_r_$OH(1SkZ^OV31)k?%G-SwN5ExfH49Nmklas)K zb+kKne=(uEBg`uLimZ1JPm&iK;wQ! z9{1Yd7oDI}MsE_{$~JV6f5N#GskI0(!zoHWS*$L4fBsDRZWxQ;5ioXszzGtVj{KbU zI1vJ*!@;}L1{}jYj}zhr2k>9tzkI0YTL#QJ%m>dBKRovq&_)FUpxsE`lon>fP@8`F zaf(}#>3zAp??N{d7ZP}r&bOTMTL6xW0+NAQtyJ{3Gkf=jWVVmpV4$gs;C?-wMhJEa zMn?1#lm_{71g&jgl=}6rO3V{S)c^U_Eb)x0{a`?xc=S%u-yT$JJ}4~v7H5d`>DiE9 zbrrOhmd6gNXFpin<2b6lm_7shI^=3yr+OY8YMwl(rXMZs*96$RCi&s)Bz(jr^=LA} zb!ywuDY0)xM&WEXb{etYb^QM5z+lH^zx%aNeLt#64PLQg@QSr-2d@~s{_4T2mm7vL zIBXd{q(hPVk&ZxWBCSo=k4mR2*Icve#^u)c_&%nWC)Lr9`a7AKI7<}&t-E&F@<}V#b*)gU3O*hk$@D5ySA5*B ztJFu``|-@K(jxW0J}&PnRjSx0XL}A%qdyrYo>Dh_(pT*5co_NMqj>cm8>T)yQL6rY zqA18m@Uag%qM!DP;X0%8i>>PZFYODx{~3?|LKxc7C2SH;2CyZ+XIUWg#4F4QoR%_~ zR#oxkRqCZLtEaDCx!fLSuex>(#>@{|#%!ej2Go~!D}Qrd`FoV@ZtW}&P=EV!fhbXP z|2V`c0W}KIelz}q&NMRRTad40Sv~c~VO_nx$ke7s{Ssw8s^gnK{#2N!gR1q^I3yh> IP7M Date: Sun, 20 Sep 2026 16:09:22 +0200 Subject: [PATCH 828/927] packaging/nix-make: Build libnixstore Add the libstore component, and the machinery for components that depend on other components: * `mkComponent` takes `deps`. A dependency's roots, public include directories and generated files are made available under `/` (so libstore includes `nix-util/include/nix/util/hash.hh`), its units are excluded, and its library is linked. Dependencies are transitive. * External dependencies live in a single table (deps.nix) mapping include prefixes to packages, pkg-config names and linker flags. Link libraries are derived from the union of a component's matched includes instead of being transcribed from Meson. * `mkStringHeader` reproduces Meson's `gen_header` wrapping for files included as string literals (the SQL schemas). * Config headers that reference derivations (the sandbox shell) use `writeText`, since `builtins.toFile` cannot reference derivations. AWS S3 authentication is disabled for now (NIX_WITH_AWS_AUTH=0, aws-creds.cc excluded) to avoid the aws-crt-cpp CMake dependency. Also update scanner.wasm, which now warns once per unresolved include instead of once per unit reaching it. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/deps.nix | 120 +++++++++++++++++++++++++++++++ packaging/nix-make/flake.nix | 8 ++- packaging/nix-make/lib.nix | 121 +++++++++++++++++++++++--------- packaging/nix-make/libstore.nix | 73 +++++++++++++++++++ packaging/nix-make/libutil.nix | 90 +----------------------- packaging/nix-make/scanner.wasm | Bin 59909 -> 61762 bytes 6 files changed, 287 insertions(+), 125 deletions(-) create mode 100644 packaging/nix-make/deps.nix create mode 100644 packaging/nix-make/libstore.nix diff --git a/packaging/nix-make/deps.nix b/packaging/nix-make/deps.nix new file mode 100644 index 0000000000..9b10be1ca0 --- /dev/null +++ b/packaging/nix-make/deps.nix @@ -0,0 +1,120 @@ +# Table mapping external (angle-bracket) `#include`s to packages, by +# prefix. More specific prefixes must come first. Each entry has: +# +# - `prefix`: matched against the start of the included path. +# - `pkg`: the package(s) providing the headers and libraries. +# - `pkgconfig`: pkg-config names used for compile and link flags. +# - `libs`: extra linker flags for libraries without pkg-config files. +{ pkgs }: + +let + # The overridden dependencies used by the Meson build, layered over Nixpkgs. + deps = pkgs // pkgs.nixDependencies2; +in +[ + { + prefix = "boost/context"; + pkg = deps.boost; + libs = [ "-lboost_context" ]; + } + { + # Also matches coroutine2, which is header-only on top of libboost_context. + prefix = "boost/coroutine"; + pkg = deps.boost; + libs = [ + "-lboost_coroutine" + "-lboost_context" + ]; + } + { + prefix = "boost/thread"; + pkg = deps.boost; + libs = [ "-lboost_thread" ]; + } + { + prefix = "boost/iostreams"; + pkg = deps.boost; + libs = [ "-lboost_iostreams" ]; + } + { + prefix = "boost/url"; + pkg = deps.boost; + libs = [ "-lboost_url" ]; + } + { + prefix = "boost/container"; + pkg = deps.boost; + libs = [ "-lboost_container" ]; + } + { + prefix = "boost/"; + pkg = deps.boost; + } + { + prefix = "nlohmann/"; + pkg = deps.nlohmann_json; + pkgconfig = [ "nlohmann_json" ]; + } + { + prefix = "archive"; + pkg = deps.libarchive; + pkgconfig = [ "libarchive" ]; + } + { + prefix = "openssl/"; + pkg = deps.openssl; + pkgconfig = [ "libcrypto" ]; + } + { + prefix = "sodium.h"; + pkg = deps.libsodium; + pkgconfig = [ "libsodium" ]; + } + { + prefix = "blake3.h"; + pkg = deps.libblake3; + pkgconfig = [ "libblake3" ]; + } + { + prefix = "brotli/"; + pkg = deps.brotli; + pkgconfig = [ + "libbrotlicommon" + "libbrotlidec" + "libbrotlienc" + ]; + } + { + prefix = "zstd.h"; + pkg = deps.zstd; + pkgconfig = [ "libzstd" ]; + } + { + prefix = "libcpuid/"; + pkg = deps.libcpuid; + pkgconfig = [ "libcpuid" ]; + } + { + prefix = "curl/"; + pkg = deps.curl; + pkgconfig = [ "libcurl" ]; + } + { + prefix = "seccomp.h"; + pkg = deps.libseccomp; + pkgconfig = [ "libseccomp" ]; + } + { + prefix = "sqlite3.h"; + pkg = deps.sqlite; + pkgconfig = [ "sqlite3" ]; + } + { + prefix = "wasmtime"; + pkg = [ + deps.wasmtime + deps.wasmtime.lib + ]; + libs = [ "-lwasmtime" ]; + } +] diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index ab09a42298..1c918a6197 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -22,8 +22,12 @@ nixMake = import ./lib.nix { inherit pkgs; }; - components = { - nix-util = import ./libutil.nix { inherit pkgs nixMake; }; + components = rec { + nix-util = import ./libutil.nix { inherit nixMake; }; + nix-store = import ./libstore.nix { + inherit pkgs nixMake; + inherit nix-util; + }; }; in { diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 0e19617af0..085ce1375c 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -10,6 +10,8 @@ let function = "getDeps"; }; + externalDeps = import ./deps.nix { inherit pkgs; }; + # Non-optimized for now, for faster iteration. optimizationFlags = [ "-O0" ]; @@ -43,12 +45,25 @@ let ++ optimizationFlags ++ warningFlags; + commonLinkLibs = [ + "-pthread" + "-ldl" + "-lrt" + ]; + + # Write a file to the store. `builtins.toFile` cannot reference + # derivations, so fall back to `writeText` for content with context; + # such a file is then built when the scanner reads it at eval time. + writeFile = + name: text: + if builtins.hasContext text then pkgs.writeText name text else builtins.toFile name text; + # Generate a header with `#define`s, like Meson's `configure_file()`. # Integers and booleans become bare defines, strings become quoted # defines, and `null` becomes an `#undef`. mkConfigHeader = name: attrs: - builtins.toFile name ( + writeFile name ( "#pragma once\n" + lib.concatStrings ( lib.mapAttrsToList ( @@ -65,33 +80,40 @@ let ) ); - # Map the external (angle-bracket) includes of a unit to the entries of - # the component's `externalDeps` table, by prefix match. Unmatched - # includes (libc, libstdc++) are ignored. + # Wrap a file in a C++ raw string literal, like the `gen_header` + # generator in nix-meson-build-support/generate-header. + mkStringHeader = + path: + builtins.toFile "${baseNameOf path}.gen.hh" "R\"__NIX_STR(\n${builtins.readFile path})__NIX_STR\"\n"; + + # Map external (angle-bracket) includes to the entries of `externalDeps`, + # by prefix match, deduplicated. Unmatched includes (libc, libstdc++) are + # ignored. matchExternalDeps = - externalDeps: externalIncludes: + externalIncludes: let matches = lib.filter (d: d != null) ( map (inc: lib.findFirst (d: lib.hasPrefix d.prefix inc) null externalDeps) externalIncludes ); in - # Deduplicate by prefix. lib.attrValues (lib.listToAttrs (map (d: lib.nameValuePair d.prefix d) matches)); + depPackages = deps: lib.concatMap (d: lib.toList d.pkg) deps; + compileUnit = component: unit: let - deps = matchExternalDeps component.externalDeps unit.externalIncludes; + deps = matchExternalDeps unit.externalIncludes; in pkgs.runCommandCC "${baseNameOf unit.path}.o" { __structuredAttrs = true; inherit (unit) src includes; srcPath = unit.path; - inherit (component) includeDirs; + includeDirs = component.includeDirs ++ component.depIncludeDirs; cxxFlags = commonCxxFlags ++ component.extraCxxFlags; pkgConfigDeps = lib.concatMap (d: d.pkgconfig or [ ]) deps; - buildInputs = map (d: d.pkg) deps; + buildInputs = depPackages deps; nativeBuildInputs = [ pkgs.pkg-config ]; # `_FORTIFY_SOURCE` warns on every unit when not optimizing. hardeningDisable = [ "fortify" ]; @@ -122,21 +144,27 @@ let linkSharedLibrary = component: objects: + let + # Link against every external dependency used by any unit. + deps = matchExternalDeps (lib.unique (lib.concatMap (u: u.externalIncludes) component.units)); + in pkgs.runCommandCC component.name { __structuredAttrs = true; inherit objects; - inherit (component) - libName - linkFlags - linkLibs - linkPkgConfig - ; - buildInputs = component.linkDeps; + inherit (component) libName linkFlags; + linkPkgConfig = lib.concatMap (d: d.pkgconfig or [ ]) deps; + linkLibs = + map (d: "-l${d.libName}") component.allDeps + ++ lib.concatMap (d: d.libs or [ ]) deps + ++ component.extraLinkLibs + ++ commonLinkLibs; + buildInputs = component.allDeps ++ depPackages deps; nativeBuildInputs = [ pkgs.pkg-config ]; passthru = { inherit objects; inherit (component) units; + inherit component; }; } '' @@ -162,8 +190,13 @@ let - `name`: derivation name (e.g. `nix-util`). - `libName`: library name without `lib` prefix (e.g. `nixutil`). + - `deps`: other components this one depends on. Their public headers + are made available under `/` and their libraries are linked. + Dependencies are transitive. - `roots`: list of `{ root; prefix; }` directories scanned for sources and headers. - `includeDirs`: include search path, relative to the root namespace. + - `publicIncludeDirs`: the subset of `includeDirs` exported to + dependent components. Defaults to all but the root. - `sourceExtensions`: suffixes identifying compilation units. - `sources`: explicit list of compilation units, as paths in the root namespace. By default, every file in the roots or `files` with a @@ -171,16 +204,19 @@ let - `excludeSources`: paths (files, or directories with everything below them) to leave out of the compilation units. - `files`: extra files (e.g. generated headers) by path in the root namespace. - - `externalDeps`: list of `{ prefix; pkg; pkgconfig ? []; }` mapping - external `#include`s to packages. - - `extraCxxFlags`, `linkFlags`, `linkLibs`, `linkPkgConfig`, `linkDeps`: what they say. + - `extraCxxFlags`, `linkFlags`, `extraLinkLibs`: what they say. + + External dependencies (compile flags and libraries) are derived from + the `#include`s of the units, via `deps.nix`. */ mkComponent = { name, libName, + deps ? [ ], roots, includeDirs, + publicIncludeDirs ? lib.filter (d: d != "") includeDirs, sourceExtensions ? [ ".cc" ".cpp" @@ -190,39 +226,55 @@ let sources ? null, excludeSources ? [ ], files ? { }, - externalDeps ? [ ], extraCxxFlags ? [ ], linkFlags ? [ ], - linkLibs ? [ ], - linkPkgConfig ? [ ], - linkDeps ? [ ], + extraLinkLibs ? [ ], }@args: let + # The transitive closure of the dependencies, direct ones first. + allDeps = lib.unique (deps ++ lib.concatMap (d: d.component.allDeps) deps); + + # Dependencies contribute their roots, public include directories and + # extra files under `/`, but no compilation units. + prefixed = d: p: if p == "" then d.component.name else "${d.component.name}/${p}"; + depRoots = lib.concatMap ( + d: + map (r: { + inherit (r) root; + prefix = prefixed d r.prefix; + }) d.component.roots + ) allDeps; + depIncludeDirs = lib.concatMap (d: map (prefixed d) d.component.publicIncludeDirs) allDeps; + depFiles = lib.foldl' ( + acc: d: acc // lib.mapAttrs' (k: v: lib.nameValuePair (prefixed d k) v) d.component.files + ) { } allDeps; + depExcludes = map (d: d.component.name) allDeps; + units = getDeps ( { inherit builtins; - inherit - roots - includeDirs - sourceExtensions - excludeSources - ; + inherit sourceExtensions; + roots = roots ++ depRoots; + includeDirs = includeDirs ++ depIncludeDirs; + excludeSources = excludeSources ++ depExcludes; # Work around a crash in `builtins.wasm` (Nix <= 3.22.5) when # copying "layered" attribute sets (the result of `//`) into Wasm: # `mapAttrs` produces a fresh, non-layered attribute set. - files = lib.mapAttrs (_: v: v) files; + files = lib.mapAttrs (_: v: v) (files // depFiles); } // lib.optionalAttrs (sources != null) { inherit sources; } ); + component = args // { inherit units - externalDeps + allDeps + depIncludeDirs + publicIncludeDirs + files extraCxxFlags linkFlags - linkLibs - linkPkgConfig - linkDeps + extraLinkLibs ; }; in @@ -239,6 +291,7 @@ in inherit getDeps mkConfigHeader + mkStringHeader mkComponent commonSupportFiles ; diff --git a/packaging/nix-make/libstore.nix b/packaging/nix-make/libstore.nix new file mode 100644 index 0000000000..8d8f07bc30 --- /dev/null +++ b/packaging/nix-make/libstore.nix @@ -0,0 +1,73 @@ +# libnixstore, transcribed from src/libstore/meson.build and its subdirectories. +{ + pkgs, + nixMake, + nix-util, +}: + +let + inherit (pkgs) lib; +in + +nixMake.mkComponent { + name = "nix-store"; + libName = "nixstore"; + + deps = [ nix-util ]; + + roots = [ + { + root = ../../src/libstore; + prefix = ""; + } + ]; + + excludeSources = [ + "windows" + "freebsd" + "darwin" + # Requires AWS CRT; disabled for now (see NIX_WITH_AWS_AUTH below). + "aws-creds.cc" + ]; + + includeDirs = [ + "" + "include" + "linux/build" + "linux/include" + "unix/build" + "unix/include" + ]; + + files = nixMake.commonSupportFiles // { + "include/nix/store/config.hh" = nixMake.mkConfigHeader "config.hh" { + NIX_LOCAL_SYSTEM = "x86_64-linux"; + NIX_SUPPORT_ACL = 1; + NIX_WITH_AWS_AUTH = 0; + }; + "store-config-private.hh" = nixMake.mkConfigHeader "store-config-private.hh" { + CAN_LINK_SYMLINK = 1; + DETERMINATE_NIX_VERSION = lib.fileContents ../../.version-determinate; + HAVE_EMBEDDED_SANDBOX_SHELL = 0; + HAVE_LANDLOCK = 1; + HAVE_MOVE_MOUNT = 1; + HAVE_OPEN_TREE = 1; + HAVE_POSIX_FALLOCATE = 1; + HAVE_SECCOMP = 1; + HAVE_STATVFS = 1; + IS_STATIC = null; + LSOF = "lsof"; + NIX_CONF_DIR = "/etc/nix"; + NIX_LOG_DIR = "/nix/var/log/nix"; + NIX_STATE_DIR = "/nix/var/nix"; + NIX_STORE_DIR = "/nix/store"; + NIX_USE_WASMTIME = 1; + PACKAGE_VERSION = lib.fileContents ../../.version; + SANDBOX_SHELL = "${pkgs.busybox-sandbox-shell}/bin/busybox"; + }; + "schema.sql.gen.hh" = nixMake.mkStringHeader ../../src/libstore/schema.sql; + "ca-specific-schema.sql.gen.hh" = nixMake.mkStringHeader ../../src/libstore/ca-specific-schema.sql; + }; + + linkFlags = [ "-Wl,--wrap=__assert_fail" ]; +} diff --git a/packaging/nix-make/libutil.nix b/packaging/nix-make/libutil.nix index c2e0ea7365..9a26ef8d30 100644 --- a/packaging/nix-make/libutil.nix +++ b/packaging/nix-make/libutil.nix @@ -1,10 +1,5 @@ # libnixutil, transcribed from src/libutil/meson.build and its subdirectories. -{ pkgs, nixMake }: - -let - # The overridden dependencies used by the Meson build, layered over Nixpkgs. - deps = pkgs // pkgs.nixDependencies2; -in +{ nixMake }: nixMake.mkComponent { name = "nix-util"; @@ -60,88 +55,5 @@ nixMake.mkComponent { }; }; - externalDeps = [ - { - prefix = "boost/"; - pkg = deps.boost; - } - { - prefix = "nlohmann/"; - pkg = deps.nlohmann_json; - pkgconfig = [ "nlohmann_json" ]; - } - { - prefix = "archive"; - pkg = deps.libarchive; - pkgconfig = [ "libarchive" ]; - } - { - prefix = "openssl/"; - pkg = deps.openssl; - pkgconfig = [ "libcrypto" ]; - } - { - prefix = "sodium.h"; - pkg = deps.libsodium; - pkgconfig = [ "libsodium" ]; - } - { - prefix = "blake3.h"; - pkg = deps.libblake3; - pkgconfig = [ "libblake3" ]; - } - { - prefix = "brotli/"; - pkg = deps.brotli; - pkgconfig = [ - "libbrotlicommon" - "libbrotlidec" - "libbrotlienc" - ]; - } - { - prefix = "zstd.h"; - pkg = deps.zstd; - pkgconfig = [ "libzstd" ]; - } - { - prefix = "libcpuid/"; - pkg = deps.libcpuid; - pkgconfig = [ "libcpuid" ]; - } - ]; - linkFlags = [ "-Wl,--wrap=__assert_fail" ]; - - linkPkgConfig = [ - "libarchive" - "libblake3" - "libcrypto" - "libsodium" - "libbrotlicommon" - "libbrotlidec" - "libbrotlienc" - "libzstd" - "libcpuid" - ]; - - linkLibs = [ - "-lboost_context" - "-lboost_coroutine" - "-lboost_iostreams" - "-lboost_url" - "-pthread" - "-ldl" - ]; - - linkDeps = with deps; [ - boost - libarchive - libblake3 - openssl - libsodium - brotli - zstd - libcpuid - ]; } diff --git a/packaging/nix-make/scanner.wasm b/packaging/nix-make/scanner.wasm index 9411db96f02326de5f124c17700815821b623d8b..d9fb9681bc003e5011a8bb884fd03b3c9a2fe037 100644 GIT binary patch delta 17935 zcmcJX3zQYrxu9!T)#>x>?&=pbbR%`DTWM4Td~_os^lp)dV0^}Cq5@H)o>n3cA2Ha~ zN@Lmt72ImjtHxIp6%}n%qD0$hMn`eQ+%XXyZ!XC)Ys|XNgnKoknRvr||E_bIhsj-c z%~~T;yJ|oG{lEWz@57pRyr2EfJHT18;!@=}j(R|iTI|Jftm4IP<>{wEweqSH$eT$`?H)pZe*SCJzfxe=0{f5IkUGIXWSL82U(tE+8 z6)Trty!1jxNln4lELpj7`HBlxCRJHmb;-pm_`XH0h+XezYFK%mmv(($Iet3jx+&MM zs`7o;b8Bi{m_<5gucuJ2ZP)HvxX z$5l?oO)2HNRj#9)ypsPsFR$|T4XWCAzrqY&D#iM)n@KU+%QaT{UM}r5c~v}JuaHU= zd^erVx~yi0D?rG4DH-k-T=~y&{FJ&(wY8a}+@l+=Je)uJmw{-TSQUc7SA6;qv; zO`khXPg9Oo`RGbLVM^l<)m=)TeDxLAsb|!)>N&L)EWD;(SN~PLs?NXQ!f#%(e8tN1 zx2QMOe^GC!g39G?x!)T$zjBXJ$C*0s8)}}p)cccKV$SoYtMkk~eoIZSlMnq4x4D=N zyc&+;enoLiWy#+jQ7qt#c3HWn^bW9H_=REK+Vn9)kBU~ZTT)5)7TtmcfS=K(Vx zdV0Vug)a5J&IHGqJ8GsL^1Nc*l&zRsaOuetjA5xQYP`zuRfb(jvTfP^Bo#HdRm)rYCxVJetFNhkKWj=e;U8sMt>TY^OV{!gPn5Io|JfyP2_@ zflQc%V}ept+vyghxlqhemDPn(-022kPBVXj5iyI>>_^pMHFes%LZ>g`H8Z~@qT-6M zFyHTH+?0|o*dBB&@TYoyOK`givmNe{)KE-`3ddKwW94;Pu}0SFaA$P7hw@g_>2^?5 zce?FiwkKZO&wr}RZKe+J8Yv2$ZU{SMO2ttEjT1G3%$|5nPye;o4GgZkcep=ZU2Fr4 zuv8qaono!m9p?OQo^2-sUagF*4I3%`_DIAfpMV`^W6%Q2wO#!%n+@#7h+47u&q~{oQK1y&COJ;Anknv8B`5=F^Ai~bHF9CQMOIFbW%uL@ z4Z)XS!PchjPT?!dh4PUJW7ujLJI)B*^~y_NtX^2Xt;=N=840^M0O|@Kj5@yBkvBhT z0fI!Iopx2O`Tw{Eg}-i>k)q*%~#vBvGDu^{`(6>CKK!GRgrg2FF0vR0wDSO{xk zz!3&T6pC5aal*z>g|+e*p5Cq^``H^dwnVksIRWN!=Jo2`YFQ0t&CNrFVk2VKpY4ru z?Esqm3G2cd(MXK(OHmE5Yhm-N<_LQV-U=|ocBd3o$C{?h7kP$?9Pu7hMOlgy1#+kh z+hv2BEd6tO2MEaqH!Hg4I@}L3^4!UYFgv%}rTgm+sMb5-I2X+U$A!6x?v6~C*PWNH zvx{|7%mZCH(Nyn%|6GqiHjJ{`&U6A4z-NQ+0uNmmYBdL(^LDbp+~J;{j6FtO#^k7* zOOlx`qEOSF^C@zj?&75F?4<3?qWAvyzIFiQy*2*PZyaA{6YlZ^v>&?DWZMb)Sud7F{900Yzx*&d=`u^0sw zucPexD63?dBW+(alkMAq@Pg=UFiIv8;iTw@s$1wKYLA!de8@ry*ch~v1*DY0f*K(Y zgQ{Wo4`po-{*jcQiaf}6gLTTT|3yYy6n~nD8d{2|;|}-pOtD$Y4>JOi?7LWRIl4Z~ zMuj;nnMIjI*+U&CYJw&GIBe3hmVvl}cBw~9qo(0#jPCvlTWiwemkFB2RJwR0U0U|^ zsU7LVBtAXNlJ-$?Q$AR)svXNefL{pfdrQSYP~KL?Q&`@PR-mPstu(YmS;1YlH@HP5 zd=y4mk=>kbhJj^6?STPpqR#%h4&$4aI=kam#>hvIjEHlh1niJTJ4AFPICl~r$wUP~ zK~_7m&#VZhY_u@RQS3DWTWmcFbk)k`ye>Mi+8Ha2E{$Qm)Y2u;1>1Vrd}$4I{Yr#& zm$OVddhtll<$0Eg=J-f#in(UYtyX$w5i;Z<8Cq;E004~AFU1S1t3Z%c_C_#-r8?9X zE*N1H7^K1|9}8C?J-ej6fI^DO+ykq`6^1PYR(3x{DCP<#g|h@tpJ()j_4q9l6P;ig z$;VBg34v}zfKX2X)D$nks$#tz!-&-utt8%J5Ik&6H@MW-ik6a{pe75TDv0qE85fio z&XjLwilg)4XpF09H^^eui)fm%)JJC92kH8nGS=69usY#m1&a)FMM8 ztPJYNb_&}bPR7lx_60IgCAY#!@LzwH%@@28hk1iIpyKaZ7YHg33jP<0h?Ixvb(`Vo zar$10K)io+ekRIEVZGeqn9vtPH9CX>y~S}j_Ccw50AF57uUZy5y~QSM>{?7yJf4>a zc?nULxp2&!YM*6-QpM7^eIj6!9N4C;D>IlgFy|Ko*|;he#H*7(Ubj~0OpM3;Y7gKG zVN7_CZh;=D1&3&J{Aa}L^+a0mXgb7` zL9;{KP{dYjH1_!Io^aZ_ZL*-MfNNoLZu^Se@m=MsMiE4rtrpKj{x$aW%GbJAR z!BXN-0iq4sMuqh2z#~KUC@-ZO>o{4A( z@d=Wsq#C!64Yl`#gAYf!;842|oi9KkjK3aFLCD;+}Y)z^4^?tbsAA_XIwo1`%P2IQAP_9%*SRkPDHCdoq={ zr=}D&1n{nnd+IB3&scHgo9SF{{rEVv6?gy==rj}f)LSsahRSzCIq<2M$s2f#d}9h4 z0IU2hh7nrq69|DVAyFj`sw>AqnXt7I2gM_S5WxbI>$GnE&jmu6av+4KB6yX@o z1r4&ddLF^ly&@D@adyRu7E-7>=7z6D+vCP+cO^2l7R!)u$KyTW^~H88`m=F(%Hj@U z!nnQ5#bd?g?W8|tRi`^BCF)^bmkl+=?Ol~vo8&3%2~O-Lb^at)&g*SJ4m$L9%78GS zmJYypD~eI4@WH}@R8h%iZ;k6J2UDxlyC679+8t#))(u@%>{AoC>5Yqj7Q?zRl;I%t;$tAJt$TPR!&s|w*!Jx7R) z2S>^fI|Gn(x?2P%^_}jvGOo5uNdU4{N*tosr4*gm>Aotj1me67^J=xb4d8jlq69|? zVhFDrq>S%U1O;^!k|^ZR;(*Pg2ZAz1nJlvGW7pa&3$XSkyJn}mQA+41934VW$sdL2 zCV1ZREtWOS+IEk^w?U_@9Kfel*uM_9QT$n-4+t$bQc?k8lJHxDE|CkcI;iNr|LkbX zxlElcR$J26TCQ)cummX!h?>=@lW?m2VpjORcF*L&w1S^^W8n$S1-iRKzai9qR5pbb|xaZ?O`1_h4ish zV>2RY3&)00FKh`L)WZh#JdYH$YUe5>^9y3MvBx`xuC&kJ)ifZqE@>$RuE5mk3OgKt!6<4jBomgv#{|8v@l7_0rv$I-Yqbf~= zIBft#MoL}}EN7$+0#ofaBUPPbq^#24laaEu8EbPa^mqvJkrCCnCn0qa38`j+r|_UI zH-_QCPIzFFn?goxmMX7PWqCr*?gg3JEh9CrXMh$lCqe*y-N3LQ4si)!Q9LVo^zU&8 zB?WHnOTI!tn_DOwOZcxQ(be1BVx1+hE~9B<1xVN)b{U^6?|KPa~r7EC`F(Jhwr{}jH z-uB6_KYnl9$I_IMrk+2&@z&~hH~shLK9MT2bK>9y_i*qK{D_B5Pv&O;PUyhB{2!4@ z7blLyxOL{Fp3@V2bMuSMO>vOFlOv0${RydLLNERY|1Nw&9&^O9vzI!x6mhlt1{FB^y6>Y|re z!hOf1w#e05vmSaUSF5r6%u*zHTB464j6tKQ94S0PGLpK)2(NQHjH6bn~oEEkxX2|P76lM@iH%}dXn%ZVk)3y4knLB-gdeMA)`VRGynRm=F)C?SRCF8Pa zlw@3;9hYNVFykcpE}n5PeSa{ci#1-Ev5J|xj~z38ri1jzfsnlHe?av1vmi0l!+-n0 z$Q(PVPL7evy>`)=W29dmJ0`w=k02&HuLvTnD1yiVn+PJs{(>k~`w60#2qMM)awt{% z%b~cqF}d;#L_i#Q3*^z5e0n0CULu`mDmZ&YI%S+4HX@fAB^Wunu~(#2s7M(EWE}U* z=_J8Ctu}NIf|Mq~(@`gU;KB*)CiEnqZmMUFR~MT}GrKtoTRO8j1>W+`A@kFouZ=_aIG4?UoNFa2mb@9?1(cQ}(sx{A_3J}0PJ`Yo z(s!<@-58qX$6ZiW{`{xo&QK+DsD(mUzK_a%DME%ZzWMfqAxtE4t~Js6Mvu@FWMZz${`QATgQmFcUJ>F3!lOeI^9 zjthfUtbiS}$&E6}ay0kjyEMz!Mt9^gf~fPd$&^!apZf_)&aE81Rdx`Ux6oYyqq4^A zV2sQJ?BFn2L4z2XJ91kHlh{(|6C+89OEpX>sM=<~F)I;%*Vb~UE#*#|%bhmKH|-nU z7?dbYfWgxeDi|w^46Dd569`pGpauc2NqtKDOV&pTP2wY+ibOSnegrlwKdmHys4y z)a-JI6uywQgj0p5oZuu`Zg>yKx=@4}@#d{fXp`1yB0XdoabwXa!{ik47sYlV=k|!6 z3m*7-mX_QpS<86wCkLfu`2m?Pp^p-O=fWhLQq3;)L?}KAF^H6 zAN0gk^};k@3m}880D^KYv46QsG@|!hqAlS%An_!5l{nk*qm|IL-kzRJGvIZ38*N-_ zkw!LS4<~LEuNnP;AWOMf2|wa=^S&`T=TLHR!v43qdL?eRAEk;pHptO|Y(*?#(Gv7Q zWE?Zjc+kX~A6cD7U_fm#BZZIVq;QHUx0@+ktN%k!Vs#ixMu!8~ob4(3e4Gi^xiGrqID>sV3&L4hw7K&EuyYP?wjJc+^F3 zngG>H?FXk$O@Y-gW9m;kD9*qpFFW%CQd4-1OhHpJg-DC)k^{4b#6Nj&Bf>=0i7!{I;@yMP zD)|<)x!uVh(jwQ29S|&l@Hy;{?SpCI>#e5t^wGJ3L}fnK*)>y6Z$2<9c(Z6r6fu0} z7c*AFBm<$~R&(j;2gevz5_?^OemChW)01=W;edrBXUR`;kPm~$YL3k z0r&eP52y4Np;wqMTCmxO2ehtDOgT^lfCQnOp5-J=ieT<_=(X5($Y_5-HO@zv;i}7S zZyD>+zE(vk+g!DPEUBE~$Rz=r@S<6?T*ISr1LPN8}9(6*B4jm!xNI_Hw%r5V~Rv6hDbYEbq{G10}2h~WLYFcxQ*;Di6F@ilY`fHJvr>?F?uV7y@B|;d{5B3 z;&KIK%nA|F;mH}gRiforQQ%~bp}x%(@@6%^Hz-tfghVChbyp={ghsvp^LW!0c5sTaaH9~tz<_4ij28H zf!Y$!SH#^)w`7!2nngd!cqUrvYs?Aa^e)_cxrXVnCH+z5V^}Z;?Kk6n?|Sk ztSL#{r!ygkgcNn+Nqo$UWCrCqV7`6ksCZHo*z=@ZGS%la$&|aw)f5I5ng1KE&l3NzOM zOpzmu#my9z)OMQL-_gJe0iy zgMBt%Ip-j*lQ6<==XJN@`&geU*`T#n{Wc`BP8>~OR(Agdcx}NN`#~G`8@ObPPQ7i5zf&r4+XUINJdb<`3{v-Vy3?o$qZa zKgLh*jNR=5D87;atYH&@SAQsNiQQ%oLTt_QTc!N1ZL;*Hi=}tsyeR!owu$Xn@`5;V z;(+g?vd6enZp$5D1qmWOXrtXau*GEOc>!I<3E zQxQIs*auO~kWGwP$;JU}+D5w&GI><)GX0C2T0d3dkNi!7yXIfdY2@l_ zBA}luM8VOEsjtxK2W{fuC8BQtl!Hb5IFy5aLCcJ;Lx7X%%8yv5p*zrZ{^lsR&KJT zf&Uo2qrM>#W6=Z2_~?6tP+UYoqg6(3xBOx2o;AioUY3QC!LB5klE?-{k&rraIi_Ig zEhavz-CTZQZLLsE2oNl?2E@F4)`TKa1rp{3i>2!hk}2YV^SV=h`a$`D$$otW40g@e z4~rLAUqCXs5(e-K2g}D5)uj970-GqlPlu8hKux@g{T6^NByWI5bL8aF2DCOOu+9_V zc$MG&$O%ALJ^=te)k|OzM?_&LG*9*%pkF+(*CiI$VeV zp~xVNA)cl$7$nz8lg&R1Fb??|ehGt42bCryOE$( zD(e)60A5o#bSLpDU|g z%Q-iTJGsflQG{;#I|Iy;=ex~Y7k!hw`oCS>&Pl`Q@1Dlbw0ggLpHf5BtEMY{UN!HU z!_>FTmDeoc7dY=-^R>)HU^SGpgK2#yP_?@6OqxIFdrN)WJjw~qGIL`8@oMRi>F4Kc zOU)hEeuHWMbnSFW-40E;_`bAyW#}I-JF!Z=V!|sgq5F+jb`@T+Uj93RZO2^uog;?4 ztKL+pZ--8BnK}HruJ(H+lirL;OzoAQ5A=Nl3`j{(L<-;Bc-=WH^)J^wky$DsodbyX zt^N)$v<$3ao%;r+@qB;a3|5}H<|TUsb<y@dXv&G(xlx$KD;t0b;{gz)1u;)=n<_r>BIUl zI=BiZ-23}GruF6pqp#en!=Bgg4E5bSUNv2VNS0ZyPjd(UVZdBvg7xj@*Xxc@*O=V; zb6T%K*sMiu1X8<_6ok`-IyJO<{Q(|F7WdzJG{2eq;MNBKz5lk+DmEK#JD&I7+*VZa zEw|67;)L5LQE}Dny*xj>{lfg^#2E4;lU#85&>25`R{5y4r%Z6i1Q=u59aBB`T4hRi z-6KWz?iMN!ygN`W!htS126R}Rqjx<4ZjRP#B}hBh%)IAGb&C04_p}uTF!a!pv+Ce` zru9dU%IVX9dE-a+^eK1mZ1oFs(!H(H&7l*jEBM~f754^8O*P-U?+5T|!~I>YrTpSV zNsb(oO5ACU+tcQg=f64bID0OKEkIl2S4j4Ow*rxw{OoY~3kR9&pZbGZZ!UZKhQl^* z?#3O3uN}DwHp$^zyUV!DSQK8zw&W)WeXU{*PBEpYT20H)F>0Op+R(vf!wbd20%4D` zNia8EG|Sxg{2}pg%W515eXFd{$a0vAM#k}Qb*)_Ra!DEN-mMnEEK=vK)aj+D*9)(} zhY|fn)T}ehM6)vg{FZ#=;@L>HBqC*oa*{0pnRr~{9N|m^ms+T+Nu}>mRLL)%1ev19 znUbF?(p|CN(BtVihJJ1&hVLvo<76J5rp*to#m*t4-X?o?F6TGl4_mGJ}b$VikU|!?jl@YLMT=8d)mE%%S@j3;$_8J?Fc|JT?mzuY$|aLdRw3sD~$ouc(FG1Cwe4a zfjh@_Ws;KY1foKOROr^_*viACj2P%gIWkSX$sBT7xIun}Ct0QyVGf3w&tz3$A;v9a zc6c`}KhE>T+R3tPGt*1QlI+6P79ew#UAdTsEks~s88M2|KUJb~WVAQ|l&;}!ohmS5#@9h*G6Y$pvas<<|0fB%u6>`0`;&*peJ2Ea&XQE2hJ>QI+* zjG0g1=c1OJu#qsA-3S=mJfuh*D7z&&3Px5dKf^@I=C_a^fTMb0Sd?pui~0Lwf&6b} z9~KLCKAbYM;QnJ{gkH8TMB1OZMNE?0PQm1S1N|V8IC)bc<1(GTB0rS_>cmlzs|+L$ z5zKp$HH)oU>5I#OOJeIRYd_5o#QW^c5B$C=29deuN2m~+0wxw81frE9_=aMnrda|o z;F}UUN4e8_rd>b4862ly(!*XdLMkeqgZoH{@=+ukCIvZ5DTuqJkU&-v#X#rx>{jub zRWK(ohbi80rF*JxSQDvLa@5yTC$Chj`gI6}fa?6Zv$ZU(*etD^01S zDfAgGjWSZk*~aoX8s%nEWnB9{13hMCjy08G%*+j@HRk6pOjq~cvh4_UpE+{dX*>tF zO;Gon&D(x&u6_292@j~E&<2x(;uU|`3#bxeu}N{Qk}C7xo}Gl-sh(?{aW||lnK^pH zap>0IZjiR3g_K@CKVC@f20vC{c#7y`66$=oL`*7pNE!E;2~2bFGY9kh>N5*?-uKLb zJYRceD$hSZGiZ9BJDa!HpF56cVe4C|bD1@7nt$@<_#udbwv$5&6e56HR||=TSDawT zxX&*!gZF>Nxc&z^c|P~RLZ1GEGphzA-gv~!d+?yD4aw`u2ai|(V4i>QpoXndu?ht* zclkHk5A(@`UFuPD$d5l$kD0cI+FBk%)CdUe?^-#zkthMF6c8RWOCDO!DxW-b70*i^ z{%+$j!XA=*(#dTN>wDF1T%tCbWgB}kAR|T9kIf4km#PQM!H*olxN{$AZMYltCi!@| zIVe9Ys0!{jH$F0E%!8nk4HhnmpoArdFBb`d2hB^5oWS&R-1PW~Jpa$*C#lDWPIw}t z)Q9G@Cy%1)h9^Hzr3O_g~TKX4Cp=bH`>$+k_=7Pg=jjKU<(Vmbd?Ufz)GLs%*F%9A?gb^{Z{L zIO;Tm?*jkq<((i+q60&fKi$0ZY82r6`Su=K;wJvgq<%hwTSPrSzwXdqDz3-TU)+bu z{F@ZS#P|W?C%Q6>sE&MV;x_(l=#M`ys>$|JO(1NkAG&T>uq|peIM3Q7;$lZM7s$s_ zM439*%zACQa?RDR{UmiRj?o;W?-@IE>gy|ADqnkJmZ~%9Hz%u`OxK$SQghCmTZx!P zZNFTdH?(&9?=l)E*cszE&UNX)X`?)V@0Pv3pW%JfUf<92K5eh>kB6T8)nuhEG5`GQ z^O~e%fxkX~NAtItzhlhT-ks3)H(R(=bK9_!e+=KtamJea-ZfcSx|P4<%&|MRt2c*o z@7uvfb7$^|Np3oLjp#-Mrx#*KsyKadM0DuDbSy(lwWNUUk{>t5>adT=@>{_o~$^ zufFz+Sxwp2T)uks%2ij<$qZAgQswPb&nVCHiYk|Lom?@mR9@u@gb`5B;{xoj9qy`;|b?p-}Tjh zP~TUNsmH;2w|ZW^pnjk(|NNDgees%Wue*ll+)un^ltgsa&Mm#e!&T%T(dhOE=68A$NXEa zMZMH}Y#xAL>OHyeQ>B)e|6SDX62~m{WAi!xEVZQfVSkOQzHG(?A1c?3D=*@CU3uQ9 zu}YAsF-Jy+mU#Y3csfS$kY3G(78z;rU)Ok3%YheFYGbbhvSbu18QW0WH6v) z16BXP9kR-!=7n&++F<;;*=lz0xpkwJ`j+Xa-%xiFXqRADD!<)?yhc08lgkk5X z_2#-^bBpU$xfF!oGP{Re6VGIdbvwajs@_f!%9Oiy@jXo+qJR+R44*Pwu*vCls)OeQ zc{B<;)|oqp&p+mD*yBReFq|C(3zVCVcXZg9s0ohO7D4@v$O-Q@9}J(UW}68k4)Yv2 za;4g8?ijh4=Wj=jQ5T!PjQqMf!`wD%%$PP|iXX%AW;dtR+?=f_$2mwp$GkLZW~zhW zoyqQqG`E(8dQ@jpGr>UuREs;6!fA1jk81R(tjdVms5r?BMl+aUx$Ta*LAss9x$q?cI=+ba6SK z{`{t7gjP44TM@qb9h!%mMpShqB?kIYi5|WlvWVq;{ULwUf{Lrt$T#7(#?;q-7sF`E z&2I{l`tXvd3~<@HyoD$gMLDT3z7qN2ve*wIA7#A#MR=uVI1YR}^aSsx={uf;Z%g19 zt9HZ{0S^php#D%r{s#eF9hp!=tSBl+m9)iejLPkItmBVrb<5NVO=AN4T3We&u?mRM+;4y#Gm9j?dgX0bvNC-e&eQ|zIEv>F*743aflQQ%eC>076U^G zU|5^sz|=n~9Xt%>4WdRnZxsFgR@MWJWN3J&iW6|J2q8u!bx{(2Qzi8sQL1kh zU41o*z{Y|FU4#ozL+HkGqa(l5QXvX;F5IXx;1CzyL{o%m@(96Lb!2RvFiL75c(Z$a zr#zd}@KP~ZB_=6$X=h$XY-PzPQL4- zNO;x|jqF?%)g$op?ZAePG#SZE(a5MtSlJ}392zyiLwa>Y_InK9GJgXyEJk5CW!4Xg zvi+(R4C{#Mj_|P|s;lCg%Oe>L+JjBt?(i4eIwBg@*^;_`ITu93v-Rtao;Q;3vUImY z%6e+Y1R07gK)2(qU8q>xK5}D{w14Q_a)dK$G%d;l(pZ8BVNtACb@aXboiHG#Y}5(+=8K zQKOEUGp-Gy2+-B7x3FxAF;PK0BnqHif*8oMam9+gAJujo59Af?{2kz7&qQ+vDZ5qE zq_zVt#uYI$q)^OIK#^0!xPnWE>^m{JPvdk53!60 z#ORmeArb=k;%$JTIdyX|G7#Ypq9K3=DSTZ<)>#K7iO^{IKfft_NnvDoMW&lzigS*b z(z81ez^+=C$S)B4T8}?jSL#ekgB(i8#5nbYXGc;2b}bh7Mv>j7Z(HMXkjr?4xWK$oiaCcaKy8lURW82H;`xg*awt&f8)E zx!GNk4Lx10K>4wb%d?4=QJ|$)Qq;7#OS861vbGDeuO$>g3$a{k`z+MCc&KL=u~S>z z?D}CHvdmKZF{DB;UU7nq=H-B&)V8?G5RIS%X4Ybqtv%J#1#y56XGyo|NqLSCga{z8 z00Y#1L2oMEQduggwS*leeWoTsw#=!vZ>8F|xr6zUo6vSTo;B&)oRpjBRTgG7#buu2 z0dTP`z?PM=kdQ{$!=N&v|1PgW@Asrk55q;v%WyM8o81q5ZApCCPa2yLY^vV(;}KH6 z>kCMNcU)&Bx(@3c%wT0Z9pUmpBwb7E(jNBmwMDQY3NIa>o!s220gAz$TZw` zkv`rh^cgbHhYq={gLtXJ>XpI3BOf&c;a#fYSZM&CAgb%^5Cc=vQfuh!aiKYtX!4yYKDD}RTUX4N z)>>V^8udNobV)~VpySGbWsw&iogGJo#6HQx_i{e;E5Qv?UT~ru8fPhI@8t9wEZV4)Umm!sG zpcoe78fD78empe5hu2$k2ZYw%$f{K{gxSEv&u}nfV&1U;sQ_yO>w2+3Ksm+ff`J5s zc2FXC?mOCs?ZJi#ahfqpVYUP0?P$YF7&3`Y_zN4-2*B2jlrq~ZriWOD^%rI)V?U;D zpiee7iCg=7EyUqx1LswU7&7DX5Te9%C6F(B1N+Es*_g@3NennLuQv+)=Tp%?O0Snq zhlkIptL)fl4` z4WC=7VFh+H_0^WOVzj3?08zD@KCTU#<>FdFS;c{yx^4UuQ=%0H6i+gr*g`;W=(HiJ zye(=K@;x75yM+6!D1B-iFL#TX7-410MnySL?VUngcd_O;(*1m z^!UWYpPy@ED?}6!57FyxUhgv@vPTFiaSm$v1$8E82$n9{>mnM8UZgUu_6s8 zNr@B71UVW;7)ZrpVlj#(;c-$djO~t9R);h`tf)IPM+ZGi> z24zj;yDVL)$e{;Jbv8Dgf(Qz#qq4FyDxavnrm~QeofXQ}W;p^8OUS&0oXj1I%b`2l zB#Af)JW~ldrLh`vhA^yAK&cZrP8<|m%8CQ_&I%%P6S0bJmO;r~4d{2E$Cfq0&p-^y=J6r5sjYKO8S*e`EGGu7!Kz5Ne`VJc& z*XRc*YYEvZbYT7~@eFyqV-!dYb$oW#k_JWX16`EJx<;?Bj@@X-7VH)S%BCN0%gbJZ zX_v-z;najs>2xRyi!%G2K?MEMRX9T11o0N3=VUannjJ|agq}oH9Yu{9Td`D)SVmDt zDv04u;?)mNmJz9)p++WOuRy-81sZ15!R=Ww+bAWjs#{8IG+pnO*IN5ZIFhMcjow6} z%nD$5iom}NHd8DocnkaKuLvtF0SNk$k6N8({9`IqLWnFYt&`~y@HMhvi+dvl+!CpW zcOveg$hs9!EL@32Ep{TH;iY(j(34)MI&WmLB|)3vOFC_tjb`dpK;ScfhaYI zYQxhY8X<&8fY5q`#APBhk(9wSIU)oud>0{zDlzuYk4rYqP^I2zO4+)6M1;_Y zXhde~8ezWJqZ;HqoFU7}zMvq~8sK!*3uvc0+v%7u5f~u}JMkc@XH`tNo`F>8KzB>h zSKL{aSLWrYY%t$gq{038G=DpBM0zGVUMnsilsZ94_PW**?RMkP;-&XX1V2n~6K^Wr z0r=EtROZM8iykGJl~+Btkb08bJ>YaEIReTY`?#dg?nrKnJaDVmOx1{vq!_98KmF{V zUfX)dZ=OyMI~!~}8j$2^_`_S9|8(!3H+Q~Ut&bQUF^;%`@S^>lcklVruMYlZ_rYqj z%s!Rbo%a9u+3UCb`mul6`KSJYIHT}kP7Z8>sf|84JVinzp4IMUzwSn9o|H%i0NZU& z3M566eG^dvQEn3R7pA#R`}AAaC!AQ&oe&UK<+W$rq;`QV0kK z9Eq@z)bcxFM4F)qHc+DL+@U%(PRl9gV_peB>v^Bby7p*<2J!{J(hWGh+`tjrdH{ z-Mv-u@$_g3=Ln5qMVq;6+IdQwcc-mZ?dGDEXwq3w#>Nm~yBsDobA&)KANS6zv0v>lzdHSNYU*cxiE*+m%Er~%aW#zl?#xB>{o~Bz=sWU^R@ON0 zj5W;k#u+1L$dS%oIm`K^vz@){N_3?Ar2XS(J@J`S>ZG>X>kyqe0lMtWk>f)lgCmxV zM=l&1Gz(>5Cy^TeB`aobn>9{dXtvI38_OIcE#=rE>I1B$oApMqEHnhOO};HrYt5v# z6IH9ZsO>m)npxjAQOz`uwSBp9(DMC2rqE{;11`{?4?Y!eNS5l|DwLHX^WSxpW5~^> z*gi|8CVn_F%i5dORI|B#CPzwdw4c{NbDaf7tJPYqgkej}$!AR*+UKlq^?dz3i4i^Q zf;2cLJ!_BJW){x=%oqfcQ${~=jtC^*WzGv7B(iUIV`Uqns2lWN1d}=6nZ2PmuGD6Cpx*X0nmAOqEicOj;0`r-RZ>Bvc*VDRy3 zCeA!LZ~QT`N0jQufof1_`)JQU&zr_!>y)z(o@`M(`y=fGLq=C;IV6P0)|+4+ofnzT z`Q;(9Q~}lslXKzx-dpFNtNwrHyL>@3=hrtu;=FF=AHI&xP^Ort%eboBc;8P0uUi9Y)OMyi>JiX1fImD2CC|Ds~O z(vRIm-(ncc=Z!Rr7vpserb6j%&V4z9)-Ow^Gz=qY>UR6%I5fb921@t19jJ!{DvX{) zmsXE}QHeK3l5Baml!sW?(HjlZ<+C+c*fl-Tre=3_m`Ks%qVsa<+3Y^x#1nz)MdoeqctXYxTAZt%bI88RO_zQo*$L{{d6@ z^vReq-zQ@!x*!K7E;d<^6nY4f#47!Bd1t3butZkeqwqv-(UC!Ta>ymK4d7X@c)kLj zxRHTLEuMWx@RYYciOu)O+d)b4g88om@nXnz7(~0-Jy;b>%j0JEkcz=p@)xunR*A)d z3KrQXASr@L3;F*QX7Q65%obXb!>(8mGPDU!UdcG_Rx;R@czKW;rGI;DX3yRvj*sxltBtDylqTj3@b4Ygn+Fs04l7xX8Vrs0jE1 z$om!$9F@$3st(A7Cp%Wu6-Z!5N9IUML9{yTAesANd^LUIY%ZjA$#4DFo`{k4B@mxic%y!B{zu;6U{_U zYI#SvdUzMFCBawLDuk0>5Udn&#sx~(T8+ZKh9@ineb&gB$@u+3Oc7EczC$!uVPj@z zV~w(j&mUW7&lyG877I|MJIvzqPs&5ps9vsENr~Z=kf^8FO2vJBhLk1|GKoKSt6P^?@O^`;u zeE_A%#YK3M69mn}OX`pD2gsWBhUdB1k>)`Yn%^urF3rrU^&M{3S7f#bHS4!xhH)%p zm3^*5QbG#3-n=pd5y`Xaout+eT`vD(%U6Kiq z>}-;=%|6Ey(2)#yXHvBJEc#YU5?V+;PItM8a{(>aAIJ;^kSA>y6|(y}IZ+joUWF|8 z$4y9F2D8y4Z_hf&bMZl*%fjXdPTwW%Vn4Vu=7DAhdQ6HkPd+Z0~d!*h~{YKfV*Ht5{8kt^1TtpSi zMglD}F`};fL`+HC?p4xMO%}3fkDK{3biK$Vm=hFBqD>+|F-`9>u0M)%-;_x>oa1Uj z)h#o#b<50rlv~Dz%o9C+vuZ7@Tc*=dZkcu)>0uimj~lTV7{9ob*#|Z#FwT-OJU?-S zb>jQq0|E&Q^a<)E-b5 z3JMSs)5M-ea)d2-<6S}*I$PsOX?Gf)l~B7uvLgXmwvbjTdL&qqdD!1zGI-l!-dQ?f zY!Oz029qN9*}Q^actIle#EiduVl0vLqQ6;h$}sJu{bix4rDuIlO4B(?=Wl-3=dYz52_d+7F)koZ{$Mh6bIJnb-YR5 zv0lIbf_<~LSvxZvK{32gU=?PnG`z@mj-1DndUBq24tpszY7DrsCznmFA=Zc3l{2N} zML9rbgY4S5qlR{6Z!Qu&w|bidp!fQFVx6t1*erUs;R3fER+e@N3I|mx(G%M@p0Zas zh6M-seGo$|VkmKO#tVYwawJzPIe@X86}ySY563eex@==o>E9?>eCsdx2t_S=z?M{( zHAuJyG>VNd15+*iw<$pr4lC<}L~VD7yl{{t-dCo!-nU?z~l?albC4_99pK583(3#aeM+feZIvH z;2_xH$gw=OT|!2Ve*}=67(ub@gjnW+i;v}U79%Wr{Zb{ULXZq$E7}CNM5Z>DWm^dc zr17xJZh(L)cQkn?dECg5;Tl3B4%{i1>iQ0hU?0wDCMd4fBK=TJ^KB3%!Id zRq|THK|3x?92c&*8qJ7mW#$MM1Igs!hdHzKl95EWN8~Fc0Le~`Byn5{^AHJz$Ko^V z%d!gRZGBQ6#1gf*yLpwULhOP@T7=*FUE5>>E?^Xv%SpAstlzOsN12*Q^bxrAZ*3c( zXQnR4;t+wmUW5u#zG>o1ho*1BFMD|swX;_50Vn`6$)Kz`-&xiijps^F7Bg~Lj6-Q6 zHOV9s0KptSn2q)E7X4|QY7{C9en7fJL{N4%u_N*a<<`4N!7RRPqB(Hs1s$_RusQ*= zUM@-3-xAkn=T{j9+*Y;n#ieIPcDth@D-bNOR^%1^^>j57 zr0XGN%`|)8PcqZGs@8`U0&5L)zl3CxkZH|0YX}ElmfzQ#DT@bU*6n}9*jmvh0`U5A zO4)^4pR&%AGnib^t&TbjkQ=BdfEZ!HR6b#g_EWMjo-S%BqDnn-lU)OM@JoHQ~UV$hj5R)1ln7^Kj&Tyd$l6-!^%+^o%~@!1Pf}n~*ukTg~>4apuX+9(7Ugrd1CW)vn%?zjAIt z?K10bTCSFxKit$>-lfWU5^lJ$$T9ESa!T(NH@~Lx#xV=GEm7DYv)tni-0pCOvxX3F->-%WE&^H&LzEea64aIz$d4?!E3D zs@}QoLYil+cwKcEzw2R^{(jdSb#?E*cjcA3+WdLt#Y}ttsu>;%U`|=zI^lri-A7<) z^8HDEzC-=cdce$!S0Y<(?p%KOZ}hDj+~t9}e`6~v{mZQv zfbhw;y=aeb?)*WYqfgMgDLB4yH-3X2n>Mzo<-Py3@qt3Uo1JUvz4x*Zx*skz={LUx zf}=JqYr2N(8+rW#P1+6R!>htl=bGT}szH(59 zBd@D_)6L`5&>K+(F-t=Ny;j%g{aXR^OQz0DFu%U*6m_Es%%!7m6j2t_)dZ?O zD%ZKa>&;k?6OKpjJ&hla{r28R0R5)>hO56f_un^%_qXqhc~{-@c`oRlLd}=EJ9+-D z`-{N}0(i+A)PyT~KXd=HDo0%Oq^bM%MA&@Vw@>oa8ngVn-{o0;u!*YX2SZhYUg+D@ zjtS=co+s5}<2^J!Sc@Xd{Uk5E$(;PqKgb!>E#|;O*%{RA<~PJPAoAQokJt!u2i>|_ zacge47L~9LF1)Gt&WA&#PBafb@^uc3CU0rwIrgzL%sV^&j4uA>nHHYmvkQ4%`s|s7 zO%lj`-8}T{v4x)O_4Q}xsEwv!=dlfaQt>4qVzovrE)vwr8F)i=zVUE_K7 z>>A1Qxm}x?ap4bc%cs!JDd2u^q8n_x$0r(9MT-c(d$+g6_RhW|*SV%Rd~3Ih-;Ko(04?e= zZ$COyZ8oE~9%s~#zp6fK(jQM%x0&rfzH@B9=^KdVaUE=XG8=fCx#Z=E=6lgUtjdhtK7~;W zw~s#eYg`vkOfEX(TkqQ2ObRLaK8#TI7W`>qZp;t zDw8}dZf4!nXYqXb>DT3;kw4j~u8Vd1&dZ}4Wp9Dx4f5C6?R$G)8%JD$aCLMXQ_Fn@s$(m0{wI8#|25;KRb`Fm?f_q|6dtfbKfi1FgpM0 zMLaKi^)Fb~i+^?r&t3L8^ff)IM_Hx<;Rvo-NO*X>#FvHeduGLJ!}(z=HBhQxqndDv zx$m`4b2z?sPpr;2L-r&?u!8nVLS_kf8og(xZNFLVHR|;y+Sv?Z0yAs3v-kMdW99Za zX3-l%M=Zs+6eX|hB*gleZBP63N8_yIoqg}`BOS-NxfnX*DTnjjH0b+j-iHnP-pTv)LEpdY zJ^1E{N_CpiZ(T98o`y32bNp}Mei50#4m=c?2M@NLnf;dAJ~^MpLI=LML;Wvury%&H znS1ae^Y+0@%zM9&dQbmDPE|%RQUDVmn|c@i@z?4T;T~ Date: Sun, 20 Sep 2026 16:21:42 +0200 Subject: [PATCH 829/927] packaging/nix-make: Use builtins.parallel Scan a component's dependencies on other threads while scanning the component itself, and instantiate the object derivations and the dependencies in parallel when a link derivation is instantiated. This requires the `parallel-eval` experimental feature (and `eval-cores`); without it, `builtins.parallel` is replaced by a no-op and evaluation is sequential as before. The resulting derivations are unchanged. With parallel-eval, scanning libutil and libstore together costs the same as scanning libstore alone, and an uncached evaluation of nix-store goes from 1.22s to 1.08s. Most of the remaining time is fixed flake overhead and the scan of the component itself. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/lib.nix | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 085ce1375c..4f5fd4aca5 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -12,6 +12,11 @@ let externalDeps = import ./deps.nix { inherit pkgs; }; + # `builtins.parallel xs x` starts evaluating the values `xs` on other + # threads and returns `x`. It requires the `parallel-eval` experimental + # feature (and `eval-cores`); without it, evaluation is sequential. + parallel = builtins.parallel or (xs: x: x); + # Non-optimized for now, for faster iteration. optimizationFlags = [ "-O0" ]; @@ -151,7 +156,8 @@ let pkgs.runCommandCC component.name { __structuredAttrs = true; - inherit objects; + # Instantiate the objects and the dependencies in parallel. + objects = parallel (map (d: d.drvPath) component.allDeps ++ map (o: o.drvPath) objects) objects; inherit (component) libName linkFlags; linkPkgConfig = lib.concatMap (d: d.pkgconfig or [ ]) deps; linkLibs = @@ -250,19 +256,22 @@ let ) { } allDeps; depExcludes = map (d: d.component.name) allDeps; - units = getDeps ( - { - inherit builtins; - inherit sourceExtensions; - roots = roots ++ depRoots; - includeDirs = includeDirs ++ depIncludeDirs; - excludeSources = excludeSources ++ depExcludes; - # Work around a crash in `builtins.wasm` (Nix <= 3.22.5) when - # copying "layered" attribute sets (the result of `//`) into Wasm: - # `mapAttrs` produces a fresh, non-layered attribute set. - files = lib.mapAttrs (_: v: v) (files // depFiles); - } - // lib.optionalAttrs (sources != null) { inherit sources; } + # Scan this component in parallel with its dependencies. + units = parallel (map (d: d.units) deps) ( + getDeps ( + { + inherit builtins; + inherit sourceExtensions; + roots = roots ++ depRoots; + includeDirs = includeDirs ++ depIncludeDirs; + excludeSources = excludeSources ++ depExcludes; + # Work around a crash in `builtins.wasm` (Nix <= 3.22.5) when + # copying "layered" attribute sets (the result of `//`) into Wasm: + # `mapAttrs` produces a fresh, non-layered attribute set. + files = lib.mapAttrs (_: v: v) (files // depFiles); + } + // lib.optionalAttrs (sources != null) { inherit sources; } + ) ); component = args // { From 565524f2bea6d91b64a4b470853355cf9322a09f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 16:39:47 +0200 Subject: [PATCH 830/927] packaging/nix-make: Evaluate preprocessor conditionals in the scanner The scanner (updated scanner.wasm) now evaluates `#if`/`#ifdef` conditionals around includes against macros known to be defined or undefined, keeping both branches when it cannot decide. Pass it the platform macros for Linux/x86_64 and the macros from the generated config headers, which components now declare as attribute sets via `configHeaders` instead of writing them into `files` themselves. This removes all "included file not found" warnings for platform-specific includes, and headers for other platforms are no longer inputs of any compilation unit, so editing one no longer triggers rebuilds. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/lib.nix | 60 ++++++++++++++++++++++++++++++-- packaging/nix-make/libstore.nix | 11 +++--- packaging/nix-make/libutil.nix | 9 +++-- packaging/nix-make/scanner.wasm | Bin 61762 -> 75036 bytes 4 files changed, 71 insertions(+), 9 deletions(-) diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 4f5fd4aca5..3ac6ee57cb 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -210,6 +210,12 @@ let - `excludeSources`: paths (files, or directories with everything below them) to leave out of the compilation units. - `files`: extra files (e.g. generated headers) by path in the root namespace. + - `configHeaders`: generated config headers, as an attribute set from + path in the root namespace to the `#define`s (see `mkConfigHeader`). + Their macros are also used to evaluate preprocessor conditionals. + - `defines`, `undefines`: further macros known to be defined (with + value) or undefined when evaluating preprocessor conditionals, on + top of the platform defaults and the config headers. - `extraCxxFlags`, `linkFlags`, `extraLinkLibs`: what they say. External dependencies (compile flags and libraries) are derived from @@ -232,6 +238,9 @@ let sources ? null, excludeSources ? [ ], files ? { }, + configHeaders ? { }, + defines ? { }, + undefines ? [ ], extraCxxFlags ? [ ], linkFlags ? [ ], extraLinkLibs ? [ ], @@ -256,6 +265,24 @@ let ) { } allDeps; depExcludes = map (d: d.component.name) allDeps; + # Generated config headers, whose macros are also known to the scanner. + configFiles = lib.mapAttrs (path: attrs: mkConfigHeader (baseNameOf path) attrs) configHeaders; + configDefines = lib.foldl' (acc: attrs: acc // attrs) { } (lib.attrValues configHeaders); + allFiles = files // configFiles; + + # Macros for evaluating conditionals: platform defaults, then the + # dependencies' config macros, then our own. `null` means undefined. + allDefines = + lib.foldl' (acc: d: acc // d.component.allDefines) platformDefines allDeps + // configDefines + // defines; + knownDefines = lib.mapAttrs ( + _: v: if builtins.isBool v then (if v then "1" else "0") else toString v + ) (lib.filterAttrs (_: v: v != null) allDefines); + knownUndefines = lib.unique ( + platformUndefines ++ undefines ++ lib.attrNames (lib.filterAttrs (_: v: v == null) allDefines) + ); + # Scan this component in parallel with its dependencies. units = parallel (map (d: d.units) deps) ( getDeps ( @@ -268,19 +295,22 @@ let # Work around a crash in `builtins.wasm` (Nix <= 3.22.5) when # copying "layered" attribute sets (the result of `//`) into Wasm: # `mapAttrs` produces a fresh, non-layered attribute set. - files = lib.mapAttrs (_: v: v) (files // depFiles); + files = lib.mapAttrs (_: v: v) (allFiles // depFiles); + defines = lib.mapAttrs (_: v: v) knownDefines; + undefines = knownUndefines; } // lib.optionalAttrs (sources != null) { inherit sources; } ) ); component = args // { + files = allFiles; inherit units allDeps + allDefines depIncludeDirs publicIncludeDirs - files extraCxxFlags linkFlags extraLinkLibs @@ -289,6 +319,32 @@ let in linkSharedLibrary component (map (compileUnit component) units); + # Macros the scanner can rely on when evaluating preprocessor conditionals + # (Linux, x86_64, GCC). Only what is certain: an unknown macro keeps both + # branches of a conditional, but a wrong one would drop includes. + platformDefines = { + __linux__ = 1; + __gnu_linux__ = 1; + __unix__ = 1; + __unix = 1; + __x86_64__ = 1; + __LP64__ = 1; + }; + platformUndefines = [ + "_WIN32" + "_WIN64" + "__CYGWIN__" + "__MINGW32__" + "__APPLE__" + "__MACH__" + "__FreeBSD__" + "__NetBSD__" + "__OpenBSD__" + "__DragonFly__" + "__sun" + "__clang__" + ]; + # Sources from nix-meson-build-support that Meson links into every component. commonSupportFiles = { "nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc" = diff --git a/packaging/nix-make/libstore.nix b/packaging/nix-make/libstore.nix index 8d8f07bc30..c8286b2c03 100644 --- a/packaging/nix-make/libstore.nix +++ b/packaging/nix-make/libstore.nix @@ -40,12 +40,17 @@ nixMake.mkComponent { ]; files = nixMake.commonSupportFiles // { - "include/nix/store/config.hh" = nixMake.mkConfigHeader "config.hh" { + "schema.sql.gen.hh" = nixMake.mkStringHeader ../../src/libstore/schema.sql; + "ca-specific-schema.sql.gen.hh" = nixMake.mkStringHeader ../../src/libstore/ca-specific-schema.sql; + }; + + configHeaders = { + "include/nix/store/config.hh" = { NIX_LOCAL_SYSTEM = "x86_64-linux"; NIX_SUPPORT_ACL = 1; NIX_WITH_AWS_AUTH = 0; }; - "store-config-private.hh" = nixMake.mkConfigHeader "store-config-private.hh" { + "store-config-private.hh" = { CAN_LINK_SYMLINK = 1; DETERMINATE_NIX_VERSION = lib.fileContents ../../.version-determinate; HAVE_EMBEDDED_SANDBOX_SHELL = 0; @@ -65,8 +70,6 @@ nixMake.mkComponent { PACKAGE_VERSION = lib.fileContents ../../.version; SANDBOX_SHELL = "${pkgs.busybox-sandbox-shell}/bin/busybox"; }; - "schema.sql.gen.hh" = nixMake.mkStringHeader ../../src/libstore/schema.sql; - "ca-specific-schema.sql.gen.hh" = nixMake.mkStringHeader ../../src/libstore/ca-specific-schema.sql; }; linkFlags = [ "-Wl,--wrap=__assert_fail" ]; diff --git a/packaging/nix-make/libutil.nix b/packaging/nix-make/libutil.nix index 9a26ef8d30..9580b14156 100644 --- a/packaging/nix-make/libutil.nix +++ b/packaging/nix-make/libutil.nix @@ -34,15 +34,18 @@ nixMake.mkComponent { "nix-meson-build-support/common/cxa-throw/is-logic-error.hh" = ../../nix-meson-build-support/common/cxa-throw/is-logic-error.hh; - "include/nix/util/config.hh" = nixMake.mkConfigHeader "config.hh" { + }; + + configHeaders = { + "include/nix/util/config.hh" = { NIX_UBSAN_ENABLED = 0; NIX_ASAN_ENABLED = 0; }; - "util-config-private.hh" = nixMake.mkConfigHeader "util-config-private.hh" { + "util-config-private.hh" = { HAVE_LIBCPUID = 1; HAVE_POSIX_FALLOCATE = 1; }; - "unix/util-unix-config-private.hh" = nixMake.mkConfigHeader "util-unix-config-private.hh" { + "unix/util-unix-config-private.hh" = { HAVE_CLOSE_RANGE = 1; HAVE_COPY_FILE_RANGE = 1; HAVE_DECL_AT_SYMLINK_NOFOLLOW = 1; diff --git a/packaging/nix-make/scanner.wasm b/packaging/nix-make/scanner.wasm index d9fb9681bc003e5011a8bb884fd03b3c9a2fe037..85aa6047ed5541b4de0c3bd289a69351bbe7d382 100644 GIT binary patch delta 37149 zcmcJ&4V)FlneX4#ea@LPbIve5yg9rmeGVfI3W9H-h&oLa5HRZcvL@;(0xB~I0wbD) za8Oan#w2K)gcyw^>nfXQLZWMki*ZymXiUN;nvlCBlB~Ni8{(2(_rKh|=HvhSd%DkI zK;7MY|Mw4{>FTPku6pWuucxZ|Jp1e5zL#<}Z~n^boacG&5qH>?L9JGEwJZHv&C`Wj zm3UX?xRN;j^17`TQNgK}m(_G@t}NK|MnygJu53vQ(>H@PYie1=HMJ%+MZsOSjW5#n z8vc2Aef{_;E?Bg5Rq498<%$h|BCr0m&^HHuBqT$!R7M#T+Rh8ll%k5 z=Ab~UbXU@g&~11w@2+=89R-M9dtpRSb7#bB*Y0KBuU3}9OH^Kr}EMB^}?}}B&dH^CJL`lI54x|o zv;Oq5^=_T}jJwZ$*}dWZhx-?ER?*1DVB@}IjuxV>(l zE4!AKzq}>)8+VNJYmwLG7b`BJ-Efd)A}?~W>zU#gmCg0U(KOF1jdiXTB{Q$>Xy;*g zRjG7}y&U=Bpybu1lg5;e@FG8U%X8g&U$?8m_O?=*2?r;mgVAnQa!PQio11(#_&2vS z`DE_66E=UL-)e`~pVjK(o#f)5dQmPO(-ZeAv-#eBFTBYsMY%4&qtYA&;oRh(bDgyv zP6S;OZv>SSJz$@e>-M|HdU4oWb)BA1#$V$D%2}JXXUanc%48D96&H*t^WDz z)Wx=Sbm7JO#yy=jK^@0XrUsBocV5-E{v&^eTRk;M+Vex*za-=G{qC^jH~Bf-A741S zAU+H`9=ocr(j6JmZby^@<;AQkm4xx{Z=t6g;{fH?B>&NL__+70`SPs;YI@0)#nCQG zK3uF$vlpP;yV|}@qo}v-l|pfVIL){(99jCi$DSx&>-(Q{$(YhR&QHda&*i$TJkgC# z?k|rRGe)fFqTt+IE%G(+DCiZO|Kjop$63iw%3bb=l#h3MJy9{N zMENfN!fH{;CDl^=5SWOY!eQWDMYG|sVpNJBRdF-Lt!lmyHS6KLC>I`{{H}Gv>D8zZ zj)l538}@{qBx@b?hS{lhX4D)@LgrZRyr`JwqL!#By-O|{bn&p)5W-Rv#OwSXiAoIm z!&yoHpyQ5t&4IDJ70gi|L6?80Un$V}TzKd$cqcwQeta7g{6I3gEpjg$M_4P!pQ4{$7?A=3_x+GaP52KEL56{l5J?U zpc~DS>xsPZrR43yj&gI7&W>?&#`)dSj$x{MbMa`}?(&C*M=6Oad3q{ZQD@b~<*YgRnXM~@y93kbR6p{$#WeS(e#KBE8NeLJ4c+y^%o-!cMFsMHKNv<@!$TG z|Lz!h_~Em}p)w#Ox!BD_lDI2Zak zc}b-$u2ow7ZXUKtpa)f2B|x>3;A@FUa+P6GTU_p~v_-9;yV444lxc*1yl5DBZ_(e` zqtGN(%l1aYMpav4Z%)Kzc5nQSTRzCgxhNsrm0=8LZA)*pCABwEJB`9<)edM>Z3k`) z{@S%S)r67`vH;t{_yU8bonng>$gO(_lk&t_S&hj@gf6nk-^=ev}KCF zJ$v4tJ>Qn)w<;}7@gG+?^|rVg`~YAnfQCjJXE+?HQ@#UrZG!}<2D=J(HdhB(pHQF| zHpxe&o=RRSyCvRehz3*POjX_Js`;~81Kr-|Dy^zPw>vb94OX)T<*Kw3wT7cqYlEwn zgw!too)lNDS@rc)9|U>xX}vog?@((!9gIq0h`7m9L3XJn-VIuIG&2gCCQCP%ozX_G z*Xfda;_}{F*oHKfB{eRUUf)li7x7fV7>G(`Vr`AjA3uHft9`n4B-VD+5TpRMfz5=II;`cSJcM) zJyN-Nrpte$VPcKeJ1G>eNGOyFL>qj0Nl$fjZHoU21@Tq{{;)V#85$sUR$=rk90>Gv z5Bfles2-k5tOTkJ+X*7nq%a$Pi1a^^0zoAn@ACTvTM(M_fi8coiBCLM46B~-R*Xwy$$LKh8>@U%HJ>uhYT?Z!m~xX~2(-mX znWTe@_tAjJ|HRt1F_BFXuc4mK zF(q@dwTiNOCE>jZDU53MWL>6*ZDrUqeM@Q~-bFVxv1ciMeZU)t1Jy6l2hJb?w&fhvkf%FF7ED4UsH{FJ?P$ey=w6?&KQUi|8<-2 z3Zgw(FN5Ok0{~LpD|K1O5KPLY34tcWsiWA4GT9(`+*KnLZFLz5PfYPTS@;u8q>+TN zo+^S3m21YMrkBtu(o2#sS}axa%weDy?_q!s3qPoKyKG=a?+F;Y7M1n8>hP#l0Fl|| zs;xb}DG1d(ejXi$@oU~pY-AWu?u}c{w!7EfdB#N{UIi_~SOAp)?=jh%;_pT=rT-bC z{;TJBD=o)<>TfImr1JT27|f;Y)5;de8q3AcE1M%5*wYu4#aVC7S$3zgMY4v)tb<+I zE3*G;arZs&|9~ETm9=qJt5g%?+TL6($-CgTzZo_SP#Ii>%u?+4PV|`O&1x-!J!uy4 zf!G49n}NZxN{xw;u8jKFZcX9LEgwZY)mg26E%q`SNnfb$l%h;2tS_#;f`De}E-G1v zGfgkKtap@iX`1JiTL8iUrV+z3${j=4jQa|#(`U674$*6}Ot*Byao~X2Z(tY{sX?ud=sss=sCW&>_;-!cCKI-)gQR_Mi^L!x5!&lkf`! z=j3XZGHWguQDqZ67kAHYwa2BX^<1M}s&}gvvb#VB2`kIP$#Fu1{;CRq9ubg(2T1~2gg~9TzKmomPYp(T( z7SXWqFP;3y@JUxL=m~SRw?sw^c{BWEjt{7fci-Jl=fZV}RA#zon+&d;7{*TuwFa~Y z^=-x}<@@e-Yjh&VF8<;FdBZu4H6250s*x4Iz{Shkn7-qT#ou#Gwf7s;yM~i0(niyR zvZfou!QN#~*O}GQdxD(w-^dn$vaXwp_9ypA44?vMo3X*9S;dUVn(nLX%_9c#tOqjNxJnpQ2HExi*L zX+BVlViv4vxR^%WePDU)dWCUXeZyp#WcG?qzQl9IT#p?;sBgwnP&yF+TzWhHS=7SWgZlf zbV|I0iMDvt{XA2M0+7jWXk<&35`>ZGXp;!dmIn*#H56SK@n|=XXlph2AzACfiArx0 z&J?eTMn{lnEk;Uwz?g&j_=wu(~xd^&#Q1Hk*HhV(mN%5;KBU?5V&Csy?JHc&v?RlH7=v zwIkd^G_Ijs!js~6Y6k*i^mroCxfFCfl2&62u-}T9wZ(oPf^sKXKjnl+XGFJGjx>2b zQPD5{-855JiH@A=Urz#YmXI_}@vkL8^_)Can3XqvS;ORXOHg1#T{XbCB*(HG-)BL- za{LG>PMpncfv>OH7UwlfYEua>k?kGCYN02U+NAhkIld=*mv|5F7Ao0f=iQI;m9SJ3 zqOY$E38yd!?^Hm5S{745xd@Fy6m)=x#ilrvN5ZwHP7qwWZ_+(~gJkU8$nrG{lQc=J(Pc&d~!2wksu$$+M7w2d~@ zY=r5L(KIwpuhdxq$%F$-asFJ^?{ZiXcpXh!Z#$e`&Y3fzDs^5ci5FBw{L=~#ScuCT zP?WEfO5thMHYRySn6{MYs5k3<1g(keL+s%Yk-Z4fi4RvLOwhiapb&(Y-w4%Jif$jP zN70CAY%lBx0}O!yhHyPuS~Kk@ZC+wIBpMNyYv(d0luAXsEbdr|N_zQD{Kz5U=?nq& z*cJ^A&j4#!05O4Z9acfI_=Xh&Ma2PQAWjbOfp0S#6iO&dRd;_=M#3R9BY!9ti9$>* zMf{-y*e3U-H9P%F$1bh%Ob2qGnwL1DIT!Yk}malrC3Wc zwS(7Wh77FeVFw1R$!yT#9w89(IC3>gZaZ*1w zRgMCYN1AnNlXbEdYDlvgC~>tFH=AuS=&=UM6lJ)VV`5m{p!T&(u;p@AkFpu1UL2R+ zK`Q)wVFJ{c2$8OZNSa>>o8@={NWp5DIl1FD#?L*jA2UX(UW9oO~91 z7-M)bYpA{!cZNnD^C(v2d%{a7f!L!-d~B+gCMgiF*|IM$U?>UvoQNv61s^;wZ6t6Y zMa)vL%B)b@8W7UcBq9}A@#Y~5t5I{!=t0~8S4>Ihyc}7k7%56) zha%?j5 zq_MRo=n&=3WsViUu@yK@iZ^{HlPQf&0&WPi6+#;~VMHc{8l+@@9;5<*&ME$E?))kK znKpn1f*4J%nqyN6m1d(bNaj_FASQQKwYe3R=YsDJq5LTKc8dr03z{JK6y_+d;=vSQ zj$Cgghv{-K6zi(JVyG5lGwmkJrgu4$l_r_EsDn;!XMH>KGsppQtqSg+myzCZ5Q4onQ2jW$JYCAJ_|OWil8A5Gm6r%k1g^G{n{`;{YXiRMP1A6O)B zXzur2Wj}OT|J|$ZEdv|GzaFUJ53ITmNf+s}=Jq@Vdww9f;`D@FJyIpQ6vOA?N8N?G z47F_dLNerx)5i|e29x@O2%RA2AvGonhan=Rr2Y6C*qn05@edE~5^0c1TH{Zv$jKdN zjA>rQo+2b_(+kNr&X_Rn0Bi=@2qULWOrKfq3)yI8OgftY9Yl} z@<{h}hab8x;UJ79Q{p3d{i1lmkSj}Usi?)<(!EV}I69KQkB7D%++}XY40m@DeWj9I zJEO}@N$#34x%FVg{cy(Y(HkE?5{`)s_;v$6c73s10n=TR9RGo1$9xrQ1RF{!6=$k% z?x89O4@V}o54`MlCG*alcG#)HR=~VA-jCjhhs3+kBDC_WGZ%EFn~ivhhSD8S>9(Zb zDuyYWQm?$^tcyQ%Bb$s&pOq>+v@z12)IO?d-50nY8*d~LJo8*oGf=8Q#p$M4@lg)G ztc|I81Sr*_%>0u#7Y=VWpWQ9<&sZ)+6OUOg%@)5#pu8J{Qs|3vD=L9m9o`G{p6dJ5 zXOM-y753IXr*MGcZe>au)pJ%o9}n)sec$IT*PB7(O)4@^VK}M6k##G*QuIT1pJCH? zdcD09<}%@H!kD6}F<3rJ?I^)DOyOdzn2J=x zf+0baSICUW4n5diK{Gw-%d9jpt1O^y=+^w`X7%Z%u(6&SHP5V73jzwsjS8day@WNh zRSX5NHQn;#H|~W~&NE};$NRGkZ2|?r)3yc>Fn|<%X1a5+Lv`?JLDTGfyoQKo3O+^5 z_EPX`#2E!Y`?~5QF~Ucs6#fEW82mKbQymCDZYGtq6+nZ-Y#a zLQ2J`MZu$DG*mkZi!twoBpCV$B-rA?k*n02A|LA;V!7C^tVM2UrA(;ULzddu0mOTX8Qb67Q4}F`e(aw^WEfCcaSOV!!gBViCRVI2 zu0!IABKaNd5UhnJTK!-7qu&dKS%|%=b})A(!_wha^~LJ zCZkpja5NET;Yl&iGSUTyxH9Y{Re{e0`L#NV1K$Jx&7~QnM{FP=QOYVIoQnHG=>BE7 z-C{qwiiY%5X>VxM*2)GzEhwoCn7@TdEca|mX|}f1Q%}XI$!0EuL196{S(IwRjJu=v zya=a>?M?KuyWR`X#oOu`jjdGg5q9xP8tG_*%?3ydQIQ#|HklS1?ui7xAvF@Ay3~9S z)00jY%r^l=$+e#wp6w)+iYXtY+&_D3zmzT0ADND1@ywO}x<`^<&iwTdDgUncb!Kiz zRu{Y3FcUwSb&~B~oy-GMd2XrJ& z0FqI|bbJ0|x1YVzeegeTY@o^NitoDR;Kp*=SC}=44rPfcxpUTp+QBa+2Se~#*YNTR zczFeRm8L|aKugM=-wkwmF(?V!R}|bJ0+lVM&S7l=8d?c%EmeB8jmT!+nt&Iv1U32I zEC?wQh{ZxZ*>D9Z(}s#;{Ad8tACAZYf#wgTTU};Oi&cDB>}EA+EWv7q=*tffO*JVg zYf>VQs~Q}bl#pjq5`;B;TdG^DydiU9?yJ1#PBLBpFZ;X!c}XXja(gMbr{>~KpQCb9I8J%RUfKGBkXf)2Dbof! z4Km)UYGHV!9&L8@IGJW+gqXN8Sq2fp6>pEH%G7{>92I%$#cww{iTC?GW)x^=EiPKR zGu>v<>s)w*ntw#QgGjeH99HX&$xSXqdBq!9mg~oDLPe%K&bQVx8{~en7hEgjo3ncX zv5EV5L!dzfs8WBFcKpVhC}<5YifX7Nc24o8=)7Gi`oCX*GnQ?ZM`>57#NxA6dOWMN zC96aj$r>T-=;F5mO~hRMGO4E5cs0eK+wZ1mpD#3l&Txzdwa*t0YYO{{a9dEX$eXZJ zWyX2{k^9)%z8OWYci#5HidqV6W9z|n-e6ecZM5TU{GbtwM@xhzr1t6Bhqu;&=@cC? z1}Cgt#3RhvcEeb_POenH5Tpe)dK6Y+lR%au?g#nJ1o0_Zjn%TpDWz^o`3ess3sSVC zb6Q3OG(P)Jo1&o>t7uZJ!l+B5xwux3R#-XGm+?BgrlAm1ZZTBqRwGjxUbMG23nQ_1 zCDE|GzD;5Z%b-d*RT1=L1q_~Dibx}ayUzyN!QV`Mk1QLPd5;?c>G zAJAhu2hx6YU|`J5q3Vz#4n~|ue4F;qK`UqBeAoqzfk?5KCOel>zmt@~VMrKItxjpo z3mh%f3+YIgU2QC_gOp=jo_6Gs#UFumX#&{uP10LI`5ImabMebx-H4ZU2SIOEjBU99?DKWX2 zm|cyqhiR=E3Zsqi?7rq@q_hD9$lt*67t>I1t}ASGNUpU~6> zb&z8e8{U`=l1(r?k3A+vdv&9Yi(Y)ENRm%g@%5$|Re1+KPCKa$id7u8xWgnH17Z=W z6y6g^=ZJ4W6;h?0J?w#^Nl+p{tHpT3Mw-s0O6vUBN|04{XPY89$}?bym&C4y zwGjZ>*e**A00Tn3q+P%eHnge_Q)~erQdf^CMNx0gkolnHrKr*3o3ctK>QqA1!88j; zD(AVO$ja?yn0ao==0$`KzFU>@;hL$FUidkHdA2tlEN zU@{?t%Z%H>jb5wSR8wYWvxfW8!(ZI%*UH`gR>OxkkK3)@2x+$ zOStQZA0e@d_lREPNi6@Y@+0HjSw2In7eAx-O5u@A01TDs7>!G6Vr42K2I3Qh1azZM zKoT3wjH}*~lYW42_fuHkJx`4|$0;!y&^A`YFYKZwHB5ao-K@74oHsSKD7Jurix0qK5C1#pv+W-ALec?fU>Wn(Io9n4fdPxfLx9#MPB@}Mmp#{ zM|$u;p_2`?_O6lA6J5N=C?yNpmW!^jQHrZ%q;dmBIs3pUVHN+tI2j&nYd%V2m@M=} zH7URPIEyw;4kekDsiV%uxn4LG6U`HO;`LIO8&&qY`3+wDKQ!_tMvmySU)&!+V>XWa z6-KKq!#vyTS8WIscvEVro<%=(8|eQf&~k-SmY*SuEiNbv2hBU1$seOd1)2> zBOZ#67JGaMtmLDzP-3kzSnW&jf2Pux4!F zl{mD_`CJw&J;gQSukPxCp?sxmEySzi-mN0)2`-9rDT#mTE0}0Zrs--IxAe}INX>1_ zRYt?2rtXWiD!2;OuU@91XS}n+?>b&!xzj{mk1SCAs`^PEvpfhqnm99_$vt5=!_nX~ zqd!?k&B7~0I}E?kazh|yixezOjl6bOVB}MF-IVb!F1>c)-;7w@M4S@W3dFcHFJ-LG%D*t)HU@0gkEtp-l#ZQ0la)UdEmkkohhl(YH(M+QZ!wc zwgi6EQnLHPku{xs(N1b}3sT*0_RUYj8O`6! zZl!gY*s+lh#3)U*m29=A%@OST(DYpz@MMUlc2J3IxMB0Pi|;e%x5Z-rf%a(9PE+>ay zJaQUiq`Eu_gho-$oHh_lMXZQ3O03xY!5X@>nYJIOgdxa(PL^LB*G?DX;ecK#O~7gc zOWG_VilKvUz_UP$7M19?i+#fQ5!1vpUDXzGOP4>P+6w!$rV9g#gp*5=(lXbC-?4yg z+^3*6J$Lz$V%h*vf+uPjzA&@ETL@33AGs@lopL884kVh%^U4Leb?kFf&1tYYy;+N+ z-XZL;yYPPwX?HZl^+qmLZZCXpy{jHoTypsrkFIgl+({?Jc|uQ z(g+fPO-fOY_xXCH{gTo|^2Af~h5QiM2Lc*McWS2wNVE4XoG*Is&`HKewO}J2RlmB( z<}_QmmnzsvTKD<<9jd@t&Zwns#R1HWIDj2w#1ZRRRlP2GrIO}%f_?|7nl&R%+8-dh zg}BD{Br_Q4ByE2BQ35Bk{|bHLxd50v3fKf!d=MPO9ShzeckFK+%)*Hn&)VLTfxatC z_h#wqN~Ls#5)A49l;DfTjc}5DU!g%LSueOVl&lRR>sWnKU;aHXy={=KRohHDx$c68 z%sJyKeoCeAXyEhtqrEPFmtXb7%U%8+-%4u!ZY>gca4|dUD~e?*D2%<3iz0-X+rbM% zWzEf^kO_ri%nWf7t&XMcgn}m0L^4?2jmMVcc;=>k&`MQKeqP+AimkbLvQC)6H%5?$TapKGT13{ zy)F}tE?Ss11QQs_Q3DD(Y!M=X@d86RP!2>n{F??B4eiu27mZ2>p`BLHtVHvKoXp6H zh36+9yljHDV2Kw^2!e`d+Z`DNu1pBM>6{lsn%)K=F?-;)D&5C)f#Vl~4OSA2F_udp z#&6goK`F4s7}|d&0ONnKM+d=}!-JQv6aUJd0jKc-$dVTbCU*Miqdd9x!Suy;iFv3%MGApI+4hqxrZ)(MIQHtAig8@Yawd9RANX;2{3Bd0`O^G|>(lecV9zD9d>a9C#gQnQs%xJ3J~;zYh;I<9)eVrR z-o_!_8b4Zh@+H*9BZf*8DEAzUgVJp1Hu^?w1THzXkR3f3o}OA@8TJ17 z9KsUqCTYydjU6`RB(rzs{1%q0(s@LrogBXO0cS!2^a1+KwkT@V4HUq!>6sAF9-$+& zplz#0CLMMPP` z*IZk5QlwdLHPgjQ8Z8|1R^~shOmmHTvIR zp4xxSJ^#nd^X2~r^VB@XY}_LcY9RM0Z`@GHfOTo533(EI>0R{ zvwU~qB)-q{=E6#CPHCfSYz&a{U^P9*1c(I3BGO0E%)h`3(z-ju1aw$g^z?cae&IN$ z{7QT}V#?{hMw|T;&-WP=4{R2jPEc88$|(wGu@{=;+<{s{2ja?@6N<7hc2A}(80UJE z%NNXUJH+En9$)bEpwXymWJt=c5Kc;NURWJ9vNYn5uX-K$o>#c&825BCebMDy?_IPq z8F}@&{$KgYAFm!$WuSJ<6zgLO(P_j|ohF}(Mu78-iL^W6fEiBZ8O~#|yC>Z`N%YC3P8i$otAr;_ z*%5A5n#C^BK-wDSq<-D4;NR>&ZY}Bwr%V$koHB7fW=v_h*SK9CS+=3%vwX)f8);4> z)sf`bM$8BEMu`&4J}N*a@nPhCHzO&e0T#JT#IYEaPv3#8SU?6kD=tHKcjAqeon~kd zgJ?s}f*qib(5v>C=+&SC#dw{mEUg3ty9cy^&p;cnr`muSX)drf8a|N*iDznH8H+}q z_WIxoGe0Br`2~!I+yU8g-IeK%q?qnJM>dQ#wW1ADd&+K-IMcHkrWiBOn8cWQV@yUR ziT06ENd{Ivxvr<<^f@s=8g$4}!*AnN0|P=fz+gv`0aUCLi`}`&_j^8H8y+7f4+&x# zuTJ#f^NC(__&bas@W@JV79PS#I*B&pb&q)87pf@%jc6IGh+&oIY>UF)rCZ=}cDIV4 zk$W^s`mSB;-&jh%-uvZ_8%x8}_YZ_1z3_*qi}RLrx>d;yOXjrfWddU8y*_zy$(Nn~ zSnqYy9DBVVT6(No$u}?O^Ciov%l@=@gGqY0D*4>9ITZbB*(E%lcKr|B>ZG`QaqHga z`vHkPdCVxEPu47-;MOExTYi@NSn`ME6Z}2TCu3HuE5AV{4cdz-_{NG^O&_Dt`1R+L zKdy+{)dGO9r9}EwrN^!mkQc0+=02W$dgbx{?&p(dSI+QvJ)ac%rj>U-*U#6~q}_Hs zm(1(y?A()raE?}(Tm5)BCHCT=iS-G^-3enb&IWnI@NUTn%Q%TiN z%VfONTjz~O9bq`Z3l&a4fTp&5;liNjk}SUQ^aXWDv})1-#XAHP9a`zDHXQ(kD6E0v z4uF#Cbwn0Wav3P_us~_K>70~20zl+ha1&FZXOmCe)UV{+k1P}s|N0|am0R}Fk2MLs z0`LzX?RLLSj{n$c?F}%s_cJd7uKn2Ym+aH1v}(S3c~#+nNXUUHduPgv*HhRA7f{{4 zrMhV%L3Px1a|t^#Rz{n#SQYCR_XF|juG-t_>om-J5qo1r2o-92fncwF0U+NQqqeuh zE*4bh-Ryb1n>#~;D!i`DE-uO_Z9Lhl%%1Ft(t5R8{FZWWWRH|K>bCx7KneED`L{It$W?oiCC3>=BVCEHNRH7$4 z2WECFqY^!NU720AEVpamqrJ-P$(|^U^;j>xrOX>yMrotk+sf?Ao+xcpTL&}6`?DuX z8`bW7ULRdr%X77@v{COKW!7g;ls2l}r_6@ziPA>32bI~FJyF`I_J}f@vL{L#)oPC` zw>f*Hv{7%HGF!4IN*mR-E3-9wqO?)%Wo7!aCrTUDUQuR8_C#q~ZLnxG@X_Aqlc#Ej z4S$`oNN9-=#GbedA1~hjeDY7VBgaZOO+qE$JXMXeCF0y_7m+0U__Y&#lKE?|U9ex( zHLVhkr2tH@(Jl23Y}UC05wMjf508b3HsCc%Vm&lq-ctp<{9H2O<_pkZYj2+3rVh-A z1rZ$ZeDeL9E10q0-#m5nZ=IBK+O1Y~8}I;B@`qc;w(^3xdmPN*#p`bwZrU`JOBCnb zw;Wa8(*U0u&);&2zxVmOZyi=sqjbvXt2waYdUc$W{qzp)EwsHq0t5ta;35jcI#Xd7 z%FHM4)tlD#2-T(iv{@OXoEe<1k(BFs*54Zf(pmuOuitvaky#s6EHhtoNm0Q4yHIdi zel=eB$CDG*O`W_&Th8snyy$WUVy=#S;c5{3vUr%32JI9K)+jPU zGIKD?8?H(I?)K5G*%Nk;%gE*Km{|L*Ys(O4f~@g25XZbFLvltB5O`1v)U~%%Xm_-w z`;_5C@f6D-`90dAEK0WAanj+t`l)zqy#5)YgRBw8Ur6&iNI1&pub({t30JQl4^wVf zKc@QM81mn)KXl0dv3^G8YDuKB(I)47qT}RjV39r)YEU0CG&!$gM-Y1sYsp?bUWb1W z)cZd%vE2r3QjrasCJ(z5?qrCNI92|}5_LMN{7J9*~Yt6wMY!5k`^ zi{r7J^XGH@9nU3y{Nxd36B08V@Z@gMf?oH0$aNPJBGt+$htX}05o2IFm47K|#C`yu<1BdbV?oGb7;po~5 zt#mNm$W@tw=_XI3gsp`Jx@h$X>C*-J#ZCZf7G2~v62Nc{v0c}yk zG{7gmm`u29`smvR)VO`X{T;6KcyjYyJ^q&ali%MpT1n~d3zS@W_t+D^d4D%CQG*39 zkkf<2@_S-7$n=JPF6UM&NJ!>CYBRchr(m)Hw zP3(BjUX)=WjGb0p|A49<_n%iC>Sg0qqWNjNpR7x^`k(pYW2fKK;k&n!hdwt&-+TYX z=l+5T*3F+kj2UtAh0mWw=C#jPlyMu+Rx)?vM9Mz0v6t&VY>eFHNz)h33U6T#8cUwd zQ46=>tuA@)3rS)BR~b@Aa@oC)6xM5WGm=4HJepbDDPL@NC$9ficUVM*qWY+zsUb+4O5lCMDsZJk9AlkcXqGzRJESp;uw=K^~##%{nKuOg~$^%L&Rua}r zl3(AyyyLkCpyILd1Di-*aq+)?nRI^;emd#>()y0y{Uy0^@wXo$xzCS(vYGTjpNFLf zMuXz62Py{vInLT|06E^W1A!!L7-8@005H4>Rx%LCW31!=kbm{SXGWnGK*MD5RXf|i zwK3$Bc(XQ!T$5a|=_lp8;8%i01tR44B*%aGONX3Y{>7Kmv&&bhB2z_3K*h&C{gu$U zladGi;*;*i>)@7nhDop4oiF3F#?~ z@c7cdIX_sFlTuJaimeU!iU#WN;xuxZt2QO0jvqA~V}eeqcmee}@H*ZjddQ}#S;u#( ziC~8JCnzyA4^M-1aRedf19}RlvBF~EwC4JGbxo3`JXkfF;cmY*TOG5o91RiWVzYug zEaJXR%hOFafue4~tt|2>>Sh6<>59{OUCHj_POJ?p-R5*KR|kkEv!jpTnnKB`yB0nK z6=W&-h=SrvWtkDJ8gb7WJKv6z%i2MmNP%WH&OOWv`h#Hy=0asPSRv#Bf7Y=!+j%yEeCcR4dZlPiWGc4W{`F+{P!mWi5#0a?YYuMx$Ka2e&Ap*vR9-7t!aEC=C)E(VOZV~ zkHD)?r{0Rbsx+XT^gG1|+S6CK4qz^85BoWwJ@NKtt9t6_p7R&eF4a;y6AsVm~9JyYqCqQr{VA0X+T5 zyky@PA&@o2ztGTL1Jj0Zp&xia|3HLC1?{W_9yN$FVjDLCZP}~CCrW4VO%OfwNfAI% zTbIzXsHlym>dnY%ebbq8v!i^i!ji3yI|HmqQFHi+(>>n{VT3Q+H=%R!t691?OaIN% zK%4gXb~1=tyLP*$B?AGfOT=iKdV63UKDynDk3$@o_GIM4N99IQ-oYLx#j%l4IYE+N z6m%}`Pb=v`D3r5Fi^Js-xk#Ts6qzJ-NV5%rF&b4M#pM#JltgN%CHy*so9NAMO_6A{ z11`>9W)I+hMdW!0jip!E+o#?Urv{}B0@exY?^qvFFXv4S&WqF6joyOvLfgU0=bzcU z@?NwQcW-rmZcX;19GvFXasqzbH=Oi10e4&TP(+>ay7(Uf2j6syW5gIGRV%0(QZ9}Z zlLZf>PI&B}XM>rcjUw7FhM1%0wD= zQPC=s=1ej#sh`#roHj{hF@mn?Uf!n2lX<_ScFHgdq{~Cr8o2i1X{AC7X}r)P)Y}i0 zoRdsC;aGh(0=ct~<^oC-uqJL3cYGlx5dS-;h1DuMpqR9@)-%pK? z%KfZd-vf~xPSoRm9L??zYTvHU_e3O(sv2Zf^Y2xanr)4m9GbSyRMVQ&CwTePNU3qD zXt2;CD)fRGIZhxgBd#bbpJ1#vTYK7aNQ9jl#UN?u`t1T+l5)JGMJby~rpvF{S6lui zhfic8rH_y_b$Us33J0S6LB8kcnDD_g&#V{IFdgALrgu=_J743Zpi+cD`j7J>`=O_f zz_=Vrm=?ENrU{BZirD#aJLVXkFX|hF*l3Oz;SHT}hXgA8SLP9>W@uNoCFIN^TP|ly z9LMwV04|fOs(ro$7P-~V@urjfdDQ~qsFtF_$E&%ER@zZM>y_CnFi2uz2N3B?3ZexX zQ&GkeP_*+|W`W_Iff&KXrsA7N}x`fqSEgPmyzwr&}vQuXZ(owFqQ8L>}ach{?@wjx9={pcS_BG1X zvCz)5^dQlIQTp|&2Q{S5YV18>sGP4h)n(YjWUD>ggA9Mh9oC1jWDf1e3Z)d3>WfR| z*VCG!+^mU8UDAGr&<0hi#H|9WKueH0uRlFR%kw zN<8rK3sWVIqUB&TtblsRNoixtqo(s<6Q@Vms)hqp5Mi$U@^eM&1r(EzdQ_G{BcpxG#T@YlRC9Xp1S(4&cV(#=A z9otdN_9XQ6?9pJKtRV zvj$1bV*R|Hg+TCUib^nAM0~Z8xYU1nuAVh+ zu-`*8_Xv6#v%wAMt&qm@14o5{;6FvhDxXNtaUgYucL616bA2<4$?;i_ruO;0-lm7f z+gN~zmoIwv%km%pAD=x5`GH^21ehnXkulvDFTm;iY)%@bWq`k`k0>2r_A$aO#4dRF z_w|Qt{b(@1(bH#MZJ!VHrT(oGxHp9N#l;;JY*r^bAZmpq=+=Rx~NxoHZ-d|GdD~$-4T_cY2byf&We2KbzsRmZJ9nH_5^_SWODt zBsEaNM=&i6M2{w@vJ2S#K|vwN!RBHZQP?K#+OD=n_u%!sN0GZ34%efE9BRUyRzcKW@@gJQqK&fi-c$ zGy>nL1~vwIwa!4iFC`dN9mhVKu{DCs*fuw=XKzvZ_G+HR|s*+j<4 zx^s``7YKRSisj68!z~Bw{WP%cQP;dn9Zt;;qdlBQq$ ze8|}#3&%5T5m>nt)rKwokgx1v zN;?I?I4`l;f)sr7O>Ehs!BuM6nZ}%WQZ2)h;+&M*>J8L+MD3WDHN_&g5M4n^LU=UB zF=Qrxa9|4`(nFWUr?B#Y!Tj#LGDRo6vz^}ytLWU2{TDKLU?5eqmOwQmh zeaH|ur8k{j_?nYZ)tj|ko;a=}(?TBe^?{MGqhnini+%n}dn%wC@N|@Z@v4NWR!f@y zd8KC@Tb)x+myboTnbBv&hI)a(+}wa7z^U&@(T09KOP@KT4sb=H!vX)&2bqk74*4Ja z_d*S6-y)e}W@*Tw-YsB)OpVZs*J>#CTTi37(Wk6AHWzs-PGt~B(I?W)z{ATc;7O96 z1q`5Lk6zT4kJ7*SsmIUZbqlJe8zLc;#m)7x5YB=S_f__Y_`R-JT{^xfmw`O1dUH2r|xh9$N^fLG30e(>yZS3l#@?;P*-?Kj++ z{cd`p-^*6@A}{+0^7}kI&Z1YEdP>#|)gfQ83ZOne3vVNB(}ziRB>j(E=f0O5`{<97 zx4wQ&^2bL{aW5uQzkUK`y(gI9s`W)I^<=RaYIJ(?;cu)++;bmy3zFJ%li9bh z<+)EEQ9tVTu1(aU@V&`p+m9M~C6U@xTbcMvTlqGd{8_T*!K>V?YOs~LWWSTHXKYcDrVP(f0$?1X2e%vr8D8FUG}nLMGMDQI zzIi0q#os)R>leSdK6&{Y#~iiORfH0p5;UAh9fTB#9Z`N6^(jh@*fNo~=4}~$&eOQK zgzr=vUPIrBlbTkJVs-;i!muC0oGck?RD1`6j2|=S4S(YF!D!GR^zl~APU`BnbnavA zw?4pi`?vZBZTuR9{Q_|rQh4U>$9^_6rp*`SZ-7pzhHg$3&*ywQY;aGdy;;#{49Fh8h=>{LyFtuhtgZ0^Dyz?@KedxUYO*s|6;QHg?WDey~)wvJJ$dHy~&5a z*9HBjyjYnmd&_F)FIg3^S#Y{Zh1NhR`ij2WJ+ESWOEAgdFOyfkSMh&vZ}RWoJN1~a zsjKGraV0PrU(J5nN@$YKhlZaLD$UxTo?O0T?8x`uIg)&S#{~b8r;_J)OdOLsj!~X| z3H4#001hYDcAUj8-kY?)*lDnL&G;o7$OWag)f@t`mR1z4$UI=17-eJ88PW9F=BJWP zFOGJ9pFH*AhqOF=s|=gS*Zku9uLOgAI^ zCYkVq@4NnF{|}C{uh~YD4|;sm=5Esg08_S!{R7 z>K}3lA^G+X=g^q{qeAC_lN&3Bl;4IY)++hc&t6i|PAV}9_T2CgwoAVA zqod~Sdfp(Y`Y~j9zs+nq99`ecrd2F4h;3%$6Kc=(vl0#$vYR}au=A4R`+=0Dale}e z*-;bN82|=t`@-_|{j{u!cGwf`XnSJkt2Ln}&hWzuzC1S1?}qZMDRL8UquGg`LUwj* z*22(N$6@+P9gU2K7k$e@CTq?R3nT~&j*t12uH6x#i1O2tmLHEABgFH3B+49LPT1uk zm`X7>EIISXN7lZ2TfgY;R%Ji-?&7p09E^k`YPt*!i89hG6=@%AWneI*6nOc_woIi+ zhmoZjyvnYteTNq za4M?Q=wU~rQCjS8WYSH3)+i*Sr5As4R&A*(DDn}^5U;k~T1COuZwqUP3}7o}MC+^E zEB5Mi-@ImZHP26kt**8(|K(+VZT8|vsoA`SD#Z)+^I^oMqV?-muZ|GSHB$d;R06>J{IgJ$u3YrAwFfMe`RdShR9wG(WoV z+7*lDFO23bTQGk~bV1*;74xrI6s?-Sb^xQqJ@iA_N};K zLEqwKOQ$_H@~1Dku`Bx)PFlHQ!K9TRSvjfC8oFYM8k)3p*%fJiW%8?^&rII@SyOWV z-yL4Me#N5eS1eP{%aVB4tYqr0-<8vPlkC0Z4?mm!*vOy1>5pu#EL?Q;;-!lgcJ;gQ zUU5JsY1(~R?eB}BH`epKrOvaop6@v%zv7U5ANf(dcl0FBtG{ytWfLf?=Z__Sa|X!hW=8due?EHgA^;12YwMX*DK9UcKl*)Beymg{mZe9 z+y|4{zdWizqw{HM5Pu{2EAyA*Ps16_-{Jf%n1B8J1&jMW5-nS`XvNh_mfc9%2chqI zSse!QKJUgmz5ttl&j&x=5&X@Mu3om{y7_&5V%y}+U(R(uNT$73IklzbYe=HR|}zc5oJR7G5o#C4+#C{!-HFc(7S@{Wr62uXm{ox z`_^y175J|7NU?C;GfY>FMndXnJboEcTwNfC!e)q#j+JwC1?MB zhW}DA`PA=^t-b^bJE-vy{@VGg=ReB5C`z?o{Qmf}>&34(iqqF#BtP)AabQRJDt}UN zIsV4-_rYaL7hSOIx!R@lSdjeH^G`PN$#37B(x`S}Qz~=y zYCCCMZPQx(&ofV{7Y7upwOyucJ?}L@8bSX0`AZis=mn+o`=V(rv6)cANAj2dd~C^k z07Rgfy>I4BbpP$2k2tbk^le%c7Dw`Ts1EE*PTu#Yepey6Yv0KBBM+%GF4?y4_Wu{E CsD;x2 delta 24343 zcmb`P4V+a~z4!Op`&c~>dsro-}%(3 z9e%KI$;!etS9C4BZ25|1iNOrd#@sn7ToMhLQ5=J)-4mfk^K zOHC%I%LRjj8lHYo%w~%jKUZ7pGn?yf0HHR>s=HtG_0RJ%S#zBkJ2o2_uc>BOP%*Z< zyU#0BJiGquN&0}&+&S5rYZhK}?Xv5E?!yaLoZGtay5-*a;gUrkyX>P-^s+@u7Ol8! z&y4mW-5&&1y{-%!8VgA#Y%E-FGL^BO4ZF%g zn->K((@7iVv?e?!3SyIapP3V1lKG>#BEB$t%4r)u*XM=_ubf$L>~3=QM_!b%V><0l zWj3tq^TIB#5M|o^)>3U0gmdC2vTcKg8B<~UU{*swDkL(*+J+dO!HX6Ht=j@GkX?t&R_waETE9y4e={Yvn!tw!>;u^%K;3tH9_y* z?Tk6XzD5#$#lht}xPtHw17`hpzsJ{ejUVQsV0LDD6^acG3V0?NQbV(zWLRUH*=2P*14nA$jo+kk{vVBi!`JkhZd^jzRhP+I(;?Ua}$b)_wD1>fV26|-Z z)pQ=zdY-v0zO;2#?lx1bErgH9kG5V`S-*qsgBWLz2AsxAH!pUQ-CCVs2QBQ|>_Iiu zgu-jV_V-h^Xg5`d*{9y^wkK_kB)}Lv+w^D=gWd`vKq`!HO=tY6VQ(+Y6N({s7|xFW zeb_MQ`SP&i&Fpy4@Nwsr{OJ<=Fwr&{JCbqQ{g$w-Br0W{sOi0}Y-}+cOWy17CrJj! z2l>f%~CsMZ!cn>+k4ssqF2B0VZhnv3gB8d)jM1=LzQsnZstdR5m) zMgG^%9TBcGQEj__Iwd5N%0)zfxL(gFHE7Uwe`<$6fwzVZuv#0{cld2l?Tj_jqh_+- zN)ceRkQ6)o2wu*2mWBu@uiP>|@Mcs#Iivfg%FU~L);`eJUAd(+7BHgD(opM_8iVOk zU1mDZu@?Ydqq;UmEhGcF4(N$U7ZR`;1GSxt#(sRG4OT{to#im>HRa>N^`_j^86CIL z4qI7np5l4Y5ag=KR`Oj-J0n+6RCc*7F77NhIWVKJnh(c&-emeT0Rg4?&UVl1AP+vf z!0^}^<`dCaR^Z@(1R%c5q8)OgBC5I_kcQe!crPf0>%+U9PRT{J6~MJMFzP)9Nh*L} zJH4yTW1#RBuEXz!5ujEAAQNRG_`x3bI&BcP`>+zELN=tmd0p?= zh8mMe7=J+D;H5D%D6NCRmLY_G>tvX@LEh-}-f-hmg zmFC=1;Tp$o%H+jFOIdMVbhj%O!&hCf^w`NhgQz3i;sGbh(<={|S*d3aV-@8Q05PQ0 z1j(E1cGJnO({!?V#xK`FkW$fBN)7&WDvMfJU8zCa034WyEl4t@7A7lpm5NaV(1<@_ z`BE+Ocu`AaqDK9}(z{LBWxJx55#>hfomYXmyg6O=4YRZX%grx9g;ERh)Lq+Eu4@C* z-Tg5MdZX}Ggc){whz|;(JAb%9Pl*xlVNJP~ zOHE3&N!Llwy}$s@km>U#`@6{qCSb1(ZvY-z&PO+aa^5Zw$n6#XT}j`w%ykToURj)s zbTx^Y;$1~j*Wq8D)O{eSyEw_civ%veAZaq+W=cb$P<3d#S6rg`!zRC?;=L$%;fvR+ zHK|@I`|{E!2oS^s0y5 zU((zle81Ay+)4o=TuaY(|8U+q6u*`)4;oQI?Y8@e@}*X#FXaW2Xk2O*Lobe+quO$D z7E@v)At@6)uRIu*#EKhi&t3}Viq@wbF)a`7D-W~N-@)1j+c8UpreW14!J9N`>~F#t z85+b(ztd95MuQ9Cou=M%3y)0u*osG9Ktpptk5Dh(sT^9k2v0tyuude|?D(ou55svJTUu@9N zDR4-YQx1qN5TAppFKSy&*~eD;nWbC|o$$VfSj-kC#ahD3p}bvxCl<%}L@78{!dO)m zBHe-rCBBr~@0~8TyAc4kZK)+0&SALG?cLzg-Xt}pm7ph!!=p-3Ml-7oG0!#Ro_uL& zAsWi=D)k0goPLq2N!1Q|qBaVrcqEz(i5ILbT2zZP4tSa9ErovUR<4f*u`e90vUa1g zgd7A;8s2S3ydAAbS>kwFX{%^(JW^G9iLQ03M*GM7L__RFwXvvvy-3$G`waTXHHyY= zO#02P&j>PUqPt8*uidpQz8LiT4(n3`PdY$6Z+nFQ`4W+|n_asVo*rc%AaOn+N}^}2iv6DGS7wvr81>Z3pb^r>^J=nrdxy3WcQd`9{_wY8r)@bOA%P! z7O=y0(X+_Thk6;D)K}(>V9%fdBF|RG7U}j?Y_(45y+jE`VJG^=o(x;UDpW-couwj^ z12G$6xi&Ex^9*mhZ(f~eyjpFI`7X$TY;O74&e#3rT~W9!sbUKuJP)wMhRLhzhwZkP4 z$fj}j4LbMO+1=O!X4j~1F?(tS-somln?_r~O2TGx<`D$NymSD{>gz`DVZkRtBP${C zN9wTySy0=IsAprjE}Y<&iR}+o7}ttqU*pI%$87%{pnj-X0{2jSF$np_) zhuTcJA*#qg#5J(F#ZoeJN0cd8i2%9H1~;hHnUJ)UI5iA#gPf&#lpBOyB_gG6P*dUt z;q!pR8S7;prf!gQLDdtgR9&G~!Hcl;yFvr}8Hj)V4p|8|Bes+=}`@4s)vO_lm}NyDwvWC zWMG{6SPCR_oaa&x6U&r5+1zFl#|3#qTTgew4TII8MCG`MiZWzqJ_wm~A}K3DTW!^G zX+rb|X=%+oD&Z*uTolcCZgm$eW>Ia74I4|_W5-&51+uh;ogv|d$7V#gmfD=8pNqjG z#{AeUAtq{@>|?QF@^;Xiw5G%FQHnB{GugSCGJ9|Ir>FakHVd(%o?4_WEMVq>-3jQR z!|o;x@dFxZ0E~B`2yGS{EGnvqLb6>AzHdAXI-8TfR&d^)I`43uYf6nk4Wf^#s<^sr ziFhL8t~I6R@KiADodpokEHwcfOww`7F=Sj7*j&K&S}!ES&5mJx4{b6i(Kl>`Vrt}A zgh$hyCXo(LR}VJ;kaYOl2`d*OA}BjkWbIap2eLyc2GK61(t#cR^LoV-=e3VlMRa)L z!SiGX3E!p#;9n0?dY{27XsS}gAa-t@EFO*_#N`OeSmjEiQ{|d#t0rLX&2G*Pf0I%O zD2^6E6)B3)MeIwkDTegS`W$n+MT&334$U0Gr!{(MkvW@DKv#C1EYJjI%+_$SR5nQM zF%|8N$Bd44oXya+rFuu&M#uFdt1Q8aVToFsU4T)YDb9ycTtAvrS)x`f zAppcj4Yr0}juq;oO0_IoB2sxKY?Y9nRA8g--0TXUi1^KpcFiJw9@68nmn2wI*h z^BP{8!znBQuc#P>;aQ-K9YL%hY`4k}5cjnEBjIqd%s`wR;RRphN0_2U?P#S+UC}Fe z4B_}m2IQOa5)Go(j)VmUi3Oyr3TT^<;G&`>5M3DSr*|?QzE(Qo+tC4H8--$t;S_Vu zOtOW_k@#*BU-pn!Lb^xCftxMkDbSmkSK$9h9C)vM!Xb8tOta~%#BF}gkZMyLsvbkm zsgO;kb3C*e3qVAq^m-iXs3MUW2ddg!NUA;wNjar|BqZfZ^BP&Y)L8_|6wzjQb{c44DsZIo(K3hX^ivh1EHekBet~cvwD=3MMQa zl^DuxcLDM9uqj@1#uPIn{`MK8#)c?s4$ka<0RcOsvH^a%ihjF}B0mh{+?l1RF7ihU zcPN9iNx(f{&K+?!$>VJ1oUf^qG<;1aZYkj{=Hsi*oHHQ@DDwC-+Ez`;N8@u)C4SA= zU`5%-93g-otSsdTRR8GAu>{IEqslC(_t2(ab)0vI#PO0=fx{(m+<{`cLS#(QF2h>k z`T5wTfY4UXbog_4=XmH`7Y$8(AeN|TYPm*+phFHqQR7BDSN$q;h~<;cFJ%*@goTUs zi@Ycbg2Agzq~>s)2mW)~!$mFHbe7pvIVamya_%-2=r}NRM|y&2IENl{yh5YgFh=GaXQK6H z7nZ-tNpE;X{6=sIC%mtm)y7HJps7!B7W=)a6FKjCed>wEJR2W3?IVLvL%MU42i9M( zJt)#JoDy%Dc9Gc`2WMOJwRq;)$D8NkYtG(lz8=q-eikJ+PyZ}2NnhP|<}(-iBZ(ZYd>s$ZeK<%!p^M{Iwu`RWD^$Tm7q^Bp+>8l*>g(}YGbfBY0;>acn4_+a zH;<}_Z=X5Ntd6(OoMg_7e?9XAb5ZiXg{hcN=RK zgiR6vAV8DYUE7#{kliX(b!q&CeNjC3ymQ)< ziv-NUyaO*D{QeGr;hsciwZr!F-c#w6N_4}PsC2YqOoEH38r%Ij-pAUzNCf^ET>=o~ zWuqh?LT)jM^mH+tU6|;ly2q7@0MsU16m76K=_chRMO;W)w@X$1qvpi9T=RJJh0Fc@ z8{_!GU$$t0?e-Dwrf?ceTqXl>v3=*D_`Y|Rj#K3%q?{A|0jqDv3?9&Y+b`eQY-VpY zv^~SMZA;tssp!qR-+I%4?r34Lljx8M=WjdfUCYfo{_$V~Dba5K^O(UhItovR)YXCc zTFwMEr6c5PxBoS80bb#k6a6NxG7+p@OL*R}1sb~_AJemkq`>oG$)A4YzfQof#!vy< z#_ppT`NekgXK9%WOUhkNtpYB*UZsO&yQ#vh1+HV*=hZ36w<6Yp!gECw>L1>IS-^$w^YEqLlnd6!;yxqh_5E{chvvLOA>dJmNT$A*fGS?!UTMqvGa7;0uRrzl0B3iBvI?FEqywM$%cHB)uk07AC zf)2uUPp9iqJV|%!LGKw)rgTo*}64;S5VY|gOBkX-7GNbHvy};)71yzi- zhpH8~3w5n_6NziMR^%doS0ycYP6fm5UJ^B6TG7t4UsK0|_)?w`)}F&?veblC_KC(R zf0jWkNUw6}UY5g=sO%MtL#+)|?jx-pa0YuT(6~_MYDeVDg$v(7BM=bIB~2jMB?uZE z1iD+nO}zh~FjPEFH5?3Rv%iNy9D`eK>|v&soc>7Auq&)O@25R9JnIK5PItY8#92V7qK2`u~5MlYp}giImz zMdhg%#wj`I;o}@uUBaniDKA{0Y5R_VY>TA;WO7dH_Li}#K11R|K(vhbj;FEws+{&# z1l?UPfWS6kYFv{wvQfCw(4&;;SCgh>YnLJIga!UhN)rHjQtjrrhr=ZL_&OzVQi9do z=8zMVa+tVL4!8wKk1s&9+0u@iUj4ezm)QZ zgNggD99qe;dE@$!WYCi&xJUkUI{0n}kb~D}U?WU%JYHddcss2Kx$id4FG&`}f{8dd z4dKWMwnbP-YJV$ZBT*2I^mdB%MW0ANP*nE$;e~pEC93BOuwxfu=$xCJrrLJaJ z08bH~x`@E6&OC8?IMtHUlYD=8OGkk#-g2NlV7z@AoNYu4N6D{T?_zXbGe9)7A0@2A6K0w=w-HE^wuDVo(DNHz(q=E-81YHNVu zJYxojR~+wiCe^$V5iN%AmK-57sofwb^Jp)wl`80YqOse>Vo{;wPyki}(bPFB9?asC zfjAsZ*2OP;_%xG=^H+~;;tUG8w};;dmX`ChFUC`^o-jxrYK~x{+vf;8!OZauR}Y^8 zur!x8LM-&2QM_w1g2UYj368Ebl^q{dkGWai_+XPC|KMs{nZo?Cn%<*JvzYDyqLn!J zM(5uW2a%i@wTW=>x! z25nM4h-j+hgC3#7IdZn3@QKqea4~$SEhKz9bPA@cjkrYqCKY%E08TU-UTrTI#H<|D zmh0i(dZ&ny2_m2fn2>Jk<~faHI2d!xUsJJeBWWj^m186Fqik~RTa#+j(4E=IM?gu0 z0W&}m)>j=WN>-;t&PkYV?$HvCaSDR5q>A^Hzoz8h!`6^1-ao++ZJpwsJ$ph)TP_o`n zLgSi~fw1%i(^}f

GU**#0EgRXj!Qgwc)zG_4Xtq8*sZo;ZyQH6|%}&9Dt|l2_sa zN{QXdpk$F9sc5$fXOPgorHt0;VGwl=gmjmI-A!aHgUYIFz*$ zymD5TKXfC5Z9>PhV1X+Mi7L?TCY`2$1~+27_L^Z8T`o@Wu*Q&XicM|@J{_EHjD&0=@uyFy*R#M$%IO9gcrl0MDP$~v5G7=AUM}zI68(HIxmRS z+-2OO^Z=GVCq04<`%5rdcuFAJmt2*g6xd=6n*wNm;i?>ORAJ0;^uTUr>`z@Ca5`oH zSuza4#QxOe+^5?@SSoyE;}L2|t=)YXxbBhydld|u(Zw*daK2o7B!WerSbePvU@7?I zj0~`vf`SEz4Ug&2Tz#>C|bDn%&K9-5e%%P;?>Aymg_@bgPD?yI4A3O?)D1zNOSW zKm!wYl_|xRbBQM3#C>(gVO#w!I2(A72-}1&%oiBk<7b zaLmUTN~`8jig~`l8BmI`)P;h0Q_Q#iBg|L&QP0(Gq23*t%_Su)5Y7elzR<6=bwdp? zk=|GTKhi*bdY}Wekaq9a4*twk0XPMln6#iKaE6ErA36085forULAkx&fkVW}=V~1# z+x>XRvQz7vC`&n97Qb`ZgbI!n@*jkktI1v(%aP!9pT18b#o$O7TW4KvLRlcv%Hp(C z<=8$S#S27E4`=O8WswT3%RuK{9xRndqG2>S2hOQd*W%&|IvCIll_>8c^i!>_b5fZ` zmgHr+9o;=}LlPB18qRZenOqa<^wABtBgm9OZo75mQG3g4+maUmO)NY=svNa)M>Mx$ zc}+ATlHg)7YXd2j=Zy*U{S2;DfQE8W#V$NLTqbS7fz}sN9LRQJ!7eJF#ihr|4YhAX zzEwL7W%J zsKR#^e3!T`3V7pvcN5hqQgj|A*q$uohrY>kA4XVckLx}@#9S7aK0c@Mm@PQ|*vG%# zFf_RptF_jI6XKaSlt&CMRKHTJeyA7Vu`_pJX54iB$$W}ruP^bu`GynC{qf^Be2{1H z#^uE#^b!W--p8wMyr8m{k4-DSqRVJ})8@g}t85Qne@^reAluCwctjWuJ?GxwH{v6U z#GwqsYo}`y`Hz(eY!LRPgF96KEsdRzfdwI9ascB+88y&2QD%m@(b=_w-A2(^;N&Rn z=M{cp;ypKxsf3?HZl&R6sX%2JDtJ~h&x5=axSrFux=zwej-fBswk@g zU?B|Nc;C%u$2Y8+@1I+U-*NL=4qpmC3QUdIc2W|5L6ks#;0tgsT}i(Md(Ot2$guHKT}h z?<+oW*{BMmiw+r48!tRxf_5SV@c<{;0I`BTkPPs>Hv#8 z$cKt|-_m9l#n0VxUfuq?`alt)d1c(Z`axs1ZC^dba3$l_Po88x6pz033XZJ`F*~b@4!uqbA@#j9>*0v`B;rx2`u6p0@PO@?lIpDk+ z65FlM#y|b^zxgDOt$A4Ey!-YA{o~;M?u@tIURL=Fw@?13bT}ykmgH7fGCqf*>9|Ne zf*7n^Q4aLMnkz>=WR^L;uv`sKAPkc5kxxn4Ps*fFG}|?wIexUm39$*`1O&)dRj5PI z^N7FtnQ4_OBnpl8gW??n3U)Z4HqPoq!YRHs67b@ofB~JNlHYUwf4Z{I|6`l-qIV4Y?}tQ$9Q0d@H`; zvu|(eU&1`3H7nwN=d-8!`<{#c=d&Y+?n^MrhOhBo5*n1mlj6Y*DPKv?+O0d|i{kdm zfuwoav$Tu+^G68#DwL{Xr$5N6ePfKq+Rnn{r+D;D>+Npd+}c(Af-+jZGD`bvUQ}jJ zTBEeutYTkMZf{zpw7=Z}W%i{tO8c9=rp*4dMrnVu!^#{?Yn1jkTm3aMhte9Qt{Eq^eB= zGuxF>iE6eE%kKHh1+wtJJqJpuHlp}LmbVOCBZCBa;=i<|PkIPA1 zk=gjN-r+uCeM9d=C6Dx8edT`D5FvNlME}qO#tN|A><;+fg_~vk5Z?o~{t{Q|=6zH7 z-?=mX$oh-WOW#>PwNVqeZ40|j8&}KSB{r^(yC;wQmWwziONg>qPa>Y0^N6S1J2u{Y zccE=>(!GQ-Msd>0w~g2%PDv-?iDLZ8-NzUA(9o$TP111BnZweF0lcwO>R6ZD=#65$ z{GO3#|05&_5D=P45K6q?L6vAZfvYi|y=T-IP07-@$6u5kawSop- z7ys(s3)#lscHiUqB|4q(upwT)?|LkQh4-I5GR2_I!DMQe0~;*0fxcw~#J>AaxN<3G zv;J4BefM>^?-uPSr^Ax!C>WMlfo`V_mx3gA=uS}U*il*BD~G*mhjgCvKR5Aao#7dE!jls0;!Fm3Uo$;)PMi-rkxV`A$b8-C_ z+fU#6EJjQz5U((mzXA(qcQUu*m>?m&i>tKun^%jz_{|XqMKe9buNeD@#zGvDnS93o z&_myrZFBO5t8!WowqAVCh6Vk0&cO}W@-sb`e`$mN^18U@;eS?g$HT`rW1s7asok>< z{s1Oy>|GB}$?YU-pI#T&Zan*x=k(6;8FLd1X!1Nf9{lYn(_}n%jKj%V@`DI+3vJ)J z`1*~JSr^~4@s#n)I1J9(WQ0%|?(8}Nq;F=rkeDU*V8694er@9knJ>$#|KRH*;;ozV zlRiqM$HMWE7LBv@F`1SF^4H`0ySx+P->!>a-ZXXCM+dY&@_Kz?TYS!!JN>nH$6LNU zQpxwfe36p!BV$jy^X}#N$S<@WJ@{ zM@s&W?}_hwWYp-rx&W!*Hm4T3PRr?|`e_Fsl##UjoohMxOQ0h{za3~)6cn7~?^sdJr8fJ9R#77@QA>_Gp zlpssl4=ZugmYGcP-Yq3li9fz&F3*42GLe%1-qOYMvaJyT&$xT*%*sb14r%FF7k)%O z9JaW;IQim4*DS6dbk+yoq6mENE;qswYDNj}F|GOdH(>_cn}l%;EF(($_B$$I7zzqy zE?CNzXUIrj&w#D?r&~W=^WwuGc~HFME1PQ$=#^jE`RWPyeDl8AWc&k<#g}dyGV;J< z(`{izWVj6=6#2`NWJS9gtlid{CQ}Bs!?{wmI+h9f&WhMX<`K8z>K9dL-~*3IINdD` zMmfCg694xb+;{-BL)F{K8YFW-07B;6FpZt#$_`rZR66kWUMR64=5?SlH!1p zk}CA`ejvxY>_8wVx{`rFPH-gyfjrTb90Bs#C+`?7XWr*_1hi^G zA3Y8jXWBJepc1@zNnH2F@c8DZo*cXyhmFmb`}W~KyVtY$p^^CKe;pQoXvdAm+%SH1 zM{>is6kqhTEz%SJ64KCf+XGLB#+(v=<(W?t4S4&rlX*^kWjMbd^xjwg$($2E^U4UE z{2#qCu5y!aa_*>s3!g3kf%A$|C_+s5R$-VDhjDN=aUHyWt^~4*C7bizR`U0|bX7xzO6S^ZFg z>v+)UR8t{oelj8LK?jb0U5A<@zn#Ez?xz!6Uz$pM@xr@w%?jI<$Zh(w67ZsnSL?n2 zE`C7tI$r(bWtFX4r>F5ceakq}BX9kse0^I&`Xp$D7A$}B2)*rsgn?q3!*E6gerl6M9xJI7c-B4e)Dw2-b%NOk5@i>vAHDv!LwtA zT;rC;#xNjC8r~9TcCLsA^|hFB@$EY|48Oz;=2)UjPen^f^5F-z&HGxLG3(>&o{P=B zapvm>*G+9DLyT`Y#$TShKGQkyGQBe|L3!-hrj@_Jb!L z|BxvONzffLP@lX6Aw}ST{}d5hbwzQ_A11!OlCS{!hbYFqr*|Gpw_{mGvhAbhcd=j<2f^StZDX*E4M2KjQl_r-BF>yy{aPiI1& z^L{#R&<+)_1+z)j+D$GK-}2MR=8^c>pZ?iA8gKjA*b$Gy?YXK&g1JHPldVu7JR1Mu zXLmBoEkFMl&%-~zq2<^Q3FH~@6~Pr|Q*3@QBM(;iSSb8bJnt7v%tP@rzc`J4ul!=< zp!@kn5nXoTLK4^D^byeg@u*)88@2(QvcTfvGL&-U%IL0nxFKHf%lFau%fI{x&oRF` zk7wtv&gZ$~R}0Mj+ww03hD8*AJ(;3&e?6%_{jT8!E%DO$re9xc9*h6*>+^U{{muD2 zZ~M&x^Vqf@|0Zv0UQ(Mu@&5lkiSl9lUo;oRf7)L%567*)J;i^-#Lpa9W!@K``TI%c z_PF!+pBY^}X6ldmq9Hl@6ud!rdpzWo za^Nj0SH&NEZKSy&zTvg4`Atvup>o2m__RNsJ?fk!Y!0s@k;zYS@K!`=>KEPa_~S`+ z2)DBv@W#IV$9I-CaP`90+pj2rk9mDEdDTrq?jU!^Q~q?azkgla^{4s%&UNwAe>%zE zwJ!ecpV~1-cO5ECl7R5ziBA)#6I!W=P*oMEf4^0hN z`gfveXy9S$7EOHc^_IG#^ho1Ui&ae1s;%pq)>oUxkNm}&JK_U>X>H#M*@3033xY2^ z0i@i9C*emMDd#w`M(gAkFX9P*eRm_Ogxw%tHj1M1Ayw({p!lJ`jt#LEGVX)4#Mby` z{G-24eUrv()sWy6u61Hsd^7&W;qRHJpR<)KlD<>4U;`wKI|1G8-DRN%(#(D9*Zya}&N18;<2L$ku%rt6##3M~FfbuF0{(;!_jPGwV;c@F~XuNBNxyQxdSzd0W-*{M*E(W_GlZqe11Cta$GFZ=4~yDZ3c!#Q+-BUm0A3DToQcgBnuBYe?dcw&%v@GrGc#@Rr&skhBH$5pH;m2{sJfR*f zA%|8BOhaZsQc^%zR06&yJD(Fh+!H?aVji*iW=NC9@vyr4GtNe^J*--$P46z>yl3zl zY7;&Yvp7}>uk1Z7FsIC1pEh)xz94l5P<4)-{lhZn_Wn%bn!8aU8$*3LKs6K|J+sck zUh4A-FE>54A;UTKQ-7%8M}&J1@_WJl@)f-g)|%P9y|w1$r~X%+`HkQE!+LXxe`2P0 zamJipS+wNJ#UHt9q33xw=R$8R=?LR_!v^HB^Su!R@=x$SWI+BI-p?42Kft@%pL`Mz z{)zqdkKfsYv$>vG_(t$gA6c|`#iAw4uj#$4(Tpfnr-pgFI{ua|yyB{J7cJ`t>7?F! z8_ii^wb>~&Ii0`WKQ(gSt6Dm_cT8yJ^=B9NE)UJv{_K?AFNEg!O11IZ>E6Jf7EAE3RC$;`->?l?#`BXz{flrR?`Lp?3l4NdB_?y?I~z{O4HWqdr*k zhVyqt^r35)UBl01^8?yZ@60AM&&=t)v&odkwAa;rV*CfAPtb^eRVaiy%`mF>_f2L^ z%~#el+G)KvHJg#W_cohh!}JSEh3^ghagYG>08MN8`*G0w{buu@=3^Tl8f0o(Ja0ud z^xjQ{@OdJCE3&;$3^VU9#lGiFB3-s{;gS{0u2`}n($8Ca-Z;>6KIIcw`3zF`8`{Z(NHw=w z3VENds`2a{Umh(Un~buZpf%5lyM vZW(K)9XE}opFo?ZXe-uvGf9bho7>EL{cmS_yV}g~rqVIZ%DwB_%&q?ymv$?+ From f5f63208b09687c4ab6d9a667303aa0e9574f529 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 16:59:25 +0200 Subject: [PATCH 831/927] packaging/nix-make: Build libnixfetchers and libnixexpr libfetchers needs nothing new beyond a libgit2 entry in the dependency table and its generated flake registry string header. libexpr is the first component with sources generated at build time: bison and flex produce parser-tab.{cc,hh} and lexer-tab.{cc,hh}. To avoid building them at evaluation time just to scan their #includes, components can now declare `generated` files: the scanner (updated scanner.wasm) is told which real file each one inherits its includes from (parser.y, lexer.l), and the compile step gets the actual files from the bison/flex derivations. Evaluating libexpr builds nothing; the generators are ordinary build-time derivations. The parser units are compiled like any other unit for now; Meson's extra inlining flags and NDEBUG for them only matter once we enable optimization. Also add Boehm GC and toml11 to the dependency table. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/deps.nix | 24 ++++++++ packaging/nix-make/flake.nix | 8 +++ packaging/nix-make/lib.nix | 20 ++++++- packaging/nix-make/libexpr.nix | 86 +++++++++++++++++++++++++++++ packaging/nix-make/libfetchers.nix | 35 ++++++++++++ packaging/nix-make/scanner.wasm | Bin 75036 -> 76453 bytes 6 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 packaging/nix-make/libexpr.nix create mode 100644 packaging/nix-make/libfetchers.nix diff --git a/packaging/nix-make/deps.nix b/packaging/nix-make/deps.nix index 9b10be1ca0..ff9550521f 100644 --- a/packaging/nix-make/deps.nix +++ b/packaging/nix-make/deps.nix @@ -109,6 +109,30 @@ in pkg = deps.sqlite; pkgconfig = [ "sqlite3" ]; } + { + prefix = "gc.h"; + pkg = deps.boehmgc; + pkgconfig = [ "bdw-gc" ]; + } + { + prefix = "gc/"; + pkg = deps.boehmgc; + pkgconfig = [ "bdw-gc" ]; + } + { + # Header-only. + prefix = "toml.hpp"; + pkg = deps.toml11; + } + { + prefix = "toml11/"; + pkg = deps.toml11; + } + { + prefix = "git2/"; + pkg = deps.libgit2; + pkgconfig = [ "libgit2" ]; + } { prefix = "wasmtime"; pkg = [ diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index 1c918a6197..f554aa993b 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -28,6 +28,14 @@ inherit pkgs nixMake; inherit nix-util; }; + nix-fetchers = import ./libfetchers.nix { + inherit nixMake; + inherit nix-util nix-store; + }; + nix-expr = import ./libexpr.nix { + inherit pkgs nixMake; + inherit nix-util nix-store nix-fetchers; + }; }; in { diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 3ac6ee57cb..29395828b5 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -113,7 +113,13 @@ let pkgs.runCommandCC "${baseNameOf unit.path}.o" { __structuredAttrs = true; - inherit (unit) src includes; + # Generated files come from the output of their generator derivation. + src = if unit.src == null then component.generated.${unit.path}.path else unit.src; + includes = + unit.includes + // lib.listToAttrs ( + map (g: lib.nameValuePair g component.generated.${g}.path) unit.generatedIncludes + ); srcPath = unit.path; includeDirs = component.includeDirs ++ component.depIncludeDirs; cxxFlags = commonCxxFlags ++ component.extraCxxFlags; @@ -216,6 +222,11 @@ let - `defines`, `undefines`: further macros known to be defined (with value) or undefined when evaluating preprocessor conditionals, on top of the platform defaults and the config headers. + - `generated`: files generated at build time (e.g. by bison), as an + attribute set from path in the root namespace to `{ from; path; }`: + `from` is the real file whose `#include`s it inherits (so that it + can be scanned without building it), and `path` is the generated + file in the output of a derivation. - `extraCxxFlags`, `linkFlags`, `extraLinkLibs`: what they say. External dependencies (compile flags and libraries) are derived from @@ -239,6 +250,7 @@ let excludeSources ? [ ], files ? { }, configHeaders ? { }, + generated ? { }, defines ? { }, undefines ? [ ], extraCxxFlags ? [ ], @@ -264,6 +276,10 @@ let acc: d: acc // lib.mapAttrs' (k: v: lib.nameValuePair (prefixed d k) v) d.component.files ) { } allDeps; depExcludes = map (d: d.component.name) allDeps; + depGenerated = lib.foldl' ( + acc: d: acc // lib.mapAttrs' (k: v: lib.nameValuePair (prefixed d k) v) d.component.generated + ) { } allDeps; + allGenerated = generated // depGenerated; # Generated config headers, whose macros are also known to the scanner. configFiles = lib.mapAttrs (path: attrs: mkConfigHeader (baseNameOf path) attrs) configHeaders; @@ -298,6 +314,7 @@ let files = lib.mapAttrs (_: v: v) (allFiles // depFiles); defines = lib.mapAttrs (_: v: v) knownDefines; undefines = knownUndefines; + generated = lib.mapAttrs (_: g: g.from) allGenerated; } // lib.optionalAttrs (sources != null) { inherit sources; } ) @@ -305,6 +322,7 @@ let component = args // { files = allFiles; + generated = allGenerated; inherit units allDeps diff --git a/packaging/nix-make/libexpr.nix b/packaging/nix-make/libexpr.nix new file mode 100644 index 0000000000..c9c54820f6 --- /dev/null +++ b/packaging/nix-make/libexpr.nix @@ -0,0 +1,86 @@ +# libnixexpr, transcribed from src/libexpr/meson.build and its subdirectories. +{ + pkgs, + nixMake, + nix-util, + nix-store, + nix-fetchers, +}: + +let + # The parser and lexer are generated by bison and flex. The scanner is + # told (via `generated`) that their outputs inherit the #includes of + # parser.y and lexer.l, so it does not need to build them at evaluation + # time. + parserTab = pkgs.runCommand "parser-tab" { nativeBuildInputs = [ pkgs.bison ]; } '' + mkdir $out + bison -v -o $out/parser-tab.cc -d ${../../src/libexpr/parser.y} + ''; + + lexerTab = pkgs.runCommand "lexer-tab" { nativeBuildInputs = [ pkgs.flex ]; } '' + mkdir $out + flex -Cf --outfile $out/lexer-tab.cc --header-file=$out/lexer-tab.hh ${../../src/libexpr/lexer.l} + ''; +in + +nixMake.mkComponent { + name = "nix-expr"; + libName = "nixexpr"; + + deps = [ + nix-util + nix-store + nix-fetchers + ]; + + roots = [ + { + root = ../../src/libexpr; + prefix = ""; + } + ]; + + includeDirs = [ + "" + "include" + ]; + + files = nixMake.commonSupportFiles // { + "imported-drv-to-derivation.nix.gen.hh" = + nixMake.mkStringHeader ../../src/libexpr/imported-drv-to-derivation.nix; + "fetchurl.nix.gen.hh" = nixMake.mkStringHeader ../../src/libexpr/fetchurl.nix; + "primops/derivation.nix.gen.hh" = nixMake.mkStringHeader ../../src/libexpr/primops/derivation.nix; + }; + + generated = { + "parser-tab.hh" = { + from = "parser.y"; + path = "${parserTab}/parser-tab.hh"; + }; + "parser-tab.cc" = { + from = "parser.y"; + path = "${parserTab}/parser-tab.cc"; + }; + "lexer-tab.hh" = { + from = "lexer.l"; + path = "${lexerTab}/lexer-tab.hh"; + }; + "lexer-tab.cc" = { + from = "lexer.l"; + path = "${lexerTab}/lexer-tab.cc"; + }; + }; + + configHeaders = { + "include/nix/expr/config.hh" = { + NIX_USE_BOEHMGC = 1; + }; + "expr-config-private.hh" = { + HAVE_PTHREAD_GETATTR_NP = 1; + HAVE_SYSCONF = 1; + HAVE_TOML11_4 = 1; + }; + }; + + linkFlags = [ "-Wl,--wrap=__assert_fail" ]; +} diff --git a/packaging/nix-make/libfetchers.nix b/packaging/nix-make/libfetchers.nix new file mode 100644 index 0000000000..28eff8e2b1 --- /dev/null +++ b/packaging/nix-make/libfetchers.nix @@ -0,0 +1,35 @@ +# libnixfetchers, transcribed from src/libfetchers/meson.build. +{ + nixMake, + nix-util, + nix-store, +}: + +nixMake.mkComponent { + name = "nix-fetchers"; + libName = "nixfetchers"; + + deps = [ + nix-util + nix-store + ]; + + roots = [ + { + root = ../../src/libfetchers; + prefix = ""; + } + ]; + + includeDirs = [ + "" + "include" + ]; + + files = nixMake.commonSupportFiles // { + "builtin-flake-registry.json.gen.hh" = + nixMake.mkStringHeader ../../src/libfetchers/builtin-flake-registry.json; + }; + + linkFlags = [ "-Wl,--wrap=__assert_fail" ]; +} diff --git a/packaging/nix-make/scanner.wasm b/packaging/nix-make/scanner.wasm index 85aa6047ed5541b4de0c3bd289a69351bbe7d382..d903bf9170238134de5091098c64ad74f06ba6af 100644 GIT binary patch delta 29224 zcmb`w4V+a~z5l=0-e+dc%sFRf4-X^5<30z3K|vu66hUXBfPm&hrIwm{71J5iKzugO zC{$!v=pqNjl!^oulZ2F%YDmJV$gtG7MOP~8YLU^6ii(@R_h;>Wh5@bn>i_yjbJkwZ z-}QZ7-?jGkf$wI%^m3-oyX4xdjOTgg@w@BJ_jBdDFF!kY!_?pX>@oS;t1fAubJ5k; z&7JGj-u;_LgWs6fZKoMtTN9WDKlFpZzG(UlsR{z)*O;mxlQEefs1CB#S!1fIsxl@U z*uR?L18ai}|2LUTCaA8i2?Ea#>iSh>g1S08&X2;ZsRFZPe0u>AQh zFIV>Bl}c+)uByIj&Q;gUzsdJ%K0If^DRVw@tv4gQ`pO$Gx&iPny7KBP7hH7xc<*)l z1AoYcQ;ip!+{k{nn+eAbe9$~>Cd@i(k?Am>GAqrO%x}z2^E2~b=2i3G=J#fg`GeVQ zuDbdo?bqFM>qoEpsrjRM)BKOgo4UHEmSkQxM;gBzc`bgnXd*@p8|>_0a^a#}?s!)b z`IHR(_|f*cpkGU$hFs)DhO)l7n@pxW+>68ZQZT}cf;iI_`7r}9#3cJxaK1Uwo}c-X zX|rcljX!3|{as1Fxzh?J-asyX#fviWu(o)WGD}u;dEo*t7iC)f0ma%V2v4+Mts3KZ zJa2zlb$WSz$)leG@7l=34Q-DXgUEY4SIop6U~jDF#r|{#Rwg!+$nM+9gqewf7w^$i zz!M$Awh><5&&7)??YvyP^BMa5uNOb?EFiqc%jH~;-1)vKo7-Y9>{l}OiR!4ZaS`Kq z@s2JvvpcKbZT8weHC<+a{ZY*nhE}3 zI(QTO0m{YRj8b-j-!EwpCJpjQgKSi;im1#4trWDh@snu_Bfr%r-{QBF>f#RmcjJtV z%O6|JvK}hNlxmcWE7gWZ;KawlHBk`NM73d&>=v&r%7!JiIkl8ka#|@DKimO{YQm;z z$!L>dkZd{1#gD7JmhytnQmdm{b!d$;;ZTUoXbyx4{(@u$a}ShU!Q*zXtyB$Z91Kt0(pJgN>yBwbm};c9Q-7?_@MkBe#@b5C)(Q~E^NBpcO%DslZ3drCNMWNpchqtgLY z6wDwgKS$JZ$j2gI0%GD!oJByM2uqV{BY%dtTBvai9W<(+A|Qeme~Ay0GU170(@m^( zNc_OpVZRG)t}!y-w__UDHoYcZtAf|UM$H4O)1i-jvvIu1+Ne+e@=J(oywqyBhpHW&bv~bRArp8R3ah_*afhi|>_cL02YTZrY$A1z=6w33|l) z_kqS1f4QNJKiba}n^f!-tD?+kf2AoNrrHOL5XQqQQ>6K53etAb1u50k;Gq7omn-@r zvE$Q%!(G#Al*E9!Z6!0>_a-7mCJNFgG+mq1ASFg|bzYpc<-UVPi-D<3N=1WAbl60{ zMZ}kQWh9RgNt4~!cSQM~AkNJ!WLfWiudSFT-A0;=GsP+&EJ#GEsE#8%*1P`3gP)=p(={9T5_=5!2kI(U}QgD0-H6` zI}Zw}6|u4+RyI;8Bh;5w7ZsxVb|$Y?ho~kzmnrKtB^9;g?`5p0DurpWx%M2@Zj4{1XOCibuyc8;P#=FReQGF#Z7&?IAyr_;vLQL2b#R#7+_Dj`h zEhy_4J;;>ZQzrbjDZw@q{AB@X^W0KNMI50im{yQ#qD9#0*x!Ry-9x5CCs<00L+;@` z(>|k6lhV`imWkJV1M!aH<=;$%HC|3WJY7;7mFa-qOD3eZC{+_RP4t)gC}A}E5(J-e z34ypeS-cj?bO8)O0jsYRXyxar%7;7s1qiTPQfuUUyOqP&b-{OH!s(uGxwpj;gh_?P!ijzB6E4RM}97`-v- z7llz@{mmQ{?lmQsZIAj5D)o)MSrI>{w#RRodG)?$d}OG=5K`NHV2{+R(Uh7iEGx|v z8}pf5@`U9t_Qy{3)6M}^l}miG2(a5qeS1a_CUauJ5}Ss&BfCH6B}@o_1t9OphBde- z9a)hyC;uU0hY-Hxe9BzhT`T2>Pgw@I7JplW=jJLTOtwQiB% z)Pb(T?X{)6n-dDO!=_bH0gEArG#8hpbdn*CSKH;LR5h&-=n+@BQB9b(TkF8T;up%5 z7$<5B2dmLiQ!0qUE&j3uRRHT+JAiz0!6se}!lWLj-$-A{Jsq_MdaJp8uua;$z z?yVI^wm|!0!GZbCRcZnh(@9?M**`j3Zx?JZi-ba{(CHR0&tmWd|0RLE5leX(dxK`D zaB}vh;^?v?PAJ!@H-{g>;F?$b+v0a=+926dg1Jm_C_UNNMpZcIrPAuCG&NehI{XH# z*Wxd7)NS!QoM6Pm#fq8;8}GO$HjyLt*Inxyo#R2bTZI!Ros`?S)E6N4&_}dfybMb? zRf8{(mTU1pQXD{ITxwT+XHWIqL>8|ohTwldVuECe#5b^*b{ILu3&c42_yP4Bn?R*e zkJHcOFwC>#3FET5NyeRfq8^2KZLc2iHqFn|&jqkrKCGuH5dk-NA@-j(r&H2 z1&@`V_r=?Kz&+K0h6v$9CEe8Gv^D}E@%D1syTv~xVV71Xi2?}kIbFh%UI3}@#rnC7 zB{-3lS_AMfq?6PdUD=@UxWSYnr5jC3v||&zjuZYw1hfnC2FlosP4JzLK|Gv}K|x5z zNL(1Pz@8JQLb^s~`>-(9xGipBv|=w>MCSC+Wl960Mgc@-lPNW}wd-jkVve7}gkk)u zCr=}c$F;|GGwqop`ojZ}c651&xhq{No`d>D8aEzLsoj%eOZyFKVUHGnaJsmD#YUDF zCcGikV6;#WIZ0o{@^+=-vlE@t`(ISR=EfbL#ryJ;$dT;T9m-dQP4JSkpHQ}%tnadS zDVrf1xa{4^5_T$NTz0v#S+Wjk@Tf*unRbzNSALc9$pqNEeEdi{#=b#}+pf{^Gnrj8mFX z@MABrZij0wv~FEGSCyNKU$)*VW#`Zy6>QDOq2&ZF+B#&Selb=)r2&o;91A+8Bk!0lOo4ZnkRcN0bEynsyr=9GGYlUdrXlyB zv9zbj>EHR6WaVskEPj{3!!1z58vS02q=+C>*q%NFG)d01af zlo&SJf`xu^e?S4VZ%S-p)IVHp6AU^ZTAyO>-yFm(3I zDHM$YKxI@)>Id~65bC?Qrc~-bANY=cG4TH}F3zY_bN@1nNUSL$0X#TiRc=ww8pq22 z)}ow(?7f;qJNAs6Pe<-q9VbHn=JNjSDNyK6ZkV%6l5-0D5GBaXMMI+U(BwI4z)E;I zDnY_Z6E!8paQwj@d0#x8lw3$HOKhgwseI?-z%#>1t^!mWN`33ek}1gncN>c@jP>$~zl(gZ znUR=e*hvA4DdsEN;-ED)MQw!hGtu99{`GB-Yv#qCp3WFl>eEioeAH(qy1#EyKYI|{ zFY3!}#U`{1yDn!88=^jKI&NxUA%y*!+N6|^o>71fgkCDb)TltPYa>f>YEG*W~YMKU5hO37dbaOxr& z7!6XvaY^;$@Mz_ZEjGwTYMMl;K)w3W{&AvWStdRxHn7&bygfIwcE@g~3=9X)+UKBI z`y4u}t7cYBr-|9qCUB)2?PQ0|ybc|}6_o`@j*UY@!uvADT106z%rllg%80jAd8qK$+!|c=MU85af(prVe$lj3{Tjg z7LqnUy2B(#@+`ZqcsROcgu{9$tUF{N#}4rzE^?)H8WlyrvSkc0*tpU7dWAEUB<;bq~+y3}y6d>#-03VW#~c9khY%33N~ z(P0Ed=cKt>dIubb6J1HfsR-fl{)DPICy8pKy6{LCFm;g}vd}2;QsPo^Q@mYT_c$*X z^_L;(FE;7VxH+AX5JL%=Pj)_S-_R8A!;5kq>e(RLFHIQTe|Z7>Gl$tF$4xTF+vkrPGQ0`pMZb6tAP`znHsGLDMZa4? z5xXHxVAW#fW#u+ejQ_f$Rb2D-LOsY-!So zromn^>55q=DA?LUu$5#4XMH4t^%KQgzv^I^3@SPCA{rw5UkH61gxq#X$cmtKX^yj=7fLN;^H2d3~lnpKee zSR(vY#A*my1@6qKYl44kYlA{mJ`1K_yPRw zRaHeKmI(=agm)RvBi!bVJ(wdyJqS%tw(m?2sCC^WBb%3;dRCWpK-~~kgHy<$Vl1M_ zhjSBEw>zY{!|57+&jZ212`n5Cndp6B2EB6u30Rw(pzA!6%mn`m66__dF5^k(-xK_c z>87hLF8Y%qqKw7taU{?qI^krYFcf;J9c(Jj>5h++anB?{9h{~?C-|q5F!d=UkZ&@H zGY*brIX>n*RXUSfQc_{(3sJiN1GSOy(@hcVpM^st~O$yssq$MP$Vjc zM4!33;IL0Bm^onnXJpnPZ7(u&Iw&ejNgSZEb}DUU$ed|Mez+*5I|vDx!A3HX8{Y}1 zbP1=f)xwly%!SAcQJ}s{w5tj~z!I=awHS`O;Sk{!iVN4Fa1DR3 zRH0Lz4f?6H&Y<(l00CXY2!*8g?#k;eSO9)99={m(_?)d;Of7&r4=ROSQri4)&z zerbb~V)L4vcG4*GD|^*RJIt@`^wtw7xuf+4`qkpuC;f;r)2}Z4fUS?upzVj^BWU}{ zcp`Ibj&Cjh(&X#VBDl&#Ky(tzrsn`=4m#?Md!6(idXM65D1>~y281npBlN(@Z{Osg zI32G08vt;j{lU~`+jZ)g`iJGCxSJI|mj!KZ<~H*ayJO}lg&kk(^5VYnBH7>f*oSsS zcIK?}!_O()B4?mdv0>IZM_k)u?ut|rD(ij0@%8%{XL!7ygmI3YeH5-kd{fimnia+l zr*IS!v_`sMCG8H^N>c%Uy)$F#3~;moOpyBE6EBRs6$4moQ97jW-6>y7$d z_)+NV77n0=mH6zw_JOmHFFe(|c>OQVKG6)?bbIfXx=MhoAbgqdl=UaP?;%sV>yCpP zWZYGlq!}(9!5D@gV_VLtFMOu=SoY*|Zo< z$$hr%nf8)lb7o?leR}aK)=!XllvLe;e=@o&*@Ws;-M9+?6s$7w4z;ccN2z|f>g6r+ z%$XcF;dJ^_LgM2+Y8HkgRkab*jpf$%Q!brO2~6B&L^0_Fx{s#K#9$3iPEmRBhR(_? z@yejhZ4()2X)>QKa*{sKP`X(x%KAHJBUi);QX&n!qKhpP|B;oLURE zE}y-~7MdkUi19?+y73bBJ2b^xGPKwytVn|i;XFfzfFo@a?*rwv?CmJQt#gFh7$gKb zZ*j^`SQRcEX;+5rU~;UyDB*K6vMy?JJFzyVpM#3CkDpCZr0fo zKRA3U7Z`QdPK|VZ$7PUFpv<(T1+0$+%V}bXZZU8TfwyEA*JZ!>!Pa`vs78LGOY9#$ z*snar`KAd~4rf5-YyUH*sn z<)0(AWdxU_aW3OHIS6NyA{-o&BO1C@RuaoB63a+3tGJ=vK}M3YF9=#A-)Ru!9f3Nq zs^pd`OV<$Xh(ofQ&=Pw@V9Y>ao*TCsOWY^!;uzV)yAmA|ILZc+jn`cXaw=N8l0B|u zd#{o$t_10+pw=ND*>Mt0^h46gP1ryZ&7Hr%32_+;AzUr2A+XcQC9M zZxKbR62;h&C@MAHfs@>q)U8UV)!a;iO}gZkCE~FukTMYGfGn6@r!cIPGgpuTsAaE| zjrT5R^h`4PjZW(#GdjhVKG3;!HePujO{$Y7A8}3C{#E8(fKtE%Ri8}(*VG*dng(7R zn#|4+fE{WtE4}2HUbn8w8OlH~lX&soUaP!;NCy zZM>vZdNW1lHkX&)L|=BNjhuYas4IZ}LN$88DO86hr>BzDC5p1kFZqe8bm-%fi7zOm ztT?U$hAqAnIrmU}V!zN#_9)?6f#jGgEt4GUG9nhh;fI@)dVuqWV^&6pDhZ-0YJr-= z8I3NTxH87avN7%l<#0h)_4SMXPtV$DU^ zqRpIX=_M3?myd;X>krz>l6)c5*%iGLKyH5^^lEU!9Iz+VL@hTEm@g3y{#IGq|R?B^ozX9=5WXv^q zSi&|i?n?7a4qcoMBo#MNvQdW8sX|P>bmYDu-g-}$v=!VGFJegdu1zAOUc5|mJE$K=dx)WVvkMxR6*LU8^u7QqL%f>I_tgQw*% zh&GKqAnpT1u4+4(3QwYs7eAzx*1v6~Z(k@1z?1%e`$`$_omVNI(zO`Oid&`dp;czo zKEYaW=7CkhbAGbUbeU*_gO^!mm#xuy)|qwdT*EpKaHf4%p4qe}YdAl+tw}w>zC5$vWIMu=ae?rZsC-B)@@H6>^%}QQ)CXqJ>Qpn5&{5E{Mjl@;{S`ZBC??{tO`tG!w^FEDacZGLi>{etZ5>bYyAc(BriQNg+o=(u zOE5i=KMV^Cnx zq1@%eWrU-%QC1qCQvk%0ch&ZCiI50ukHb|5NfPon=YKiRPC@-6(N)Vqj^0XeP6K*B zuNlz$XDfxSl_Lq3C_j!jYY5yjgT3jpTzN|5C6Q#p={hK3v#XF0n|}!-uLiWS4uU$0 zGOH#29L$r0qi;|OdBc2*qY>P0`-RI-8m5IO z(gl&KidiT3NgF8Beb0wW1K0=zQh15Ud`)}Tq}$GqH`kH znI;rS(S#06a%v8gJKaCo`-tC$7|?dr3R6s5Z4@0M9PXzHr6V+7Cc+x!E7I%~zJn z?NRtKE&34x$3_&8tD%G*mx;l!gVVML}ML^*q z(M$w}!wU{zDB&a&xZ@-g`un|nwVcyPZ_89GX0F+QR;OCd*^i1f5ZdEk+`Xa8i$}$K zNZd;x4=CRpZ%p&j4~eg28`_#jOFAT38T zMKEpU3=pk4B!(3OM2ez<5!l!ikfs_L91!k!FZ@u+1M5a+YmZbW*z0j%(L21$C%q~C zUeZ+x9N9*9wNG5C8F)p_J?u@FvRG45(UGRSFz511lvXf>o%ko1LOgmfwJv=Yrq%>H z3nn70A0XFemM9pLwv_$lntP+$-|j^gae1VH;wA2Px%)-Ahext_U+P3%_xj1gG5&g! zP0RMt#$J)zCq;+=8Nb%!#k0qR=iD+y6Ms?9K6r5eLMZ77@Tj!O-667Kz8o4@ zUW$ftVhFrS2r+HtkU^`1kbzr)BGdN)(BT>0YjSw%k`y3iyhz}-Z*n_1l?yMC?!~=S zj1N%~ps61(I*KV#kf!Xm%QfMD2w3$;yfcZ<)uq>d$n!!Z3P(@N(nT$Civk?CC>D3f zqI#{Ve@b9y;TYx6HI-@L7KQ4Ce-2#~Z)*V*hxPw@Inw$EuK+yBoi2LptQwJ?#0+B7 zLhYGU#O|mns@G1_iC37>5g(^yRUDybqJCkuxB{c?h1jw^4)3Ter7uDM3I6Lm%6Pn{ zB3XN=j*w}qV!9&06?X@2nck;%@&p8b;?i{1!2NLOU z_kQK`IG2VuKzzDp-I*0@fQ!lC)bfnNmW(SnB-c!-qD6^|hQeSWEoguf&AQ2uryKuLivDK8N3m=6>RgNhbba72_tE+yZF2%SB| zvLG%FTDlSx%1CFy9X{Mq?H;$;H6JO3O%a+hz-z{nC26uh`AAf5V#!>HNMmDBle@u{ zUi=B?;Dm7qUX5UJx2kDjv8kd!M5_U@UZ5x1cDbh881P=6n>T1zvMC5ZELF1s9W8vZ zab&B^X^2t~!)VCaH zk2}7Kg*jO!W^HO>Sok|G#%W3%wu&u*(vE(ZB%#b+cda3%W4I0&fTi?0 zpzJ*7x_k+t=Qdz);5MK=`TCDla|oZ|5dMYhPsudp_6K(If;QW6L!%r0-S+t#mR5hM zGQ*;G9-Z5-?>KtlreoZ>aIs-ZuHK9HKh)*LN1~!Hw6ku1-2Be&yM0{nJLB0)Z~CA) z%6{Xf|1ekB;?2bg_XIO>ARxFtbpRWt*O`*n{rR*^vQLi>@gwYwH%HCl#4NH{!=LU^ z>8cDDaNx+V+}v8=4Q}#qt`A?@f82b0{Sdvb2fsxMwT!pZZYlL2lxsG;TnQS_(47R; zg?sI-w~RNN?U!%4kl*~R<7-CHhC4!b$f-l^lv}3`yVUhbFL>x%1C=i(q(}1~Jaw9# zF)cdxzzM!{@|)r5$iU=>zSNyNigJP*P6~M)-+ik@ZQTp_T*5ebAbZ=bXNGfnzs<}I z7yGMeNBDOe8%{4(Cl?azanr9lMsJ`0iz^9@4y4#Dh|PRLZ+vX%r^ar-byzuk9(5+| zp?g2>8i#xj$z{<4)#;{Dq;v8uh*Qe3xmxUuK~d&}H%;Eb1I$R9q50$idN_aBeK0S> zrX%NKb#M&2Uev^BDUaD_JA|dvC3@`>vD3QagCYc|@ByQrq{Z!GJ#$TALp*`F=k%p1|x)d395^;@X-vxR&*s=zFM(U zaaSJH%4y+jh-VOnkee9@ zW=Zqy4)HQWx)?;iR|4aD0j7~NA>>qvUxbPQ=sQq>PnaOa<$y`eh+FD9tFtTYd>uLC z112X*Y7}sj#gTtCNHad*U1<5)SW5d9Stsyojl!#e+OwAG_Q&VPiF%m}& z=p9Y%DN(Fb!P+AGJN1+xMee136OV8QZQ%+dw2tU8L1XAiK}(9!48Ds4&=&)NX>pJY z%K;7&1C@oOl!KrLaVXe_Thd~lOu{^zNjU1zmzc7Ql|tg~WHhy7OdIt!HE**tZwJ`{ z53EO5d=5wYQbIOIlyq#Sb2Q-|xXwlFd+u0juC=e;(JEVATsXcSr5J!%7O&5r*ozkq zs3RP;3?>;AUTg1LIKs@g-&i=SZaYUG5flvPuJ5xb=bLNzalRi1d@7`*! z_~fGeb{aV`iz(Rm$r&{_FlfAOtDSyl)Tj}doNELU0F~ZwrvQ84&PnDb`{O&`mD%jr0Dy&gdD3P{J0w?yi!`U%G4Jf26}nf3YODx}x!0HgmEt&+vz=c3*dgDAC7*{48sM)Ui^r)c@C-TF=2?$^>;&CnBLC=qU z^3x}lE08G0+XISs2q^3}0Hx*tC@4Dzia7vECQ*_q!RJaqL8Jf$R7#Y%%xkoT(=}-l zxC(OFYd$kRA)SB|>DGRRZy`KqU-?XzlI2S;6M_G+bfa<`@4lr*s2A{)?rAl@wYS_e zrLkuTv*E)k5%)XyyvyIQ)&B22gAdz5b7n_wH2zL<5OwEPJI?m6Px(qpGG4RAzSp*t zUrU;oyqI>8fBGO{UxiNf@J=_6@ms?*)>;*FmvxwWgoHf6MYWt8^RyrfKb zTBEemtR~*6+;)5BXZzWCpS|Kk857<$6?J<*SrgZZEmF-}~0;ab;S*rgvt8GAdDzb-gp2lu?OlHulbJQD&pv zbngk(JN3BT?zy*L`87T6;79G1#@o}(Ze@0*8KphV_A2vwTBEe5*?wjAq%}%=nk^F9 z_N6sSlV)h&c!{by_z?(7d)h5iW^r1hw5Qnv$}CN5l=d`RrOfiQMrlv8waTnaYn0Ng zoXomGxix8((w=sklv$V7DD7#sv3HGAZ?cV8m`vQlZXRa|#{*e_Gqtjas(7n%9(nZL7VgbD)nD+|oH#$)lZDT-vLp7x&E3Yf$IRuY(QK`&HWQGPmR* zDT!6-0C-1n*}#rw`Fj!L?<}9(pb4Co4YoxW3wdkxy*Ml8A z{Mbn=hTFCGC>$n}$xBHA(&b$2X@F19(H5 zv=?C)^+u0eeE;B+{uL73J!O%cD&R*m>JAS*=^gA*-EIbt8`AP4?Cm?`i(R#~4KZB3mXLW4w1<0lxFO!v1o_ zS(vwXKk()34Z7~`VOxLuz)e`pIiEXeaEdB9^YZ9QK^)AB13p(H{oD1qBQCu;Cl?Du zrq#YQKW3$eL^{!`jDm}$(2guncylB(OZ*6DK@6xYab?GVl@94Vr$0Eh>9>ZRgJUcw zeL{gyw)DY)?@C8c8dK~VsL(ypP0tBu3A=)P>4VL7?t``V9}kWj(zThgqvAaz?(pLF zB)`t(?d*q6{nto%;-S&--hVzctn_~`*$!MexNs1-_T-gQFLBFqv`d$jus~{wA`|uI z1Tlv`cjN5zA!~(tPG)IyIwxpeJeqfAWv_m{a%`i+f}5p+g%O`0d0H135ijBG(*6)P z0%AGajiNIZKLSVB?Qm2Cize3Fr!S7gy`P`$uiIj$e{pEe$$~yK6u)lsUl@Jtn&+_X zdZov2WhSarTa zkI!T8Uv);0z2CFy8a_mO@t0Pa-`lE(7xVk+hsV`zSOFI>bvV~{KRl^s71{XnD{SrR zll+HQ*jcO5Wh?AWtB3g;R@nPjj~{&tS1qgJhtP%0+^1dz3n_rWv~UvnJNuh8Q<`t@ z)!>d^&kJqqBUky$9<)z9GFZv?A9=5m(xb!2fA+yvZniQC?l<}(u4ePm7T57@yzt}l zuC(()7;cN};vH+@^^@)OkB;znK4c$wbV&cD#GPVY^Cg~G!!=vi%f9gFk^b&~)nM#n zFPT5C_t&1u=XmXVzr4vTw&4?3@O#q}v-sWe#IxpuwqxD2!fl))ChrN}7MAVH>umMT z$AF>PUj3D|)w}h0`uga}$;Rxq7p^~S(C#N&nSJE50 zF|MRHbVs_99_XHV>cJ!TtY^mK<0Ts)0+wMg;UIRIp}sPWmg)8G1mA7`%(M~aGj_wY zlD&N6AbZEt@gUcH(>pe|OJ__q*8XV5DMLDpsVYYbCM3tz5QP8;8zjW>Gxq1xFNICM z_4J3v>iduxeKAhekWs;pYPErZiL~>|QTSQ+sEMpBA!+#Q`$}KXQDS_pqa42yc?d z`<=4S1C(=t3w?ajy}Rb#WC;}1w5T>#liX{iAA4oTncDBzH)mAWEhUa8CAiqP7k;gD zlrTZ?iO?sju>y-G(CUKe zvdz*mC3b%9o1|`krF)c$^FR$ZB)Tn^`??`ULpOI(lE8#pvAyt__xkr6`;})-8_#>Y z$$MEX!aGBN1^$xH^GVYZ73NG5D?AGr`8=PkdUhfKfhq3y%&ubHOrS(7^672+=E;NX zyydW4LPz1e#b} zdaRfYKfHeS*9mrh-hSj8)~s5e`Q~s#mzJ(k!;Z;)%=o=!nV8}tYZ7#*~*a*~LU8zdVHiZbD&_IH~XoBj6sZ=K$>`zfU6G47$n+j&w1usYuM zl>N_d9d6FHZ+&ZwS!>5UHCSy!xQ^+^dZ!lBcW`wfFiSWxtW>bKmlx<5S3OL;Ort$!8@Ki|>+Hy_lTS^- z(-HPY<7aNGe2B7Ip*-;S6%+AVOjfMPjd|TK^2J8qWgIN&lae%eIQ)uzbn6D?P zjNkI}6M4Pp>F1~Sdc9}^B`&!FOeD9*r z)p1qDRH{Dmy#-YL>U$qI>+RfaL;G}BCRgwdUJcjV2ev)Yk`S$ikpR{ouCBe?g!o#p6@<)fY~E_czr|Z?5Yjxm9u3!P(8p zpq1L^(yYDU2cykX_TxW@##H)C^bZ*Tji68QR>hT3><@o%f_d83Z*MwmO)}4u#;J^W zb(*y&Y#&C8^S4jtclq}BOx^gl)knOg>L(=s_Q8XsFTg*EG=q=lD?_%tSTY;!>=#?i zpX`DckLG#Ri|^<6Z!azw@o;4X*QEjlaWw|0fN1ae;k#)5?H?Y&@17sNKYB!+YPgID z@nlJeAi3^etIRm(M;D+8zV)NK(KPRSX}0;CUHQ_`na}jhRTFtrb<0R!m+Bb;vGe0~ zYNL;$i2=xOP@XTG2}{y>l!afjH7}3Prjs?<)|W>XP$nj>jBl@f`7);8@v^3Gd!^N^ zv5&nn#oe7mSTzcQ$0++=8EGE1!+u;cpIblm$5Tv?WUSb6t~q=CA9wUO=9{+mCr1qa zvePN(Q8w#a6#Pt7jE<(iY0vn{>c;&~wWbFX`rgx6Z~yw5pJojQLgCH{{La{U8o$eS zo>;wqBf|bIyJP3b>Lp)ukC~rMGpp>ZpN;IhRt0+Fu{NrWA8;AF@MjavbN1Pv{l#pt z>;G%`pe^8q!wu)ti?ejKS;6ce++zRmUw1Rh!dGwPcmJ!mjPB>ug{$Pn{tk8+HP`1( zkSWo1pE!Br6HMrJ&r3{~H9w!4<+T|hr`yi{`PJq-_L-j_!(eazd~n~*d?XD8!Tq)@ zuh8;7O1Rk$`9*W{cMTW~fP-)IF)}}dHTbU^eAk}wix1HEkzZWL@33Fa;$9%~hLz=iHIbrIel@Q00_tm8_j~N^zq-bJ&;H?8v-q9->(lwY^VesX@2&sI zud}9lui7-**Z+GQ<;}0}VFa7>-e*gA+9rumdvwqZfOi|1PQ_5_x@*t#0>JJl=B7-7Z z{fDuaZ|0-n@?v0VcbtvC?I#BZrA)Y){iehwuN{QO%jJfNihc-Tu%(R3 zk23T|%Wg9{-h76jVrwXJ-ybdkxWE75WPVTjpRUIJk0CyMx1xKku*zumYrD(lVtf4W z#+e6f+wVR-v~modoQ!i^flIH(Z0{Q(rvkW8?aM z^Jo5aR2@E%ERavA=J`LJUEKdHSS-Y!+$3ED85-kHCyzSWi~EG1w3GgPG<&ADKcDWe zT4BHT=cD{5R@gWGJW5;L?tR5^A~tuCcVU{{S1OWyEu+CIysOXTY-`;KsC>L|m%a2a zMgJ=+?8pCd!ckUJp%0Xxy?94^wJTvrChr%1QoNwOXYNGkWH0{f0K0BY_FrspTlS6h zmwm&YyKn5UM2ALI%ueWt^EyCG9TeVoSYf}oZ+NnVmi#W+1b}rL^8Nb;3{AAHa|l!T z-Hj}_$6opk`>wxUuZ{Dge;qaW;Ei(-=U^_r7nN**fA`m6V6O1D(F4*gGVG-HMXC3+ zvCbR2=db;0IcFt^I4|C*rj^~aYg(u@wU7L5YW%WsQUpB*pq%SqlZ7WenHU;O)Uw@q@d z&L&%>H|#5apZpGum#QIwXyR?jX5tOIdH;8bt1S87OA0&hfE7itQVo%kzy0qD-gm$E zALCk=J>SKgoM`aB8eYzeR}#^o081DzN$POYm+}-V$fR@rZ}I$w7w;xTWmdm6tKZBX zQS}g2M~gq(%ibCbW7xOG9u$m94w8DEX(5Q|LDGR>l%E`G>B{On_V`IK3O*qi_1U~R z?_EjMsmgDS;X+Qx4vIM;DlXdS4xKA8r_bJNUcOvwjnf5VROLZ z_>W78*r9&em-A}K|n6P$Y_S$Q%n}6w?XjE7KQPDM*M<1SZ z^_=;aESPgyborHY)8Z?yjjq0CL3Byfe)TmsTl3#g9e$65IN68Ojj3bWryh@+PC>ujrC4U+D!w=+r@3PJn^=9InO8pJg zw;ZVVZXz%ED)s;1c}TDP@ziV1qxd_nD)i1E{Z*Cc^*bbARK9aYgE^*H8EP6ub^Ohr zbIE0=TsgmIis_vz8q5i`m1Z+LUu`fG$5pZyr@e&RYkN?A0Ki+RNAr4}M~3F;BYGOl zy!z6)*IhQ}+Br8am^1(COXgPcm0=(5yf!q$dnUcK^P$j;>d9W-`Ge3LV`g;*jb^03 z_WsTz8_gW^ozA-(%}I@wWq-EX^ZGOMah*SJG|Q{M`Z-wTn9ke#n8BSZ`k3Yc`ZQzC zHve}f^v^)WxLZhWyi^xjWiC^?e97dtmJoAVoMgV4K(>;-}6fmd(w z)6qb)J}@tKzBI%fX-3^U)J!~lR88oePo)nCh>v`d`=JkYif>?VVlW;3}p`}97 zCf1I8_vAm%^9cUt&zW=eg87$Ry&%$mXVCK&rVAef@uqhEIWq4xTRUeJ&C$iJsGI@x zxQRbWN+o|Y&mzdy&ijjqX{GqJN^#olJLFxr&Nqu@lv&f+R3geYs&jhDoK&nZ;!{8= z=C0(QspLB!E}4lv&E8j&Fmd(x-%FWHg~?H4Ji|B|DP=0?U0IP!S5>8&GO21qql~F! z4LLiU`u~_q?YWu#4_x)tp19xTlWR=0g6JEIYg{m+W zOt%{|WBjGh+jlcFN>>*>`njcD6Pl>5^Pxg2^d8C;($P|IIM(wbe+mOD6Pbx*7xNaT z+f!b&P;V*T=ooYk_ws%&T2XH2<)RHw(C6=7wBSjWaG{sWxgNPQd{Z(vMqbcRH1_F= zux9-N#_^(^Ppg^TQE{;OlPy;En0@S9mD70NFFVdOul^-?k<6_1wrTuRf2(Q*Hm}<<_l|EYHl=UkX=^0 zf9X#q0|!jbEl8P45n{gSDrTTWCNvZLe3;SnOLIh3s~;3A zTPFD9DXW;^k0Gg?;J4Dpo8S*uF7l=pvlILfL_%Iy|hR}@W)-b`J8-s8; zC9ip%#R_N=&zy;NdYwTrtO$n19RLE@K_|QdyEED`VU=U+X)bqSm{l$xObe^QkobL? z{bVq?U$*E+;S821+NF_^*X_eWgPCeK2E~cl(4Q&}77kRU7MUm=WrJy&Hr47c^1((r zm<~(62P;N{qes`nYDe2beP~{^lj>LZ-!8s#Xb$p5l9-}Hy-oMqzf@UFMe}{l?nf*A z&Z5Ro1ur40(PFZx&;+k)`c$+@gSSN6#O^;q1>|V5z!$4-HmuHX^%wejE<#O)DX3Ro z&JwpA*f}0jL(?xFwq(lVu2id^xKY>mpfLAljbB^xZ-he^!=Y(7)bmSz2meZ;Y4yEg zC5uiA-*5m0V2%e&LXr)Z0!XVr-w3caKV9gjVy}=1({27jQ)p1_0+Vk8hrBH^C@^Xk zn51EuK*Dj{-+r>8<K~tOWwKJs&Q|?U?p6{eG+gbF=B_J)^8E2q7prylx zcI1SWN>1o45lM&r?Ar~)OHcSw?zoyPAnouv3)Q4Bc%jM%$X0($(Tg(2fzxy$!$?9y z=oK?jrmNQX4Y-BJy9&|-+mg=P)j6tg%_Hw4sxpO)gT1wo3pSWy389_fuQ4FgA7`HM zi#2de=}K7sFmepM&~z4S5jeOH1Y{G?P><;_D|tE+V1s5Rz)9pWq+FTa|koPdD*SIvTG>BlD*MYY1lGGV~m29H*G5=P4vIjMjWt;*R*QAsyl6xO&fJ#6>tPeBm zlsXXdIb>rT1qC{2@MB3B+o`>-Ff+A=b<5tmHP=SDnNDCGcl#a8o-nlU7}`$TXP=Uo zwi>2Y5!gjdMR8H4;OB(RiUf2tm_4)4_j6w69>#BK2F$ZH?zW>Glf#eto6cD z=|kMAVlbI=+!}=-eaJ@g=RMWGVPBo<_^aq?A8 zk9d*k%jc*>R@TeO$Zeb7*dFuWZq>Qo3F4qqrxQ&nQy|@~J@Z+(0leU1$%|@+*L%eD z@OrdULNGpBQ5J$#w1^Il-cUmcg$!Sxe#+*frR8C^&@F0>cE!Cf7LH@3Fx)AHSSRF= zL&O>L-8q4$hnkV5T`WtVYsb^|QO2x%at$jN)=gQ5xO^TuQ5*y_7zde)0u2Ad0|#K7 zUm7^dOtbF}964cZqUU3xner%@nS>o1U?<5gP+xO-8#oP>AI%U2G`?;lM(D zsw1qI5EgP_eWG0QvkP@msZis0P+2EuueMMl>s`ueebpgJy3iEXMft8mU05TYu8|Zn z2=jPh6U3?3-*GL$pG?tZyTYax+))pYE%J`-iasz`)cT(Bk$_C0NxpM+SFw6H3uTCU z29-W-W=B)3XWcOql!TIIvMjb5_%WiOZhrRtoa^Sg=E7P^f(i`!{H#b+9j!njYEddL zIme+o+EgW#gQvwPt^VdJJ-1W^=pDI-)zK>2S4Uf_$`Vu^R=4?^s@hYf6*wL=UMZ)C zbd`}R5c8~KkV!IjJ!t(kWVE`lfoX1KL>aZ1p4HJVCa;dx?K$`w2H#ngPr1<_H3F{{ z`%)f#Lo#}4e$ei(*XT^SqdX;L+mfS{t-YN~K&f zz9liPYLe#oS5O?zirX!c!%zE#OwbaqA>;+0jrn{Aehsv2l6%O|WSNE?#URH#+nADe z;_|LiP=_?tptfm5glJ3CPjSdHf|ne@Q<*^Ya8?_3{My&p;kD&J>?+V>G**e zgOeF`toN!xLUXd&iQ-r&PXz7D>Po<{qslES?#3E|lLgtXD!?LruPQc9JF5yBOaAMg@R}u0>2OLb7M-vT5WGFwv`;#g43z} z7aXn`#lNk7kLFFq8!s@IE)1ck%tZzdw>bR%aQLKf;r)SlQJaexs`*YbqG8TmVInrE zjsYVRI$}TRTF-NqpKjL+Cr&ykwUF)< zvtwbrs1N{Z|JbC+vWRieH8Y&XIK=fr^Sojic1*k~je3Y4hX-|ecrA!Z>LwwVGpu<~ zn%nyHfVXLWnjUAuYPEsZudR68)bEVmQ|pgWx`|Snov6nt(SjvqXpVJDLx!S`uDF|8 z9N&o|jTV;FjNWXgV|M9qlBfl}k<3yXZRoQgH9bdHwn^8#pXiwO)Emv42>j1(8{AiZz{GdRvE><6SUe5WVhA!Qcha_^zn>ID5{BCU_w1 z)G{FEQsDz?;R)ywYTT%~T)Qg4mV!9cHtR7c0j{x719(Bq8wv#AVWS~fIx7RbHxl`I z4W-ieUx;5=8?9sH$dBVFvR6+ip9z}aC1syhwt}qVuxO*QX|gF-`+~9rVQSc;(Rhop zS+Wjk@Tdy6CS7E^m48k7c!HAhZzf|*u8~>+t_@YioIC?wd#d2d6ynZPxXVR;*H{l% zbZSi=Qc5d`?BW8D?2H3GP;3YsrScf-4Q?pByenE5r#Q97kG$BtjnG`E-I###)ioE- zF0FuJnJ&!PeMSwDwu{?3WTJnuRaLmgEa~v6HQDIM0vmvw*i4H-pKJ%Ln7_R;i=>Mv zEjYokpkq4nj_HDA@rfl&n9nhxEMl~?6>h-;@YLNGf~H}HLf-n)@aX0Qm(}ijsdSvmyV}Jj6k4AzJj2h>@McV^6Z0I zaaKfjWDR?CnRO(uD(r`?jXlW>USLK(EmWXWO;{tBiRcRMG1edbnl08uH{qU?BZ2vW z$nbPe=1TEcjreF)p)nUWhW(u91s|8%m`giX+&zE<$F&nZ1vQx{Hf*#73;pE&fC6S; zAKS#RDY)0j!wa4;`3hB}CV({PgsH89Yn*EWW~0}>+{5G%)+|8w5{05scrT$+=qq{d zJ0R4@nZk0QQvdnDKlm2||1aaF1!_&$R04?D?jRB@11~J&fclPcto&~Q12D z_}!kl+`jtY61EbK3vmGwwuO_+Y?Q>J6J1{~H4@tt?O2`L(1+Dwy;5O(NnG%sJwvvb(bmC5#0rDh^0CWl4v~_s z!eF-LK_?!F7o|F*>#qpCuEIWHzT|?3L)lR+z~2e6N11TY6x6O{rdO%o2p%$Hy?pHY zO4Q_F%A@J0C7D895^3#7bwmcLo3QRURCyi$_UVc$uZTRo&1@+Ka*yl6;5f{{KD?AD zp58*_nFgJOeo{1=m381tbm}YyNkD6c{W_(e$4$j=bX6$kg3L87*+><0#5!r3W%y|R zXBK1`FV~3z%XWe9V!!BHzgquS|H`hg-<-HQh|;d66!caFbFPTy-}IRU^O6J=#=b2V z)=jTTX+4t(Pr)+i1+A#Akn3HL_WN}+Fe3t@w-lUWgcYBwi&ow4jBsA|w>BghF6jaj zB`R0kNu^>GteNcM-fRwqBnr?4~4qN#BSi- z6T2ZlYGTc(x#LM`5olXYe5~X))J*L_U~O(Gxxxs5&{lJvtaH5GuM0=q4}Xu4&s?kx z#yRlN*TMK=jTof@c9Km9_LmkXC`sswU3vxK$1E|h(SVHQaatAJi=A0jDH%M8C4?bu z(voY^tWxHwYD)!rG_x-{fR*KC8CDp@g|iId%RqHt#xOCtaEQ*tsL4ZT8Sy-pAQ0HZ zw`q&dr`oja7iCc?b#+uZoA>N-EtoHJHG<xc7`MGj|N;_95B)I!v3^wL};(*3|+pCTjIgj4zpIrs z7H~o#Z7yo;4DNEOTb^Rj8C6Ej!QFC!bD*4khf~JEV#7Hu43UqBzf6F?q5TB^9P)VB z!m1wW5T22+GWJ0dgJxVtNf)vT{~^GMyz;uhs9+&%T4vOIeB`+oQ^H1-B3dmB2ytp9 zycDD%3H=Y}REh+?5Yb?0a()|Z>W7kSwidJ zQsQ5|fvthkCwY|T|Qb( zinHf7woEj)SjQ;>Dy2>(bSjZAABolJNURQ0ElD~Px}kGLA=&$UxGGZ!-1;vsGz1fX zffJk0>Y9VHp67>H}!Op7%mY zWM28!j$4C?j{mc;C*`q3A}*D&>kZE)3(kaf!C|tP(3rzmNnGEt(T>`*uuF0@4IHNz z%B+B7!hyLc!$~8YkQ1w;k>jCDgwGDor=8QGHf3JOi5JvH3{(RTxM&rE)?1V*=Vl+kU?O{iBU(dzc!Sf?Su9GHpnbhgIY*S7e+ku9jExwkx^O@^tP6>O z0UBU{2A+pVjk>Yw;_)Sx4dDO{T;)`(&VCl-WEMDdqVFPJ8-gQQF6yx^>=zsb(y$0( z0--u6V#HQ+ipYDT0^HEGj1=KyHy`+J2ZTlmWYWrRa?D9X86U;AHxr3KY(Yi*Mu;!v zln>w-=3uko4vs}-_@?ABf?`|z3wh8}wmJAIK5ZbT3yYU){(@490W!D0k&Y+%XF- z)$KBWx@SMzKCHOZz=a&Gc{ScZrS81cZM1X-sQ7}t>Zq@9JuBZa)=XPFv14Cj-mn)$ zS2eZcy;ZtXarO`5En*)>+MUtK=1qIdq{#f*-ZE*FdCPut(pIz0e(vbQD0%MaYv@P# zBk5P~`c>0!<}ovAd-pM;Y5U?a?abj%zP|LP$yZBEg1+qED<=Q+>~rr60Uws9>b?Ww z&GeP0-DjZ<@zH5A+#63pxub0`Wq>{Yh+CRA8a#1M<2YB>0j09R^Y*t#967Wp4J(u@ zLLz)!QbNQ5l?F{~k2$hnj%gLCU>FABFFtG&?NVK#*TO9qZyG%~Y|9NYri9uHu69=| z`d+;Y?cFyPTX!$lt#*K80bM)VtHR#$#Bg()eej85-Fbb_F@q!QywYfMlwDaWR>UZ^ zn@g7r-upm#52V@`o*cqp+n+pd$T{%}*MH;+J^S{Jg_=E9Vvl)plKHaT{8YiNdU9CB zFu)5Y*k9f-WaxRkhQ`@}w6k*Mn;i+}&%AN+&{{8AdN+cxZ|I<{chE*=Ze->l0vFm* zQ}!GBfYgUh%~j&%%69>23V#O!?Rir+o2_>I)MILPA&gN&v=z~$?@d#$3*J`jNqV4M zm_POtqpvB;Lasn1s9o_q{RlpKUM^bjRo;h1OGy}G(XsoXHJ&_n2+yA%+k(ja`PjUv z;3PFjci6k8546Q;&opdbM&lDv0u_-T?Q)s5f1Y-sU$I##>O_0qjI$eFbFZ{9v{@gQ zzdGZ{-F=C+CVlP56DHN%{#ge#s#bWo>V&ywf&JYHZ`W+YJYa@eIY_7cjhQoQp6FY$ zcH)UgnwFlM`ZUtI;6$-!Iyh;qJ?U;!xa~80Hb}b*HgPjHq`^sy+-O(N9AM|4Tw7!N z4rlK;`C4>n(1U_QuW$%aB|~rE^xiAiiVsK7rAZ}ty8lg z7@=mX++Br|Hs;deI9EIgQz5bPf9T#uoAwm0C@rb=qW61eiFWy&ZVSvn^W*tYsv35I z;o>OiYDFVS*BCDIkaGGgaD|n9< zrvya`?dZ4YeqywOf|}stbg{!f5te*iKE=EefyH-skvp*O4gk>ka*`;g9;ezw@1;6f zkcl>vLe3~Xr;tr`_(JPWF5Wnt1w*y+oqpMo)>65m$b~8+1QcaOdU3V02U7VZqK~rQD=yH#HYYS<@Hi8+#Lcd)KK$CpD?i-B_s46b@*JDe`3hF3Q(-l^zb|m z+hwOWl{Sm~lDZ5xvNs8l@ODos)e*W|h>~T5r4L3@q8VU?TzKn(`BMtnX%Ga9Cx8X} z4s?5ZHgA ziZntL9j1|br$rr_;0~u0rGoufEh3{_EQ`4l4En+FeAY4lqq-Jqq@oD&vY6Grlr@&X zC#>y^&ir{0lr~zElPS0inR3{X%8>@ae(r!tzxsT108SKw$)3lR*4}RybUM zONl1YCXJ^%xK6u+7(9hGO|ii#F4;9_BpXYU!du&=@Y%pQPrWjYS8I4Fjkrvb#N{Q&OPj~;9Mx2iiwUFd7_1~r%P3~%~@|R`ne#l z1ad^5%SC=48gt9YF95Bsq)@lRFFFt?0MJbm3h2|ph93g`uZzZcP8RJ}%YB6ufeD6% znoc-2rOXxLH zOoLi!xoYIR2I|FAln@S`;BQl1EY%V}FF1pk8ax=UZ&iX4PHm#RW6?v1db|6lEbb7P zSnkH*Sbn*QC~J?cG46qLkYcuroKcL3H7t^PV)}(jA#B;TfdyN2!5Ka5$i)*849*bT z$cob~PVR6k;c*??I=zBEXmj%~OcyxG=UzJN*Ft19w3%@WK~{VDA0@)Ac()R$iRTxa zIhvMP?y@t3t(Tk2SyLDsqqV{MpNB5(* zGbbBTgol`6Qz|%1DYPvg7XqT}d!<%U$qe>8=jBT5t_x{}O!K;`pC!R(OOrql-D;q1 zHPI-NylWY|S56FJ3QmH>NSIOVqGJRZDQbl!r4Bn_OSBcX9Ic5c^RZu)zqII&joH8v z_;j$53G*>M8Fi&ZlXSUK>_%|By?oZb=0EK6StprR``@$nD}C2MR$^67P!Kzpi(bXu zj-K(33%&~$`CKR@dLw6Ud}q+XqNQn_Z=*xD>c|Vu7NRhh4sp1|AT>maq2nqMndLjv zShk9!_^G-pHT5Q#abBTur*k7N2)KCE) zGdD!a6tYhC<2F!ch3&Y2(=;(7Jz^c`^n4$IsD#lPb%xqFNoS~h%;vth;IfgEznG-X z?@C@Jee5zgTLo05vDC_Dc?v%ee9(qb?p~;H=z)#{IoseqjER;zC$PHU1A0oQ=Y0IZ z5fs8*IeYzuV;U3zt0eU8OlMIM$h>{>!hweZ z9nB@CFrCkPT0vTUUd1Vb*pdi)SJ9U|>kLjMEFKTi)?PHJ>PSc~59}O!`bEW3E7vlx zl@mqc)?&42tFNbOAa;%Ot|T5hKZ^mbp6+f17*0hUudu6-nrrpLLM31lauo%Ii&!=3 zqol)jV<=`Nv^e!ntAE_fSIBkYxKK?e!kEnFMjRxsQD3Q6LTOJ<%(k^XUNkCNM8MO1 zs<2M^0nsZY%1c4SUXPu5@qjjohj?>^QdGopUt+A2Xo;nKTX8y!ZEw4{p(G8ejRzam z-d%PIWbEE#Ck&8Uilyw)n8GOUeBk56a0_YKXYn_w0TarDLCC&=5lC2Vc$l{C; z0yeeil@cpND`$mhwJB-ktPrhxQixH7PH_AVP#teF&J{bc6q&78m=nuOP8Ja=<4V>- zf8IEx4tV3LKtFe0%I5@Hz#GZZx5{Z#!H%S0o6<5$un7MICGy>b2BBo3R6&B0`Ki!N ztT`D7?_Fzp->q!EhGx^mb1P1Wb1p(6m-rp3!21{toW4|sI&Jlf9vsEn8~Qei^fe!i z>%pPiQ7S-DeQg|~E&0a7!Qjj$?HXl`oIWKJ6oC#UOT16_3u1dn#_8lwF+WGM*dmRO zZIeb2%eWl?=ZvcoQaCjyg(O(TQc5A&9?rj0KM*LTd6+?08rMkTV~n2X1e}livhX)a z8g9?F+QViaFl^-$FtWbUc{F*Uldq#jq>TIYA!1_}&o1!I$s@BzWI4YgAdO0RefG!_ zSCeFcSpFJZqq3QnRdE2h6bDVBCMz!o4dx`>27U<(v3=SDeeN3>b(dj5Kg>!7f_M&7 zh^F?Uxp>!8e8({O;-mzj@BSG9V@Xde!QvMWS_aL zsk8}q+MSYls4MMU33m#@V%6?&Cwao1RXVIees#X$ROV(M_eGz+@nM_jFC&SM#yje# zyba#%X~>yBpT*$72N#6;jRDA4l1#<-CQ^|H4jF~w*KpYrmfxF@l35LWNWfZWzjXffy#rZlWdS$9SKsJba+%dRF%+m#3 z8g(V6@w0`-_~K(svgU|GG*B8`tT`5^28JM6#-dBfl2p(Mx}xPVKD9;CW+~c5hvR$I zmVEpjCe~Z(wbT52m{p(mz%Ce;e(M#qvgUCbCib5IIVBB24~KFnvp6-_g*=?hW8`2) z7ftcGkjUF|d-7a0pIVfe&33sH_v!?Y?vS!xnY_-`N6b?ZXyz0%Q-isCS~q zF43{X;-8uIPbB8^A-odu}s>Kz{4Y&C2v}3PcGG>MBMDL9&nI+DZ zYWv){O6|UJWq*J5%@rHj5ORn(@0w#wlYQ!%8{M!QEAA-|Im+&L?O7i^EFbjsUgYXK z?BmxS$Whcg*A{rb|LISfvG&yKK4rS>6W0|cJmDV)`8k1E1ES#%2?)+p{4q13<#=Nl z?Pb4g{q^Ahu?U}l7C68zR6$*R)y`MRu*$USJ8HNUOdgiHoluq-(? zhDfhH;-*o9F%86{vK;if{gJlG7}bnLJzO*j3Cx<5$o21+r?PDT8-h{(%^~UtI5jvD#pEK z0aTkBT43C5%cmbQ@7NbUJ$~dn#>*9Bnm9TnDwH{45FFV!Z@X?@Y_75I-`pWpe&j9t z*CLvthbMkr=iAqxZZ6>C@hb#=OK?rDe>`Hme-;yY!4D9KyKfn8uC^O)nO4063+!-s z`P$~s<$QCc9e(S%+(^0Q)>E@rIYb3l+uz(ejiT|NKaJ{JKmUDmy*>2{7uW3A*u%p3 z1`D@oqy7FD#+V!J-@Y)}TxUBLjq%^uXfIu~AitePxQ6zWt` z^y4zAO=e^Lpsc`^J_KDt4|hogz#AM1F21dLh(j3h2O$jX$lK#<7zY{;yY99lTguB+ zIIMS_o?EA_|8oBWv|ProbjvckmlbO=#FAz4y9QcJfr*8L)4x0;CY9zBsUG<<_WxPS zH#|!2va>|c^pbVTeRs)ql|r@F({X!;dE4G}`w{iMtC}vE7cpPH{eaVUYArg(xc-VF z{%lt$M0tFLF{Onys1|%6TAKDfwT%-=qoJ*t&ORE@04`xS>G!CI6@R6*^nTo(f$10e zGg?Fx6)F~l^v-{tMSd_;V{ZUj7u}(6QXjp>n?r*dY*R)AQbuWS%?@SWNNSXpo0X#X zl-r(EDeZ0dzA`(L8l}C>7HlB1E2&Z1+ia0)=JU`Jl=e1TqRhgiMyabNc)+QIMsPir zt9)_NNNI16rG0BwDx(thUePzRS{aq7W>w$Jqspj6HEYV5G0Lr1PDRSB>zmo2j7n6~ zW1soTfQn5j>alNprLkd)UX@qvYsze~jop{oFLn>CTm%WD`5Wy$-3<*(^uCaX+9{3Z z+c&!#ON;xKuT*Af(nx8fYV}yH%!(wVw71!#%B)Iil=e1TugsdHMrm)e4a%%jO%-G7 zC0b)IUwW8bx3p%Q(`I5k%zQd(?J8nx5J7%wh<2e?6H?U~v1v?4yEfWGmyH-EW_27Z zhF3@ajuXpk40bZ&^qZEA^-0z&yXd@KYEEp`Db(Ki`b^+F$W5?>3k-UgY5mZW=lHtKrE-cDokij1a5)`<+8Z zB&#T5o%q5j29@^@fqF#ddTiDT91j0VAfT(M8F7xmiRYjFu3bPYj9CAiU! zzGqNPQo~PMNN1mX&)Cx2rY-^7v3~+~j8Qz#1Vsrp9a&vnJOU_ky)>+QV=RXmS1PnQ zqa^Bl#8{Mt_Jey49=x@OhWkYepXrI?wO@_%n@G4HdD7R8?E~_+zt#rGuYYam{|m_B z*Y^hbxUWx2ya;ITs3amU|9bNw32H+9YK2?%UZ6n*gwO#$gBZORwejVi#BImd$JRTb z9aASj+tDjW9=|nij!DNf?d=9Y8E0=9)s+1Jxw7~Ht_W6(Ev`=&+KdVwp$6c{2?w{E^$e-`Oj2~CLVAV?V1)KZELY_;$F}`{Ww+%yP4KBAo z_{PM_9%WT~PNBPe#vnM_<*59zqmL3@DZ&_wnJg|RliT!sbdREQ+pk!ZvV1WO< zW%h3m%x(CAD+~R4&jp(>Rm<#64<2yGd0L6z#fd^T+UCvXcW5|Q;qK=+Z_SCv!`03#wD&wTr0FA@*#p1%iuupA*>9c5?@HR!A6{=3+5V4Q#Iy9sG@dU$@}xP- z-m-Rb@EQG<9G~ii)w~9uHFoP-Te0f_7Sn96cyx8eLal4k+F{>5+%VxekL}a4@R5!v zH&@OPcSTZ>KPOJ+67#jI9~(d`o`?2rg)2;@>sDl2G6f1}rW4@RJqj3%L;pv_b6FvL zq5a2WXPR5>8IK=+_QJb6bg|9X*B*mpo}$L9$9k@GsLQ6Wj_R&RN`|?TY@d>%E6MjM zDY%j#DY1Wkd`|Ooy05!mboUC9S4{NoJ*2x*!I$j(bqkx{xtH9?=<)kVmiy6b>CkAE4wq2kXV^ckA85Bf z5!o5*&l;->T50`~l6LXnQ9E>Y^}%hh66co{M9!ADSN*hCw=rgb{r(dzqtjqmaVkuV z`yDhPRKJI2;@>MvvPI0bo&40Wb~VR`2HQys7<7*-(q3+XiEr+Q@vpnc)N3QNC&Ndwl!QcbgAjWbCv?#?QxRL3g+wXue!-y&>?xa*X>KDlq02 z`j_vXY!0@^KYe_bZ!p|`*qWY3`^%??v9~on?L04e=5%wh{oXS}`+u5)4|aFl<$E5= z#Fwgr#rFMY7Mf4ko1a}d@M7r^wooqc!80A7?`4C|wMRcU+?YGkl!XpF+1_4((SY;o6 zexX@vCw^~6|AmjiA>Yt%IqvwjWJgymw}0E-`MnXy^>g1FV;-y&%H3+thPUX;f&EzG{Hj( z=hLa#9VlsOvN6(0a3H^3=GxfPUOcGZS^AisA0*{ok!q@gZ`!+FY(D6lfH&4Vmd&Gl z5yvR1<_uI_fhg|6$ZStRDz~b?X@B?P;eaygrJ=3gf;%`S5=(HEQ71RP==)!E#saJn z=fXR|x9p`aJvxk^boCV}BAmto735LaOAZ9{>IdF(Gwcfo?x%e)Fmp@%y3JGmw_D(Y z>|Zx8<-+)vzyBxmq`mnE->H8W4eG!&V?9O=*4ddaUuD+WjW3@axVjKoD_0$~WiD0U z*m8?`*pC0OMRPkvM?M8`rNwO{}7u?PP`-SiESKK16;b@$BvG9I*C`+AbK;VW(CQ9JFGa7?*B zd*i(az_6X_%F3hIue@^De&0?AjMp!C{Di<8;)&N74P>I{lC1sUm7(U_cIc~z^E~s_ z6U<}w!B+>3cyrIz$;V1-kYoD3Rqngv)gtZde$r~*wTJvVW9Gw>?eMD09|hV>1dvh{`BMFMs=#>#!x7BcHgAA?yr_-Zrge$@_)6&c%=Ob%SD}4rP|M@Hyu>9v*z<$5zK=oYqizD0>Cxlj`Ad`%; z{ud+7D*M(iistKUD_%Ro{A=Mj`iNI-kfY!*xFx@Y}lj%eXhdkX0*;_?9^Ykn`dn4*MBk3+H2k##`BK12Jrm$TemUd z)NS*4_H4VZwZZW^f|uf}>iM|Nl~?Bu^OWbnK8ioU0TcTACbSEF6PfSY=YKOP`*Mt+ zmu>K0mzpiM^j`;l>S_I?jBXKEVupZ{vsPWQ(_2OGG{5g7Ka1;Je3wQmO2XWgErn`{ zvE~y$DaxA;eux0THTl{6)D*=$($O1y?fFC7_}kN2_NBkQis!c9PUAUl`wX5pY@cbK zUiyfcI6%y(v*AFaLnovf)? zqBhO;p&jEX|J9DyxX$(D?+fO~_Lslk|FCCF{0-F(|D(S0@{$j%KC;pV@AYru`gjWR zPW0*=V0HQhyiNA7_bxHtTlr)uI+P}6CHy-`hk%#K5Gx$)k zi}-hn5TH(lqH{y_QJ3o2YLjI7^-pIXbV9Bl0A2gvqrm1b{(Cgfia$)}`LREY;5qLP zIu&~04{JwlF$KvBa;MV(sFeyRk}g*aB_<$6cEQfEC3pmt?xZ&65e55W9!bLyQ}kt? z27!xcB_64~vIWJ?I>o1c0N;7~RWx6>=E%?+P5on|ugE|GZ4A94b7k&NzR9}Lbi<2# zS$sK@vE%=EEUUiuk4N)->W@A3OCLmae3gTGQn{NsJNYl0hi*mHg9@FN@Vn-~<0A3; z4dfKp^7u4u*SYq_zYfMnU;fvJvrGBzkhRXUAOG7?gQv#l9w0;L*5)@p55e`~-00!I z?N^OIC6@WUj zKmVm=HuwH0Z2*7w_d;wYG_y|1oON!*^=$ zqho)JJ($ly_09-z*#z3U4~Bx2aUZmmcYm}{po)4=^_P4wRJ+LWAGGe#pr*IMOCL;% zHXFxlETowWfBGTqG4AUi{pNc*3iat!Wwc>E2pbk{C*gPBDaWyA&%t_w{ck_Pe7p{B zk>Ie!Kw++U{`N=A1h*k^AH2nzlHcCG%N)LU!|tPvS*(`CRoU0X4*zev-!kS!Gr#+r zzB#w%jnBYZB5$!WxV(E%%G_ZVb+1dA@yD!wzDHup7W;z_Dmmqg);;SAu!+&ixDJqxItS$By!a>p7UbKLQ!G#=wP0`k>drI1jX`F@g`z zc2EFuI7J7v&8~(=_v0D!YDths3HaDm`4*nf&yw&lu6~L)S`U0w61^zIq@^iIBi_j`@zu&Q#igSroCGVM*}?DVA91sBgg|H|&Wo6NWY z=g*sa{+vtCoqgP;=gq!q*7;YKo6hgv)ntbC4s=5I@cw30Z}z0_Q~H|&%?aHL`2JD@o5z(lbaQqjzSKPD;{`C+R1W^q3?)I!Vt>(oZGnd2w3wz6(0U zmFMs$<|^ku$@?hfyT3Es3@Y_J2J*7Lmg=$K=r76}UhaHcF9wxUe)%e1dG2cl$(cdh zKY_PF>XhW)-7EhW@?oEP<%RyMSl6+n5A)aG>s~s-Oe~elmQf}$$s*{#HG=|T(cfX( zx3hE3KmXFXbI!eVZm9pulIML60%;v%z}h6zGA*9xUGQCU@vQL3aQ69^>OG;g+7IoM zpUV3_o;ReobARxFWcTLp;eB}b>m$t3HT_xle~|6suco^+(wtoLBA_>uy`H~AyKft2 zn#M8T0pSrxB=eRVTuXy`{>u4J^Ddga*gb5NIj~SJ-c~M7#(jaj>(+hEC^O2e?!J2z z!LNP0Umj(SDwKKgF_tWLFXx{q=ex(Xn)cpir&PvlUT(G}9qX8VyFb@zrq-8>Qxt2o et=+G;np4f)-R*7WD3WX1%yp)<``tG4$o~aCkViBC From 0f7e9eceb0a7362338ce4965f17abd784be36cdc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 17:02:59 +0200 Subject: [PATCH 832/927] packaging/nix-make: Build libnixflake Also fix the prefixing of generated files inherited from dependencies: the `from` source has to be prefixed along with the key, otherwise the scanner cannot find it. This is the first component depending on one with generated files (libexpr). Assisted-by: Claude Fable 5.1 --- packaging/nix-make/flake.nix | 9 ++++++++ packaging/nix-make/lib.nix | 6 +++++- packaging/nix-make/libflake.nix | 38 +++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 packaging/nix-make/libflake.nix diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index f554aa993b..ba61822c5f 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -36,6 +36,15 @@ inherit pkgs nixMake; inherit nix-util nix-store nix-fetchers; }; + nix-flake = import ./libflake.nix { + inherit nixMake; + inherit + nix-util + nix-store + nix-fetchers + nix-expr + ; + }; }; in { diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 29395828b5..4502315b73 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -277,7 +277,11 @@ let ) { } allDeps; depExcludes = map (d: d.component.name) allDeps; depGenerated = lib.foldl' ( - acc: d: acc // lib.mapAttrs' (k: v: lib.nameValuePair (prefixed d k) v) d.component.generated + acc: d: + acc + // lib.mapAttrs' ( + k: g: lib.nameValuePair (prefixed d k) (g // { from = prefixed d g.from; }) + ) d.component.generated ) { } allDeps; allGenerated = generated // depGenerated; diff --git a/packaging/nix-make/libflake.nix b/packaging/nix-make/libflake.nix new file mode 100644 index 0000000000..5bd4e7116f --- /dev/null +++ b/packaging/nix-make/libflake.nix @@ -0,0 +1,38 @@ +# libnixflake, transcribed from src/libflake/meson.build. +{ + nixMake, + nix-util, + nix-store, + nix-fetchers, + nix-expr, +}: + +nixMake.mkComponent { + name = "nix-flake"; + libName = "nixflake"; + + deps = [ + nix-util + nix-store + nix-fetchers + nix-expr + ]; + + roots = [ + { + root = ../../src/libflake; + prefix = ""; + } + ]; + + includeDirs = [ + "" + "include" + ]; + + files = nixMake.commonSupportFiles // { + "call-flake.nix.gen.hh" = nixMake.mkStringHeader ../../src/libflake/call-flake.nix; + }; + + linkFlags = [ "-Wl,--wrap=__assert_fail" ]; +} From 0b915d7a57bdc0a29785bef299ba90e490d60410 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 17:04:27 +0200 Subject: [PATCH 833/927] packaging/nix-make: Build libnixmain Assisted-by: Claude Fable 5.1 --- packaging/nix-make/flake.nix | 4 ++++ packaging/nix-make/libmain.nix | 42 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 packaging/nix-make/libmain.nix diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index ba61822c5f..9a659aadae 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -45,6 +45,10 @@ nix-expr ; }; + nix-main = import ./libmain.nix { + inherit nixMake; + inherit nix-util nix-store nix-expr; + }; }; in { diff --git a/packaging/nix-make/libmain.nix b/packaging/nix-make/libmain.nix new file mode 100644 index 0000000000..9bbe1adce6 --- /dev/null +++ b/packaging/nix-make/libmain.nix @@ -0,0 +1,42 @@ +# libnixmain, transcribed from src/libmain/meson.build. +{ + nixMake, + nix-util, + nix-store, + nix-expr, +}: + +nixMake.mkComponent { + name = "nix-main"; + libName = "nixmain"; + + deps = [ + nix-util + nix-store + # Only for the NIX_USE_BOEHMGC macro, as in the Meson build. + nix-expr + ]; + + roots = [ + { + root = ../../src/libmain; + prefix = ""; + } + ]; + + # unix/stack.cc is a plain source file; there is no windows/ directory. + includeDirs = [ + "" + "include" + ]; + + files = nixMake.commonSupportFiles; + + configHeaders = { + "main-config-private.hh" = { + HAVE_PUBSETBUF = 1; + }; + }; + + linkFlags = [ "-Wl,--wrap=__assert_fail" ]; +} From bc164876f8aa59e79fbb9c267dc5c7bed301cc9e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 17:06:44 +0200 Subject: [PATCH 834/927] packaging/nix-make: Add a `root` shorthand to mkComponent `root = ../../src/libfoo` is shorthand for a `roots` entry with an empty prefix. No change to the resulting derivations. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/lib.nix | 18 +++++++++++++++--- packaging/nix-make/libexpr.nix | 7 +------ packaging/nix-make/libfetchers.nix | 7 +------ packaging/nix-make/libflake.nix | 7 +------ packaging/nix-make/libmain.nix | 7 +------ packaging/nix-make/libstore.nix | 7 +------ packaging/nix-make/libutil.nix | 7 +------ 7 files changed, 21 insertions(+), 39 deletions(-) diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 4502315b73..6fe5d08ef8 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -205,7 +205,10 @@ let - `deps`: other components this one depends on. Their public headers are made available under `/` and their libraries are linked. Dependencies are transitive. - - `roots`: list of `{ root; prefix; }` directories scanned for sources and headers. + - `root`: the directory scanned for sources and headers; shorthand for + a `roots` entry with an empty prefix. + - `roots`: further `{ root; prefix; }` directories scanned for sources + and headers, known by `prefix/`. - `includeDirs`: include search path, relative to the root namespace. - `publicIncludeDirs`: the subset of `includeDirs` exported to dependent components. Defaults to all but the root. @@ -237,7 +240,8 @@ let name, libName, deps ? [ ], - roots, + root ? null, + roots ? [ ], includeDirs, publicIncludeDirs ? lib.filter (d: d != "") includeDirs, sourceExtensions ? [ @@ -258,6 +262,13 @@ let extraLinkLibs ? [ ], }@args: let + allRoots = + lib.optional (root != null) { + inherit root; + prefix = ""; + } + ++ roots; + # The transitive closure of the dependencies, direct ones first. allDeps = lib.unique (deps ++ lib.concatMap (d: d.component.allDeps) deps); @@ -309,7 +320,7 @@ let { inherit builtins; inherit sourceExtensions; - roots = roots ++ depRoots; + roots = allRoots ++ depRoots; includeDirs = includeDirs ++ depIncludeDirs; excludeSources = excludeSources ++ depExcludes; # Work around a crash in `builtins.wasm` (Nix <= 3.22.5) when @@ -325,6 +336,7 @@ let ); component = args // { + roots = allRoots; files = allFiles; generated = allGenerated; inherit diff --git a/packaging/nix-make/libexpr.nix b/packaging/nix-make/libexpr.nix index c9c54820f6..640330e7c4 100644 --- a/packaging/nix-make/libexpr.nix +++ b/packaging/nix-make/libexpr.nix @@ -33,12 +33,7 @@ nixMake.mkComponent { nix-fetchers ]; - roots = [ - { - root = ../../src/libexpr; - prefix = ""; - } - ]; + root = ../../src/libexpr; includeDirs = [ "" diff --git a/packaging/nix-make/libfetchers.nix b/packaging/nix-make/libfetchers.nix index 28eff8e2b1..b036607b27 100644 --- a/packaging/nix-make/libfetchers.nix +++ b/packaging/nix-make/libfetchers.nix @@ -14,12 +14,7 @@ nixMake.mkComponent { nix-store ]; - roots = [ - { - root = ../../src/libfetchers; - prefix = ""; - } - ]; + root = ../../src/libfetchers; includeDirs = [ "" diff --git a/packaging/nix-make/libflake.nix b/packaging/nix-make/libflake.nix index 5bd4e7116f..60b2a76583 100644 --- a/packaging/nix-make/libflake.nix +++ b/packaging/nix-make/libflake.nix @@ -18,12 +18,7 @@ nixMake.mkComponent { nix-expr ]; - roots = [ - { - root = ../../src/libflake; - prefix = ""; - } - ]; + root = ../../src/libflake; includeDirs = [ "" diff --git a/packaging/nix-make/libmain.nix b/packaging/nix-make/libmain.nix index 9bbe1adce6..60dc1865f2 100644 --- a/packaging/nix-make/libmain.nix +++ b/packaging/nix-make/libmain.nix @@ -17,12 +17,7 @@ nixMake.mkComponent { nix-expr ]; - roots = [ - { - root = ../../src/libmain; - prefix = ""; - } - ]; + root = ../../src/libmain; # unix/stack.cc is a plain source file; there is no windows/ directory. includeDirs = [ diff --git a/packaging/nix-make/libstore.nix b/packaging/nix-make/libstore.nix index c8286b2c03..893df7ccbe 100644 --- a/packaging/nix-make/libstore.nix +++ b/packaging/nix-make/libstore.nix @@ -15,12 +15,7 @@ nixMake.mkComponent { deps = [ nix-util ]; - roots = [ - { - root = ../../src/libstore; - prefix = ""; - } - ]; + root = ../../src/libstore; excludeSources = [ "windows" diff --git a/packaging/nix-make/libutil.nix b/packaging/nix-make/libutil.nix index 9580b14156..1138001aa1 100644 --- a/packaging/nix-make/libutil.nix +++ b/packaging/nix-make/libutil.nix @@ -5,12 +5,7 @@ nixMake.mkComponent { name = "nix-util"; libName = "nixutil"; - roots = [ - { - root = ../../src/libutil; - prefix = ""; - } - ]; + root = ../../src/libutil; # Every .cc file under the root is a compilation unit, except these. excludeSources = [ From 95de80f0b5c372ce6774af18d8b51474e1f25639 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 17:10:56 +0200 Subject: [PATCH 835/927] packaging/nix-make: Build libnixcmd Also fix components exporting their dependencies' generated files to their own dependents (in addition to their own), which broke as soon as a component reached libexpr's generated parser both directly and through an intermediate dependency. Components now export only their own `files` and `generated`; dependencies are transitive anyway. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/deps.nix | 10 +++++++ packaging/nix-make/flake.nix | 11 ++++++++ packaging/nix-make/lib.nix | 8 +++--- packaging/nix-make/libcmd.nix | 50 +++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 packaging/nix-make/libcmd.nix diff --git a/packaging/nix-make/deps.nix b/packaging/nix-make/deps.nix index ff9550521f..c2f529b33d 100644 --- a/packaging/nix-make/deps.nix +++ b/packaging/nix-make/deps.nix @@ -128,6 +128,16 @@ in prefix = "toml11/"; pkg = deps.toml11; } + { + prefix = "lowdown.h"; + pkg = deps.lowdown; + pkgconfig = [ "lowdown" ]; + } + { + prefix = "editline.h"; + pkg = deps.editline; + pkgconfig = [ "libeditline" ]; + } { prefix = "git2/"; pkg = deps.libgit2; diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index 9a659aadae..882e73e21f 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -49,6 +49,17 @@ inherit nixMake; inherit nix-util nix-store nix-expr; }; + nix-cmd = import ./libcmd.nix { + inherit nixMake; + inherit + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + ; + }; }; in { diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 6fe5d08ef8..141cb2136d 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -114,11 +114,11 @@ let { __structuredAttrs = true; # Generated files come from the output of their generator derivation. - src = if unit.src == null then component.generated.${unit.path}.path else unit.src; + src = if unit.src == null then component.allGenerated.${unit.path}.path else unit.src; includes = unit.includes // lib.listToAttrs ( - map (g: lib.nameValuePair g component.generated.${g}.path) unit.generatedIncludes + map (g: lib.nameValuePair g component.allGenerated.${g}.path) unit.generatedIncludes ); srcPath = unit.path; includeDirs = component.includeDirs ++ component.depIncludeDirs; @@ -337,8 +337,10 @@ let component = args // { roots = allRoots; + # Only our own files and generated files are exported to dependents; + # their own dependencies are resolved transitively. files = allFiles; - generated = allGenerated; + inherit generated allGenerated; inherit units allDeps diff --git a/packaging/nix-make/libcmd.nix b/packaging/nix-make/libcmd.nix new file mode 100644 index 0000000000..5842bd0b01 --- /dev/null +++ b/packaging/nix-make/libcmd.nix @@ -0,0 +1,50 @@ +# libnixcmd, transcribed from src/libcmd/meson.build. +{ + nixMake, + nix-util, + nix-store, + nix-fetchers, + nix-expr, + nix-flake, + nix-main, +}: + +nixMake.mkComponent { + name = "nix-cmd"; + libName = "nixcmd"; + + deps = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + ]; + + root = ../../src/libcmd; + + # unix/unix-socket-server.cc is a plain source file; there is no windows/ directory. + includeDirs = [ + "" + "include" + ]; + + files = nixMake.commonSupportFiles // { + "call-flake-schemas.nix.gen.hh" = nixMake.mkStringHeader ../../src/libcmd/call-flake-schemas.nix; + "builtin-flake-schemas.nix.gen.hh" = + nixMake.mkStringHeader ../../src/libcmd/builtin-flake-schemas.nix; + }; + + configHeaders = { + "cmd-config-private.hh" = { + HAVE_LOWDOWN = 1; + HAVE_LOWDOWN_1_4 = 1; + HAVE_LOWDOWN_3 = 1; + # Use editline rather than readline. + USE_READLINE = 0; + }; + }; + + linkFlags = [ "-Wl,--wrap=__assert_fail" ]; +} From 8a723697f886a49b5282cb9f675de18ac4da790c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 17:20:37 +0200 Subject: [PATCH 836/927] packaging/nix-make: Build the nix executable `nix build ./packaging/nix-make` now produces a working result/bin/nix, with the legacy command symlinks and libexec/nix/build-remote. Every compilation unit of the executable and the seven libraries is its own derivation. * mkComponent gains `type = "executable"`, with `exeName`, `binSymlinks` and a `postInstall` hook. * Dependencies' headers now live under `_deps//` instead of `/`, since `nix-store` is both a dependency and a real subdirectory of src/nix, which silently excluded the nix-store/ units. * Sentry, OpenTelemetry and the plugin C API are disabled for now; NIX_BIN_DIR and NIX_MAN_DIR are fallback paths, since Nix locates itself via /proc/self/exe. The resulting nix evaluates this flake itself, running the include scanner under its own builtins.wasm, and produces the same derivation as the installed Nix. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/deps.nix | 11 +++++ packaging/nix-make/flake.nix | 16 ++++++- packaging/nix-make/lib.nix | 81 ++++++++++++++++++++++++++-------- packaging/nix-make/nix.nix | 84 ++++++++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+), 20 deletions(-) create mode 100644 packaging/nix-make/nix.nix diff --git a/packaging/nix-make/deps.nix b/packaging/nix-make/deps.nix index c2f529b33d..41e6d246e3 100644 --- a/packaging/nix-make/deps.nix +++ b/packaging/nix-make/deps.nix @@ -128,6 +128,17 @@ in prefix = "toml11/"; pkg = deps.toml11; } + { + prefix = "microhttpd.h"; + pkg = deps.libmicrohttpd; + pkgconfig = [ "libmicrohttpd" ]; + } + { + # ``, ``, ... + prefix = "mimalloc"; + pkg = deps.mimalloc; + pkgconfig = [ "mimalloc" ]; + } { prefix = "lowdown.h"; pkg = deps.lowdown; diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index 882e73e21f..601e0fb16d 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -60,10 +60,24 @@ nix-main ; }; + nix = import ./nix.nix { + inherit pkgs nixMake; + inherit + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nix-cmd + ; + }; }; in { - packages.${system} = components; + packages.${system} = components // { + default = components.nix; + }; # The raw scanner output for each component, for debugging. lib.${system}.scan = pkgs.lib.mapAttrs (_: c: c.units) components; diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 141cb2136d..8c0376512d 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -153,7 +153,8 @@ let $CXX "''${cxxFlags[@]}" "''${flags[@]}" -c "$srcPath" -o "$out" ''; - linkSharedLibrary = + # Link the objects into a shared library or an executable. + link = component: objects: let # Link against every external dependency used by any unit. @@ -164,7 +165,14 @@ let __structuredAttrs = true; # Instantiate the objects and the dependencies in parallel. objects = parallel (map (d: d.drvPath) component.allDeps ++ map (o: o.drvPath) objects) objects; - inherit (component) libName linkFlags; + inherit (component) + type + libName + exeName + binSymlinks + linkFlags + postInstall + ; linkPkgConfig = lib.concatMap (d: d.pkgconfig or [ ]) deps; linkLibs = map (d: "-l${d.libName}") component.allDeps @@ -180,30 +188,52 @@ let }; } '' - mkdir -p $out/lib - libs=() if [[ ''${#linkPkgConfig[@]} -gt 0 ]]; then libs+=($(pkg-config --libs "''${linkPkgConfig[@]}")) fi - $CXX -shared -fPIC \ - -Wl,-soname,lib$libName.so \ - -Wl,--as-needed -Wl,--no-undefined \ - "''${linkFlags[@]}" \ - -o $out/lib/lib$libName.so \ - "''${objects[@]}" \ - "''${libs[@]}" \ - "''${linkLibs[@]}" + case "$type" in + library) + mkdir -p $out/lib + $CXX -shared -fPIC \ + -Wl,-soname,lib$libName.so \ + -Wl,--as-needed -Wl,--no-undefined \ + "''${linkFlags[@]}" \ + -o $out/lib/lib$libName.so \ + "''${objects[@]}" \ + "''${libs[@]}" \ + "''${linkLibs[@]}" + ;; + executable) + mkdir -p $out/bin + $CXX \ + -Wl,--as-needed -Wl,--no-undefined \ + "''${linkFlags[@]}" \ + -o $out/bin/$exeName \ + "''${objects[@]}" \ + "''${libs[@]}" \ + "''${linkLibs[@]}" + for name in "''${binSymlinks[@]}"; do + ln -s $exeName $out/bin/$name + done + ;; + esac + + eval "$postInstall" ''; /** - Build a shared library component. + Build a component: a shared library (the default) or an executable. - `name`: derivation name (e.g. `nix-util`). + - `type`: `"library"` or `"executable"`. - `libName`: library name without `lib` prefix (e.g. `nixutil`). + - `exeName`: executable name; defaults to `name`. + - `binSymlinks`: names of symlinks to the executable to create in `bin/`. + - `postInstall`: shell snippet run after linking. - `deps`: other components this one depends on. Their public headers - are made available under `/` and their libraries are linked. + are made available under `_deps//` and their libraries are linked. Dependencies are transitive. - `root`: the directory scanned for sources and headers; shorthand for a `roots` entry with an empty prefix. @@ -238,7 +268,11 @@ let mkComponent = { name, - libName, + type ? "library", + libName ? null, + exeName ? name, + binSymlinks ? [ ], + postInstall ? "", deps ? [ ], root ? null, roots ? [ ], @@ -273,8 +307,10 @@ let allDeps = lib.unique (deps ++ lib.concatMap (d: d.component.allDeps) deps); # Dependencies contribute their roots, public include directories and - # extra files under `/`, but no compilation units. - prefixed = d: p: if p == "" then d.component.name else "${d.component.name}/${p}"; + # extra files under `_deps//` (a prefix that cannot collide with + # the component's own directories), but no compilation units. + depPrefix = d: "_deps/${d.component.name}"; + prefixed = d: p: if p == "" then depPrefix d else "${depPrefix d}/${p}"; depRoots = lib.concatMap ( d: map (r: { @@ -286,7 +322,7 @@ let depFiles = lib.foldl' ( acc: d: acc // lib.mapAttrs' (k: v: lib.nameValuePair (prefixed d k) v) d.component.files ) { } allDeps; - depExcludes = map (d: d.component.name) allDeps; + depExcludes = map depPrefix allDeps; depGenerated = lib.foldl' ( acc: d: acc @@ -336,6 +372,13 @@ let ); component = args // { + inherit + type + libName + exeName + binSymlinks + postInstall + ; roots = allRoots; # Only our own files and generated files are exported to dependents; # their own dependencies are resolved transitively. @@ -353,7 +396,7 @@ let ; }; in - linkSharedLibrary component (map (compileUnit component) units); + link component (map (compileUnit component) units); # Macros the scanner can rely on when evaluating preprocessor conditionals # (Linux, x86_64, GCC). Only what is certain: an unknown macro keeps both diff --git a/packaging/nix-make/nix.nix b/packaging/nix-make/nix.nix new file mode 100644 index 0000000000..62f8275eaf --- /dev/null +++ b/packaging/nix-make/nix.nix @@ -0,0 +1,84 @@ +# The `nix` executable, transcribed from src/nix/meson.build. +{ + pkgs, + nixMake, + nix-util, + nix-store, + nix-fetchers, + nix-expr, + nix-flake, + nix-main, + nix-cmd, +}: + +let + inherit (pkgs) lib; +in + +nixMake.mkComponent { + name = "nix"; + type = "executable"; + + deps = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nix-cmd + ]; + + root = ../../src/nix; + + # Every .cc file under the root (including the legacy commands in + # subdirectories and unix/) is a compilation unit. The `doc`, `misc` + # and `scripts` symlinks are ignored by the scanner. + includeDirs = [ "" ]; + + files = nixMake.commonSupportFiles // { + "generate-manpage.nix.gen.hh" = nixMake.mkStringHeader ../../doc/manual/generate-manpage.nix; + "generate-settings.nix.gen.hh" = nixMake.mkStringHeader ../../doc/manual/generate-settings.nix; + "generate-store-info.nix.gen.hh" = nixMake.mkStringHeader ../../doc/manual/generate-store-info.nix; + "utils.nix.gen.hh" = nixMake.mkStringHeader ../../doc/manual/utils.nix; + "get-env.sh.gen.hh" = nixMake.mkStringHeader ../../src/nix/get-env.sh; + "help-stores.md.gen.hh" = nixMake.mkStringHeader ../../doc/manual/source/store/types/index.md.in; + "profiles.md.gen.hh" = nixMake.mkStringHeader ../../doc/manual/source/command-ref/files/profiles.md; + "unpack-channel.nix.gen.hh" = nixMake.mkStringHeader ../../src/nix/nix-channel/unpack-channel.nix; + "buildenv.nix.gen.hh" = nixMake.mkStringHeader ../../src/nix/nix-env/buildenv.nix; + }; + + configHeaders = { + "cli-config-private.hh" = { + NIX_CLI_VERSION = lib.fileContents ../../.version; + # Fallbacks only: Nix normally locates itself via /proc/self/exe. + NIX_BIN_DIR = "/nix/var/nix/profiles/default/bin"; + NIX_MAN_DIR = "/nix/var/nix/profiles/default/share/man"; + HAVE_MIMALLOC = 1; + # Crash reporting and tracing are disabled for now. + HAVE_SENTRY = 0; + HAVE_OTEL = 0; + }; + }; + + linkFlags = [ "-Wl,--wrap=__assert_fail" ]; + + binSymlinks = [ + "nix-build" + "nix-channel" + "nix-collect-garbage" + "nix-copy-closure" + "nix-daemon" + "nix-env" + "nix-hash" + "nix-instantiate" + "nix-prefetch-url" + "nix-shell" + "nix-store" + ]; + + postInstall = '' + mkdir -p $out/libexec/nix + ln -s ../../bin/nix $out/libexec/nix/build-remote + ''; +} From ec8a63bdaa95c52da88ef68530e780fbc94fa2af Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 17:43:59 +0200 Subject: [PATCH 837/927] packaging/nix-make: Reduce evaluation time An uncached evaluation of the nix derivation goes from 2.4s to 1.45s (2.0s to 1.2s with parallel-eval), of which 0.38s is importing Nixpkgs. * Never use `lib.unique` on derivations: Nix compares derivations by output path, so deduplicating the dependency list instantiated the whole build graph of every dependency. Deduplicate by name instead, and deduplicate external include strings via an attribute set rather than O(n^2) `lib.unique`. The include-to-package map is computed once per component. * Use minimal `derivation` calls instead of `runCommandCC`: `stdenv.mkDerivation` was a quarter of the remaining time for ~340 derivations. The lean derivation sources .attrs.sh and $stdenv/setup, uses the dev outputs of build inputs, and sets NIX_HARDENING_ENABLE as mkDerivation would. As a side effect, units no longer have `preferLocalBuild`, so they can be built remotely. * Update scanner.wasm, which now resolves each file's #includes only once (scanning all components: 1.8s -> 0.35s). Assisted-by: Claude Fable 5.1 --- packaging/nix-make/lib.nix | 124 +++++++++++++++++++++++++------- packaging/nix-make/scanner.wasm | Bin 76453 -> 80152 bytes 2 files changed, 97 insertions(+), 27 deletions(-) diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 8c0376512d..2ca2cd364e 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -91,28 +91,83 @@ let path: builtins.toFile "${baseNameOf path}.gen.hh" "R\"__NIX_STR(\n${builtins.readFile path})__NIX_STR\"\n"; - # Map external (angle-bracket) includes to the entries of `externalDeps`, - # by prefix match, deduplicated. Unmatched includes (libc, libstdc++) are - # ignored. - matchExternalDeps = - externalIncludes: - let - matches = lib.filter (d: d != null) ( - map (inc: lib.findFirst (d: lib.hasPrefix d.prefix inc) null externalDeps) externalIncludes - ); - in - lib.attrValues (lib.listToAttrs (map (d: lib.nameValuePair d.prefix d) matches)); + # The `dev` outputs, as `stdenv.mkDerivation` would pick for `buildInputs`. + depPackages = deps: map lib.getDev (lib.concatMap (d: lib.toList d.pkg) deps); + + # A minimal derivation that sources the stdenv setup script (which + # provides the C++ compiler, and handles `buildInputs` etc.) but avoids + # the evaluation cost of `stdenv.mkDerivation`, which is significant + # for hundreds of derivations. Unlike `runCommand`, it does not set + # `preferLocalBuild`, so units can be built remotely. + mkLeanDerivation = + attrs: script: + derivation ( + { + system = pkgs.stdenv.hostPlatform.system; + builder = pkgs.stdenv.shell; + args = [ + "-e" + (builtins.toFile "builder.sh" '' + # With structured attrs, attributes are not in the environment. + if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; fi + source $stdenv/setup + ${script} + '') + ]; + inherit (pkgs) stdenv; + __structuredAttrs = true; + } + // attrs + ); - depPackages = deps: lib.concatMap (d: lib.toList d.pkg) deps; + # What `stdenv.mkDerivation` would set for `hardeningDisable = [ "fortify" ]`: + # `_FORTIFY_SOURCE` warns on every unit when not optimizing. + hardeningWithoutFortify = lib.concatStringsSep " " ( + lib.subtractLists [ + "fortify" + "fortify3" + ] pkgs.stdenv.cc.defaultHardeningFlags + ); + + # Deduplicate strings in O(n log n) rather than `lib.unique`'s O(n^2). + uniqueStrings = xs: lib.attrNames (lib.listToAttrs (map (x: lib.nameValuePair x null) xs)); + + # Deduplicate a list by a key, keeping the first occurrence and the order. + # Note: `lib.unique` on derivations would compare them by output path, + # which instantiates them and all their inputs. + uniqueBy = + key: xs: + (lib.foldl' + ( + acc: x: + let + k = key x; + in + if acc.seen ? ${k} then + acc + else + { + seen = acc.seen // { + ${k} = null; + }; + result = acc.result ++ [ x ]; + } + ) + { + seen = { }; + result = [ ]; + } + xs + ).result; compileUnit = component: unit: let - deps = matchExternalDeps unit.externalIncludes; + deps = component.externalDepsFor unit.externalIncludes; in - pkgs.runCommandCC "${baseNameOf unit.path}.o" + mkLeanDerivation { - __structuredAttrs = true; + name = "${baseNameOf unit.path}.o"; # Generated files come from the output of their generator derivation. src = if unit.src == null then component.allGenerated.${unit.path}.path else unit.src; includes = @@ -126,8 +181,7 @@ let pkgConfigDeps = lib.concatMap (d: d.pkgconfig or [ ]) deps; buildInputs = depPackages deps; nativeBuildInputs = [ pkgs.pkg-config ]; - # `_FORTIFY_SOURCE` warns on every unit when not optimizing. - hardeningDisable = [ "fortify" ]; + NIX_HARDENING_ENABLE = hardeningWithoutFortify; } '' mkdir tree @@ -158,11 +212,11 @@ let component: objects: let # Link against every external dependency used by any unit. - deps = matchExternalDeps (lib.unique (lib.concatMap (u: u.externalIncludes) component.units)); + deps = component.externalDepsFor component.allExternalIncludes; in - pkgs.runCommandCC component.name + mkLeanDerivation { - __structuredAttrs = true; + inherit (component) name; # Instantiate the objects and the dependencies in parallel. objects = parallel (map (d: d.drvPath) component.allDeps ++ map (o: o.drvPath) objects) objects; inherit (component) @@ -181,11 +235,6 @@ let ++ commonLinkLibs; buildInputs = component.allDeps ++ depPackages deps; nativeBuildInputs = [ pkgs.pkg-config ]; - passthru = { - inherit objects; - inherit (component) units; - inherit component; - }; } '' libs=() @@ -221,7 +270,12 @@ let esac eval "$postInstall" - ''; + '' + // { + inherit objects; + inherit (component) units; + inherit component; + }; /** Build a component: a shared library (the default) or an executable. @@ -304,7 +358,7 @@ let ++ roots; # The transitive closure of the dependencies, direct ones first. - allDeps = lib.unique (deps ++ lib.concatMap (d: d.component.allDeps) deps); + allDeps = uniqueBy (d: d.component.name) (deps ++ lib.concatMap (d: d.component.allDeps) deps); # Dependencies contribute their roots, public include directories and # extra files under `_deps//` (a prefix that cannot collide with @@ -371,7 +425,23 @@ let ) ); + # Every external include of any unit, and a map from external include to + # its `externalDeps` entry, computed once per component. `externalDepsFor` + # then maps a unit's external includes to its dependencies. + allExternalIncludes = uniqueStrings (lib.concatMap (u: u.externalIncludes) units); + externalDepByInclude = lib.listToAttrs ( + map ( + inc: lib.nameValuePair inc (lib.findFirst (d: lib.hasPrefix d.prefix inc) null externalDeps) + ) allExternalIncludes + ); + externalDepsFor = + includes: + uniqueBy (d: d.prefix) ( + lib.filter (d: d != null) (map (inc: externalDepByInclude.${inc}) includes) + ); + component = args // { + inherit allExternalIncludes externalDepsFor; inherit type libName diff --git a/packaging/nix-make/scanner.wasm b/packaging/nix-make/scanner.wasm index d903bf9170238134de5091098c64ad74f06ba6af..b5bf487398e1b65e9b384f758298b2c117142855 100644 GIT binary patch delta 36151 zcmb`Q3!GJDx&PO@_GR|$Ju@s|z=0WL?G5T+a`C8OYB~!Q@Pb-dT5-Hoo^nPlmCTad zSYT*W*s_L_jFQq4PZ*|@m}8NW+KCdC%8pca4wa{F38pEbyvTsqkDGX>K8b!FDUYe zf7!chFDTe8FM{BLvR&Ixa6$gsYbkuK72qnk>BibkcON*?P6um%ZzfOM=o(e?9crD=xb5t>^#S#g|+ZSoZw8FTCQiAW+H5Ti)@`_nd#_ z!YeMm>}~#Ki#@q;;leAfyl5fSUF2$Y--x}q^gLOvhlg0`G%3W}v%t`L+v3R|f* zPwlEcAC!QhR0OWd=t4eMskmHFC>9Jnmo0trD7o{6benHQgah`;9PrpQ#gL+9ghqIm3ok@Q`&T5p?K*e>ePa{>7JFyzu<14iEmA{xqC0|0EYAu5xhu4er=uPrBQ^=Irrr zT<(^+)ZOXUxj(sIxnH~I-T#2!-R?#AH}_}vuB)!T@_iq0OWacTu3x&v;a}Yz*W$|M z{x9bLke-q|!}X>sb62_J(-HYMx!Gx7{txbo^uog7N38nPKoGk)j252U>XJ>|B~Jx$ zKAG5?+@qUSw+sZ)qM#DzyTguZbDWE2r{69d=2oP?FPu?-Z!Mr$46xBPu}enu-c`-T z!CjSVK3PGprv^b1&Y@o2B<>h)mvI&6XXS!qv95AlQ84PA9JGX$WMv~?P)XK(oiZ;5 z$xhPfoS;&%5|uZF$j7xw5VhA_`e5;p;*J5mPoFOy6s0b`{?@4)^X0c^r*I1^ZUWJlj_%n1#o|I<#o>0 zCk08=S1U{o!k(C(25})aY=MS}aiKe;Q*-pRJ8Y?1l8+14nK(!$U(%Z_bG;)f$%@7F zWNe&|;-K?UcN8BV5DFf)vGmjrWbrI}$D*L8Aq_#Ev%N5ma3Tj+WrnbFHnOK&tO?4ctxF{4&a z7W41wxp})ku3CaAd2wW|Ny*W*=E&eldivs!t~b$fwf#7dm!gA|=d2oC=?>4XRgy2% zh4H3n?8&~mlR#!Et|WKqaWjuwL;72c4aM^yOcmy%HUnX1cr#5mu9tDG{w^N&Y!R=w z1--Q*6etKH!(eo9T#OF!6&*q6zz~>*u}4sKH~nl*9OzB#?)13i?;v55ze?X2z25yc zy*nC>n0`At{1v}-R)0PzMTx4-b%)DCh?0+HBOcFhq%RYawGY7kGt=tG*nKCRJ@Uh) zyip*@r%#VuSbr4BPF94ywOs6?UpP9QO9oW1GkJz2`Y)73g)B?LURe4u2hzjta9OC| zAJ8^?<>D+3ES%7wea0`_%1J5vcG!l z5TIHK66KSP4m#>VUQHtrN=nh~prAY4=*SaJK^R8qae&<9r-Ykab+q0NI3b6N&8|xJ zDLlyX#UnE$D}yhezHxNtNuzE@{4(*ZA*j32+lCsu5=Qdz=vmHQZg_$Ij#RhqNY7UB@wTrhjnl2pzF z6`o%!>Cp#FsW+G|ghxRi#wGEO3e~V8e4*YAXi_m7YT2r41%YC=qP;AL+1hxdZZ&&m zYaY&Q;wR5+;V@<^CUv;22|fyioo3^<*0{N^){1z*ZAG|kBnni_rblA7qL__dS;NL` zag?E|$v`t+K*@`fOE6Jrcxc9uh6`&&`rnkSq;9H6!urDAWR)5c7a?32ak^MG>Py}+ z3CME0v~PD9nM@wt3lCYl4F(qn!^vV>S!Zbheqm44Zz;NEkZBrt(YvZp-hXk?T-#(kI17)SGlGNyNrABT~AZ2TcEN&k2J!KWT7eM6;*yWFG! z@I>V36-r`R87Tp-CsTV%gdq^JWft;Na-~@^2}yS^mUU|Smdo^i%K05A*$HgE!+46D7 zcwns^wiv=JcWRF$Uk!`_QMoh2vnh$jj{VB=(QX?G=%J=R~0 z-0~zU*7RaN`OC=woC|kFkD8ZMPVV47z)Qk&iOb#LTKiq+qGBA(#a5P+&15Ji>s&+V z%5iy0c!!&n8*mksz-1Y)6i-Fs2VtIRj?1&vg@IsJxRo107mufz4_ob9a2W-BIayX# z-On^rcb$TiSLIuqXXO@`TXI(YR;T`Thua3L-`%J_niamF+9|nn*s~qkvt?8^2*zS= zr0idyGBM9ib&d3Sr3~VB)OAsw89tX~*q&v0CcEFJ6z1Ei=j7YsAR%x8pSmqHT0f%$ z!n&T{(dcA1ZKw-+xX#ro)+X#Co~gvH52{jfb$zgcS1IRer%`B5mup-F7O+O|P~&(Y z{L)};bW9admolw%H93@At?~f1&IAPVVesHH2l6c+Iv;SmQR%gylWGAZ<^pA@v0zAn zXTZuGF6`lzTBRKv{0T;`9nHNpm9JoaqX@}u^T6|5PkB^Mc9KU7O_u{9Sjd5SggP## zggct6lQX(NKhY|lJS%qx{Ksg?*nt+KGiiEi@CV26v_cyVZ+$UpT>2X5=)=xtgUXiz zhUiQ^T~-#h7~z#g7{+{M29d(?{$J_2pr}J9jH@5a7^T;VX!JlLp;YL!&dU#&I5WIQ z=-mb#J8^@%ay#l!KDVYkOxH6;!K>8rqK+wVF1-A#73_p&19f1A@KMbk^uoH z7Vi#s3AUUcGf`{4T2)c-8sLf4Cf^=Uo*gf{J$js)8Ra_DR}Fhkhm`TQ<)U)oK zWw0T5$>v5uHqO_eRH8FF zMb*x*j**MuTNT9UV!Nx2i7Ud7`2QKSQ*WOKp%!2ZX|OGMHjw+#mdxl&%BR@XvwQXg zmnRPeeaUzps8ezn@L|U!+q1rRGs2|)ZZ(v|jkh}+*zYjp$I3D}>ukNNTA{yfo?SX5 z^g$IxQxPl2boe=`3SpsH>^StNmXpPJ07>X$Bf5XT?hDaaxJ$Pm)NPSl9$T}r8RH&D|KJuwjeZyGNY}@>a2{SFS=i&`=J)Ts9ObRS}PQwU%Q)Y(q6jU z)eN8JP#XLSmo5qWrUnel=C-zgJ#}h}yI2>{BC{~K>c%UccI~K;O;(rux=t#p&20^n zAhVIQh4MCv(pF$?@!KZ|sFmhoXARDgYD%kzv`gB3DZ?uId?xpJ;{bQu+}2WZd=*tF zf|TVh0vPIN9etGW$d}0&sHHZ>^QHLHSdx1)m2Fu^WLt+=(^wP}TW=_P#5_7cRW8O1 zYsksvl5!1E2VLMACU7lH6>tg!@?b~^zT)i1!C`?*dgis-TRa_&{zw{cv9O1Ma@(X)wXuI+h+M0z?bbtWY2XX5$ zL5pw58%y$ng|Tv|Kmo14;O%TYD*B2`_PEQV&90@W7x=YHfSarB&2W(VAVrg`z^CLA z3mJ_R5TL_3hE)gym=To1eff&t|8f=o<9hRQI_ec`)0-)HmSVVdfpzK; zXKej{>eNemfYprHU}g*^Wld{9=Uj1aa{G+@n~#F zK#*HV7;y1;|2V|C$B8gsYRKOYqPKYtG5*bXcNi~0Yq4Qw4otClv-6DEet6!O!Q1i- zSeuNCF8Q>)@#Ncm1#?@?%t|`{gMKi~Xop2fB2&jBXA-}p(1e-c2BimjZzw)sCN8e8 z><&xmJM5l$JZ93(@A+Pxk)mhFB$UiR3WUjQM&OwuFXo&A$C`+E>BHHoRi=cDD#XV8em!*3}KJS2=T#fV}Gl@6_+`jyjUgFofPD_&hlQ;N3{|_sV-GEl5%x)W-NA<>Vjw)VgZWud;a!G)} zgZ6umGe<;kk4MM|mPqI?VK8-9SQt)kVq;spudBR!c?kB(BMEW`^_P%&3*BI}P)1#O z+3>Vamfts^ko+nVO}CIoFTmZtFBoWdK{JwAG_Q`8$4F0HJXRFee3%~=w?N=JO0QJK z+Q5#yWF8|9qUnf&YspL5cgS?wK->oTdOI;D(0h`fn<5|c9Nf=C8PH$SP&fKm*PC{F?B1B?1`slF8ul{O056j|h1=Rz%Wn51g zwH%un45GuOs@g*4dx@E~Qc&^HQA9lz4joCwBeB*@jbyLP+i!(~$ZSXx>AS!k{Lw*l z1J|(KL+Ycr@EBfA-P5K*6u%WW2KWO^dIiTqA^aeO87$P&?JJt>D~dCcwBXW3|02XM z01%RM=@ZFGHHXK*#k`s^QAM)=ZAAUy8!VgV8w{?=aoqrWX?^(9RuS}7wY9Rjght~9 zwUHHIY*GJ?B*DR$;lVnLTJfP<6ps@kM@2`(cs_(jy`N5UmL>@B;;XID4W_G~TZ%R+wLtQ<+(9>@1vgqcGdx$qOO-_$F?YyCN2$IN;PMQHtC>_1G?7#?Zv~Cy z%>>3G?j-^A`rZ+A8dVD;eT&@W@DbX4=p3@BPHHtyD0}G0BffyTM8jhWF3a@pxh-Xw zE*`1ixSP&nP#Fmd!2o>X!%<0{n3#8NK>z6|%h4-U+1@M##q=%nYMAC$tk^ z)#Nf+fy#4!0l4zj{2<2_>J64-ilCMsVmELi?Pc|J_-kYZ4z8#D``05c)EwzW?g#!|OW^1gJdNAvwL4r@k^+S5A@Q< zxG0J!ctUD=8TQ6;h}IBo1eF;dK_}g1R1-54;H)O8mSQFPoCB-qmh*sK)RAHE1b|aT z0I(+Hfx|#!ri?4F5(pDqfjhAujZ#NZXx?e{z`*c%DA8mPMyYW)W>9(L6trS4>G49? zTF+^gB<`yf8C@0yLqq}$nAs^g*5pjdbr=9AVxZAMj3nd-s5&!y(l0$s;ZZtAk0R!4 zt*mA%5GHz|0+ECs2EkAg2r+d7$4Y}FxvP?SOFoMy-c1N?!WW^(4hyU9AfBz zuA-k(gAAgB+4ZHf7hm`OCCgT<9#~xeKvile?ya_B(3%iHbQizTS%8NM?1d!=Y1COs zu)m_F(F|@X!NbQ^y!UoG8#iH(qq|*gRJ2x6vUt?($+)X(BUt@~u`A%EmHIC4jqM&8 zVfI*}WiS#t^LJB%U>4a;kJd|ZG~2x|jsVx%gbMN^=pa&nz~%dd3t|GqslE*Ur2XzP z-yVtqSyCiI_3sLjRX`={b#U8VCA!(vwFur0{W>5$%-W>{Z0RSDb5cw?>b=CSPWDbM z0BNew@QLP@iotCzObxV5qY#g@o)*+o0}||5XXOalQ>(WHk{bu99;W@NlOBkbq-2Pp zB>6tCVB`?v|06CHAc++oDOI|u#28-@xg0!mU0A~@igSI;PEa^FRdP44ukxtFc)Dw` z_&G0*6c8h@P5@sV5ukMhRqqKlW9mE*lc(q`i$&me#rmmpfk)dB5z`8q%kJY!@wj% z06?knoaUHK{`WzPN1`F=SVPbIuey@^sv=Wr5Z~ipa+W@ zZgl3sFpuxGg=kQQ3s}2Q!KDbQj0<2vA-%@(vH<6gB#%?7r&20L4>U}lXa|7M0~f;R zAz}(sq3A(=!&2VhFsDarO>F%SRt;VxfEu$%Vqr}So@}8Kh3XZ8X2cIdzhpuRgJy)< znp}QxZjTHXjxM@Lj<{%Hmehz(R>V|3AiNh2<2}25xy1$bpt8k@DIkMEZoO`j zD)-C~SQqucm_fq^txmOdR$+)In6NMoh?SJ)YqZJ~qk3s>T8Q?6-&L8V*~*Z{Z9#r9 zO*eVefe*y@luq0kjW*)IaARH#CFaPtH1b1j8-L)H$Uxc{S1L=ONx6BJT>dgF8G*Gy z^_0s;o#}`pA9Sxuzjx$8?TbGF<2fSnpp<}-;CL}D9d(rZaeDkw<5~0jhNIRWxI+<& zqmtDdu&J;(4YTHf6+IS5J3c*h=I!o;^u?JSqm!q;YSs0`c(jnUInwh9t|#rBHG51# zK{K!{ZNwx^9uFhUPTxG|yKYB1W9~_<1NT#Sbh3@sSebj>+@;0rvSr@ukGMuIw8*XL z3L+ZA#V*9u*JSedCnBLnn z)$J{QNam2XSbw7?v0@5xg>b4&F3I2%mMgHdXG)I69Y=*n&p!J4C9KNIvx^*MjBE%0qiJQ4%lsmiihNZP&QLUNFk-Fpo z@7?U^hPbq}hHE@KzJZ*JYQfT4i7dPtb3@!z*Slu_E-q1CZz*a8oAffUmzan}OKVMo z1*rwqYOko9in&}$UYc`Dsd;JKw3MF8Dpp>JJ~aL^GrTd5qWQur1@4v~;8^;KB2 zKAgM$kMmkw5vvyEd(vA^8=uxreQ5MEx4;pn!Lmqg^1R)w|Kq7|3yU|y_t9zTTjrlX zdW~I@2N6t`_~%c~KYpkz$}cOM9(=|bS`Plm5wk`s*lc=8Lj^OopGvPj<8rqv{nZ%{ zjMhr(qp>6?#NNAXo_9ui;j0grzJ2k43}rV;&w_o*(cZ{GnVx;hiO%Hb;%CpZi)`o?P|T(<^itPIE07LKUNDGB#ZmW7DZF^ zYJde`b%h=0XhJd}RL$?^!4)}1n;oWsbim=r<)blbD%YEAL4!@n)lF^Ctle?5POg`mlG{wfo3m>HF~D`Q zE~j4P5@fzp>#mal9u%TiQv^q6I|b$2ph6Hwn*bz`ZIck(YY9D>wq%)DqM%H5z@r{= zZPEiW4CsChQ3)cC$?jY)Du^J(6>{Q3p`0b({DRMDl6tiw^r2fnqdJh8J&;pFds8j+ z6dqTEvvM0koOFt;3RzVT?O2zVQv`OeOe~wF_v$S;;Fl%Zuq?Tv6SaVgavMTi+$}P$ zLI?H@e=x(q3l!k_mOMFja&1loRS2ow;XS#S8j_tLzX`9KCpSCxMd)H(PT^p!$MKw1 zjG30q$@lh+D=0IC4zQGrO3W3{$rZOk)XOk{frs1_drVL~n{jZ-E&-__6RJ`f6uQhh zjn7q2-p;z{4i>M+6EvS9%V7q<-!S%pN%Lk*JSxG5Ogb|^*4OTEgO+=mo4d>SCDs>sdZlpd8UIH*0`yn2bqv zJ6LP5N3fJ?%L)0yD&bw61|w*-%agP^(bIs6xTuSf5D787-avX+?Mem^hia5OPEw&K z!|Kmj{VOD}l4;g#CXe+al+9kIH#w1NAE68P5xQYASI4aWW1&Of82o0~s1=4}uQ_@x zfY?a26Ij6cCAA#H*-if?rE!66ExSZ1&*Av@1acLRc1gKLNFKE0eIbk0Gzzo2` zdrJVKnIB|-dOeK&86M$40faaVNujJ~e~~iL$|R$Melzw*e9)>VOO{!RRnHjSa``dO za#w@-^J0GB7n>9^x=er#JoD$p{3bcn%0ND2{_Jr%SqM|Ze#4pC#z%m+Hv?-13J%T& zs&KHl!X{w(-2`xA#0r>VA(!E|&pWfi788v^GxE#|E-ZlQZ&a(16nrK-?Fl+^@X0Q( zx&{ z5;?MYU+dLYsZnB)%#1Dl$|viTrRcM~)3Q*z^rh-vi27AXK+V;2vLi+VXIpNnzV(K< z$MVvuTDf0AePIYt3!oZ!f@I0cq29fnM$95-ZCv7QAl*S6eFMmYb*vU56YD^dShl`a z7m)If40~(rJ~LLUvs|mPOe?cYH|ri-wIT#dW)b+sFnN=-A!LARn#-^UABBb*%mOkB z3>!l@lXM`A>c>&i^3Du53xC4isM{|<+HC8_vFO7D z0}FwI0(oY8(F;PZS@Xx`7Kncv{1ky%v%Hz%(49mBrDMNa?6P4r<^B#x9$zsy+N`@Y*dM+Fb+5L4r6q=q!a@`P^^YbQ3;BQ z8fT%bl>O#x3l2AO`4FeNvvCJY#L1H6*U=Nv*GVls&9OPida0Jx%i^$Jo;B1<#moo$i65isBI{?@@P5kr z@7>Rx^pi;LAm%{ZtlW;FZn6RoWh$!g(nELw=?Rht)bl2Kj&c)!4x=&hrhQlp7y#d7 zg0QUik62gD)4(*%Gtdih@b`fx@>ZugL2Y#D20mu!sTOjYaPP{=>PB-iv%E}uM4^{< zaC&vVB&_97=PO_vA!WHz=&?*}M7NjNy|g(ui?d8}j+9Bz)|E?UTtTayO5qcEIBY*_ zD0bMa7DR7gTFYSC5PEzt;z(MIKYRMHOooK+u*W;}ClMKeEm2qSMoC^U=xwNs#71UX zxwNe=n3zu7n!(l>SAdpo1s~16z>@+CP5%b+hC)l~|Y}&L&8;Oo?a~-Sh@BXUe1@ zdvLNiY9N;^Bx1|FuC2Bp&L()l6Y&vMzbmXU2B_Qqa2pQfYi8>S$uVFDizImlff{>N z%V7_gKutp*S?diUys3eGTvXRv;u5)-CA}^r%b>gFk}`yZTj}!#!DMbbD2po0;P^pEpf9NjjdPgnu$PQuH1KKl)7>mo&{QA` z8Kf54RD#W=N?K#(R9RehQ?wT8Hb%t-!>2MfO8je6)MPWooZU1U43f>4Mdh(lLYV=L#)X45i|vw)Hn{bmq>n)t>=V#i*-E^ZF=Aj~WuO|6m)2&v?;bxvs-*V9fbO`T+ z7YvQ91iuC>1Vkw*c#tUyGD959t3ndRn)C(xvc2Usbs>0(p*a^_#@A(M^OkpHZGVo6S)Ep6t#G@?VSMy-uKWF33{W&5LTtl zV{`P&e1iZF0{#dGt49qQkK(Onjzku*U~y#}N*Kk&n!}agiL)f|OfXkVfGgL9sEUrr zKv00H7P9Cl30UvR?3!>S<%*pTq<>I7Xga~iJBv$ZY)VhJHn`#8^na4xBO`*_&_|`5m(RBb*pVOAtN*^7p3Pzh#U8 z;*a{2=Xg?K5wrz^`GcjidQum|8pbCn#pWhzNt z1gG%4jijTOJptgbLHcBc6q-c~vJ0dMhuMZW_z@cONbl~<_#VsRV@hFkvEF2MQOiO# zTeFO8=0sr>Xdkc2qJz+|DMZkQMN~9#03%~o%yJS`8Ef0NT0IBcoN>bPEM1+YE3?$uA&*^ZOeVE z8^BTM4A7r3%5Ud4A*Rjma>Sa!k^QdWH;kqYmz?Ffx+zVH-kkC93Jq6->bS(KFQ|b4 z4~~X68*JJSfv>qKyXi1zQ4$?$@rw<*6K>4)H@G%4U=n-`wB>@yVKrqsPLpLBnoOHE zlq4BUR>uvORSNRnhjER`X~HW!5iV1C(gZfz!dvlPs*ScniH*)&k-4w~lg_oC*QLTx zalFA^Kq;?olKt2Vd6FAJtsks;Uaeh zK-j~_x-2N?NibfdSY=KKg9YYxb5M(PC02z&khdVx&PF37Z7&jqS?FOvEK4S%lg6(Z z9Ss;$kRSu+6*+IjQiLsWHe0VZ2yvjSj^~<-`h=m4nhJC=+bnHd=;K`)8j7+tcuk1L z@x4T_s=T7k)lQ`Y7DQnd+dLONS)27~O|CYIb(~qBaMi3&Ecws+RQFbV)bLJ`%Q{7_ z{dNlaadOVSo#KQMfMT5U_j|+OaCi^ubYWE11vcfzW0B`UEg6|(#v_7=;gGyXEI8-T ziPoq9!*&uP2&-bUQ8W!5c8Ctlu{@{rIq1(Uwn#<#j2^MWZJW|3>{WfM-eojTGW;Zu z**l>~%r41HGRaIf+bhFQ!;q4S=*cE~Wd;(8F-ar%pv97$sK-7?#dH{cv+>j~^#PFt z>vls`MjPrlK?D%nSl|%&v8l`j){_Mef$coi5+NHd%Yj3j44svrKBIS#qD-TA z_)zh3s?thClt!ZEGC)9q6}dqd1h9m}L1z&f80L7MaaP?>=hd&aC94;pd<&&SF%vRU z?=fah$o}0$i}Sq1U3{nSsN6;K>GyRP3GApXVIg{Jde;@T`i(>#hdc{O=de0V>V|n1 zKC*^kVj#&-d(a6i<`}y}(OPIupca&B&}vSg*5SL}yzM)`H2Q1r%dCIn zf~vhcEPr*F*JH@1e}3?&+BP}qco~Q8@NvfOnmD%uXTmtb?C!g~x+{*VtY5=@#rY=b z(#q-_mel;W{T2vO<{scU7kAl$YFjD5S~Od6l~3B0X8tUWME2B{3}jDj*)(Aet(xxo zz&X{f##l?7*#T!#d>Ci4H%CLX;k?<+*)Fw$XX@>)8u%VR&#GbdSUz<%Ory)k)VpHu z`}2HQ2RV`mLDky0mttgp9b_gN=YD@(fw1gjXmn-op0G%RWmJ20#Ynz)R}`r>FMoR; zA&;9_WZIy6!ob~7vpk;;yt_9&>s=$=m(t^|J}T{h*NWnu>{pD|rcgWUyGFaSksmbnJg=7} zETONS=ZxF+y5tQPOQSUm$CdT&7;0Nyt`;3KeGo{kbCI?!88;#TbbBdFRH>wHpgsM` zgOl7f>5EHGNN@PfE7HF$J)RL@di?cM(r{ojlQHSD4|dcyb0i%;erI$|Wo%{yhsH?J z59rc<9)F+5=Vd*yqYl*}d{==E%ofxO2X%1>LKV}ipwUbcJmB zG{T~CrNk1fX(ygvs4yE7kwRV&_Jln^lNaCxTMEMk{vZcU@K^y_+Uu*r!or%u546c0 z?dBOjSg*};z~?-a^&3{Gfody6K@J4ALW4S4I*3TQ!63ZHfm{a-?dg0N-C@#eV&r9) zrX|i^Q^g^i>W1Mr$Po?{ve>Q73J<|4R`szXdMr+@3V(vf7SIgF27o(jH5lHiwO;}q zHdvBI!E>C27W3R8t}=#MhD>YSclwJ5J3FqaTJMJW?-L(9*tzw8`QQ}i-knaq;X;=4Uw^|nrT3VN z7`-?B#SN$OXvU4_^7@7w?{EKf0gBT-R_n@X;`p(}Ek3@PN=k%skniKK`Mp z;rm_smk*un>S@pN!`!y?(&bAl+sR`pDcP33xO`r*PAkbX>1iLXMk)??1w= zAk(d?3wx3+S(Pvn)XrVCu$H?VX13?}RQW4_oEKMLo_N?8whE1Uf^%ulOeVGna#c#u zmC=UCi41EB$VRE@ikr$43{^yof<(}0Jg<$7)~jS|df!bajBh|Gy*~(MATZPDV+R~A zU^RzlK)E5HyiP%}0#rT&N~4i2>A4>}!&69&L7}@pHU(t;_+t<8`)N+)-;lILA5U1!e%;4k6%9gj8hpxTsq{ZSe#ALD1cMe(ExNp>u~aPd1aZ(er<1nz z7A&B1)ZX*T>k}T<9C_VotStz_9{MEqu*aZ#LwfEfrkwmj_9Re+aX7zIl+Z(2KmxeC zxJ>qNLR^q+1m9+MlMTAk`(!Iu$>#JgpSU6YaR2zG<=^IZ@wd~@_K%*hNv{m1oAq=V zKPsU#S)6X|zs(J#eJjU}-mC`$+1stU*}Q(u$~U5S#uDmCc7MTWM`FLsM?tKy-By8?Nmu@o=m-V2fK9Mr^eQV2}6glMrIW_ zMd(zO*bK0^_*2uM=9fQp@tcQNVW{>v8fs12C>a1RTB1rF7Tg2)!6)|8AlomtH>K~r z9A-iSRK>VewGdTt+^SifyPR0ux2hR-^~k_Uo=Z=-b#e>&wY@{f z18h&fvFZThR!^yI>GoCK6)~X)eS13k)5o~y(lb6ic7ZCSLAot%!A>l#)JAQ4l>WWG z3jj*wJ*F`z9ywk^3Bk{#1D`(dKsBod2>mn&W#Vi3wxzp2J#lgdLk$hbelXuhuJ`TZ z;bYrrFx$5x)r$9WovXYtU2^NmQ?*|tNQ{&ezgvC4 zq(Sm%W1hN^(Mry*^jJi`LqFTqlD*=1QdGrjK09s9E{F4~(w+^}WAO9S&yN3>Mm3gK zk4-RCLFSfP2476Lq{5$m_O;b54{SFbtXh%I^rQcXVx$u8Hh8s1%?9v;C@-fSShXy{wkoR01} z?F$nt#S{j98l^?$g1jBDK^CSxsv=dc65I2#AzjbD6abCqJRF-WyXZzUtiv9BhzFF=YAk{MqOWjB;`ZD* zZQKpR^4~b@`a|iQwUFIZkANvs*X8D0g*0o7F*|8SZKP_E!*W`51T@$-jGsgttU?R4(^d@Yc?u#g5uA?%! z-RWoVI&jQ>v!zG;`_}LStJXJv`Aoi5lD^^YjU!j!HpOT$yauMIg|~yha;~02CwrQa z=c8XahL5y7`jr!*WZU|Mc^6dvHNAfQf!uz6{lU@oEaYG*42OYn8GX>D+t#P<`Sg-| z?k?`W6ZmJXpY+w^9I#z*?*W~=ztWRb7Ru7IqA8bzSNLQh5!iI?z2nHm?<p z6=SW9n*)~>PGbbFHLCDEG1{+46fAPl)TZcZ4_% zA=Q3v{f8I&opa;T?|yxOJ0<8aEceYJUCEkX2s#V^nZJRsK5ztg$-;V1I_CxmWH zxGJCC^^FcH?tf?K#=rVZMrI*hq0d5}c6sZXJ@$;F8+h?M<%I~w*2D2)Lv8t_@<=QW zGe;=xpObfoO&g9V)%EugmHzVD>b~Kv*+WzoiO<`c!gQ1q4UDMg8BOb-c41KmI($y& ztY8<;<*k%ljp_UIgevid|Mmwfq0XZ}*?A5uYD*Rmz*?O24^6DU_=t#~Za~b?*$q@_ zCk3FeSPI5#Rb(DO`&l3XT(E3DW@<`z3bcYVfJ^N0>fB4PQgd)ET)ov z=7I5}x1$F3!y2UpA~t>0Q42?Y!!w6C<2-sOh($+_BI5;fQ>iKpHLzg+C3w!7W~V+s#svO22d0a~ z3(%?|CHMoGIh0KCZHWf&CAxpP^Be-tKPz75?1LY={d_o!uhd*pEpd)l6Bt9#p^^^m zVWn7hz7->p;^`m|{nHE=uyofBTv3G0R`(XbvtR24{%n4U(68vPQ&rehtK%=4JGJHD zRC>j!Dtv4!c1`J}4^EzJt2k{DjDxD@?#t_UZ<^#Nc-A-LG5g!5e_}fC;R$Yi`fm@<=)4C_IwjXl+>b<&RKjRo zRTNJ2)%5O%mxb##q{lvT`k3AKqBicuw@9{grB9X>8857Q(K5`(x zBQ`%#d|r7sq&qfu@dc~Gcg{WZKFSeYVcj>CpH;;#wE5PYKLnd0t}63|KKPx}+}F}4 zzVoI-W@HKEvfc9j^jD7_M^_GcY-0CDSRF=}iD4E?ORr!Q zm;A_lx#$5Yd(>03F}>`udnP4{9uc)vAe?g!ERyW~Q`%)Cds3O7e&>j35?cBk%}=j+ ze9k}Gbl;W!`SF$P&;QtWfA7AVF8zw zI$h({ky{q>>Wf>hci%{7eE+~v+Zupt`T;njZ=@G||H~CmQ?_+M)A2ueSLH$Gm4xVg z^kDj_AG~75K!yeeOhN;C+Ha>%9viF=Ot|E`*=_oVA53KIGv}lW3dVqaU55kO)8+=m_=-h1Jzc0H@_iUZ$zL`#Z@;H7MJb8xu zR(j`?T?ak6ckb+Lln3N!1D?irnz@q5R(S^y}iU=|vMNDHh*qMM8@e|4yU8lx+(T`|G#; z3B5TlfIrY**#fbi1~Nn=W~@QR2oQb{#aJ8=dVgio?R2bk8~M!U;suv*_FD zV?R8cujah?!>O&PjA1p*`qAO(F#|E}zG>j8^zA=7f~!?O`T^?Y9Y21RZ8AYHRSj}S zH5-3?u={HI^B>pTruD^VUgiG20G$7m(_6lmeXbC;DWt1@f6OS{N?Y`8E3#~wAl>o% z$vU0DK9b@m;@BZ0k|Nh$Q?`ho?eJKzhoIXsO3hv&u8P)!Pn)dP(GuA#g45I+LlXt5 z(6-uxga=gdC8s+n4^2wQsvt{~^U)XSBxYG_BJh~35u4|-$Z|#VPw55!d!Wmw_5VHe zqRgFCe3K@K;-cN<&8a~`$$6cEH#kGxShRp)%bLp#Z_V0eZqd(103?-a^c|%i-Zg>G z58b)z&8XtGKX)=aaOj^8PM`S6pWRdGcmMOyuCLJvf!st;a+Br*bgNzDP0>^7=x1lu z4?)Mk0U#lI6J`O|kAbVV@gR=*)z~Resn8Y{YS{KSx$rj6xV9-A>pxL6!>`W){cOCw zgk$zJc*mFFL3`B(>MefFl5Jy`gEQLZQ%bcqS?&=MTK=x(KSo}EFx6-+;{gGv+*-Sk z$uVr4vG2c^qgh7Tc-2+X!SA-_-xfxU#SZ}fMVH4M@>$f2e2tGQ<$4mwd3gD_H5o$B zwCfC66BK>uQKYIh@ik9|rO1s1m8H|#+4X&}%zV*19Ffqwt(GyPT zO4gcfS6w_U!t3PH_qUiw(E_?nFroqr&m=?y z+u4c1>DlsNW@sw2Ae-;Yp# zAr`*N>zQ%(?*7XB{)da5U9^_4n|wcm$#mYcSG;cBh92{@zzT!X{ zUsOg9wRd>3n8+#io{&!qGJ_*#BH`^Q6g@|{0kn?CdGpQwN9 z_L=-vetJ5;=l=Bg;^O;hV|%*lrw12T_{+^dJ=uLZZTs25qu1%dX^37ko3U2gO*;2y zv)oV9`p^F4ewM!b=acxo<>%x0{p!zeqQ<$;UCr;nbJw&V3r=DwqQsxvhQZdcZE?Ef z7m0f|efSr%OCU*h{D*1uUzfR`r}h6j;>}NUx+kr%=?4yh5g0FNSEH_q(bIgPL?K3Y zowCi7Ts(;@S9aP~OVNNoKB}jMUlE6PpeVSh6=YEP$unT;jkcxj|NRC)yzIZ<#qV?f zeJa1xcg*MaeLK!{Pp|*(j+~=^-~Z(-9yR}JM&sPM$JP6D(mB6+2XjmJ{_0eI-LL2K zd*-jtbkD8-%&$u>eCGc2i_ag)v;Tbl8Mh$4|9`6PN9j-g=kV~SEv_2;`wE50#+T8@Cu@~pRybAgK6^p~ zbfle*u?^APVp$J_?mqbW$_KHE%O@_@jB z8_E0K7dxFgHs!3}$Wz~ccZ3v>7Gys55CR!KiZu3a^$;i?<8S`k+nG#w>~Gy&pTAY1 zZ~3aYgNOqs=Q4Q=CYdfsN9>tM^m+Q8DUEpoa%h5-_krGDx@TfKZO@_gY!<=Zwl?0< zxJJ%s_ov&!G5%Z3E940<7|YA%O!kQa?l}dNs_i`B?1k!>Tu+$&u?Lf51mtru*RAcL z<~|nuN2BdS%!kE=dQ|3h4-Pv@_7|{gawH}!+4umcoRn-Q;oH#hXvL#O(-e-A)uY2p zVJ&_2o8)M`G7@%#$cy4Cw!L{4#KCKj1CH~9AE^V}%jR-rhqssQTQ#!k>^l^EY}6*| zCC80#py+E7Xc?P;LlpUFv1z9kAMCN)zPPQk)}mk!Kl7ON8Tz~ga2(mN@!IyQEr)*J zAc9?1T#-ezbygCY$uHkEJHEa;Ud#`U?VGGv9y$z4CQC<1TI>_M{@7FdqBDna0-_qK zWj=y*C?A5xrT0zf?9^o@=&aYD6`l>x;scyTf)jRnoFA#Zqw~0BSaA|bZf5%Zo(@I8 z3xI2=vt!R`dMD(SfA<1uCBZ2q6HT8BYIPX7$R0NX$KZFWn{}kP&Y9z2AuIX3l`I-} zCpral-&d^G$H8Pv)Y;XxXeWz@HLGK>B>HN5X*@3bZ)aqG@!3&V&+ANS>L(Pm4UbmA>_ zSzE^ZZwF7DO#4m}aJi5Ke!C5F-6)H+zvew-wB7TuCyK!05^tX!qXrSSLu}HR(s19qBV3eM!abMP)qG?(u}x2g@rC7SH4%TmlzW_>|9+r_VmX2{xX zbeFI%5r-mUFdBTR@id>&7KFp_SO<~FLB^e?YOk)EWG>MTwLxZOj~pK`5McTah88$V zYWLIKS~!WmN$UKb+8RP`Qll@8X2YD6#xxymnU^*W6qh!?9d zq?t(ZeU=0+9yWV5_$Bqh1=#;!ue-h*P{;c0^#SYF;WcU-+{@2NG(P3N4a04F-uWmN z_Jw{Du`wJSOQ_oKK?<;^((#OkHekae1_2r;)ruMMEdD0fzYLPyq|rGr)TxZ>hLB}0 z(e-N5+#^9m!}tV&rjI!aCJya8f0|rkm@eG1RH6T#yqkL3mWQovmTWp)rhS%q;#FiT zDu-6=gcfxnuU71EHq$0eXW;WCTlx#-3qy6t{atDp%Ywj8P{hGRa`tNLVi>C@TQFt?H1OmX4;TkT3;WM5xx-Ih1voymrvL-Z+VL?OD~p>4D44A%WHI63O-1AR z#A*d$RP|`deYNuMNjX5%31J~9XdS-8f6~i*m!ITbfHVs4;aWg8 z?yI~%bJ+bG`WLplw~c!ZkIMYN;-U-RdeX&L3}%0A{|oKzMECmsqsO=t+@<|jk8u;- z4gD*}xbxjt`ge_SuW;||pFEb;5m)w~J=VR^ZM^TxV;zeVzTLm)0C!AL5wgl7_npw; zepsn*8y^KBjr=G}AIsA3X6g5{^ocC}9x31kTe9@tEd5TFKANQ)vh=c<;5sCpPC=${~`R>KMBA74Rl5<0lW{m-_n2MLGIiW!l8e>*uaZY zsBQ#wCi&>sxOZH9)h-HpkUOitc#!MqKkpznwg1F}-NgIef3TaB F|9`%qcftSw delta 32586 zcmcJ&3!GJDz5lp9!$Cg4A- zUwT=I$NVq2tZ1bk2A5Ur+W!QXMN5}9oNu{QEeqUJap$e|3m9!FJ(u#oApEr(5nXZ3 z4ef8be9@vy-}3fr-n8VZw_Otis%yRa^1drBz4r1YSNi+HJ1)QOn!xRUbj{lrEecxi zeDR3!)^gER*Dnb|dwks$m%sVaw_LU8ifon}-D#s$-}JU?-+AffOO{-B{S`}sAiLPr z8-5ylV4dy0c*rsKEc3T!Xem5&LWFEEb}+wjgrFlB@8@1$5v*9T|D*IaeurSE`PF1_lStCn1P!>r)>?C0U6a~8NDb(O<9Z*g;G zPxzR-&&@ge70X<|`#)~2`=t9f_pJMg`>Fex`&akx?gjUIx7%HP&9!}Rf7i|LzWT@R zMfW@R2Um4%ZI7%hJfEEto$oHqJ{;ZPj?YFGUhhuMt}gu1^=5A>&N^o0hX;eiC1Jei z^mdnS;x7F`kQCC%z3DpLto+De5HATTNuehkS8Gk8_~h(s#iLw*wx@W0nj z+Xa>M*)P!N&p~?kn?k;ba-j!pq^*HD;*o|aT^b9X>%%P zm7j7g^hydzkj-zKm`!M#9G)L!Z)~e&m$aSg&fmDU?dH(^^TtW-<3U(9cf>3Beea0V z8hcz3$e&)hJ#sCLDM8xr`s&50K~hZIoRA_d^d>4?9K@58VozAEm3YWSHKo2MEYw@N z=Y&(KE6oWLik3OyL^=d>!g0DwgEQ;pIbo-75c>vI-=Lf{iU|)3(R?a;dig2z#z{Co zuPgW zy{zKgdL_ND9}u_1U8noeP6WT@Mp8*1RDCP;?IBw&C9Uc(KPkkAfFwq?lk(gbjdExuYW- z)d-hDaT}nx!qh-W=U*dnJz-F91xJN+jq3#)R1`FIq28*Z9DfK9^n`01ZNh0`q1L79 zpjJ!@)55i`HdasXb^;C;>s*cI)2PVXC5;>dYH&dRG^o_VCVJ0oXSqSrwH}5Yx2U)7 zriH;=IKm}SeigvT#?5s`v1dVAR!zCL9yDtJZMiqUfkf#ttznIY-$F8WZrDQ+ih~Mb zxrCx?F02Dd!8JKlJZ$VY3AOE`-3Owyvar1jl=cO^wJPOy%1Tt5?xsufvP8obP-Vy$SJcrRTQR0YuwzZA9B(5mawq4IpNI|pyynQ);Zyg6v(-G6jjzUC!Eby z#ja-3O$1v=XX>Z37wxgyu~CyBXSBCciFl}YG?Mn_oartq;$%eg9ztdQxzWW*8=C~1 zP&Kj<@?H_vO9(DdY5WmE1mcVMtF8`N%?VdWKrOJXQVKbOQgnK|gcB{|0b`l~0&mz= zp-wMIPMZr5ZBgp_&TMbV5o*Nc()+)NR7lb_U-CSjuHimDPt30NNIpLyH$qg4D0fwe z7)GYAL|jlW))CeF&1;`wL-GbjDQNdrLxpyJvG^;5I>ROKz3r(rVTx6e2u*vcR+#!| zD%ghc$%Pb5b}nMv^(DnK+lxtvQZ5-)j!Y{T0H>To09^<}6y|3yj!hb(_-b0Jg`VO? zw7#lX3oC*uipw@fX{7FndNleWd^w`-t#=Nm>6Uy6qi8M0$cb=>(i@`f3jlY?NPP^= zUP|h!1;}ki>Z6jjzWOLQM5JDX)JxEaNS!+FA$3+}D;|-I&grQ==^Rd3Id!)HoJPd3 zP*{;cScljuH29(X0Drv*+*;DrQLpV6e57l*F79g!WMl|?ElI^_3f&6@4pDGAzpvi5 zpTNZ|;-H+a;Sq_Za=n3-054%humo|EC_oa7-<|xn#z=!9)~O^NNn5@X%!A>3nki-h z`7lcYHj|cGD;VYaWd^LaRwTRt>In_Wshm-e*r0fb6`-tXVdvbOfSaN_T@Ba^>BL^+ zb;wjO7%tmI<0js{bT7lZKoB!dyfYrve7`Q>opdKm^oDe1x}wQds|Mk+W7BOE(xkGh zhIiC$HSO0^jkB3n7FzpDejR08Ju}x)9 zozWiXJ_d0VoiVb64HeQjJ}K+&9*@m%Z1>^rr0fmdhn;;md;^~1)Gan%n#eTjgWBo| z6DeKe<(_aVqgUf;+yhNR^t7mK>P?oNi;u|urF({}XZyNe&CnN3yw3eF`|89CC;w1$ z)8%dn48v`>nhOu``0{M_q(*yg*6i|T&ETX-*!Dl2)QxTbUz1L3Tw1B@nih_26l6~9 z3i|3JFknYWn~tMk=uWi4LwI=QHQ2;?!MXlSL4-=B0HZpOwIFgi|a5|5> zYh#iLwRR+MyV$PQk+hp&83jioJsP!2I6s=-NUAa^N7X8_C^}>!k5*Brbtcuc+J|z^ z3A7`vG6 zs{|oi*FWZi;>vLgWZ5?Zfl(zt;O`AEtXlA%KIHu1l7+8H#5cpQRpCQVgol?UDCxg%NejI z+<<}XSG||XVFeqzAq8fGZ(#5jTC0%_zsc2XVVj4B-`X5Lo*Qn_=uEkLSlx4Z-QH%M zmhpmBp5QQ=G1=#YJG3=AZdZwux6R3iz);wlHyF$tY{~C8t8AYWZc;t%HaO^s&G_{h z7;T^NSl~|mHq2AoBrvt5t8zzcy=rrUfj-Ezn6x9jWkXidhWOtPah%$&arNTq?U61q zdPk}W({9r~FnRa|b9Kf^M#dA>XqBtC3&TC(Y7Z(FYpt=Ez&O<8+pWa+QmYk=-|g3s za8Yak7x_+jAHZc>E8AdfKM$VHB%_|^n3Iu`k?83-hW$qP#Zzx9iD|!^Hn>Oucr*VI ztcrXDJT9k&&$ZU-Ie~ydVJoT6jkYzkvlomY*jaoA8=f9~-@(8YDfe7ta9?Y;Wxo6)HDMJEisy&CI>Yr2tqo!kQ>YX;ZSzv{ z6X%3aYss5YT4S--$D6>>^ND8`%rKCo5g_W5a$bj@I&iuxencTKmdWF}pyAozATH42 zMC3i;pwt zmO3MdC#>;}W+!O>Hw$pa!aOh*216U1WQpHh`T`>e{VP_Z#};U+B@&E1;kC7Kbj32Z z$F~nXUgX*L%32J{#(DK4EhJKdD&;3s3P2bcF3Stz_?tJkk!O7og@kA zil4#driQC0y4dV;9M1NDohbkcCz41487|eF(8LlPWnu}LBC#Y6W5jk`$A%C_H&pi6 zIMuK{wl7-YQr2KoT@1SVxMZXj#7$$1tB>sM)8%GFL$6I3r#}m1X2j{tzO)TDl&hcp z^{>I=^vR$voj`>)pU6|Ig56#D-Z3LIPNIp@am}YMjH%GagrxIs)qOGUg1B`1&$=yf8`|w1x-D=U z+3lUWZHXIQ?FGAEqx&+C4UAB&hFF{Tk%^%Db-Ld_!>9Zh1fO!rw6(R~F581G4q~RH z5fX!6!4g93n;rzIJF~qC^dwkB2Z#)Ckt-OiQm+aOcPR1xtu_{6rp4V1R6nylOoLoS zG8dA&EuX7sbMcEyty&y`zNFnIJj3LY%V$S8SB?*YYMar*)`pE@P)4|;2wO3RRph&nm3T<&SD!}h=qz(W!3)fNGm708;QAW&N^v#6s2m(D+{-JZ55BjqqsG@rsW zM*;CKHO{4lhb(Y`72|jmw+P^Rdm|akX7-C;{C61*FllSji6+Mii}RqWOl8nRuQ-Y6 zWRbl%{+>lO%Pja7ndf=9vpRuU2Fu<0fO`^-d;cyv0>ZdO$p2kFbc!Y=xZ6@By;vs%((S z1?$+fqsbnPq6J|)VE>5ohYtw$EtVTzc@{=eFk-_EdEy&N4HxtfnWv^&gwc8C4>vT%F3?ftK(K=vEoFz28+=QQXc@xZMl zlafQYY!FROI#4J4#ABBuO-sjmJu$ahGKQJFr=DF9eXE-N!&2QD5J0J`3le(tAR{uEx{mvmj{ACF24_GHx1lTp_#PWv0T+`8;W zN)^a0yv1tL!A_W^<82yzt_ zRk4&HQB9Y9jA!EmbZtUBvzqStX!BmOUfwo>1d{`B*AdqYQI9#A8IjKJ)kp~j7sb?g zhKh*{VB#W~kaVkJo_~B|e6;Rn*E*y;JDRP8y%E#Gc`Bv)8&R!;y;fxy+RXC3%@1b{ z2hKX`fLTW!JgYrsR!ygg@tFO(ayQ=l@62o9U|qJ3Mg_N|!_&Uhv^G&+53!nEU9ZM7 z>#_VX6ZrCOg(X`lS^$$Eo<%S%j{kw^LFh@==2yg`Dpa6p4uQA};#119O8O8Gi}f=O z_{Yc8rV{GFoVz4x1K11@O7F~P2sB4E!RM&zN!2Ks?I7S$K~fDJw1pA$RJc;8B8_fH zTNZIyUf5ly;s)pmkW{faQ;aY55<&CdIo>cbYYB>6%o7*p z^IN}=HL*s8{<ZAr()pO8^o0jRhZCQ0L! zjw50)J_4T`(9PzuO~(==l7_%$$V?y!_>F;&6*VJV7mM!1Z~Wv+M3+7KIVvahBBl~r z+X7}V@Jq<947L3O0j73^n+!$9%7r?jD@KkG6r=M|I^2ORMx;~iYKk~zV(n1G$5exQ zET|hWU{wP=7)-3TO~WD)*cT0SQx?rF22xB;>nj&QhJ7%{q0(bjb)4d_z(@nFMD#Dk zIA&imf&>O^o5aA?$*AmKaJg;ypc+n;Spxuo;V{XAk2wq=$UOrHqRjZUB*_B5#B;4C z;iQAe_86khEsdm{v?guwkq}^R1sSvu8tEzmTWMFiLxOc)P)Wu}SBw#vjA7u4&OHdi zM8H<6e5n0i>?CVHLf+!kpmJ9jw)n%-q^};r;DKZTKjRYDknrPZs2nfkfICza6itQO zuH(x19BMFCCsKrvnj)kY17-;IqXS+S9?!Aie2a<_|FgKS4mBYmP6lZgaB!^1X}!Ri z4=C5MywJ@NcGB7~^g5xJ=qb?8NWc@mQJD(;;JA=M)n-)jv7=;wc*sS*=u!W7F*v zhLuR#j$aFAzG1n!Hiue?7X32wHLwdTT94Sv6woAQpE+ z_^q`fi=>pDu=yz*r$+x|^~r z#+Y3%q($OW#P&$V5|JuD1)0bX(&l|~O>CG#qaF<%%d>b+V zEc~U9iZuii6B6l(?{MtMyilA;L&FFFnjgyFo*%6%R|cjP9u$czHFUJzk(7W3Xizn( zUFgF~OGS*ffqatdm#l#IahP?^?m2sWahnmNxDmXTt^uq>CWw*i_9eee$=LFp47&77V3iEMmhPRSC^iA#c% zc+5@`+LK6qSIRw%U&|Iyzpd!UJE0K46^skkv0x2-;JQJ>!srl88USTK37F0mFOi4> zjGcFA!x&1_(p?g0=S^9jgEa-8;;Pqh#3^}%DnYFg*$EYLmdG5+T1a1V&`RiH_|fWt zC$=AV&Pi@!M0>e8uXll^ibN`ALRdt4dQ>cU@f+Xy=K7UC{QRC^`COE+s($;^J74_Z z_Gce`PF1m}aBh^SSn%`b?t1YDyMMT4cd+~)=SC<8>i+nRf4S|M2Y&qMi)xLG){fzK zsqt?)f6KwQ&RumIf14k(1uGXjl$R);|JR2)Wk0(qsS~rK!gm97=2cRf4;gnUh_`Xv z8IxQimJ(5a{S@Nv%0um8>@|W8Q`x(SmVbVtdP&+qyj1mSpvC&t+TdD|9XN(CA0ZM9 zh1FV7127BETE)f1T3HeiQY@oIC<-;)_g1L^vhgGUl+lD5Ky^TkSX)pi|M^o=yX{Nb z703+E#=JV{l+<)XouC`mQD<96XQp{S9lF@UqG)X;?r3p%cDD2Qs(ge`fb0e&fs*dE4WdS&D&d8(a9re0<6DSJ#Rqy-Tx+ix@$ zka?(}wMBwJ%9u_qL_V1V`4S^v+IhwhJY;?yFUH6XhO)^btz;fx!b4>1g5+rw9D1Ed zK|auh6nF!)I+rV!U>OYEzsoD!-(NZ~jX##~4^fa4qCq2gj07U|gAPgbysM zikutch>CJ;I3Z7!1xLCL2+*A%$w1S>qvq0^JPK`Pwk&Q9qZWc1DsP3_hjs^CEJn^K zc7~%Ft0RcIFlAC8_AZPPpDol%;@+ggmN3FAD=Y(98K}Fawleb!Q-FyvRPeasU%6^W zZl~-gIx|w#2yhQ4xBXQjv>5P>%)v&fO^nPJASx1pHQH?VmVVTVQ-{71$p{jblsQ_$ zOGgA9MtsViBQL2%Flo{j`3x8;1O-K7BSm9v&{!)d_B6&}BP7-aiJ|;#wvjm+AOTK| zEd{GBSAl1?V@qlZY&-_za27~+aPY@Pp7$iqV=?Yc63L*gMW{OZ&5q&0t4gexi zmYfvPmn0t{n`(k2N>8^%Aiw6H6}Yfm2x>m zP8`Axk<+*)IjM2DqyeLdo|ID8WFd~@hB%1Rom7W*1pbw2q*z}w`|rS1rgT^))4}|Hb_%`{aVt+IK!S7^I`qWpdii&#HTqtn1V(O8MxbN>8q;uQ zG(0>1`47>1O1h7N-p8CdBRL5!M6t+iL|aJ(usqQwt_^8WCuO&s+3il+_}w$BO3=@C zo%OTP+do3@Q`rUyj_I>@v+=tN-x`*deNYdwQ_j6~^lH1Ljib#T#R*Q$9y<5raYKEP z1bN?}a=zo8fM0fI-RB)!Ao+h?cK3OU-SgRq^Plk0mK=ZYxV;cbkHtbcIQP?SkS{4z0;kR9sSDl-S@Jkube<)`_Gi{kT8f4-f`L!0Yjq7xCq+LX?dDyLY0n%+$w8Ur81&e(r9pzq;_~axic2G)` zlla8FUhAOIHr>!@t5wcvusq+2V_X{r73I%c~95r5Ncx6^F@ zoDgEp(2D3-m&Qs5L)?X}l9C3F6a{M#z>D2y086(PXt7uDkvb9RxdtH-nQ1QF3-DWs zuu`+qVO01A9#<=J%EId*jaQK}d>ba0X$ek3UR7x04Fg4?+Y~S}1T*gbh?!O64Oj$1 zAn8X>V$BK?Udkcr@=c>Y8_n3Ly;%SSNf27*EbSEy(k2SMN}%0mg4 z)kA|^I?`~B_lVA7_{*>qJ1Qf*F5+A!;*>Nf@}gBgH{#B{Q8eFP1u+?8v;Y}cDx?{b z;(f6xj+N5z6JjUHXF6>=;(F z)oRu@q9!pK05-#>CPuK8BnWujC6XB7J%Pw12MB_dk!`qJS_I77FJMsarCq8lwIJOp zj1;{v?)Sn`jdx;hX5z|16hxpkU zgLQLEZ-Gz6^Aw*z5K;wznIuT}4BO?EOkzvT_qHh40g<-lCx9YsWAzZK*a1O zHI(uoah0gS3?3L=d$QGJm?)rqedK0JsTH;h}!09+8FvF8J>d~E{f1mxXu-?3pH?v+iovf!q53iQx zsN&Y6$i~`I@TeNJF{MO!N=EzebuD9EhM!^hGh7L0&@uujT*HPGmuj_tX`dssVulR3Vx(Xh%=l6-(CZvP;J(EMa zCIcx&Ow??aN;XkwV3RmZpZWl)m|7|dc^PAxdF#0}NLOo;N_-eGEG<47M$ob@=!Ovm zG-T7jSi+Np*BT@2yYm3N0D)v$OWDj)r$t)e1z?`&wUq{vHlBts12d+oeayfH)ay+K zUls@ykI)?ZOpnQc+N_OrElA#rGz8)NV<@~^+d#b#U%=HG2t?u2p(XBRiM(%REES(Z zAKt#cM>`$y|6->IfS2DX>1-z78`9CU?hNXK#niid9OaC7ijeD3c+LkL0Q7qtE$$=BiH9!sd&F$xh}SJ};ucr$T#-gAwj&E;VB) zCBU^^OJVA5DnV+~d=~xn=p8Y%tq~=^n~~ucYV4+IpKB5NL=u)JrH~`=11h9YBS(?9 zxLCRtBI5vX`-Q-#L&PtmRlLedKqTIUw`P$44B@pB=aa}_G>kD)o%o^YJ&vrT_r!BB zTCVpf@z_LC+oE(yjerVavkJT5gjVatrSBE}kkg5>BjN+i0zvO-OP%iXlRlfa+Iat^J$3Ve43 zE@>cECMu_xGKV8du2D<%#W)oMBi=Xw76}e=Eh|O=4}01$7^|7V7Fdfk1ZyJ|!VR1! zq}6Nc5WR@VsN-CKLO|FP1EX#cS5t0MiIowb!i;UKK)_3@O0kO4|F%3{MK@|8Fm6-w z7yd^Wzfy}=#6PEtfM6PjIIGB%2&M+%{Pfd1d!VpTCAodj57+L7J`6Wb_s*dv{XwWK zV52&1Z5o#>XIug$>g15!Dff@-vc7DfPa5Ef+nHA5$zWkT=;%PMg|WY$=`Q`g=JAh# zDC^P5JR6+X8^4PE<-ib$&EwU~AbzzjF=h<8o1+?tPw)&NQS;*gt=V=0ZBE8`a-<@* z`?Q5xi9sVEp@09V z)O^mVL;n#PuGTt)Q2z`P)VYs5G(S90+m^lNE%ip1s+i1c1yg;Y?<1~*sql``BIv~x z>-pFNne|L-rWI1jld44r@=M7i{Ci%76|YfxSr)-SfzC1&ETvMs1Omw%G+7zHM)}W> znt4ENMo+vr%~W>&$u~cMoS~7VPD|3!TS^?r+L(9{g>#UpL{)iX@2}BfNQ%cR3Ex`x z(mAC5;qP+JT5q*uRodod1J%VQr2xURgi5+kBb3>EmF<`P`?f_Lkl};top}aomFz1q zngB!)l8}v}-a4Q+Ge)0D5bG+EE@Vo`c>S!vy2XT;O0rbBTAZXKz|3hcFd(N2Kt|5K z{N$uH?+Y;TrA-<~%AJ=`=G?bNx)5t1YF=hNteVl}TeUV9Ztrs239%ThKqwww!j*cR zq@8Q@p(inx%!G{Q17(F!`N|3-IlbP}hN1L&F~ZDxn>awZ743kk-Et!Gh94|jzgyh)sozkLtzzF_?+>22~0^y205;!a@k)jBiRtgI5j%@3Ys1aDyiEhs5a!+kbZPOCx}{z z*1!mIiCF1sX2rPC+?y>3vIoh8U5tpq#j@Wu_KPZyk7V;9|1m0WsDur_aV$iAv_RVq zQCvZdGs>FCLT^B!@z6lF0W-U3HBrWlDO7I83HoSRl%jHUy*VWfDb;nU2HlP+qY}$V zqEpy0A9xhu+5_VRxh|oBnM$0)%2X08O`V&d25ksOw#yWH+h?bQqlfEdCb-DinShqr=S^p&l?%CXmkLwFgG^~d8iC>zM3R7 zXcLkxI#(V%{3q`Tpv0F;SK=foC=MHIrMWvQLKEAdR*bk%H~*xyDu!Wxm6d~f+nswp z&4JL=)#`jtD{W6K$M(eD_U}pVI_ow=#x^x3r$Ac0?522bn?lJz;0JFC6tO50YX1Ii zq$>Rm2v_?p04ODK2|rI!*_ex+g0p4Brf7r`gRG!lH$$L*2OE{YX|l*7DDt@B-$(5`qu z=>UmXoqQXM#T9u&eF*6pvXznZtWDGG|qI!sN%)nbC172uA^DwhS zcz{(*c%~ijR)Uwez|kunh86;^%zpXydV}w|l=Nz+x96a}_$rOOmpM#bv3mLC+POrx za4v4Qg({#e)gn^Mzds}=+w4YDCIx*82Uge2^$1Nu@|@`huSYnX$&zad4fU1T4X^O| zBb+=7oiv?s9_kYwX5xlJ5{3z7<9Ob)@4|`G_CFS{DX~umP^X&lwHCFo^cV=0Z?=H^ z9*|LcgKaDAe>m5k59Hbte7r1FOiUXzJYXRuBM?vKTS?>=ncLeqQPPoyL2-1D`3-BE1>>Qa|ND`T` z7+(DF+&C%!+Mbdvs#5yRv*GObx76kwlrJ^KfIBJvlhKaZuOu zrFpQ*B*+$aOFW^{lAzZ+xndx?OQwkv@$lUVZUt+U%fiAH0i?5_Zka%pD;UnJ*=TB@ z+--Ug*43_j4tI9RtucoMXkVTvVH%sbT63&l;Xw_`iUki*sU5e|WvUxt&f+3v$+0bT8Y>`iT3@RA zJLz2`ZI!88lDg}N^pnx^W5-uS#fJ$wlJe2`=%uX*M4BzmXi<|E|@;j7L{>RD=~xeFG%Sncrg>Y zCgdT2wRrCU)h|*Ey;KLI9uc@;ypmMt&5l@p2zA|g#DTR-7Ljmpx&tpiAbE8A*Qe~tEyRzHm+7U!rHIZnW8Mmum(UDeQ6(T z!D}w?C9<0->Gx|Fx}~L@+`_{#>wsLrf^FCzm#ODxZo99*q;Se&dFcN-Wm%_0UN&yS2$ zwtx{MtWN;fl+`}{mSw@{{Q>s#13pBYgU>bmZURfnG)tH_+=Lx`4Ua3kV?w`aM94(0 zDXOX{*MM2g0xUo*@lRr31mS%BW7|zz1-NVIV85YaBnrwnE%J5>2Pbqm&`jZFqrYdS zFsq1OAogbj@SKPqW)pToTY^Ozlfc_AQ!P6o{)kK5HRNj#*)K=cSfRpMo(2m0trxHb zV7>Jcyp-CDCdF{T53*hgp3aF!899OSYZrAjh~z7@_4jOvVJplR$Xfl%a+laBvu>p5 z=`p7d5uz}|yRkZMbazwst@oYo-jQ{_|0uUDOW(h|vV$fjJ<@I2ciw+i z=^f0GZqFvKs>SNf;z$S+Z&P{UszX`#4XchXzB6%^_hpZ)n&Gx)zg%^y8_cHNc|v9L z)SalI>icV5-qAQ4Pc0R+(Vb}*ZM*ZCd=IO8lNd(U0d0LO;Ap5*ts4_*P1c7E_N zu0Jqvb4l=HWu0qM_nYiBYfg%X)_0Cc!|mB8)*Rz@X3wmdblgtALTq@A`z&{0MFo(n zy9B(RL%cK)HK#e1NTR3rXVX74t?^vmFi7c^gy(f@bE=fQ!lihg%k;04HS_~psWtkz zUsrmbZsp2$shjP(8RSPdDuWy7yGX-l&g%w)_sgZy6wf zw|)Gbt?y7Iqug7$GdueebGviM%FdO|P><4RaT@U^7EFEjCyxB5n=0Zpz@9%1B{105 z5LR{Q+8hKGu8@?(R(w+;Axzi{H6%#smViA_VRv~4Kh2TrW{mlb14~_$@^%(ya!Vz z?~zfiDe$qDq^qq868926-VctXBie)YIcyz!Y3H` zwq&{~yXnDy81XI~f?_^G{H|=vgX81R_~D2Jcl!1H^1-)sf8JMyBLj=Z0s~dBVAlVZ z&mNoJf-VP(a`spsa^Y4}?!zi>qkYy^7z$hN-je`q+Gu<|Qw%+tK?a?B?E-#E-uCJ+cFG$9ZYmmhOB24m&k_ z?S`rD*=*^CL&mHkMh4WNvfi|X1b*w3J+$G-?A;qC|Ba{dLr;e;_p zv}NQ93@{`IMv@N7h0W;Ui&d+SajVwjDqHmM@oq)7{^3)hfZsg4WD2W(9X73x5b$;0 z9BeFy4!L~1^^wEcmvZv#zb3LYZe*Ju$yoilk3Lk|{Ta4%-o|M2iO%iL=07&Jd-vz( zrsJht`bc@E1pAyMXCO9iA#5XUW{Yj z>hA2Pk6l9a>4Yzwc%@nIC538wLO?6mJ z&1&MjCi}q`t{b<0E!dfn-gYm=(=L7H4`($_`*h}IWAqf3CN?? zCW1b^s#t##25~s-U9lvnoR0r*bYABG1du91qbP&->4A0_zaN34Xv^&m`YF!Ic0Jjd z-SfmHv5mWB|G3$kPdLwI*(<(W<3uvs`Ni7wewTYCC~*t@z|;1Yj2~4K5sUKK8HIt5 z#%|&~@gBtJPibVT6M0CSPtj=JT0{YRJzo+@WAW>{GVLNPJD8_>&wq)GG9Y}TZfZ!i%vmr z$k_jh2qcPZE$*9tN#lTco(7lFXH-8PTKB-{-h#?9=Rbi$5*PtFg(g1`;j~0)aD1TG zv@wx*OEZ#OWP(q zV_V2h>8Q1_NZv`)cRjM+j>-CBie>Ha*_7;#j&mpSIUyf~=@I-D;zZ?8*lph6c;V59G>lXsy2nsJjKUUmb;L5@^bx3KQzx>6wW-q*`olupZNbd)= z#T%G@dT^Hh*jDoaZ;x~2lRv5*|$3}hu;3}6VJ+*r?AXs7Z#Q_ zC0-OGvDvhS>+cG7qM;GW3h?>#p1kDO}zgzM=6IE%lK{pfpdcVEaZeEQPZo{?GQ znvcHp^b#J;{{DO1#_aL$A2MoNa{-E*LACM5>^I;4T*c!}2XWi8PyXQTl})IA*@T67 zQ+DVxN4kfzS3EOr?Bhe7S-wl3$Zxa$XC~8P?K4O4`^{(09)D7$@*saM4T|@wxFrY5 zBiW3dXU+JMy6Fp1(&oi<@W9(A{h-aKU&+?(oar9PzP~d$wAoR<$U!|ZIP{l0is_Tt zgdZ-L@n{ZxoKHbN%%Ok3pX&39p|k$=`E7Rl4=1}vvrqrf}>;(1LVHpk5ROu~_T=ShK^XXD>lAjQq(@ z+()y2|H&Z>zc4gQOA^=@R`j9Tq0Vqnm~K!TeR)~9=6;jz`HHdt1R1L?ek^;;K0%Ril!{oBuuVb()-eHWws_q$HAoNKsBonak1pZxQ~+^4fEeqMJU z-MHfCC%GZo{`vnsw|Y1P%p{Ovd{H+4Ungcy{PIukOWC8pI%48yO-Ar%(1=-VRaDT6 z3?&qNDf{!U=4RV|b8T(kBlCT#}d|Ic)R;s?hu7t)ztBZ8urrk8|BuvJ?OK zpM(%+|HqW>t-y;D4s&%xb{u3R`2BF>*6h0fxRY55|9K<7@BYtsP3ts?U=R7^RzGf6 z%MFziq%)LRmijm70hL~#zuXOG*X&(T-sYjVE!({J8u#_=@IM{HV6Xqv#M7VTJIwGu z*_LI#(7^j~@slp8T;FYp+(2Er&mT_};%_)ABG$&o!>Q(n2oZls?Hk#aKfRXmy8i3! z{I2}3v-y4Izs}?L(Eq-`J-PAi{~bAY`_6rHsd{MN%;rn}&j}X%jl6wtbKlI~_~*0v z{nVe&$C2acfHBg-1fVcyICiG#d+gyYxMHQ zomptBU;d$j*4cM&J2HFU?!#Ep!@G~*ch~N7_&w^khw^*TZ)fql?6)iKddwYp_}5)c zjW1PF`HR-BtL~56GmAqgRU7ul!EOw^#k{RDK`)-QeVXpMkADM0Di=L68wE zmHzw>1M>B_LU!CQjvobm+bM`t1B3L{T(t9tJ4#B>ZVl7c$N0`Zg_35Q0th73Z+%jRCHRF_0hv1%>1jmN+ z7R-CkGMqMEg;NXr3RKC&y1oZRrt%KZEFq6FQbE#{3LH-rN<6U-AwoC{{_2+BEMEgm z7rDaRThRXA<#l(nO!fsMbxDObwjh3QQs6d;c=7D!dPw8t^#V<3H|D)bs7FP6^a=*1 zzn-eAI@;5`BM+C?qnGOly)i0dEN}K=S@ z`wvUA{(t|*k-#e;Q8bb&VL7k2&%}Dho>hghmoqMR<&p|k> zwq)RSp*zJLKk(tuG19>B#hb{!&N0 z>8SX<1D8kc=y zVG0)B$--NUZu;L2(+pg|dWihJlgr zrCDFvg_Hme2^Oc%4$N+G(;CCOwhwicW$y)yJWWhgzC>4mMhlWyu`lGAL`&>VADxZr zFM!Dty;UZfXvcRYd2->C=q+BQTG z&t-HzPmUl_Sb@NWoK63(9m~j*qRly%^aSq0lch%u%<6CxNBjs)B)+p`kQD>3?{IU+ z6Z$Bm`jUnZ82p-B1q>c>3K$G*?Qjzg?zX$b-KmI4VBNe^{p*1{Vz=IXf8f}W{PnEo z2caH&Zv*#CL|207`bN4_!+tmLxsh(nxJ85?a2r^n#GPladU>Q9cy^>a-noIXquj#A z9t{K0i3pK=VwwEL4oToIOSwSRE)2r?;rw*<*9O@iHXZ+K;f?)t?N_VKmV(_Rb3x`Bf0J@E4|3gbN`pu=l ze~xm8xW>TP(Qf)tgHPG4^)Pj)A0JaNzx;-D_w0FnN(Z zu@WNj2L#CxV?wMVRYO7Yf$^R02MqzB2?Z=*S^Gc+s={mKB3%bO;1|WnbSbtD3`k8# z`7TXhu>(5t=RX-1fY@*bK7cGoP->KAxRDN>m=up6xMmE3c-6q|W88V}>4BYN+@U;z zmIIrEN!E*e=w-r`{-}S64q$U3xc4F|BgQT=p%ks zj)N&&uXBNw&HJ?n-QR!E{inI_rr!}q2SKyXXQ(@hy5{|xxj*!vd%jRUaCO}sH|rfd z>>2j(o!o07&CY+}deX4_;|Bh<>l|2hm@DdM zVo84{pprfM`NP}~D~)Xv;vi(^Z{+3UdHFoGSo$UXrx&1;akFq@ehB}i3EG_-9?j0Am;D%v^6L4-`$T6#2 zFmTmuH-DU3O9JSBgA?PxT&2pBtplH$?WPahc8vSSf$_85h3$(=ad0VZpW?s62fjPo QO&K`h7?(WyN3IJ04?LYNod5s; From 0bc5da4364cdc488c19e7e6735a91e3bb143a540 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 17:55:14 +0200 Subject: [PATCH 838/927] packaging/nix-make: Move component files next to their sources src//make.nix now describes how to build that component, alongside its meson.build. No change to the resulting derivations. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/flake.nix | 16 ++++++++-------- .../nix-make/libcmd.nix => src/libcmd/make.nix | 9 ++++----- .../nix-make/libexpr.nix => src/libexpr/make.nix | 15 +++++++-------- .../libfetchers.nix => src/libfetchers/make.nix | 7 +++---- .../libflake.nix => src/libflake/make.nix | 6 +++--- .../nix-make/libmain.nix => src/libmain/make.nix | 4 ++-- .../libstore.nix => src/libstore/make.nix | 8 ++++---- .../nix-make/libutil.nix => src/libutil/make.nix | 4 ++-- packaging/nix-make/nix.nix => src/nix/make.nix | 10 +++++----- 9 files changed, 38 insertions(+), 41 deletions(-) rename packaging/nix-make/libcmd.nix => src/libcmd/make.nix (70%) rename packaging/nix-make/libexpr.nix => src/libexpr/make.nix (73%) rename packaging/nix-make/libfetchers.nix => src/libfetchers/make.nix (58%) rename packaging/nix-make/libflake.nix => src/libflake/make.nix (66%) rename packaging/nix-make/libmain.nix => src/libmain/make.nix (86%) rename packaging/nix-make/libstore.nix => src/libstore/make.nix (82%) rename packaging/nix-make/libutil.nix => src/libutil/make.nix (92%) rename packaging/nix-make/nix.nix => src/nix/make.nix (85%) diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index 601e0fb16d..2b423dee92 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -23,20 +23,20 @@ nixMake = import ./lib.nix { inherit pkgs; }; components = rec { - nix-util = import ./libutil.nix { inherit nixMake; }; - nix-store = import ./libstore.nix { + nix-util = import ../../src/libutil/make.nix { inherit nixMake; }; + nix-store = import ../../src/libstore/make.nix { inherit pkgs nixMake; inherit nix-util; }; - nix-fetchers = import ./libfetchers.nix { + nix-fetchers = import ../../src/libfetchers/make.nix { inherit nixMake; inherit nix-util nix-store; }; - nix-expr = import ./libexpr.nix { + nix-expr = import ../../src/libexpr/make.nix { inherit pkgs nixMake; inherit nix-util nix-store nix-fetchers; }; - nix-flake = import ./libflake.nix { + nix-flake = import ../../src/libflake/make.nix { inherit nixMake; inherit nix-util @@ -45,11 +45,11 @@ nix-expr ; }; - nix-main = import ./libmain.nix { + nix-main = import ../../src/libmain/make.nix { inherit nixMake; inherit nix-util nix-store nix-expr; }; - nix-cmd = import ./libcmd.nix { + nix-cmd = import ../../src/libcmd/make.nix { inherit nixMake; inherit nix-util @@ -60,7 +60,7 @@ nix-main ; }; - nix = import ./nix.nix { + nix = import ../../src/nix/make.nix { inherit pkgs nixMake; inherit nix-util diff --git a/packaging/nix-make/libcmd.nix b/src/libcmd/make.nix similarity index 70% rename from packaging/nix-make/libcmd.nix rename to src/libcmd/make.nix index 5842bd0b01..97eca2b4be 100644 --- a/packaging/nix-make/libcmd.nix +++ b/src/libcmd/make.nix @@ -1,4 +1,4 @@ -# libnixcmd, transcribed from src/libcmd/meson.build. +# libnixcmd, transcribed from meson.build. { nixMake, nix-util, @@ -22,7 +22,7 @@ nixMake.mkComponent { nix-main ]; - root = ../../src/libcmd; + root = ./.; # unix/unix-socket-server.cc is a plain source file; there is no windows/ directory. includeDirs = [ @@ -31,9 +31,8 @@ nixMake.mkComponent { ]; files = nixMake.commonSupportFiles // { - "call-flake-schemas.nix.gen.hh" = nixMake.mkStringHeader ../../src/libcmd/call-flake-schemas.nix; - "builtin-flake-schemas.nix.gen.hh" = - nixMake.mkStringHeader ../../src/libcmd/builtin-flake-schemas.nix; + "call-flake-schemas.nix.gen.hh" = nixMake.mkStringHeader ./call-flake-schemas.nix; + "builtin-flake-schemas.nix.gen.hh" = nixMake.mkStringHeader ./builtin-flake-schemas.nix; }; configHeaders = { diff --git a/packaging/nix-make/libexpr.nix b/src/libexpr/make.nix similarity index 73% rename from packaging/nix-make/libexpr.nix rename to src/libexpr/make.nix index 640330e7c4..d7d80e1bcd 100644 --- a/packaging/nix-make/libexpr.nix +++ b/src/libexpr/make.nix @@ -1,4 +1,4 @@ -# libnixexpr, transcribed from src/libexpr/meson.build and its subdirectories. +# libnixexpr, transcribed from meson.build and its subdirectories. { pkgs, nixMake, @@ -14,12 +14,12 @@ let # time. parserTab = pkgs.runCommand "parser-tab" { nativeBuildInputs = [ pkgs.bison ]; } '' mkdir $out - bison -v -o $out/parser-tab.cc -d ${../../src/libexpr/parser.y} + bison -v -o $out/parser-tab.cc -d ${./parser.y} ''; lexerTab = pkgs.runCommand "lexer-tab" { nativeBuildInputs = [ pkgs.flex ]; } '' mkdir $out - flex -Cf --outfile $out/lexer-tab.cc --header-file=$out/lexer-tab.hh ${../../src/libexpr/lexer.l} + flex -Cf --outfile $out/lexer-tab.cc --header-file=$out/lexer-tab.hh ${./lexer.l} ''; in @@ -33,7 +33,7 @@ nixMake.mkComponent { nix-fetchers ]; - root = ../../src/libexpr; + root = ./.; includeDirs = [ "" @@ -41,10 +41,9 @@ nixMake.mkComponent { ]; files = nixMake.commonSupportFiles // { - "imported-drv-to-derivation.nix.gen.hh" = - nixMake.mkStringHeader ../../src/libexpr/imported-drv-to-derivation.nix; - "fetchurl.nix.gen.hh" = nixMake.mkStringHeader ../../src/libexpr/fetchurl.nix; - "primops/derivation.nix.gen.hh" = nixMake.mkStringHeader ../../src/libexpr/primops/derivation.nix; + "imported-drv-to-derivation.nix.gen.hh" = nixMake.mkStringHeader ./imported-drv-to-derivation.nix; + "fetchurl.nix.gen.hh" = nixMake.mkStringHeader ./fetchurl.nix; + "primops/derivation.nix.gen.hh" = nixMake.mkStringHeader ./primops/derivation.nix; }; generated = { diff --git a/packaging/nix-make/libfetchers.nix b/src/libfetchers/make.nix similarity index 58% rename from packaging/nix-make/libfetchers.nix rename to src/libfetchers/make.nix index b036607b27..f6d28c791e 100644 --- a/packaging/nix-make/libfetchers.nix +++ b/src/libfetchers/make.nix @@ -1,4 +1,4 @@ -# libnixfetchers, transcribed from src/libfetchers/meson.build. +# libnixfetchers, transcribed from meson.build. { nixMake, nix-util, @@ -14,7 +14,7 @@ nixMake.mkComponent { nix-store ]; - root = ../../src/libfetchers; + root = ./.; includeDirs = [ "" @@ -22,8 +22,7 @@ nixMake.mkComponent { ]; files = nixMake.commonSupportFiles // { - "builtin-flake-registry.json.gen.hh" = - nixMake.mkStringHeader ../../src/libfetchers/builtin-flake-registry.json; + "builtin-flake-registry.json.gen.hh" = nixMake.mkStringHeader ./builtin-flake-registry.json; }; linkFlags = [ "-Wl,--wrap=__assert_fail" ]; diff --git a/packaging/nix-make/libflake.nix b/src/libflake/make.nix similarity index 66% rename from packaging/nix-make/libflake.nix rename to src/libflake/make.nix index 60b2a76583..724109304c 100644 --- a/packaging/nix-make/libflake.nix +++ b/src/libflake/make.nix @@ -1,4 +1,4 @@ -# libnixflake, transcribed from src/libflake/meson.build. +# libnixflake, transcribed from meson.build. { nixMake, nix-util, @@ -18,7 +18,7 @@ nixMake.mkComponent { nix-expr ]; - root = ../../src/libflake; + root = ./.; includeDirs = [ "" @@ -26,7 +26,7 @@ nixMake.mkComponent { ]; files = nixMake.commonSupportFiles // { - "call-flake.nix.gen.hh" = nixMake.mkStringHeader ../../src/libflake/call-flake.nix; + "call-flake.nix.gen.hh" = nixMake.mkStringHeader ./call-flake.nix; }; linkFlags = [ "-Wl,--wrap=__assert_fail" ]; diff --git a/packaging/nix-make/libmain.nix b/src/libmain/make.nix similarity index 86% rename from packaging/nix-make/libmain.nix rename to src/libmain/make.nix index 60dc1865f2..1b110df519 100644 --- a/packaging/nix-make/libmain.nix +++ b/src/libmain/make.nix @@ -1,4 +1,4 @@ -# libnixmain, transcribed from src/libmain/meson.build. +# libnixmain, transcribed from meson.build. { nixMake, nix-util, @@ -17,7 +17,7 @@ nixMake.mkComponent { nix-expr ]; - root = ../../src/libmain; + root = ./.; # unix/stack.cc is a plain source file; there is no windows/ directory. includeDirs = [ diff --git a/packaging/nix-make/libstore.nix b/src/libstore/make.nix similarity index 82% rename from packaging/nix-make/libstore.nix rename to src/libstore/make.nix index 893df7ccbe..6eab615226 100644 --- a/packaging/nix-make/libstore.nix +++ b/src/libstore/make.nix @@ -1,4 +1,4 @@ -# libnixstore, transcribed from src/libstore/meson.build and its subdirectories. +# libnixstore, transcribed from meson.build and its subdirectories. { pkgs, nixMake, @@ -15,7 +15,7 @@ nixMake.mkComponent { deps = [ nix-util ]; - root = ../../src/libstore; + root = ./.; excludeSources = [ "windows" @@ -35,8 +35,8 @@ nixMake.mkComponent { ]; files = nixMake.commonSupportFiles // { - "schema.sql.gen.hh" = nixMake.mkStringHeader ../../src/libstore/schema.sql; - "ca-specific-schema.sql.gen.hh" = nixMake.mkStringHeader ../../src/libstore/ca-specific-schema.sql; + "schema.sql.gen.hh" = nixMake.mkStringHeader ./schema.sql; + "ca-specific-schema.sql.gen.hh" = nixMake.mkStringHeader ./ca-specific-schema.sql; }; configHeaders = { diff --git a/packaging/nix-make/libutil.nix b/src/libutil/make.nix similarity index 92% rename from packaging/nix-make/libutil.nix rename to src/libutil/make.nix index 1138001aa1..726652e3a3 100644 --- a/packaging/nix-make/libutil.nix +++ b/src/libutil/make.nix @@ -1,11 +1,11 @@ -# libnixutil, transcribed from src/libutil/meson.build and its subdirectories. +# libnixutil, transcribed from meson.build and its subdirectories. { nixMake }: nixMake.mkComponent { name = "nix-util"; libName = "nixutil"; - root = ../../src/libutil; + root = ./.; # Every .cc file under the root is a compilation unit, except these. excludeSources = [ diff --git a/packaging/nix-make/nix.nix b/src/nix/make.nix similarity index 85% rename from packaging/nix-make/nix.nix rename to src/nix/make.nix index 62f8275eaf..de431fcceb 100644 --- a/packaging/nix-make/nix.nix +++ b/src/nix/make.nix @@ -1,4 +1,4 @@ -# The `nix` executable, transcribed from src/nix/meson.build. +# The `nix` executable, transcribed from meson.build. { pkgs, nixMake, @@ -29,7 +29,7 @@ nixMake.mkComponent { nix-cmd ]; - root = ../../src/nix; + root = ./.; # Every .cc file under the root (including the legacy commands in # subdirectories and unix/) is a compilation unit. The `doc`, `misc` @@ -41,11 +41,11 @@ nixMake.mkComponent { "generate-settings.nix.gen.hh" = nixMake.mkStringHeader ../../doc/manual/generate-settings.nix; "generate-store-info.nix.gen.hh" = nixMake.mkStringHeader ../../doc/manual/generate-store-info.nix; "utils.nix.gen.hh" = nixMake.mkStringHeader ../../doc/manual/utils.nix; - "get-env.sh.gen.hh" = nixMake.mkStringHeader ../../src/nix/get-env.sh; + "get-env.sh.gen.hh" = nixMake.mkStringHeader ./get-env.sh; "help-stores.md.gen.hh" = nixMake.mkStringHeader ../../doc/manual/source/store/types/index.md.in; "profiles.md.gen.hh" = nixMake.mkStringHeader ../../doc/manual/source/command-ref/files/profiles.md; - "unpack-channel.nix.gen.hh" = nixMake.mkStringHeader ../../src/nix/nix-channel/unpack-channel.nix; - "buildenv.nix.gen.hh" = nixMake.mkStringHeader ../../src/nix/nix-env/buildenv.nix; + "unpack-channel.nix.gen.hh" = nixMake.mkStringHeader ./nix-channel/unpack-channel.nix; + "buildenv.nix.gen.hh" = nixMake.mkStringHeader ./nix-env/buildenv.nix; }; configHeaders = { From 4974beeaadbfd7016c6fea8f54006642de229f26 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 20:30:05 +0200 Subject: [PATCH 839/927] packaging/nix-make: Add build variants config.nix declares the build configuration options (`optimize`, `debug`) as a NixOS-style module, so that more can be added later. `makeNixVariant` in flake.nix evaluates a configuration module and builds all components with it. The variants follow Meson's build types: make.release -O3 make.debugoptimized -O2 -g make.debug -O0 -g (with _FORTIFY_SOURCE disabled, since it warns when not optimizing) packages.x86_64-linux.default is make.release.nix. The `lib.scan` debugging output is gone; the same data is available as make...units. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/config.nix | 20 +++++ packaging/nix-make/flake.nix | 136 ++++++++++++++++++++-------------- packaging/nix-make/lib.nix | 23 +++--- 3 files changed, 114 insertions(+), 65 deletions(-) create mode 100644 packaging/nix-make/config.nix diff --git a/packaging/nix-make/config.nix b/packaging/nix-make/config.nix new file mode 100644 index 0000000000..a5eb845ca7 --- /dev/null +++ b/packaging/nix-make/config.nix @@ -0,0 +1,20 @@ +# The build configuration options, as a NixOS-style module. A build +# variant is defined by evaluating this module together with a module +# setting the options, see `makeNixVariant` in flake.nix. +{ lib, ... }: + +{ + options = { + optimize = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Whether to compile with optimization."; + }; + + debug = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Whether to include debug information."; + }; + }; +} diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index 2b423dee92..9e8b81bf30 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -20,66 +20,90 @@ overlays = [ nix.overlays.internal ]; }; - nixMake = import ./lib.nix { inherit pkgs; }; + # Build all components with the configuration given by `cfg`, a module + # setting the options declared in config.nix. + makeNixVariant = + cfg: + let + config = + (nixpkgs.lib.evalModules { + modules = [ + ./config.nix + cfg + ]; + }).config; - components = rec { - nix-util = import ../../src/libutil/make.nix { inherit nixMake; }; - nix-store = import ../../src/libstore/make.nix { - inherit pkgs nixMake; - inherit nix-util; + nixMake = import ./lib.nix { inherit pkgs config; }; + in + rec { + nix-util = import ../../src/libutil/make.nix { inherit nixMake; }; + nix-store = import ../../src/libstore/make.nix { + inherit pkgs nixMake; + inherit nix-util; + }; + nix-fetchers = import ../../src/libfetchers/make.nix { + inherit nixMake; + inherit nix-util nix-store; + }; + nix-expr = import ../../src/libexpr/make.nix { + inherit pkgs nixMake; + inherit nix-util nix-store nix-fetchers; + }; + nix-flake = import ../../src/libflake/make.nix { + inherit nixMake; + inherit + nix-util + nix-store + nix-fetchers + nix-expr + ; + }; + nix-main = import ../../src/libmain/make.nix { + inherit nixMake; + inherit nix-util nix-store nix-expr; + }; + nix-cmd = import ../../src/libcmd/make.nix { + inherit nixMake; + inherit + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + ; + }; + nix = import ../../src/nix/make.nix { + inherit pkgs nixMake; + inherit + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nix-cmd + ; + }; }; - nix-fetchers = import ../../src/libfetchers/make.nix { - inherit nixMake; - inherit nix-util nix-store; - }; - nix-expr = import ../../src/libexpr/make.nix { - inherit pkgs nixMake; - inherit nix-util nix-store nix-fetchers; - }; - nix-flake = import ../../src/libflake/make.nix { - inherit nixMake; - inherit - nix-util - nix-store - nix-fetchers - nix-expr - ; - }; - nix-main = import ../../src/libmain/make.nix { - inherit nixMake; - inherit nix-util nix-store nix-expr; - }; - nix-cmd = import ../../src/libcmd/make.nix { - inherit nixMake; - inherit - nix-util - nix-store - nix-fetchers - nix-expr - nix-flake - nix-main - ; - }; - nix = import ../../src/nix/make.nix { - inherit pkgs nixMake; - inherit - nix-util - nix-store - nix-fetchers - nix-expr - nix-flake - nix-main - nix-cmd - ; - }; - }; in - { - packages.${system} = components // { - default = components.nix; + rec { + # The build variants, named after Meson's build types. + make.release = makeNixVariant { + optimize = true; + debug = false; + }; + make.debugoptimized = makeNixVariant { + optimize = true; + debug = true; + }; + make.debug = makeNixVariant { + optimize = false; + debug = true; }; - # The raw scanner output for each component, for debugging. - lib.${system}.scan = pkgs.lib.mapAttrs (_: c: c.units) components; + packages.${system} = make.release // { + default = make.release.nix; + }; }; } diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 2ca2cd364e..550d24d19e 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -1,6 +1,6 @@ # Generic machinery for building C++ components with one derivation per # compilation unit. -{ pkgs }: +{ pkgs, config }: let inherit (pkgs) lib; @@ -17,8 +17,11 @@ let # feature (and `eval-cores`); without it, evaluation is sequential. parallel = builtins.parallel or (xs: x: x); - # Non-optimized for now, for faster iteration. - optimizationFlags = [ "-O0" ]; + # Optimization and debug flags, following Meson's build types: `release` + # is `-O3`, `debugoptimized` is `-O2 -g`, and `debug` is `-O0 -g`. + optimizationFlags = + if config.optimize then [ (if config.debug then "-O2" else "-O3") ] else [ "-O0" ]; + debugFlags = lib.optional config.debug "-g"; # From nix-meson-build-support/common/meson.build (GCC-supported subset). warningFlags = [ @@ -48,6 +51,7 @@ let "-fno-semantic-interposition" ] ++ optimizationFlags + ++ debugFlags ++ warningFlags; commonLinkLibs = [ @@ -120,13 +124,14 @@ let // attrs ); - # What `stdenv.mkDerivation` would set for `hardeningDisable = [ "fortify" ]`: - # `_FORTIFY_SOURCE` warns on every unit when not optimizing. - hardeningWithoutFortify = lib.concatStringsSep " " ( - lib.subtractLists [ + # The stdenv's default hardening flags, except that `_FORTIFY_SOURCE` + # warns on every unit when not optimizing, so it is disabled then (like + # `hardeningDisable = [ "fortify" ]` in `stdenv.mkDerivation`). + hardeningFlags = lib.concatStringsSep " " ( + lib.subtractLists (lib.optionals (!config.optimize) [ "fortify" "fortify3" - ] pkgs.stdenv.cc.defaultHardeningFlags + ]) pkgs.stdenv.cc.defaultHardeningFlags ); # Deduplicate strings in O(n log n) rather than `lib.unique`'s O(n^2). @@ -181,7 +186,7 @@ let pkgConfigDeps = lib.concatMap (d: d.pkgconfig or [ ]) deps; buildInputs = depPackages deps; nativeBuildInputs = [ pkgs.pkg-config ]; - NIX_HARDENING_ENABLE = hardeningWithoutFortify; + NIX_HARDENING_ENABLE = hardeningFlags; } '' mkdir tree From 5855dae96841fbe8b9700676e1284ef9915f097f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Sep 2026 20:35:20 +0200 Subject: [PATCH 840/927] packaging/nix-make: Build the parser without assertions in release The parser units (parser-tab.cc, lexer-tab.cc, lexer-helpers.cc) now get -DNDEBUG and higher GCC inlining limits in non-debug variants, matching the flags of the Meson build's parser library. mkComponent gains `unitCxxFlags` for per-unit compiler flags, and nixMake exposes `config` so that component files can vary with the build variant. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/lib.nix | 11 ++++++++++- src/libexpr/make.nix | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 550d24d19e..dc29ccedb3 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -182,7 +182,8 @@ let ); srcPath = unit.path; includeDirs = component.includeDirs ++ component.depIncludeDirs; - cxxFlags = commonCxxFlags ++ component.extraCxxFlags; + cxxFlags = + commonCxxFlags ++ component.extraCxxFlags ++ (component.unitCxxFlags.${unit.path} or [ ]); pkgConfigDeps = lib.concatMap (d: d.pkgconfig or [ ]) deps; buildInputs = depPackages deps; nativeBuildInputs = [ pkgs.pkg-config ]; @@ -320,6 +321,8 @@ let can be scanned without building it), and `path` is the generated file in the output of a derivation. - `extraCxxFlags`, `linkFlags`, `extraLinkLibs`: what they say. + - `unitCxxFlags`: extra compiler flags for specific units, as an + attribute set from path in the root namespace to a list of flags. External dependencies (compile flags and libraries) are derived from the `#include`s of the units, via `deps.nix`. @@ -351,6 +354,7 @@ let defines ? { }, undefines ? [ ], extraCxxFlags ? [ ], + unitCxxFlags ? { }, linkFlags ? [ ], extraLinkLibs ? [ ], }@args: @@ -466,6 +470,7 @@ let depIncludeDirs publicIncludeDirs extraCxxFlags + unitCxxFlags linkFlags extraLinkLibs ; @@ -507,6 +512,10 @@ let in { + # The build configuration (see config.nix), for component files that + # need to vary with it. + inherit config; + inherit getDeps mkConfigHeader diff --git a/src/libexpr/make.nix b/src/libexpr/make.nix index d7d80e1bcd..b0f734c662 100644 --- a/src/libexpr/make.nix +++ b/src/libexpr/make.nix @@ -76,5 +76,23 @@ nixMake.mkComponent { }; }; + # Build the parser without assertions and with a higher inlining + # threshold in non-debug builds: Bison/Flex are particularly sensitive + # to the compiler failing to inline. + unitCxxFlags = + let + parserFlags = pkgs.lib.optionals (!nixMake.config.debug) [ + "-DNDEBUG" + "--param=max-inline-insns-single=1000" + "--param=max-inline-insns-auto=1000" + "--param=inline-unit-growth=400" + ]; + in + { + "parser-tab.cc" = parserFlags; + "lexer-tab.cc" = parserFlags; + "lexer-helpers.cc" = parserFlags; + }; + linkFlags = [ "-Wl,--wrap=__assert_fail" ]; } From 505125d0e27b950e237f8893036d15b0c003706c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 00:12:34 +0200 Subject: [PATCH 841/927] packaging/nix-make: Add Sentry, OpenTelemetry and AWS auth options Three new build configuration options, all enabled by default: * `sentry`: crash reporting in the nix executable, with the crashpad handler from sentry-native. * `otel`: distributed tracing via OpenTelemetry in the nix executable, linked through pkg-config's opentelemetry_trace. * `awsAuth`: AWS authentication for S3 binary caches in libstore. The AWS CRT and its C libraries have no pkg-config files, so the dependency table entry carries the defines their CMake targets would set and an explicit library list; entries in deps.nix can now supply `cflags`. When an option is off, the scanner's conditional evaluation keeps the corresponding #includes out, so the dependency is not pulled in. This closes the remaining functional gaps with the Meson build, except the embedded plugin C API and LTO. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/config.nix | 18 ++++++++++ packaging/nix-make/deps.nix | 62 +++++++++++++++++++++++++++++++++++ packaging/nix-make/lib.nix | 5 ++- src/libstore/make.nix | 7 ++-- src/nix/make.nix | 10 ++++-- 5 files changed, 94 insertions(+), 8 deletions(-) diff --git a/packaging/nix-make/config.nix b/packaging/nix-make/config.nix index a5eb845ca7..a1c6280791 100644 --- a/packaging/nix-make/config.nix +++ b/packaging/nix-make/config.nix @@ -16,5 +16,23 @@ default = false; description = "Whether to include debug information."; }; + + sentry = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Whether to enable Sentry crash reporting in the `nix` executable."; + }; + + otel = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Whether to enable distributed tracing via OpenTelemetry in the `nix` executable."; + }; + + awsAuth = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Whether to support AWS authentication for S3 binary caches."; + }; }; } diff --git a/packaging/nix-make/deps.nix b/packaging/nix-make/deps.nix index 41e6d246e3..a66604cc29 100644 --- a/packaging/nix-make/deps.nix +++ b/packaging/nix-make/deps.nix @@ -5,6 +5,7 @@ # - `pkg`: the package(s) providing the headers and libraries. # - `pkgconfig`: pkg-config names used for compile and link flags. # - `libs`: extra linker flags for libraries without pkg-config files. +# - `cflags`: extra compiler flags for libraries without pkg-config files. { pkgs }: let @@ -128,6 +129,67 @@ in prefix = "toml11/"; pkg = deps.toml11; } + { + # The AWS CRT and its C libraries have no pkg-config files; the + # defines are what their CMake targets would set. + prefix = "aws/"; + pkg = with deps; [ + aws-crt-cpp + aws-c-auth + aws-c-cal + aws-c-common + aws-c-compression + aws-c-event-stream + aws-checksums + aws-c-http + aws-c-io + aws-c-mqtt + aws-c-s3 + aws-c-sdkutils + s2n-tls + ]; + cflags = [ + "-DAWS_ENABLE_EPOLL" + "-DAWS_AUTH_USE_IMPORT_EXPORT" + "-DAWS_CAL_USE_IMPORT_EXPORT" + "-DAWS_CHECKSUMS_USE_IMPORT_EXPORT" + "-DAWS_COMMON_USE_IMPORT_EXPORT" + "-DAWS_COMPRESSION_USE_IMPORT_EXPORT" + "-DAWS_CRT_CPP_USE_IMPORT_EXPORT" + "-DAWS_EVENT_STREAM_USE_IMPORT_EXPORT" + "-DAWS_HTTP_USE_IMPORT_EXPORT" + "-DAWS_IO_USE_IMPORT_EXPORT" + "-DAWS_MQTT_USE_IMPORT_EXPORT" + "-DAWS_S3_USE_IMPORT_EXPORT" + "-DAWS_SDKUTILS_USE_IMPORT_EXPORT" + ]; + libs = [ + "-laws-crt-cpp" + "-laws-c-auth" + "-laws-c-cal" + "-laws-c-common" + "-laws-c-compression" + "-laws-c-event-stream" + "-laws-checksums" + "-laws-c-http" + "-laws-c-io" + "-laws-c-mqtt" + "-laws-c-s3" + "-laws-c-sdkutils" + "-ls2n" + ]; + } + { + prefix = "opentelemetry/"; + pkg = deps.opentelemetry-cpp; + pkgconfig = [ "opentelemetry_trace" ]; + } + { + # No pkg-config file. + prefix = "sentry.h"; + pkg = deps.sentry-native; + libs = [ "-lsentry" ]; + } { prefix = "microhttpd.h"; pkg = deps.libmicrohttpd; diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index dc29ccedb3..685b531e14 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -183,7 +183,10 @@ let srcPath = unit.path; includeDirs = component.includeDirs ++ component.depIncludeDirs; cxxFlags = - commonCxxFlags ++ component.extraCxxFlags ++ (component.unitCxxFlags.${unit.path} or [ ]); + commonCxxFlags + ++ component.extraCxxFlags + ++ (component.unitCxxFlags.${unit.path} or [ ]) + ++ lib.concatMap (d: d.cflags or [ ]) deps; pkgConfigDeps = lib.concatMap (d: d.pkgconfig or [ ]) deps; buildInputs = depPackages deps; nativeBuildInputs = [ pkgs.pkg-config ]; diff --git a/src/libstore/make.nix b/src/libstore/make.nix index 6eab615226..7c3ca4bb1b 100644 --- a/src/libstore/make.nix +++ b/src/libstore/make.nix @@ -21,9 +21,8 @@ nixMake.mkComponent { "windows" "freebsd" "darwin" - # Requires AWS CRT; disabled for now (see NIX_WITH_AWS_AUTH below). - "aws-creds.cc" - ]; + ] + ++ lib.optional (!nixMake.config.awsAuth) "aws-creds.cc"; includeDirs = [ "" @@ -43,7 +42,7 @@ nixMake.mkComponent { "include/nix/store/config.hh" = { NIX_LOCAL_SYSTEM = "x86_64-linux"; NIX_SUPPORT_ACL = 1; - NIX_WITH_AWS_AUTH = 0; + NIX_WITH_AWS_AUTH = nixMake.config.awsAuth; }; "store-config-private.hh" = { CAN_LINK_SYMLINK = 1; diff --git a/src/nix/make.nix b/src/nix/make.nix index de431fcceb..d2dd13afc8 100644 --- a/src/nix/make.nix +++ b/src/nix/make.nix @@ -13,6 +13,8 @@ let inherit (pkgs) lib; + # The overridden dependencies used by the Meson build, layered over Nixpkgs. + nixDeps = pkgs // pkgs.nixDependencies2; in nixMake.mkComponent { @@ -55,9 +57,11 @@ nixMake.mkComponent { NIX_BIN_DIR = "/nix/var/nix/profiles/default/bin"; NIX_MAN_DIR = "/nix/var/nix/profiles/default/share/man"; HAVE_MIMALLOC = 1; - # Crash reporting and tracing are disabled for now. - HAVE_SENTRY = 0; - HAVE_OTEL = 0; + HAVE_SENTRY = nixMake.config.sentry; + HAVE_OTEL = nixMake.config.otel; + } + // lib.optionalAttrs nixMake.config.sentry { + CRASHPAD_HANDLER_PATH = "${nixDeps.sentry-native}/bin/crashpad_handler"; }; }; From 6985f94fb70fb88e80916d7e6ba866d92a290bfc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 00:21:42 +0200 Subject: [PATCH 842/927] packaging/nix-make: Prefix component names with "determinate-" The derivation names now match the pnames in the corresponding package.nix files (determinate-nix-util, ..., determinate-nix). The flake attributes and the executable name are unchanged. Assisted-by: Claude Fable 5.1 --- src/libcmd/make.nix | 2 +- src/libexpr/make.nix | 2 +- src/libfetchers/make.nix | 2 +- src/libflake/make.nix | 2 +- src/libmain/make.nix | 2 +- src/libstore/make.nix | 2 +- src/libutil/make.nix | 2 +- src/nix/make.nix | 3 ++- 8 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/libcmd/make.nix b/src/libcmd/make.nix index 97eca2b4be..eb55ff988b 100644 --- a/src/libcmd/make.nix +++ b/src/libcmd/make.nix @@ -10,7 +10,7 @@ }: nixMake.mkComponent { - name = "nix-cmd"; + name = "determinate-nix-cmd"; libName = "nixcmd"; deps = [ diff --git a/src/libexpr/make.nix b/src/libexpr/make.nix index b0f734c662..3af9d5f8dc 100644 --- a/src/libexpr/make.nix +++ b/src/libexpr/make.nix @@ -24,7 +24,7 @@ let in nixMake.mkComponent { - name = "nix-expr"; + name = "determinate-nix-expr"; libName = "nixexpr"; deps = [ diff --git a/src/libfetchers/make.nix b/src/libfetchers/make.nix index f6d28c791e..af4c157238 100644 --- a/src/libfetchers/make.nix +++ b/src/libfetchers/make.nix @@ -6,7 +6,7 @@ }: nixMake.mkComponent { - name = "nix-fetchers"; + name = "determinate-nix-fetchers"; libName = "nixfetchers"; deps = [ diff --git a/src/libflake/make.nix b/src/libflake/make.nix index 724109304c..f35007865d 100644 --- a/src/libflake/make.nix +++ b/src/libflake/make.nix @@ -8,7 +8,7 @@ }: nixMake.mkComponent { - name = "nix-flake"; + name = "determinate-nix-flake"; libName = "nixflake"; deps = [ diff --git a/src/libmain/make.nix b/src/libmain/make.nix index 1b110df519..dee739ddd4 100644 --- a/src/libmain/make.nix +++ b/src/libmain/make.nix @@ -7,7 +7,7 @@ }: nixMake.mkComponent { - name = "nix-main"; + name = "determinate-nix-main"; libName = "nixmain"; deps = [ diff --git a/src/libstore/make.nix b/src/libstore/make.nix index 7c3ca4bb1b..88352c857e 100644 --- a/src/libstore/make.nix +++ b/src/libstore/make.nix @@ -10,7 +10,7 @@ let in nixMake.mkComponent { - name = "nix-store"; + name = "determinate-nix-store"; libName = "nixstore"; deps = [ nix-util ]; diff --git a/src/libutil/make.nix b/src/libutil/make.nix index 726652e3a3..b10a3f143f 100644 --- a/src/libutil/make.nix +++ b/src/libutil/make.nix @@ -2,7 +2,7 @@ { nixMake }: nixMake.mkComponent { - name = "nix-util"; + name = "determinate-nix-util"; libName = "nixutil"; root = ./.; diff --git a/src/nix/make.nix b/src/nix/make.nix index d2dd13afc8..44ddc733fb 100644 --- a/src/nix/make.nix +++ b/src/nix/make.nix @@ -18,8 +18,9 @@ let in nixMake.mkComponent { - name = "nix"; + name = "determinate-nix"; type = "executable"; + exeName = "nix"; deps = [ nix-util From 82dda8099db3152f16083fc1b51a950b16538feb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 00:23:55 +0200 Subject: [PATCH 843/927] packaging/nix-make: Include the version in derivation names Derivations are now named -, e.g. determinate-nix-cmd-3.22.5, like the package.nix builds. The version comes from .version-determinate; mkComponent takes an optional `version` to override it. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/lib.nix | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 685b531e14..5101ceb922 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -12,6 +12,9 @@ let externalDeps = import ./deps.nix { inherit pkgs; }; + # The version, as in the package.nix files. + defaultVersion = lib.fileContents ../../.version-determinate; + # `builtins.parallel xs x` starts evaluating the values `xs` on other # threads and returns `x`. It requires the `parallel-eval` experimental # feature (and `eval-cores`); without it, evaluation is sequential. @@ -225,7 +228,7 @@ let in mkLeanDerivation { - inherit (component) name; + name = "${component.name}-${component.version}"; # Instantiate the objects and the dependencies in parallel. objects = parallel (map (d: d.drvPath) component.allDeps ++ map (o: o.drvPath) objects) objects; inherit (component) @@ -289,7 +292,9 @@ let /** Build a component: a shared library (the default) or an executable. - - `name`: derivation name (e.g. `nix-util`). + - `name`: component name (e.g. `determinate-nix-util`); the derivation is + named `-`. + - `version`: defaults to the contents of `.version-determinate`. - `type`: `"library"` or `"executable"`. - `libName`: library name without `lib` prefix (e.g. `nixutil`). - `exeName`: executable name; defaults to `name`. @@ -333,6 +338,7 @@ let mkComponent = { name, + version ? defaultVersion, type ? "library", libName ? null, exeName ? name, @@ -455,6 +461,7 @@ let component = args // { inherit allExternalIncludes externalDepsFor; inherit + version type libName exeName From c72daf9b1f2028eea4bbbf97d9cd020ee12f0b0c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 00:55:02 +0200 Subject: [PATCH 844/927] packaging/nix-make: Provide a flake schema for the `make` output So that `nix flake show` and `nix flake check` understand it: each build variant is shown with its components as packages. Defining `schemas` replaces the built-in schemas, so re-export the ones for the other outputs of this flake (`packages` and `schemas`). To that end, move the built-in schemas to src/libcmd/builtin-flake-schemas/flake.nix, so that they can be used as a regular relative-path flake input. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/flake.lock | 12 +++++++ packaging/nix-make/flake.nix | 32 +++++++++++++++++++ .../flake.nix} | 0 src/libcmd/flake-schemas.cc | 2 +- src/libcmd/make.nix | 2 +- src/libcmd/meson.build | 5 ++- src/libcmd/package.nix | 2 +- 7 files changed, 51 insertions(+), 4 deletions(-) rename src/libcmd/{builtin-flake-schemas.nix => builtin-flake-schemas/flake.nix} (100%) diff --git a/packaging/nix-make/flake.lock b/packaging/nix-make/flake.lock index 8bf1e31120..017d68f335 100644 --- a/packaging/nix-make/flake.lock +++ b/packaging/nix-make/flake.lock @@ -36,6 +36,17 @@ "url": "https://flakehub.com/f/hercules-ci/flake-parts/0.1" } }, + "flake-schemas": { + "locked": { + "path": "../../src/libcmd/builtin-flake-schemas", + "type": "path" + }, + "original": { + "path": "../../src/libcmd/builtin-flake-schemas", + "type": "path" + }, + "parent": [] + }, "git-hooks-nix": { "inputs": { "flake-compat": "flake-compat", @@ -123,6 +134,7 @@ }, "root": { "inputs": { + "flake-schemas": "flake-schemas", "nix": "nix", "nixpkgs": [ "nix", diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index 9e8b81bf30..1d92d96d6e 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -4,12 +4,15 @@ { inputs.nix.url = "../.."; inputs.nixpkgs.follows = "nix/nixpkgs"; + # The built-in flake schemas that ship with Nix. + inputs.flake-schemas.url = "../../src/libcmd/builtin-flake-schemas"; outputs = { self, nix, nixpkgs, + flake-schemas, }: let system = "x86_64-linux"; @@ -105,5 +108,34 @@ packages.${system} = make.release // { default = make.release.nix; }; + + # Describe the `make` output to `nix flake show` and `nix flake check`. + # Defining `schemas` replaces the built-in ones, so re-export the ones + # for the other outputs of this flake. + schemas = { + inherit (flake-schemas.schemas) packages schemas; + + make = { + version = 1; + doc = '' + The `make` output provides the Nix components (libraries and the + `nix` executable) built with Nix as the build system, one attribute + set of components per build variant (`release`, `debugoptimized`, + `debug`). + ''; + inventory = + output: + flake-schemas.lib.mkChildren ( + builtins.mapAttrs (variant: components: { + shortDescription = "The `${variant}` build variant"; + children = builtins.mapAttrs (name: package: { + what = "package"; + forSystems = [ package.system ]; + derivationAttrPath = [ ]; + }) components; + }) output + ); + }; + }; }; } diff --git a/src/libcmd/builtin-flake-schemas.nix b/src/libcmd/builtin-flake-schemas/flake.nix similarity index 100% rename from src/libcmd/builtin-flake-schemas.nix rename to src/libcmd/builtin-flake-schemas/flake.nix diff --git a/src/libcmd/flake-schemas.cc b/src/libcmd/flake-schemas.cc index e0d01fe1e9..6725274137 100644 --- a/src/libcmd/flake-schemas.cc +++ b/src/libcmd/flake-schemas.cc @@ -22,7 +22,7 @@ static LockedFlake getBuiltinDefaultSchemasFlake(EvalState & state) accessor->addFile( CanonPath("flake.nix"), -#include "builtin-flake-schemas.nix.gen.hh" +#include "builtin-flake-schemas/flake.nix.gen.hh" ); auto [storePath, narHash] = state.store->computeStorePath("source", {accessor}); diff --git a/src/libcmd/make.nix b/src/libcmd/make.nix index eb55ff988b..2b36eb7953 100644 --- a/src/libcmd/make.nix +++ b/src/libcmd/make.nix @@ -32,7 +32,7 @@ nixMake.mkComponent { files = nixMake.commonSupportFiles // { "call-flake-schemas.nix.gen.hh" = nixMake.mkStringHeader ./call-flake-schemas.nix; - "builtin-flake-schemas.nix.gen.hh" = nixMake.mkStringHeader ./builtin-flake-schemas.nix; + "builtin-flake-schemas/flake.nix.gen.hh" = nixMake.mkStringHeader ./builtin-flake-schemas/flake.nix; }; configHeaders = { diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 680b8136dd..15567ae77c 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -101,7 +101,10 @@ endif sources += [ gen_header.process('call-flake-schemas.nix'), - gen_header.process('builtin-flake-schemas.nix'), + gen_header.process( + 'builtin-flake-schemas/flake.nix', + preserve_path_from : meson.current_source_dir(), + ), ] subdir('include/nix/cmd') diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index 1d677142da..ccd0ca61e9 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -50,7 +50,7 @@ mkMesonLibrary (finalAttrs: { (fileset.fileFilter (file: file.hasExt "cc") ./.) (fileset.fileFilter (file: file.hasExt "hh") ./.) ./call-flake-schemas.nix - ./builtin-flake-schemas.nix + ./builtin-flake-schemas/flake.nix ]; buildInputs = [ From 7b1b7b0c71b210d6e8b31f67cf9fb59290953f41 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 01:23:47 +0200 Subject: [PATCH 845/927] packaging/nix-make: Add a `boehmgc` option and a `nogc` variant `boehmgc` (default true) controls NIX_USE_BOEHMGC. Every Boehm GC include in libexpr is guarded by that macro, so disabling it drops the headers, the package and the -lgc link through the scanner's conditional evaluation alone. `make.nogc` is `release` without the garbage collector. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/config.nix | 6 ++++++ packaging/nix-make/flake.nix | 6 ++++++ src/libexpr/make.nix | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packaging/nix-make/config.nix b/packaging/nix-make/config.nix index a1c6280791..8228f6fdd5 100644 --- a/packaging/nix-make/config.nix +++ b/packaging/nix-make/config.nix @@ -34,5 +34,11 @@ default = true; description = "Whether to support AWS authentication for S3 binary caches."; }; + + boehmgc = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Whether to use the Boehm garbage collector in the Nix language evaluator."; + }; }; } diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index 1d92d96d6e..7ad97c5c15 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -104,6 +104,12 @@ optimize = false; debug = true; }; + # Like `release`, but without the Boehm garbage collector. + make.nogc = makeNixVariant { + optimize = true; + debug = false; + boehmgc = false; + }; packages.${system} = make.release // { default = make.release.nix; diff --git a/src/libexpr/make.nix b/src/libexpr/make.nix index 3af9d5f8dc..6580b17db7 100644 --- a/src/libexpr/make.nix +++ b/src/libexpr/make.nix @@ -67,7 +67,7 @@ nixMake.mkComponent { configHeaders = { "include/nix/expr/config.hh" = { - NIX_USE_BOEHMGC = 1; + NIX_USE_BOEHMGC = nixMake.config.boehmgc; }; "expr-config-private.hh" = { HAVE_PTHREAD_GETATTR_NP = 1; From 0471e7ea1ad098910a4cad06911abd1a1f86bef8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 01:29:20 +0200 Subject: [PATCH 846/927] packaging/nix-make: Key the `make` output by system The variants now live under `make..`, e.g. `make.x86_64-linux.release.nix`. The `make` flake schema gains a system level accordingly, with `forSystems` taken from the attribute name at every level. Since Nix does not allow several attribute-path definitions to share a dynamic component, the variants are defined in a single `make.${system} = { ... }` attribute set. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/flake.nix | 66 ++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index 7ad97c5c15..7d7a9145fc 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -92,27 +92,29 @@ in rec { # The build variants, named after Meson's build types. - make.release = makeNixVariant { - optimize = true; - debug = false; - }; - make.debugoptimized = makeNixVariant { - optimize = true; - debug = true; - }; - make.debug = makeNixVariant { - optimize = false; - debug = true; - }; - # Like `release`, but without the Boehm garbage collector. - make.nogc = makeNixVariant { - optimize = true; - debug = false; - boehmgc = false; + make.${system} = { + release = makeNixVariant { + optimize = true; + debug = false; + }; + debugoptimized = makeNixVariant { + optimize = true; + debug = true; + }; + debug = makeNixVariant { + optimize = false; + debug = true; + }; + # Like `release`, but without the Boehm garbage collector. + nogc = makeNixVariant { + optimize = true; + debug = false; + boehmgc = false; + }; }; - packages.${system} = make.release // { - default = make.release.nix; + packages.${system} = make.${system}.release // { + default = make.${system}.release.nix; }; # Describe the `make` output to `nix flake show` and `nix flake check`. @@ -125,20 +127,26 @@ version = 1; doc = '' The `make` output provides the Nix components (libraries and the - `nix` executable) built with Nix as the build system, one attribute - set of components per build variant (`release`, `debugoptimized`, - `debug`). + `nix` executable) built with Nix as the build system, per system + and build variant (`release`, `debugoptimized`, `debug`, `nogc`). ''; + roles.nix-build = { }; + appendSystem = true; + defaultAttrPath = [ "release" "nix" ]; inventory = output: flake-schemas.lib.mkChildren ( - builtins.mapAttrs (variant: components: { - shortDescription = "The `${variant}` build variant"; - children = builtins.mapAttrs (name: package: { - what = "package"; - forSystems = [ package.system ]; - derivationAttrPath = [ ]; - }) components; + builtins.mapAttrs (system: variants: { + forSystems = [ system ]; + children = builtins.mapAttrs (variant: components: { + forSystems = [ system ]; + shortDescription = "The `${variant}` build variant"; + children = builtins.mapAttrs (name: package: { + what = "package"; + forSystems = [ system ]; + derivationAttrPath = [ ]; + }) components; + }) variants; }) output ); }; From a55a4eebd57b3f824cb5ee7585683d6dade7ec46 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 09:06:58 +0200 Subject: [PATCH 847/927] packaging/nix-make: Use callPackage for the components `makeNixVariant` now returns a package scope (`lib.makeScope`), like `nixComponents` in packaging/components.nix, so each component is a plain `callPackage

/make.nix { }` and a variant can be customized with `overrideScope`. Since a scope has helper attributes besides the components, the `make` flake schema only lists derivations. Also drop the `packages` output; `make..` is the entry point. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/flake.nix | 83 ++++++++++-------------------------- 1 file changed, 22 insertions(+), 61 deletions(-) diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index 7d7a9145fc..d333589b1c 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -24,7 +24,9 @@ }; # Build all components with the configuration given by `cfg`, a module - # setting the options declared in config.nix. + # setting the options declared in config.nix. The result is a package + # scope (like `nixComponents` in packaging/components.nix), so it can be + # extended with `overrideScope`. makeNixVariant = cfg: let @@ -35,60 +37,19 @@ cfg ]; }).config; - - nixMake = import ./lib.nix { inherit pkgs config; }; in - rec { - nix-util = import ../../src/libutil/make.nix { inherit nixMake; }; - nix-store = import ../../src/libstore/make.nix { - inherit pkgs nixMake; - inherit nix-util; - }; - nix-fetchers = import ../../src/libfetchers/make.nix { - inherit nixMake; - inherit nix-util nix-store; - }; - nix-expr = import ../../src/libexpr/make.nix { - inherit pkgs nixMake; - inherit nix-util nix-store nix-fetchers; - }; - nix-flake = import ../../src/libflake/make.nix { - inherit nixMake; - inherit - nix-util - nix-store - nix-fetchers - nix-expr - ; - }; - nix-main = import ../../src/libmain/make.nix { - inherit nixMake; - inherit nix-util nix-store nix-expr; - }; - nix-cmd = import ../../src/libcmd/make.nix { - inherit nixMake; - inherit - nix-util - nix-store - nix-fetchers - nix-expr - nix-flake - nix-main - ; - }; - nix = import ../../src/nix/make.nix { - inherit pkgs nixMake; - inherit - nix-util - nix-store - nix-fetchers - nix-expr - nix-flake - nix-main - nix-cmd - ; - }; - }; + pkgs.lib.makeScope pkgs.newScope (self: { + nixMake = import ./lib.nix { inherit pkgs config; }; + + nix-util = self.callPackage ../../src/libutil/make.nix { }; + nix-store = self.callPackage ../../src/libstore/make.nix { }; + nix-fetchers = self.callPackage ../../src/libfetchers/make.nix { }; + nix-expr = self.callPackage ../../src/libexpr/make.nix { }; + nix-flake = self.callPackage ../../src/libflake/make.nix { }; + nix-main = self.callPackage ../../src/libmain/make.nix { }; + nix-cmd = self.callPackage ../../src/libcmd/make.nix { }; + nix = self.callPackage ../../src/nix/make.nix { }; + }); in rec { # The build variants, named after Meson's build types. @@ -113,15 +74,11 @@ }; }; - packages.${system} = make.${system}.release // { - default = make.${system}.release.nix; - }; - # Describe the `make` output to `nix flake show` and `nix flake check`. # Defining `schemas` replaces the built-in ones, so re-export the ones # for the other outputs of this flake. schemas = { - inherit (flake-schemas.schemas) packages schemas; + inherit (flake-schemas.schemas) schemas; make = { version = 1; @@ -132,7 +89,10 @@ ''; roles.nix-build = { }; appendSystem = true; - defaultAttrPath = [ "release" "nix" ]; + defaultAttrPath = [ + "release" + "nix" + ]; inventory = output: flake-schemas.lib.mkChildren ( @@ -141,11 +101,12 @@ children = builtins.mapAttrs (variant: components: { forSystems = [ system ]; shortDescription = "The `${variant}` build variant"; + # The components; a scope also has helper attributes. children = builtins.mapAttrs (name: package: { what = "package"; forSystems = [ system ]; derivationAttrPath = [ ]; - }) components; + }) (nixpkgs.lib.filterAttrs (_: nixpkgs.lib.isDerivation) components); }) variants; }) output ); From 0f811a949f9ef282dd5fbe97809fccd83bbb8970 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 10:26:35 +0200 Subject: [PATCH 848/927] packaging/nix-make: Add a `wasm` option Controls WebAssembly support (default true): NIX_USE_WASMTIME in libstore, and whether libexpr includes primops/wasm.cc (i.e. `builtins.wasm`), like the `wasm` Meson option. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/config.nix | 6 ++++++ src/libexpr/make.nix | 3 +++ src/libstore/make.nix | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packaging/nix-make/config.nix b/packaging/nix-make/config.nix index 8228f6fdd5..2525fe5cf0 100644 --- a/packaging/nix-make/config.nix +++ b/packaging/nix-make/config.nix @@ -40,5 +40,11 @@ default = true; description = "Whether to use the Boehm garbage collector in the Nix language evaluator."; }; + + wasm = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Whether to support WebAssembly: `builtins.wasm` in the evaluator and building WASI derivations."; + }; }; } diff --git a/src/libexpr/make.nix b/src/libexpr/make.nix index 6580b17db7..e281ae001d 100644 --- a/src/libexpr/make.nix +++ b/src/libexpr/make.nix @@ -35,6 +35,9 @@ nixMake.mkComponent { root = ./.; + # `builtins.wasm` is only built with WebAssembly support. + excludeSources = pkgs.lib.optional (!nixMake.config.wasm) "primops/wasm.cc"; + includeDirs = [ "" "include" diff --git a/src/libstore/make.nix b/src/libstore/make.nix index 88352c857e..5e826660b9 100644 --- a/src/libstore/make.nix +++ b/src/libstore/make.nix @@ -60,7 +60,7 @@ nixMake.mkComponent { NIX_LOG_DIR = "/nix/var/log/nix"; NIX_STATE_DIR = "/nix/var/nix"; NIX_STORE_DIR = "/nix/store"; - NIX_USE_WASMTIME = 1; + NIX_USE_WASMTIME = nixMake.config.wasm; PACKAGE_VERSION = lib.fileContents ../../.version; SANDBOX_SHELL = "${pkgs.busybox-sandbox-shell}/bin/busybox"; }; From eab419343413d35c271b17be8a9bbb0322aa0e92 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 12:45:09 +0200 Subject: [PATCH 849/927] packaging/nix-make: Add a `compiler` option and a `debug-fast` variant `compiler = "gcc" | "clang"` selects the stdenv used to build Nix's own sources; dependencies are used as built by Nixpkgs either way, which is fine since clangStdenv uses libstdc++. Warning flags are per compiler, as Meson's get_supported_arguments would select them; -Werror=weak-vtables is clang-only and, as in Meson, enabled for libutil, libstore and libexpr only. `make..debug-fast` is `debug` built with clang. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/config.nix | 9 +++++++++ packaging/nix-make/flake.nix | 9 ++++++++- packaging/nix-make/lib.nix | 34 ++++++++++++++++++++++++++-------- src/libexpr/make.nix | 2 ++ src/libstore/make.nix | 2 ++ src/libutil/make.nix | 2 ++ 6 files changed, 49 insertions(+), 9 deletions(-) diff --git a/packaging/nix-make/config.nix b/packaging/nix-make/config.nix index 2525fe5cf0..ca4a33332b 100644 --- a/packaging/nix-make/config.nix +++ b/packaging/nix-make/config.nix @@ -5,6 +5,15 @@ { options = { + compiler = lib.mkOption { + type = lib.types.enum [ + "gcc" + "clang" + ]; + default = "gcc"; + description = "The C++ compiler to build Nix with. Dependencies are used as built by Nixpkgs either way."; + }; + optimize = lib.mkOption { type = lib.types.bool; default = true; diff --git a/packaging/nix-make/flake.nix b/packaging/nix-make/flake.nix index d333589b1c..bd400ee61b 100644 --- a/packaging/nix-make/flake.nix +++ b/packaging/nix-make/flake.nix @@ -72,6 +72,12 @@ debug = false; boehmgc = false; }; + # Like `debug`, but built with clang, which compiles faster. + debug-fast = makeNixVariant { + optimize = false; + debug = true; + compiler = "clang"; + }; }; # Describe the `make` output to `nix flake show` and `nix flake check`. @@ -85,7 +91,8 @@ doc = '' The `make` output provides the Nix components (libraries and the `nix` executable) built with Nix as the build system, per system - and build variant (`release`, `debugoptimized`, `debug`, `nogc`). + and build variant (`release`, `debugoptimized`, `debug`, `nogc`, + `debug-fast`). ''; roles.nix-build = { }; appendSystem = true; diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 5101ceb922..20d0fe71a8 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -15,6 +15,10 @@ let # The version, as in the package.nix files. defaultVersion = lib.fileContents ../../.version-determinate; + # The stdenv providing the compiler. `clangStdenv` uses libstdc++, so the + # dependencies built by Nixpkgs with GCC can be used as is. + stdenv = if config.compiler == "clang" then pkgs.clangStdenv else pkgs.stdenv; + # `builtins.parallel xs x` starts evaluating the values `xs` on other # threads and returns `x`. It requires the `parallel-eval` experimental # feature (and `eval-cores`); without it, evaluation is sequential. @@ -26,7 +30,8 @@ let if config.optimize then [ (if config.debug then "-O2" else "-O3") ] else [ "-O0" ]; debugFlags = lib.optional config.debug "-g"; - # From nix-meson-build-support/common/meson.build (GCC-supported subset). + # From nix-meson-build-support/common/meson.build, where Meson keeps the + # ones the compiler supports; the last few are compiler-specific. warningFlags = [ "-Wall" "-Wdeprecated-copy" @@ -41,9 +46,21 @@ let "-Wignored-qualifiers" "-Wimplicit-fallthrough" "-Wno-deprecated-declarations" - "-Wno-interference-size" - "-Wno-subobject-linkage" - ]; + ] + ++ ( + if config.compiler == "clang" then + [ "-Werror=c99-designator" ] + else + [ + "-Wno-interference-size" + "-Wno-subobject-linkage" + ] + ); + + # Meson enables this for libutil, libstore and libexpr only: all vtables + # must have a "key" function so they are emitted as strong symbols, which + # matters for dynamic linking on Darwin. Clang only. + weakVtablesFlags = lib.optional (config.compiler == "clang") "-Werror=weak-vtables"; commonCxxFlags = [ "-std=c++23" @@ -110,8 +127,8 @@ let attrs: script: derivation ( { - system = pkgs.stdenv.hostPlatform.system; - builder = pkgs.stdenv.shell; + system = stdenv.hostPlatform.system; + builder = stdenv.shell; args = [ "-e" (builtins.toFile "builder.sh" '' @@ -121,7 +138,7 @@ let ${script} '') ]; - inherit (pkgs) stdenv; + inherit stdenv; __structuredAttrs = true; } // attrs @@ -134,7 +151,7 @@ let lib.subtractLists (lib.optionals (!config.optimize) [ "fortify" "fortify3" - ]) pkgs.stdenv.cc.defaultHardeningFlags + ]) stdenv.cc.defaultHardeningFlags ); # Deduplicate strings in O(n log n) rather than `lib.unique`'s O(n^2). @@ -528,6 +545,7 @@ in inherit getDeps + weakVtablesFlags mkConfigHeader mkStringHeader mkComponent diff --git a/src/libexpr/make.nix b/src/libexpr/make.nix index e281ae001d..e601e368bc 100644 --- a/src/libexpr/make.nix +++ b/src/libexpr/make.nix @@ -97,5 +97,7 @@ nixMake.mkComponent { "lexer-helpers.cc" = parserFlags; }; + extraCxxFlags = nixMake.weakVtablesFlags; + linkFlags = [ "-Wl,--wrap=__assert_fail" ]; } diff --git a/src/libstore/make.nix b/src/libstore/make.nix index 5e826660b9..76354905ee 100644 --- a/src/libstore/make.nix +++ b/src/libstore/make.nix @@ -66,5 +66,7 @@ nixMake.mkComponent { }; }; + extraCxxFlags = nixMake.weakVtablesFlags; + linkFlags = [ "-Wl,--wrap=__assert_fail" ]; } diff --git a/src/libutil/make.nix b/src/libutil/make.nix index b10a3f143f..0682a08a79 100644 --- a/src/libutil/make.nix +++ b/src/libutil/make.nix @@ -53,5 +53,7 @@ nixMake.mkComponent { }; }; + extraCxxFlags = nixMake.weakVtablesFlags; + linkFlags = [ "-Wl,--wrap=__assert_fail" ]; } From e2069dedf7c98e3cb61d498eee4537327d3a4ccd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 15:28:16 +0200 Subject: [PATCH 850/927] packaging/nix-make: Pass each unit only the config macros it uses Configuration macros (NIX_USE_WASMTIME, PACKAGE_VERSION, ...) were written into generated config headers included by many units, so changing any macro rebuilt every includer: nix/store/config.hh reaches 70 units while its macros are used by at most 10. The scanner (updated scanner.wasm) now reports, per unit, which of the tracked macros occur in its include closure, and only those are passed to the compiler as -D/-U flags via structured attrs. The config headers are constant, empty stubs so that `#include`s keep resolving. This also removes the `writeText` (and the IFD when the scanner read it) for the header referencing the sandbox shell. Bumping .version-determinate now rebuilds globals.cc and the link steps only; disabling `wasm` rebuilds the two units using NIX_USE_WASMTIME instead of the eight including store-config-private.hh. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/lib.nix | 73 ++++++++++++++++++-------------- packaging/nix-make/scanner.wasm | Bin 80152 -> 94477 bytes 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 20d0fe71a8..9a5e75b6de 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -80,34 +80,24 @@ let "-lrt" ]; - # Write a file to the store. `builtins.toFile` cannot reference - # derivations, so fall back to `writeText` for content with context; - # such a file is then built when the scanner reads it at eval time. - writeFile = - name: text: - if builtins.hasContext text then pkgs.writeText name text else builtins.toFile name text; - - # Generate a header with `#define`s, like Meson's `configure_file()`. - # Integers and booleans become bare defines, strings become quoted - # defines, and `null` becomes an `#undef`. - mkConfigHeader = - name: attrs: - writeFile name ( - "#pragma once\n" - + lib.concatStrings ( - lib.mapAttrsToList ( - k: v: - if v == null then - "#undef ${k}\n" - else if builtins.isBool v then - "#define ${k} ${if v then "1" else "0"}\n" - else if builtins.isInt v then - "#define ${k} ${toString v}\n" - else - "#define ${k} ${builtins.toJSON v}\n" - ) attrs - ) - ); + # A stand-in for a configuration header (like Meson's `configure_file()` + # output). It is empty: the macros are passed to the compiler as `-D` + # flags instead, and only to the units that use them, so that changing a + # macro does not rebuild every unit that includes the header. + mkStubHeader = + name: + builtins.toFile name "#pragma once\n// The macros of this configuration header are passed to the compiler as -D flags.\n"; + + # Render a macro value as a `-D` value: integers and booleans become + # bare, strings become C string literals. + renderDefine = + v: + if builtins.isBool v then + (if v then "1" else "0") + else if builtins.isInt v then + toString v + else + builtins.toJSON v; # Wrap a file in a C++ raw string literal, like the `gen_header` # generator in nix-meson-build-support/generate-header. @@ -193,6 +183,10 @@ let mkLeanDerivation { name = "${baseNameOf unit.path}.o"; + # The config macros this unit is sensitive to (see `configHeaders`), + # as reported by the scanner; `null` means undefined. + defines = lib.mapAttrs (_: renderDefine) (lib.filterAttrs (_: v: v != null) unit.usedDefines); + undefines = lib.attrNames (lib.filterAttrs (_: v: v == null) unit.usedDefines); # Generated files come from the output of their generator derivation. src = if unit.src == null then component.allGenerated.${unit.path}.path else unit.src; includes = @@ -229,6 +223,12 @@ let for dir in "''${includeDirs[@]}"; do flags+=("-I''${dir:-.}") done + for name in "''${!defines[@]}"; do + flags+=("-D''${name}=''${defines[$name]}") + done + for name in "''${undefines[@]}"; do + flags+=("-U''${name}") + done if [[ ''${#pkgConfigDeps[@]} -gt 0 ]]; then flags+=($(pkg-config --cflags "''${pkgConfigDeps[@]}")) fi @@ -335,8 +335,9 @@ let them) to leave out of the compilation units. - `files`: extra files (e.g. generated headers) by path in the root namespace. - `configHeaders`: generated config headers, as an attribute set from - path in the root namespace to the `#define`s (see `mkConfigHeader`). - Their macros are also used to evaluate preprocessor conditionals. + path in the root namespace to the `#define`s. The header itself is an + empty stub; each unit gets the macros it uses as `-D` flags. Their + macros are also used to evaluate preprocessor conditionals. - `defines`, `undefines`: further macros known to be defined (with value) or undefined when evaluating preprocessor conditionals, on top of the platform defaults and the config headers. @@ -422,10 +423,15 @@ let allGenerated = generated // depGenerated; # Generated config headers, whose macros are also known to the scanner. - configFiles = lib.mapAttrs (path: attrs: mkConfigHeader (baseNameOf path) attrs) configHeaders; + configFiles = lib.mapAttrs (path: _: mkStubHeader (baseNameOf path)) configHeaders; configDefines = lib.foldl' (acc: attrs: acc // attrs) { } (lib.attrValues configHeaders); allFiles = files // configFiles; + # The macros a unit may be sensitive to: the dependencies' config + # macros and our own. Each unit is passed only the ones it uses. + sensitiveDefines = + lib.foldl' (acc: d: acc // d.component.sensitiveDefines) { } allDeps // configDefines; + # Macros for evaluating conditionals: platform defaults, then the # dependencies' config macros, then our own. `null` means undefined. allDefines = @@ -454,6 +460,9 @@ let files = lib.mapAttrs (_: v: v) (allFiles // depFiles); defines = lib.mapAttrs (_: v: v) knownDefines; undefines = knownUndefines; + # The scanner reports, per unit, which of these occur in its + # include closure, with their values. + trackedDefines = lib.mapAttrs (_: v: v) sensitiveDefines; generated = lib.mapAttrs (_: g: g.from) allGenerated; } // lib.optionalAttrs (sources != null) { inherit sources; } @@ -494,6 +503,7 @@ let units allDeps allDefines + sensitiveDefines depIncludeDirs publicIncludeDirs extraCxxFlags @@ -546,7 +556,6 @@ in inherit getDeps weakVtablesFlags - mkConfigHeader mkStringHeader mkComponent commonSupportFiles diff --git a/packaging/nix-make/scanner.wasm b/packaging/nix-make/scanner.wasm index b5bf487398e1b65e9b384f758298b2c117142855..a2934082d6fd7aa82b4236bb72883983dcc8b2e2 100644 GIT binary patch literal 94477 zcmc${4Y*!sS?9aH-tTuOtI1B2G<8_-OVpjQJ?&h>G(Euail<@y2v_y8Fg#w|?{8vz2I2X0F;rFGtP z)56U++E@)Vg6;3W zdB+XP72&ch2A36S+8tJ9$nNzHbvxjIeDR?#Z-U;Bbq zLcz4tF47K?TqaQ_w+S{E*BgxwNx`LXsJQu9RD)l8RsG9#)XU$f-)$6Tlwl8e8 z*SC^RCtFoeN6Q;(Hku9klCEhKMXCa;*3dECWZ8zIk=~wu`?ojpG`XO;G2fkrrKRPh zw>t?ZK0k47eOIITrkmgN19$vzmb4(qD{j91&g4hR+unHB4L^j)-0;TR-niq2o&O^F z#jJb74R5$<$4xifeA^rHw0FY|x7>Ww?Kk|NH{E%2o^@WGCL!&8+q#dXzns3};@eKA z|6%vt>5=r2bU8hh{#E+n?1R~dvIE&qXTOmBa`wsWSF&Hqem4EN^r7^p(}&X!r-##z zqz`02mHle=@$6{U{qy%UKAQc@G}~Pzb6IsfRpSaEN~+?D%-W}MvKy=oHCaQM2}GJG z=e;#Rq9fE*KCYIM-ly}lSp!v#fXcS1w%93kXi)Uio~5RjCdsgv0gyq1zb5IrROz6l zq5lr7$INBEYV^W;ucs-smkL^=%( z8nhAN?*Vqz2d z;1+6zL+@W+a?1aVYIx;L8lF&6_!5K+FPaaJ+S8#AkX9a27Ff#YvZ1C+Mb0f*u~n=a z+Vfs-dmdJX;yhK6puJKVsDhf_tKZ%r$+m^$(&VwvQ@aD3l&9@gIHI)P)oHp*d8+gR zl9v+u8K%(mz-M@0{={2JG$v2CH?{m~G9iTieq&bJzQ~v;$ zT0!Mh^hD0)FG-T{X_&y+B!gQlb&x6B!hlC=bY6y6lirJwMfj*ty;Ux5NmQWxlwf+L zXjAmQnx(62o<)!{XKhMJXj4c8Nx8I#4S|z}+4&*-d1 zQEu23!vXCzw8CY9%-zbm*n8zn2E;S-RkLb2s%DIS6cO=eKrkTWqDd+~?VHrn@giUd zAo&Lbg6lQ$S2j`@y`Rg{271*0@gkoOZ(gjjg<%2N={)Mj5P|FrTHr6M8rJRRRWk+1 zM8f8W4NzbV9`2eSwy&KrHpqDxw5b7_S{30fZ{5?rRVpMbaYs!Bgp=}a+&OG5RINKa zy@`m$)aC-hj#{DQUGW0zp$^*f@IV`F-xDd2P1B)Ov4AE}y@PBL^laNS;3=xZ0lx}Z zMJ?AJ>Z0qrmiRX6;FkFIi&-Y$EUI@jY2jQM@diqb7j;Pkyu6e=yuG>y>?KwD8mNad z7p-7iQ5n3^iT?bF0q}!-CLPYU#3&Urx=zp$U5da8| zucM-0dfSQ+O;z+99a^dQENC%fuAg@KQ|e~#Xglp0D_Y_(hj*8j&G6RznL%r%E=6;E zS-J{al|t(b-sdfhjriz>`cr4-VFPNxX(4yfmK|%u5U7c&RqHQljm%8jumv~w@g6Z3 z`Qo5M?k<0-9a(_S^ia7lU~Z!RnbTm%R+K+p2dgmUF~&sJe~p8re%m-9nJ zPHIAWqPalN(3Wx#x}m)@JoA>btwnC_I*++4ZVlzmsx7MY!C*l0@NyPMBik{? zWJ}FAVt*PF{XxSEv*sRj!kZnl&)Z;M`U zhUv%_@AAc}XP56mu+-hin{Zy$!T4SPUX~IGVKS3Td0Hm(O7*^ffoVT~pJK3j3s&18 z6Rf#dHMWu=sR1YY`5JKzEht3@gL=A|fsogmLkG+0z{~!SfE%P0z6zL(-bbMLCaLnd zDHlZ6mP)BQUy#OquN)EgL*~gC6+UiAW3%-M_;L6o(+*Bv{Zg^s@B*4aB>DEA-txXghr! zdo&jgta>5FRLY)@>OCExyOTsm?JYXOGi*+ zQnh7ZK#6-6(RITTYP6HdyOL&YO91Dx2O5J^9Pr&!h#C!i-0`E=ayjyFw*Zh;ta91C zb~t_ySl|VYQdV2KujFZ~M*87&gmusjC?J&~#ysF*y-TXYvw2XzLcyMtQA$I1doc*Q zE~Ud36yHp0^NOtEf+@e4vH-L}Pa(N_NPlNNL+Kg%o&gLB7X?IVgt>Wsiuv;A)R@yk z4a_l)5fpdH`{4N{8Yk)I66uDy?DAo!G--i4jc_I1gc@l1@;^Gz;n4(_u6)RARf$*O z$#Qu$dUoFu*a@w2zutQNFUJNn(GKskR~>&P-!PG#P(l3L@@G}R*u#OlB6|>^>ihe+ zukai9Or$O8{ur>xp94(8a(OTU$&2#m$8~~+Ux;rGr;)Yc4I00)>XpB!SN8M@dS*|k zt(M=3SDs1Jq9*V@+{|nN1`AmT*$n@>9wNIT_obvjHC`|tft%O~NN z#5zg6#ZY4}?-)vjj`nZGwW!C9;30QG+>3A^wUQ*cm8c?wMg31|vKtV!ChEN_9ZM{} zY{O{)V<7hOO{|t-Q@~xQPT2z&Ht4CD}UV3C3vKc zJbD6?j7Pcm5K^nT;_#R&GKLBk^e}qUne1#Ync{(^tlX>0!OfDn;8}POiaYm|U(_qd zKEJ#u_lq+C!zdEY99>>Y!lrP)QR^qq;hOTdR6H|&6VF6aTYFV5Pq3!;qvunb7GS++ zY43?~dt#2mBj)(RsyyGF?dsOJkc1SQ0mGR>R1iE6c-Rmzif{sb2*F?$fmfhExU|dT z&PP=-mz^*s%HSFejwp(Kfrn0|SP5RAOwD=L?NjLh*K&)JJZCM!;`FHs+*(AXf|%fE zK7{eYx(#rM!sV}2xWUkTxreG!i8SnLkK#;*Z(Fm=B|leWmvDjQ=D4)BqNrg(*vmUR z{aZOV$U27JH8XS|9KzgnxSmSzr$S*ikG{NQ*kjla5Iv=Uv|jj;`BjYKkc3|NVfpJt zcrcbd$h8}qW)SJx`L7xZEm7;`gMRs6g<4!~L||XR5I-|py$W{TeJ8_=mjN3z#f}eG zlO~BRh4c%m(nw(F8Ig|Rw6NxQk*C(j?W$CeK}4z21Rn4{h(4%F2=XV+(^aW*ib|07 zy8;RPHq`jI0razTNL_s)pdWzA$@ib;Pq^t1DGXPsgT!b=ocXI~;A$h5 zF;5y!e`yEgcRc>TJJ@q=MmXljlFSTt>M+wDE2N5 z4-I4QHKYj}eP{1}Wbev(z!g(A$=&et_do-cgM@wK?$uR)0YS-)yUWPk4aR#yBbtx?d=^ZptnBZ{&ye35w8KQNjk(?E(nez9hr3D)nyWyg8<%nH~vmq&! z7$PT>n8F~9m22yFdQlgF&M67O|D9ldIBQFG@mBbh4nKenDZ=LHBA}>2(pWFI6*O;# zUO2KW8HiECaN6V}N<=uIr*L^VHYGL3RU-VWusEwIrj?G|8qm3Pmqa_2`p}pMCq;UD>}W9`lf}rUjd6#X6{8eZAY~XxNezIb z#|k}o>eqSX(DPgl@vR|#`8U!Al*r1zwYwy2m@ofYw*WJjZ9b>moR_<#{JZILz23hc z`#EU!v^MQIT2QWQ;trtM^C7m57Ikl;GL(IXgchV3JJb{$`E{Bs;8|l}^H6&aXX81k zsyr_r7Kqt45Fq}Jb3&%$#rEZYvL0|-$!#zX)BD>mKkTo>Y%=79CKF`D zHC#%|!*#*SJ#H_-!(kP3Q`;p7A-Y2YNuV`;vL@t|*P?xFvgu zh6D2MBp({$YQ#^_40l7}4uUmk$1U01cw$9GX^?SW`4NG1u3JnfEvbF;7a9HxEkCBb zLnOMO#+;EN_4NKa8_%X4SkXi$o1m4U(V@h}VeB#?6d#g?zCBdaBTntbRcy_h-CO(5D5VhlTgJ z9!D`R(XLQ0j#4zMtZO(>`@V!lYIpjeS>UJzzHdBKL1IS11xX8fE`YGfSyjAGP1yk* z!A7A0Y}SJ_8rP$@Z+<~O7MMICEA#IX!AV< z-kW%zFhu!F#sjO7EH_2ml`z6!HF)kaDfDJ(=8Dv$d_-9!h z2lPPRSxTEwvf=`>UCaecZr}num|eOFO?IG-XL;!3FDd}jx9CD&L&1*;I8!&c4c7(6 z`U$Vw^RR0fH3}~TE0_LoO@&)=ouqk<Vq3;7mx!2ywK*Ud7-_Xu6qC&v%$jL}H=RF(9)Ogd?||#oH3QEQBrNDeHKSL~ z3eIDzW9$^XUFSg<@>akym%VmK*17C;J_Z^B5bid=4;1AQ!T3sMA6J_fx=!43LB*vQ zTu3W&FYq0Qg2nqMfZNrD)ddkEi|{M^dhb;1n$06@5RBXvgN;b!hRf1>)8R!S-!WKC z7&fUkIFr!KYVutKtw7ys&2&NoX+)t0{a6|*25WdixWzd%4_{~`cwQv7+CW(i5!PRk zy^aw9xi<=ojfnhUWBryeQ##lHNqdz6X*AphJ|U$P0jm_QbbvKR017o5yO!8cI5g-* zGPtOMsc2Ed;UueB1Q}9Ik|An}GDI37%JP%KxX5t}J>v2P`p4zEApq&-wZqkejlx8t zp~giFqzMT2tJNl%&tL0nL7u7uDpfu#z1CZXYv?^VR$YL0IhT#%dV_Ed*i6EaR^fJi z`6~t`!4Bq(=CbSI2`;Z2T&T;9gAHO~QRVt!633lJ@YN%NC*@JqAilo>Tthm>o%h;R zq9gJS$=#%VvXsACBSqM3sgYtxK}KNb5gw;i_PQ?9^qeV@q@K+!-FIQF9DG>LR?DrlP44qB!d ztRC@7>b#N+Ho`9#Rjh0P6BE2bAEoY5C(`iH1zs4d_vIQZ*9}&SBT{k1MmXXkQSqV| zR`w0Fxe>*OVoM?Ig@`~|ex|N@nhBxo1f3(*YmSmz$S|o_{+bHoq%%$$Qpic#_hUZ#JRU!=XZ3|0n30*TBCqP6+ z?K-F%wN(~hEwxJ=W24&=t4dVjF(O)4jsb=RcV*(5VQ*I>4hz=;ENe>c>*{x3*DJ6E zOI}*0*z$8jFC>7> z&CYg6q#qC=DSung(rvOIhu73+UUp>nxjAb$lZ~G3Y|W2%G{a1ZCncI(PoxZOtf8ZI zL*>@&5$@IaQ?zG|lXg;VllEAhPXH}j#+RjM<%Z*$o#CoXPwSe#G&!wnmx|*1Q}O-D z_)qYkjoqE2AtFoQROpw3L&K@&uf;|khHXx< z<(iP?=TwifFQ*#u=DK|pT!^l;zgPV^V!72e!{PFSV>BO0hql6kGDjV6gxRc!!b1*j zhY^AmQO%DC46cXMVRsWSF*90mgQlWwb?`}WwYQsXqhFfobY6*5%FeYXI|c^Iz1-)p z2a?Kn>fSpSb4*c~ z@(&8@qd3OHrX!QXChGwmeai6@yb1woRR{-I zlgjX72?8Y!)NC8X)mizw%I_j@z}VL~`9UuPnNLutrs(lhG_f&8GwfH>pykjmL#DB4=0>&k2Itd0GaRwJL~qHPX14j!l}avAyXM?SJdO@@5uogu*lLHBzu^fh z{-xM3R!s%(($Aa2$?1OG9>H&FGRNRHb|Xg1Z11VbdgiiQop*1ECI@MX0Iv-Xt^_a+ z5l!9~0d_f+-Ud6~2RbdaJy=Hvjysf3ro#)BdLkV&#_I;_m2^FBhv`-6PyCbhHkdnU z4^DQu6H=mb%HW+19G)E6P4b=Kz@D#hFP&vFbpI zTSHi}9zn5~YwugcqBQ~WgUCZK7zdBB@^Yufm4M#?L0ucDAm&&>7hxF!nFZ9tX`7)C z-`4-sP}r|#LN;3OptSc}S|g@`MAANFY0a3HS=!H8S`pK7OZ$0CtJhBymUh_E+A*Ka z3kn8%cVwy#hI+Jh)Y7Kg|IB2)ubk0n4O|@UjttQeY)Z%(fp)@wn3u`ugkAHOCX64h zp6NpX;XukntOqU4&|VH!C6r^PcnJ>Hqlv1%9QN2mWkbg4pmc5Y7ibZaW&A;lEZn8V zj25+Kpi4eYd8DyfDcIc+1}!MdfT-%Qi$bFVhn~8OKIFTC-rzL@_sg%I!Rs8X4d1C! zO+`kew9@dHkWljk!2+?2D!ODiYuss~VbZ|!33V1zZas}Ii0p*El+4P(=(9dyar4a+KL^S zg6D4Pk#Aa4thSVaiqN>xg`a^H_cycd|ABR1Uh+5E{gpcI%5KzDXWY~TBIJHNXQR)7 zh;O)2Twjwi1n{`Y*^@#5x_e!9VeNPkO!8a#NH)?lkvoG7%e_3UtNcsCbnB28oOl

aH&;GJ$R9hizD0#Rmq^%2QCaG=&J_N&aC1=2+Ax>j30>3 zjQF-N%u%(PkBM~?wbxXY(gZ?$z%Nw?qi4kg#PQT!xu;Ozc-7syR5P5Eb5|GhCWc_Z zkZuL5-a}rStzUBU`xW)t46Gr+9#@W=SVzF4tP7LGGD0@QXFN&jg2w{X_p==+#S%!} zFEJw4bx)Vcu@h>`{65pp(t$srB&(TBaOjK~JFNmxDcEkKuCSpE65wm1nk)<=;Eb)Z zDC2sVVNejO#{8Qwu@RwKkG-^590RZ}oJRgxM4G@pOMN6NSV^&53T@U_8>?C9V=I5k zKNdg|-neL3%`>Q4>*VyXzD1)q;5XnB&rlzMNQ+hb;)RawgAtplokxe&p_uG(MsPTU`nP8Z*e*3bRW;n!}e>vudxvpoZSJKQI8~47y1|6foQ3svTl)%hX&h5HX|>az6DiShuJ{anT?t98X6VRk z*C`AS;T)j^wwo2j)cVhO9P)yk?~wbq$LsTl4xdPR_b%>pt9 zzjPzNbh&eW0Z-nZH6!ntc^j*rrl|x$wFtv{^c8j7hr%q&Q;vMrw*cU5`4^EEU|zf< zGHeh{#BMCAKsdx>=VEKjG?`)wrf$GV$}Evv9LyQsj9HQLIoS_6QzZU`xCbxU$jS_U zt@hjV<$sF+gHsz}0#;iPnhT_Mn8%4NeN|aiBYE%xH2ONzQ7B(7&AE%7rj`MQXItLX zJ+Isf%bVa=OhhIfeZ=HdwV-+XB3KOJwfhoYPDBiwde?;)96x$ z-}Yt0P2Z~>VLJx4Wu#(nPqYCo38!xvX=$!DXOsb zy6k8{fSNpxCY{)>tEC23(FU(*mM20Hd6=CqAA5`8&I1_u7O$c&T&^6puy5SFe2CvB z1K5_PZz83PwAhI8g6e!Z_)%pbO;rzjUMD*=R=gcC5TC{a#%(BWdGd z@=_`81)CCTJh0KK0{!v`wS|Ry?O{cp+mK7Tuq9UMyzmK0?My`Fw5@UOH6Mx3pt^B) zRJT}4GmtDGz7gIb=j*u8?DZOy6Jyr|B+b4KjnR%Q(6k=dqHPBGkqg{T!vO`;kF0#8 zD{iY3O;sD!Ucv>-ayb{er5zg1)4RZQN{f3ZMUls6k+KzT+RMyMB1$B5)@ zSO7-V4n1@3m}L86-H>8tPD@=PWyLdczL7fy^}X^pjE104_%rRV>11dq(!kmx4OkR1 z!D*u5cjO;U57hjFxAjk$bJX-8j%f{hkCx%EIHxF+t~S}~6E#t%cW*npaJ=LB^3hHm z#WJ)oohd&tE8;-HETrPnM{X_Dfw`4hvYq&CI8AECl~GOTNN)JX)|0@~>=A`~l5AVm zx-{vtJlKYnn)t+YaAD#=F%V77F`16uDG8qBwq1So0<-;^x?{-zYgJHEvpBMAuEpJ2 z8xn3WSO{z|<;tenb=q0Kwpw=$^F0@EL+Z7gaC~re=521mmWAZoFi`7>-!U&tW5e~; zDhr@p#*jrL@%60eCgh5_w{hAG87_Q{Wk$iy_)hlK3Jb48;|kx@H zM)!5T+KU|Zn|0@Gb_X?h&g-kS^aOy`OV&#&B}i>RttHmF?$RU)1{EGcwMN5x^<;r_{2 z@X&gVyl3aDS>B3Zta}srhBI-)TgC!2HMjv}ak;qsb+dZhQX+;kWQ)x7D97AztuB|8 zez2=BM3_|b~+^P#aurl5%x z)6jFk0|8lW47*tvJfU+k`nS!(-FB>=?U^JY@qZ7$VIKY@QDvy9eSK3SvaikZRb! zr2J8p@GeB`hYWFMqquo#S-3oEgWmr!wFa{hb%X^6aGhuo<6ge+tyD(`ki#~S2tf9( zvOob=8!f^dZ;0*b*dGeW(v1g(u9tiPb^rVY)E~@p9mKH0p*8>k1LJQ8_?sK1bQ$tUvM1Bu#FtbLr+SRWyK8>YW`Hf#`mNmy zz5!Rt8C#JqF34+Qfd@u+VVIyY+~6jZHftfhU7r?HHG1En4oT62hAO{ilOGoXI(b^C z2zlip`EH6NuUhE+VVbwVWx6{KVl($E$SV@&j7m(K4}F}{u+2^Pru3L?O2j?T0_W`& zVBqp3iyv{8+#%E|3Ly=TL4#clJaV}bU8PhFow>q33>L*miv`oE6P{~%HuES5&Mwz-jj~^xw6GF z`te$Wbua6xMSzzk;w>PB(}Z}IQ)&zr1Ai9w5+|0A8%Qu2#@gly*^!ArA!1<4xsABM zUX74o{Oe724MFlQq%o+zW{}=G>juXnh4c=@;I1^CtP&MP0DCnifC;=_QtrR|gxJ_2wj*9JWtW_95 z>3o~sXvw*83W~sZlL3i_EEao26NxIepSu@DiHZ41QQ~ndN>`7q&x9nInwPexVn&y8 zt|d<>25Ii7<+&n3pLD6L51j#vFjzcv=IyPtU9>=N+rHDVKfmg^AQQ}G$Ns}hJ-wA=7JPfp5#4C=^_5d#III`FRJ2YTa(0=txq5dCJ@1h zh{P#ySd`~eatqlrv|rjF4n&pyjt7)Jnvxd_sX?81MKjpU&#QovI6#d-f0SY*MotYn zTrwbDwT-nZZt?7k#f*0V1C=ZblS|Y5Wqalc>0Tn8!7ta2QdiJlvBS*Q`hj19`FkrL> zG9Ih@X4Ju3wJB!uK-yP3AvSBhv=R9g^3#$L8O`}^mVS&OeTl=-S4+27CT;e^853CZ zTGDhh4>joEi6(Z+_H62?9LgY6gh|yg+aH3)#E(;uM8sxbhx1YXMh)*AeWuOLJ~Bln zV6gDFA5(|Z@(Z3T7Q?+SY|7d3a5eqWcq|7xLj}!B$A;)1LT$hj3EcNJa7xH z%jn1i03MAob)bj=TDyinfW?HsC}*&gUYdL>ydbuQ8ZS-0wNBma0)cj^wE0Ux$ii}w zF!Y3>_A9|eIuFd%H?-F}6f@*9i{ecDgURz zJFyS%lj-DXVOYLr%IfpJPRLp9-)e^-)i6Jd~( zaKgl7ZFox8+F6Jda4{0o6hONPL_y)cwtN3vb?|_q&`pG>O3kD|cV3mh=1O_z6cLIs zo+>$2$pD}w7r97GU%j3DR!fH!>4I4RwZksIz9vQ~V*ab{XbG3*{c20QLu+%~yI2J* zFL`QPQ~sjE^n>F~lT%yHV2sRkgKyYFusFt3Ec|R6_Gm-oh!+GqefFa}dy7-8*8Y~w z+)8EhZ+w6W3$9fu3ND%l-H*pudH^!I<{~L~n5T1mlHOl7So@vf|3?4=v?N4|6rt2G z?wTa<01FmI7Aw!UjaXQ;zLZ({=JJqu4O1?Mh#7cdI@=Sf zV5C8mTDSfRh3T(4#5Yz(X&wZeV2ReCa<|nn~|9HfodehgYOaLOHa< z|CE8VGE`>8kSioXQ*Ve6<5sOhu+w#994II2@mhixs-&+U;ui~{f17LPJo$Kr3N|;NYA%O@`v1;@_HRg7De}(&4;S zlP(tOUke*uFzW&8`O`VP#=5MNvr_tHGP62XC8vP+XR{O^mF_@c!@n+iMO z23e+dXztJnyhM}XBR`0jDU}^6l2L@Sc`dUYa;I6G{tnNapOZOPX5O#af#o}$P*7E3 zr?t*a$2Jf>$&lYp`R=zuA9o9d0D1WgvWVb-YLxG+{2+pg;#xL&|DANW5(*qdt)bLd zghxexII20;h10}PkRY2Xu-^@K#GY8Pg*7wf2?+9F^YQSR=!#4(oLBBqJA<^s+5l}- zu+SwybvtM7@1AiW0{+B@V&`eR}mUp7Cmf0QPyWE?;Vxs!%DVlVTYV#D%D z9b3Yfolp$b5CVsOy|7CG9u$xgl}Dv`k;gDNoI-Rr=X6TJY%}Srs-95Y81!vQ6(?24 z;STFlbsStBN!h$=xC$_7gyVEK;R9$*O4)cOu!3Euh)1xUj&ganp{U!cRhcOVEnB#(+Z014~tK!0{ zEKZx1n&mJPxs;c1K?VxV_NS&z8Go>9t^6NCd@<3asiz&DL1rX>piI~NFAKjQ< z)7yIdjgtjje>Nxdr{-`J zPn7PGtM>v2IOR7m!1;g%I9YuhGLwUOq)9G*0{!KGcQq_7lfp*L_O&X5d3)Vk+}6yQ zOc=A|xxxv5?Pa#)b8@S<>H1`8P;S;~_1P=Z1!_+SI#G zp;xoY1IT5bGA5ci2@MEfiJ+zOjkq|!;fwPbNMxYiY_Y6nQtp+dh_x7Ejq-MEwN*f1 z1ffwpKM6}2>}!WyZxSB^&9LCW`8!^Q@_J{AU!QHG_Bam0r>6Z>OZ9C4V&|8b>PIv! zaHiGyw)7b&ldG!)UR$IgbVN!ujlf|9=;ncO<07|MVw$D&B8{5cmtDo?Hu*yWu%{6h zbt>enSZ|d?@hHkih0&zPD2!$lg_rBXv@h#2 zr2FwnDDN&MidGT^GlO$!6J|JUe9jY!RugpLu);0%8DinUWUi%x<^VD0PYh*MN*Ka7l<~BePoHV+py`N8E z6!1~N0b*Worm0G(L}2>k>C}m}`8G1Oz&opCr1Pbl@mjb%$7!|F+AOm;Z`~0Hw*@AF zZC+?ZnGoSavNK5Qmn@A<2Z=e>#n&|v_P-{83PU3Kx&gnDwW+xdT7(tPs2ZpMnIw@b zX6U^N@pqGAy3uAvHzDeBqfY! zvYzHv0Mxn_``{}V;Vc_IsZ3u=tsXw9Ih$8NsDRu0&JiHH+12fvBUCmYv$4ZGolNxl*9qMFs#&!sf>VpP*o z5~fcyyK;WJ9rJ@zur-$t%ziET!0Gm{ec^Y0^P&5G|D#VQZ}aahq}%`UkAD1X?|uBq zPn=W|-f%^kN|)__eDXbC`_fatfAlH6LA8_gH(&VT-CsWR2cP(wl3-kDo|o@1|DY(} zsp~+1FOODFrM>qR4Na|Q_BR1;G_6wP7Rof+$&)rfKW#~ptz?()ftS$PzWdEp2gt2q zIvm|pLkl`VqbTNi%m3!c#K9@Qs#`e_LOjPy*(w@l37%4V&u0fTpv@-up|QGG{<_{L zz5fmNRWthEP6QyM#fXf|f)xMIR(ja7F-TxjKIK1H^`FTaTKyun!m7ZA4=F~IZ{N6g z`!mVi+iO~+yI5Q60iTZHUm-ktKIAn2w+(aXMww^+;qU+MufFo>XLh~$?kOtfPP*$@ z4X;(fc;ZfiCi;7E$noeG0LoG#HsK2ZR0In1yQ6>CJ+~_3VOB0+ATt=^Iv87!$%0ST zJD8ET+TEVsbGNZdNVT+9E0VQRKrLISNy9KX>M4uwEDHFq;r|&T4ss~PkrV7_P!HH~ zn`mlYE*m4|r1#}CW7SbTbRoWZcH%O}A#C&8v1s^np^r$lKVW`cD5)g~NxkgI7e|B4g*$llV=FQd~?FI#i_juYi3 zOtI>WQ{v&+CzbOblCW=bUM3#?sAK>9@W&TNRwFnm=4y{UI+!eabeAX29|pL%PwpJo z7_qGVD@82LSlIWdfJ__bypS4N&KrjxEwtBH+ahM**&0?XLQbJRm_Rp0(kA+47t!2T6}^pRu1{_ z1`BmH%;=gAhmeT(ShzystUeb76`48RG$4e;js0D_i3FVpO?;=p+zP5*n1|n5@G^p# zYd#>=AY&E*=@JPu(di@Q7WCV@J-z!nX?v|Zy`Wm{lMq$|yE^2K^^{}@tMrj8)iy(P zZP>M7sV3hc95}gHB0*xC3u{-f8#Kqzfxw{vVTQZ`ScJQD=#=IG!BkgqKH6TREAtgF zzs{A)!+q}saic(8iXNsIwa(HWA!TZ2-h?k%v5Q4@>EI)_%B-;}S*2^cQo7ZwYG1<- zYC_9c`Dh3(o4&M#R^VG)hM=ko5;?nfszu-pXWugkgZeCWLhl_68^IAqBl<_@4RY)M z3k{I;zkvYBL;tqb8QK4A*XVm@`}i19>Iw&993l3bghT2O7G!i1Ypsfm0pO`42*2r{ zRA*|7%Z{eugq3P}DWx6uT&eCKNl{)b?8sIB8OvFgha?Ajdnymo-Yi5_re(H{YiO_B z+#vk&@Ti6};-!|jvy)PiaP)lwG&WfScW?}xnxpzdY58E?)Kj?>;9Pb(XRu*SriN5vzHf=tv$Np=`1ND9X$(laFB1*J);LQJ#C3>A>yh`PwAOTPU-nDXOW-{)r;dq z`?=LXLf{v7445dv?Eznk4GE$C$q@DYU{)htAfKidvu&1Fw~WKah(E}u_)Zg^RKZ8G zcqXp54XMt{dYIm9FY}C%kDl)pyhRd2M(6M-K2sYZ$wi%&0p`w=e zp>W3T8y&C~^wl_D6l-1_I@iVlVg@w~p7099pZZu8gd6dz=$|iy4V8%lMtg^Q zaTBm9cdur&x!5}PB*4;R_LkWr+p(E1(YjmTWj3xEkDEn`G9l9~o2Q0N5M5w%CSq!! zVSMY^)A5{X*c6_u*G0!;l+5>ZZt*Xx)O}?fI_h6qwN+^@x(L%ciV&=b>N;&wbjK=K zNn35jt=R8=7Xb;5UXv~7n}!D^RLr0}VpmN7i(}8}7jj`O9uy}?w3F~zQvjW@CPBR6 zClhaELz+cmF8-=pj#}ke1zR#DxOBRPgi`m%xR)$iS;#uV+OWg=Wzx@Ra=QFS?^}?Z z@Pb&e0yhe)pA^V4TODTNPVgX+Kc1OzYE0rF(qEj*$U~Y=1)s6c;`l8F!xn?ec(a(2 zCi)nq3EY@8MR*);w21Y_rAb_1XFeH@gei22q)B?|3@*YacerPbJN%;$_& zkfmd6OI<;hsDHe`++<07++vm*34x2*;VKEyP|O929-}sL69rR%JfZ-jD4ezhI3(4& zz`ZgpLo#KUYwv@Hcn#T7C6d^9UKb>jDv8V>fy^b+OQt5F<|t*ar7RWW6zM8LM>U5| zXi0b(EeYJ(87jD{3!-u0P~Q`JhK6WTX|B}P^(NtQV}3}^L@TjLuO*XaUJKO5dda|Cq+nnFeY;RcBri1n|2@VXI!D3p7gh#SVn#LO3u!oEIZutd!$OIdFe)crq+d@~L(k3t2Oyaj^a z_Mt3sqYmlH!?NXUj&t5+pva_2|F4pvZ+yreegPM|@8-e}{t`#TMt7X+L3opJFioGzhYjTVF)%7?{iv?U z_j1JaQG1Ri{YcF8NX&FNW;ME;;F1rtP?M+=TE_Gvy*Qmbn?h@%|zK{vCHdU{$x z8LnrFxQ_{b=06UnxKH?v_a;g}DIcOvlg^WH@Cyd{#p3cy-{VKOTy%ckFv|ppV>Oxy zfoNtzT3hF&CE*>0(axBNx13AFTmQeqXTA-%P;2~-u<2Aj9gUyHD*n>?p`)g7XU!q< zqR88Xm}5c_XQ4DQ$Jt`!5IrvR8B<0xRFlHku~82lQJ&Bnx5R#r+3yosx%3tkuRbM% z9*9WwH4xo?YwHa-ng%4gZ&1sPiNAN}W@QC^Fo5C?& z>jlfQYnMD`e~UE>FR(T6uE zbsE`stxKlt!qKp^CmWE<#;te*FY0^7Bv;M?+&& zbXZX(|La^->~~aX61T3}P_h3UQ~(57v6o6m8Gx407+PA>XgM+AK|}wOnjyyFq)$AZ zDlmTaiB5(CbN+Ax_=JFsXhhf-&>J?4t<)QiM#*%jJ4d5r9_~3|U%U{&PE!jCO3}y< zR>lW2H(GpGBgbf+fjwn+p4Jr#p5`i~KV{D{T4J)8Np7q%sT0a1r%Ftyo{+-8odK-g zxw@rqfrq=94$Z3|y1^6C|jbX?F$ z$oLW1H?D$XudZ}v5AmWfXX6pq(Z)}6HsxhLw=q3sW#J4$VFuxpoa*xw8LJwUv`3!^ zCLi{YaHoGkhUC2strms9Y}RxOB}cBS)jR?ba9A?MB zqPdg*fFsngO}rpk+d_!Ta7hM`*OL3k;CofC$a4-e_-4?%Y| zX|o*Bimvi?6U+!GbjSpUg6d?DSg%zX!gjiF0Ak!hOqkdVLPDU=82EHf)VuJ|Gbxe- zaU@qmIV9L4mIC$0S{z@_)(O_t>N`AQI%6O^0oahm`w_pm_`&t-5JB-%(80oX^Y^y? zif<^tvkEM6_=$#A)lX`x{3{_8f#oO4mpOimpHGoqb zbHgU%U+H9c3z95J1<)spG0`UrFL7m%l?*QAZ#K>rp?2*Am(-tyGxcxs9|JYoTDnX9 z?Q5Z>NUtslHB`*!2Ut84$l_r=p4(zoaU6o{8Cm%XtNry@Rj0p{=cs->;F+Y?*eA(Rl5c5Y^&%F-~SCxC;0*YtQ zvpO@=beMFGR?zT-ksLos*Vt$pM$t$*swueuwja_LAh=7-aucttw9|fcmWHtnsET9J zb;bqe_4zhc)`c0Y;Fth-raAv%Q=`JT@tOTgEZ_^Qsr64;Yeq^#v|-cMKlHTro>>rqm?#`z zK`m{+RSGhxK#`Oz>QuxlZ0Q783LAijg$*rMiKz=zh(WD)wZxSZ$}DxwI-XPTnTjW& z8COYArFQ;RjHM)PQiCvxO@}5_nCdW>bvR#tY9*1aD~IbOR657!LZs|M29``7+9rW! zq<^e0@sQGyibZQa6UXFbF@Hu)s9AHwA7dN?od7Opkz zfonk<{TEXf1T9mp^Mb-Xs zoMc&NZcms8ZI1TzDo*1QSq)poso1bqaUNjK4>32NY1q>DJ&_eL=MY3#I8M7tz$s^G zh^#>8rbZP=cha}}HJIB!G5rm}^jcq&GCs}S`bi-QkrO%1<6!N`ID=M}+nf#=XQh4P zHzcVD5!dtI8oxByid*NaEZ^5Woh6u_?`b{91~yMNFgGfAi52|2y5Q!C2H+BWU+7@c zG`$kl_}QFD5F-at_FJPy5tLtzRNvvWsVA)BOZT|zn7NI}hkQW=<81RX@GPJ2A1RXNW4BCuaCy-@pwHJuP5X6 zNnPodxlzez+R^=tXPuoeq+kp+?YJxixnazY2oJI$c246e$jKeHSZ2&WPtKWdL7xoM9OZQF7b;iEZ6!xnt5 zsTgdM!&6N}J9vcQpF~ateMLB2BKrhG>`{aRF=LCEU^}jZCOLsLDkVm&Q%cF5O?z+3 zi4QuBm6P+2d{+b!|BCh?rR>Ar!u(8E6kmshkXb=suK&g@ga#`%mXMl7+lj+%B;iA5 z2AU%dj}b`uPZNwy5E#Zju>xbu2N7GNHDGX*5MLXUe(`5N@J23F^i*mwG{en&QZcwV zBHT_%d(0-vmGU?SOV>vg<`yKihJW_CYy9I{{To^6>=apee@aZpHXkhyiX(rqg$?Cl z3r?dFdMxhAnno9TK8w^CbU_-2RGHu1wZNBL@u(rdV)-u5tkL`6TLd_T*6Mm+WAM*( zePYzLo7=gmq0L6&_B;-aTo(!|R_!^qDAs zd?ME-1I2%xafst-LN==74Dm`3uHcHUaO*#n>U^$KZ0e?-5457WP(f@r%eOfh%Y7oC z8(v2BW;+<-V55kpl_!?;5`Rt>PvbMhL)u-+T;OphXqCYD8n@ouxagk7IE4-F)9{Gm zC`5TbHW44*W0B_1su2yhHDB(&f}JZHNVm!Ia*uMD5z`CI86zX9R#UidvzIz$t6AP- z_bt{C{j=y1Hd&`sjlDxgyM^Z>fFe$2Rp3Eg&nVnT08)5o(0f2>d|vMXT~GNtzCp>d z34QXNG`Nqt|=TMl@I=+{_;r6%B=lL@M-=bRU|82L%cbGd;)P(Y%T3MSMQs zBf?bmEgWCxcNcTClD+jd4&j=bx2#B=SziIdiUg~#{0WnKVAWLMf-VA9k&yD;5t4uv zEAO{_*t>~)ES;yjRbYV={c$5NOXtA{>N<*J`YZxK`uUnJOjt!(F)&_BQLP?N5dorq zv<8;l2GA|v)nt7kW~VPm(mEnZX0pl7qvFf=dRK%GWg=Q?^ADq0?`Y~Mp(@?XF|=VU zDHa9QPlPA1S!zk5FGq_yZ z%Y@S{Pni7jgjrLEbjwoN%m;%|Aqsp%KM?~J?mL1kgbTy6uA}LpDev)|Jx)_Qo(ydk zYCqq*$150n^<&3opSk`(q1X41heZG?>5;9nAsV0>;YlBxxR6R~>Lb?E!y*L02+y?^ zC7(NW7Jq^j6|lHJ@M#nqhptA;p~qIUB;nlk?R=owSb}`1(XZ>v+VilyOVNPFie;oW znc40AfeoV zP__{AD3ta=-j#*YyZSVtt0>y1*$wB3X9;&HQ4T!)Z>`vumi3=}=k))8t)~CF`Y+e1 z6MNcW)fYXEJyjSEdal${z7%c#RP3pRMON-9EW`ZBXdu3?F~1pox0F^3!WDJs~PR+!rSS8_h>x~6CS8m|{ z{<<3wx^(8FwH)^Zcmw|)Ama`bR9K(sjaPX4lsmNKjp$SOPu-Mjdvu_H8M>TRgZ1SN>bUMZ`am{4= z;F@uTH$j3VPTC8xXewI>K&o^S-iU&j+~I3$Q>OOGP-h~2M)wH%q4$(itNXm`z0(;- zx|w`J=W0lt5~V`P$=#+NtsvMyN!QNuci?*s%?DYI{;V>ORA>-FhxTy>pt8%Fl8=}B z`3^+Yv|#yj#;J@J4PjbEIeC)p4@Awi6Ilvdp$i?)p&c6vVF)9`wuLaX52iz*p*}W6 zUkRp3Z|5Gtu;Z^mJZF6=UTyPE0Op7vX5j$~P$;SxqZ;hpAsG4*%5_4<1bgpSw4*{|A*pUks(FcT9ma+gC_k9b% zld}kRK9RD~Gs@w*E#d2!&n}Z#x-AoglNd$fiR@q@C+Xwl-lt!44Ls zBS4y&E7hkK%wbBl!v@;V*VvS98*wWU&_Yl0f2Jm?WhMR#H;Xl16{$%z6|XBoU?ghn(}QQ>&4)!QrHP(TxLjw zKh$67Xg*^G4;bciM*%gXW63X{(Rpc#Bh-3tO9a-)pdU_FT5vlmVrfMFaWP}Bw{Jo` zI4>fr%~aSdX_n4`Zkv5FOt3q+#%2S0DH0YYWELbS51n;4If1)c10_m{)au zjP4tw>ecHgQy9`8*fqT?MX8oC7o>p59z~I&hduD zc0b+_ll0J#WxR^U`xmn`-8j_96tpfZ4%wuH>@z7q+1$+v=}hVYcjSAhCm%+-)@+Z+ zeEv=E!h|Y}T;xRoxkU87nvcC0xG#0?1D6?pIYnUIc=&_`#q~WPT44IkS3B5~5KTU$ zQfHc=d*cpXxFV!FUeb~(EoqRCK5B)R=qC(@?6fid1>}~l6 zVskaSc{BlY38hJxa|NZMm0&?*N@tn!NKAQ%H4|9UP?(1kdNerciYX5;gQb=7u=QnH zG-t9%?G>HSEZ=jCEUDSR#8gj$>0X`Si{tOHU@xUn0Ycu0X$Rk?zN}YZcM9oIlNgYo z$9XNo96U?DOvu&iqub)uid#TVY&v`2Rs80T#RN?Q;<6%Sw zcp5r*2XHDzEqw8rAE8tFXjF;kkS=IXot~4>n-6y_>M9>Yi{__yrRq#vh-hjZ=s7s1 z%CMy}CHONw!_`y`yfnMFu!whq@Pjcrn=Pn8No~eC^b4pR&g_PJK_n%hCEo4g9U;3z z$RkOCad969jVlF>W|O_6Ow>p_)0s#qe^RWdjend`tXm0|#%ylIRkJss(%@ecV`>~Y z1&tW=u0>o(jR$0ikV%pzA!8Mp+Riz&DC7}Z&Vew`s)iQYIw0=$^^U#Rh7_U(?Iq!F z^xZ$Zd;FOnp&7RwPyi3~uEjwX5Qh6czl`b7d}eqN*rL!g&$L9XEPNYOXz%=f(6|%O z1mBK@@SW~&dK><$YW7++Oec~J3ZKbq5@+m-AErAeI?fJvYGb&d)6_{{`gI@>#aMK2 zw^DL4W5bsGEJn}LU>3Es4cP|vKDGB6_HHHsKI#TOHz~Iu@Zt1lRjaCEY6b^*5r!C` z%Vn292?63#Lo?mX+6~z@$aw3Yc%TpdZJ{4CqjZZD@@=vjTej)k95Rx1Oh2;Muc!>s zk z1J97DKF!7#nq&v^h)u226LqdpM^b7f9HYe;n0`5?2qdRgRm~q-B)%zXAsqD9dD(;t zc*c^KVQW_-CV+)F><9>~n1ehEcP!q*NE2RRYl8C%txoU?wX}y#p}=O#`v9f$jvz#Y zp*;ZuLKR`)&Mdt&1#u~pF`D+pUD-Bqm2sIJbD^jG>V>qEzjT&o-V9*!ot)CIL#_lv zo2u+3FYa2bVdfk_v4)aH9fAH69g+Xaz*g!v)uj)@=!h=T$<`b8?+6Oofi4CfI2Gnt zy99_P%{A8T!B7fPQN9|x;eyBRbLXeac+Wv8V^A1x+;6R|9QQk!?l*@eNVU@M!uyTg z9Q9iYy6!iuuYM~D7Ip$^h(9Kv&aJ}YsNcwgOA=EP5~xT=zy>4I^3Famnbad4j;hdU z4N7Y4&=bRsWKlETbcOX;piNppJ<&f4ZA@wkGhd2uXk5yLCW9=06@GUz-v4~7S07C&&dlO2vF;=Q%)R9w>735lrgD8S*o}*M7=PK34SgAH#sS4IwsnSE#Y7)uM zC+5yotdM3>v0{PHmV(3;D{jbyVihU%4r~e6bUP|mss)QQL=xbmpp^~9%8OaO5Q=y) z8($c=J%{|((kp&+3E)l=KuBS%wCNyffoF9; z0PMn5n}z~_iOi_hEdPVnl#oOahRIGnjzN3C--6z7`cdAD{dzhGv-4E~Sww7%_y{A9 z%vt{g${{NZy^J<|3r$J^Qdm0o;h7gP3#m+`Yt_A4@GRCCIZTJKC_%P}M{`=3ZHW9 zWwqz@I$i+ z-&!iz*ynE_n9gjeyxs3_AB=BfW=rMm0e^cazKxkJmA41|?IZDR%xtN=J?w9f#fZ)0XlNk9f1V@J;7mq@2F)?@CdYc*IYXm;HCRHDsa}#K;BL;&J zx!XTZ^fApYMKAn-c@-Obz)gLZz)>JFV<>Hi2-vLl6pgQvW2GnaD)~-AW|ZQSd5q#? zFdszX*tW_KDheQ0(9v<+06T5!8@@b5i1Fhiu-v(~E#E%E+mI!a=cE&ejrW|B+}!a3 z0@v(HJ0r_qy973&kl3l_^;jhpgL$2NgaIa1jUy*nw?2db;^FSf*%XD2vPqOQVMOo59bOJs%Jj#G{OvX0x0O6JY>X@z`K+O;aE5?6SavM zLFUK0t)yb_Id8=sMIuM3Z<<8n=oOcRPj_491%eDSc%%TBq?{@_H>IQA(#-2 zy2eO)6&Ufx2z3)R)83&VXBZ-A_@l*RiXk7 zoHU9M`op&q6eKY4Yv`cxzl0;Y*4uHyZ+ym6&*)mG_P*No<7E*h7{Vc-4H_cQooCx< zHgm4YBG(X(B+Ldc4-#HFvreD}iivAbE+eiPNq{EgY8@P9Me$2S0)ZXs7BLDk$`C}e;{hiIF8_|F|LDMK`gX_cxf0pDI z(_NeLATw!b6Hhf{?X?VmtQZ4Qt+0&&rx>kGHqi!&(qG}*_)`cu_YgAedkERSRgg$u z2{`i*6vqIhL7EG%ZbswyO~J&lJ33xK%^CKrbLK~b!^7r{p~F+nnQKA|AUMo_94&R# z=}lQo3@i{$RNP9^7EKo8b5StH&@7l`nK2KMeNm&#zR5Cb0z-1=C3}wxc*v>2p>{;SOdl9^O78OwS2oPYXvz*nY?_WZb{?m9xL#vH zVXXUrwNJvFB7j=PXR1K6GGcd^L#^{+hj*2}#xDzqBqv_mM>y~>ILFZBI4#@N4aAZMWz(Qa`i+pl8A_TVAN)V023wY(?fCzWwNNE4mw&b zgmCKl7v9B*Ad3sNte;aAPy&*SN|Xy-jWuivWd($G~E&~OBQ6GwbmfCTEt@gGd zd!(3K5giJu)V`c|Q-nRIE&wnL6XT5v*INcEcjz=Ku};A79$%9_!zZyBw|X&zQo@_V zUCw0k%r}`&E{d}fO|y&&_M{cs&KGV(gIuM(`!(V2tVK1>jES`FhFWLnp$`rHBU~1= zB!w?Vcx*AZpr!FC$$X8v7DIH91ue%@J}CiLDFwtK#}Tp>U(5u-9J8FcfjS^f5>!`g z?j=&om09G>n8aYac&HV5F>Gy{@)V>*IWhNU3zGx`YXUH6u%$sn_nE*KZB5^yBu%#H zvvn^wL!<9eqS9Q4vl_&7xN8x!9!x7}qGL3d;&DTCAv@KN^s@D{5?;YuQjoQRw&YS@uzQqA^s-!G!y`CviTh+mr?zm`5m5Z$c@wpv)Yist<(9;?8 zob>ntFhLG&IRTql`7`D{K3$Bz01Q%!&1xQCj4yKJUp#PW!gQyxr0qVJdE*u5l7pHH z2nsJHRw`~@5WZ|lCv1sX8fjUpvauZa6dxk0FSQIiVaXP~b;2@h*kBIzP)j5PFdw)% zK)p?^SVQ8pf=32Vyu_uLIEY)b%p3Mfe5V8>Kb3&*wbI;XV6+M(7<_{+t@_SlFcWBk`I@;Tnu>r1!0HEy}35sU>>lL2J5qZ>9DeFqdzLvyEn8!Og}waQ+a5c9*_E~j_A zG_f%$cvcwIGbTY=H z{OPXzdEuM7T)WEEmBQ7}NG)2*AO$9ZByfmpgoy?2ZxMj-RFQ2{O(EeFe2B$*OA{jU3z8zYhx(T&aMvL(^ZR9wSV97bDyIYg7pHNuZ6&%@iR zd$_O9YHx(Q$EiBLy$ny>v#hE#X<5|G4FO7y=Ft!EF3=3LnAXoV+#k_AUef#JJnf%P zX^>_cP@t%M3K3RQx#DuoV9P)3@Ym%JB%&t+_fkhqb3fZVychU*e}NG z%xb5O7J!Jkn&Dj!2+wAVCXJv^*mf2Ms{pG{2@d~>v%TJltMpZdwCWHb!7=S4fYl7! z6u>dZ-C)=zE0cRF)5D~|SOThc??wyXi9&=hL?Nj}(P)@N`0^2acyQBIf;bJLA$0dD zwT|m#Zr+4Jbzk8Lagts}U*S^|>qy5O4R}&cG;GVyFHKmDpl4}>_N7S|jVjYeE;;o{ zc+~jwGgddYf*>~^uM3!$O^mlR+d!sikY`dWDM&Qaz4F`eIjdKnaJ0<~=T*Umk!sRu zuyr#Lc)$9&)c-kI2FL34OkTZy5x`k5!I+H27_BNb~RZ^n-DFgx8#`%;@ALvE%2Va`zRbf^+{ zfQ?YYR09#+G6GWVdO}+Y2}U^WIB-l4j5(acFzJz;u;YJ@V5Ad1mb*55=HA|=)$}_t z9i|SAieN4F_~vB2Vp^P=1yn)_1b|}ql|h3C{4wFnd_3_jV3GE5PXp$0+9y2C_*?xU zXLyH6DW>2&ZD<`=VY+Hshb+$v#kes8VEkxp_x?B=e@YPJnQf*^D2c8jTtG8Kqh{Lx ztq7+y0mTb884VINrgnr4^U&4DqqD$DgQgZTveO2Fgnhr2+{T;*ms_=_%-pw=BrzD@ zaar=n2bY%;KDJ05d^(Xac{oWv&o%X)MF@xfq_`HhLI<{Ky-iE&Z49DD=v*t7L=D1c z1_2>_K+^7CoOOAOYBG72%DomXf{BY(Vikmlr~;DgxHS2g05yjLvITfS{mf+%@`dtO zL_9#sV1GwjrBxrhLN}Z_M7jDa8@r|kaW#fJzSjl&q6$F+^FWK2CTBn7EyAw0!G+Xo z84^SCoiv5MLKBB&9aa^Db}zkqKCnf2Wr%uTSM&EB_+~eMvCAi_5PmZA4%(IyV>2mj zOSe2l+=F)IqTw!I*cj>F{PJ{G*sL-Kj(O#F@_|i&$yl1J0J1Nt$zs zl(@F)+J$9ffdLAMI0Le^3{+62(`uFho(=SPtz~KI&9ACA$fV4q9pg1=OG#%>vWuE| zM9o})*9}X;#lz3vE(?h)m!N0fTzwg8xH^VF%j*OIuO=a2$OTFuI2LEf2f?7nGe5i- zba<#D@Ff!ud3SnKf^O6 zPlhL*OADMl*3v@Tq-~m{EnrU>D+onEP>~Ya6q2?{Nzz&rA*C&Xf`IeI;l=x^s92}# zixWz{3XTZgiy(?py*OS3yr^)U_pYuFV(iHLH@Av!TEj&4A@3Y6X)?RDvwb$Oe zxCV4GINfdLD$lt2IMhiFkc?d>&Kunt14pj~7K68TYEw){0zem}4eVoc&?2E zv}(e-g1$ajrT+kpZfJzEIci~{hi#(Y)JfjGG76D&N*=nvTxaJMTYZtVyJ6au5OL{< zv~H&KV(qf_(Ps2cv+vd^KK}Eq`(Ul_f?OE!u2Ra6>fJ6;aN*Nhc(^hzDeDoW2oIR!#FOCL|OXEG+7#c zpos&vF&{|y!!)HHa#v*pFJW9=YjPnu9q%!%R3`J*_NwS*7EirW1u$2 zr%%AzlCRD`w4WO$3vz_7M*WgKpat9ZR7v%SMM;6GL_K6tprji0S&IUM)u``S@y?u4 z`m%^qk~+XQ2JqX}$S#0yL`VZAJ1fEqe693nYt+S4wpZd0w}{O4+O#jQxWbsYvc)xw ziEFjE(wMl&8bBTX&_!{;-VP>r@t>qFKvan2-`lHNOd1puoH8CtqW;43g=A-PsbSgX z(_V3jV7@<_TPy`aXRxs?c6u#H{P^+LB%fIc|KIP^^FBX*>Yw%UbXdB>fhZ$d_YpV{ zXE}^N_8!ij6@2z9BHn*Dm`OKAJ$}M>7|Wz(+V|XAv=cZe+UFXkYS2F4BF3V9p~a0w z`vQv_i}s@|ZY}Dd}Bx``-J%p`fW_|GW|bMKv*L^8H(mGTQ+^GmB~>EuEkTg zx5AJ);UR!8Qsv<`H$W>v{X>6d;RTAzI(HB_zBDl7EduWq!~Hf30KQCn+;Jwr$wM57 z<6O5_l_6vp-x1bKtpx@$n>WuCG;EtH%ili=u5rwUpMN=5j+hX1MDNb!31- z&U{vi?|CZ~62`6Y(J%0t+RJ7zk|aRlf8xj@lO$i!IPr?R8iEkTH<2axdbPGYt-m4F zu-0V^G#py3y$SIs2j-giMA@~wInih6T28VvN5237yglpMjncqcRxD3-go5pJV%v(~ z65~M0UI?4&tE-8JKT?nt6%MOW5#*I+0avvtYC`RQf{!o-Z40o#LFxgr8xU)S8}$)W zW}@;Nlxp{*tpg*fI(3N|Qe40v*EV#b18z1xiDywvhH#TWHm~B5Ap$N?ZKPofEZKh?z}FXd?=k z1VqV(2A!K?VeOg}!qJGFZPHQ7Dij=4FOq;GCE?-jn2f5}I#mOXBC$zYQju9%iZe5^ zD^69praQV_^Jq{Q{TY0@_;9vks`7Z9seo~ABnpR|Eh$t03s6AXA=8p_@m9%28%iVN zQiVzd<|mCHKA-T&Vf0L&=NtrCCy#+GkdCiSQOf)5ltSX`YGUVD?7=AIv564tMGP}w zmMJ}?g3!CP=~P2NrWATpAzdP$1HC7sxSGwUf>0E%3jiMD^lh0`o+u zCOPP_R)Ffjp_=9i)jU}CbW^yr##7~NO_3=0|M#If9m99*nqY`!N{OxyK!tRqO9kow zXokU}&JCZY4e|2$qk5)N;{?o4$#f#haFBoq!`52ZDhd^brLUoK*%I|i|9l}{0)ud} zdV~l?bQDiF%O{;MR+@ENwM}%|epDwvvs%9;Z(vV9aWZj1Gz=A_8Y#K>2|vstvbeDR z8>+q>XvP0raS|DCFn$duZKlOKZz4zI0_Zhvknp0FaWVevK6Nvh3L=z4RVyPA{zq;A zXtS)$frU+!oGX3PdW!FSawb2itk0^OZEa$NC`M7evWTRipRK#=L`M0f$TBAfOLFv(m$3kG9GO~|@^UO;BJZKSN zeXh+-t=|%~#4y0))c%g2ZYX`x@o#^jR{+Zz`8GF&5yrOpgrZ5ME^;bAg$sf@X^>KF^XjIEn`$In z1*)>J2~Z&-D;uB*3Q@rU&L$n!UOWg(-!iI#P`F85uBDiXq|Razam)gqA;fBAGul&g zIK+-gaQr~ExQ6)aM*hj*op?|rBkqROqsWaog4GfC$}wyyQ_GgPThtcSVP!hmj^?3n z>%OZhThbwY;24&nZ9CRwjU?bM3O%RFIODhCaRu|d$p|$?BgWQy4KQX^gL3~Z4Xq?4 z6;8}b&p3FS+iTj)>3-H`JzblFRdKsxw1Lk4gVNirF9-0Qw3xrmciw(bzH^gK%Y)TS z#CUWL*8hz0yc0b{eCCGCv$LN29d|Ie!y#^BgL#ETBtQqe;xGdkC5i0d)L=uN8iQJYC7jUo;E?vcrarV+0zAW{fPJcyQb^ zBrEhP4km)svxZ(3H0e2B(4=|)Ho{@MIXga1pcMl&;Ob;_oF%SkP27vaRlzDPwM9C! z0Od}u9oS0kdaQaliW!5EFyb%nAZgMg17u9cdw@TspGbg7#cv8cUlwRR|x?V1W49M$N!on zK?q{85obk`BGR3T#Nagfi{GtYe>FiwBFeEB*p#9cPdPtR2g+@L0UXj}^ zGTU09d^?yk59XAvAiD-9P$YH@({F`rNUb8d(9msVQP5V)F<4}b~(YNAtAwIJ(M>}Yfr`>SE(_AKan#?QN zVVbt2HHUkn&`~7`dD?X)JZC3ickhX%#$j8C_c+sdjy$#@2y`}@;SWpLGSR>{HmOE8 zZdieQ*0%|(FXv{_IoGa478~19MsXpYuAMmHloOv1%1cX@B*rZ#$6*RdA<3QWN#|IZ zA}x)AwauahmQQ2L*Mg*}0p^j8Q;zN^pCAD%&uGYeW+ipP1lpw{p+Wmxk#oqg0DfMn ztoWxu=#VbU{bIV!%V5aFpRr0)SSiLxA`pefX*F5`=ozZl)nMkVL?UyJ6~`CE^J~f( zb}iScGPz2HF2(685(Fg*k@<%yC$kk2h>f#0P%M2#6oa}BWvc9PxR@Aeg<0jg+N$fS zRkH*QIFj3UQWVk;hhm+&QAH^Ltxt-vi_Yt-#mKuUZ9am>sH;`*K-}0R=}h``aU1^9 zf*CScEQ!gELd-e^Hz$H8R|npx4Lt9pV%hjPx*9LQ5e+PHGnlkk1M-iwV)CBjl#CkVMaC&5bj7DZS70-s@yCvSF_X

L>W-ivJK-HObGcahrMgsZDE-Qwjr|cG3NyEto3yBhsuMM`jq8&B$ig{7 z9N{*DaoT7tX6-x)7w3Ck7g-urgI3flVVIAW;*)S9H|umreBCa0C7pl**2>_|KoDMdQ|bnn7^Ytlo~CMt4K_5}&q1^^|!QAY=e%cn*hB5Md+^1bSwT3l%Z)rii6=xn&^Y1$FxO?+Pb zqXK#HHF9iaJlul$I!4|*762~?#j8hmUb*|~8*bh=dd04Cu+}Uk6zAv-Co|lWoDCJ1 z0csV!*W|EdvfYO0!YEowFNI4KZfZpnd4P1L^cw-Lcp)uGI6>8G3SiN%Qkf+cOAEn+ z4*dZ^pfSM_VFVk0Ht8=cg74LfK`KhiGnud+j*~@~AQjoMD}}gM#raFPtkA@lMs6-S zTM#|XkWG_CPT&|XK}Y%`U84z&cztoR@({7c_z&aRmoVQ5;{cy29yn;2;ufy5@^WP? zd{~=d|EAp07`FT1!L`tn9)*!Zqnoc+3^O=!D(P=+gje{0i{qkNY@r6^je6Vztd79G z$jz_mmU8nY<wM3Kp%O&dK-lS60EC$hRtNvt)Ccz=1 z>?O9aZ?P@3WRkT+q|ep6&r~EsYl~Gx!-=ev&_ct-WdKYP&H8{8n+CUUPYdF5z^`|B zkBUMqek=X0EmE$j1NSCDr%@Ym;8{@%FLJk}Lp)fe$8=;1qKyowa zjY>R>TS8_4378Aq$uL52ubHLz2Dp>9IMeEW2aA?UPXKD#lSr{v8pS~96snnZC)8UuUtLJsn zYLsL>(wQS#Mk`p?!z%y8|MyZ;`Xb{54ngCdkJQv!)bo&t*<{$wnc57{4%h0%~0@ewc~s z9r!oRG$VYHV6Ax7C9!tXuxl)F%j8%@2BhYtB7e@k|IEHouk<_93i9!&LekOm@t<$A zR~f763c4fL0P9bOZEz^FRbBizSK6(tusg{^EJvqOIBL?#gwP(>N~x0!3ge@(CI~R4 z-z!4{UI@q#B{aXj2~y;W0v(OC*-(H{!FFZ>rv!F2{R0H+90b(knfS@2TL{J9#+Rdw zTXa7y#{n`Ik1a0+X;L5z-XxnH9~+@J%@2;2g<8`H>m5%3;%q|uQ~eHp=(=Z<2M2P> zxz>RG2Ln1FWiyn6jnm6mnTTOfrLSvPDC*Q!Y=C+>*$ENe*4lO%TV}C8^1sa{@>)9m zStI|RGCtR|-)00MGX=EDr_^)Tr8!uUzwIbF{$BEYGAxL@7otlVnB6y7zDhOU7?IK+Y{5`% z@#C*rYuuo8d1#Eah&^yMIk61@<8Xeswkup4BTxRz(C{2e{`E4N`F!ssXhqSB!iZxpctE7 z=PY^Vmc&jHuBGE)aWxi5=n#GpWtlKyLJ1+sYAoi})jmn|ac^11MWD)YgDP!RD>MV( zkvIxfTPjB(<*-hX(Gv1VoQ{|YCS5X*B5)CfV>+#(5p1fnDAyQI9DbONJ85AjAK(0%2v9WPzg6$4FnSW33;c0yy~Pc zGe?R6DV(4%m9zBkE=qH6(5ury>5tngMeX~6el;rq&xv~{C`p47IqLy z%olvgSYJ@e#%K38g@GRkz(km}-!NeZDA-p&@E}%G(piDh91b|{H7OIsEf5L=GS7q! zX~RTM`U8`O)h~K|lw0kU?hsiLi+HyRFrdHQMIp1ysFs-@c1QeaFBi!4QjHMf0vMx_ zZ+px6)!x}iq5SKZ(iF}dHa~Z3Lo&~BJ6QcB7Yr!IA=k(Iz1l1p7A3amU~T>Lte!CK zlPizL=Q!45Ua(;A>l~YKT?aeXbu0L#mnB`VfZxrS7!}$YgUV>Re7dfk{wPD$R*7UZ0gaP?FqlNKcTpfz)+5YysyUUXQNH+d zX$ApU&7^E-oM!OMM0ib_=Ewh8i&NU>$G;g913@2DmK*+E@p9CI21vWBoLQ@Dy_uw# z^Gn2g{tVLqYtjo+W77lXO8UB&N+rs2$5NEJ1t$Wb`vtIV-~v!Vg#5*5nCP$eo;|zj zQsP)&1!Vpd5?!JARA7Lrar+(zs8QpI6RJ=mCk8%63X>9$;7?Lrvn+>H-8%@2;DMDk ze%yfQrKNNTZ=&`kDaW=@^qzTnqRB>?b9iI05_hJMPDL zD@&9ofk_b9=5y{3!`4;g)s%biiYOY zbOF9*5an6cp6a3NnOCoV#@ki3s9)Y{ee&4c%6T*3u7Sg4SbixtMa4bkECzToJ(F16 zFTZS8?c!6i?|@Mw%|>cYA)=Tl=N7N(F?(F%mc^QrYm}RlYm^=HM@`V8W{#0)GubAr z`~Fp9s*o1Np4vuKhcYXzb#y{W1v|ikimR^fR#!=w;Cl@2H4)~B#`xE@z$XR-i!wAL zsh~;oi2ar>lZr9TIu-qhMQ4*}Nn%N^Olgm*7JXTH1s=YN73t}%62S~jh%oYrQ$k2Y zV3JuyCIpjA(-Z39L+QUvr5WE1KkbIPZZ%iSpo!2zfi^g9F-JVcD}KPQP6p)*kj&ob zVt2oRM3SK^Y=^7S{&{+=e)5~&{njnlKKQBMd3(Avbs*|DKfdwFw}1Pg&pe_is4gN; zq{r%?Jo2_D?|<~c+aJ{zZP@fy|NfhI?|R^)Km5#-ilSMJ;IMR^AO{5BWOOPb-RAW4n^hRL8&|pS82RQn1_TT)RC+*K0AnfV*?2k+|% zNE@ppIyj`wvRHdRVotXS?i1ypl2J9xV4>4b{HO| zpbFX!P8hTjoeenA^$Fkvrx2X%AS~d-!3qW^)<^fiB!N@zpl~v4TCj>-;s~miQz|Ol zfazd9;RcNfH)tHg4cL4!h8r|Q4F`bC>~oL`YsB)hprt$=xPPk=VvrC&tokbw;-C*w zCZ-5p3$2ZylNYJDrdV?n7C>@}wS=wk%(`bXycOX_7i8ncluSd8g)p#bPU%TiFKeee zMxk6(cbRa(tW=j+m8vjS1sNsmCoLuA6Dlm7={%=Bc zZ0NeQB4GlZ@Bn?bB^uT#;M6{=4P~^Oxc(HJ>j*B_O_Yfd345FDNH4Fl4ki-5D~zO| zmf8TZRte(pf6$=2iYUy%mUE+;a$H^$%lz3WkB02z zBmx@2oAzBFKFop@=XuKNfM4ebI4x}NCk7uH0B^~-u)6@`AY@Z~@XxInq3zjcxkAzwC?C)-gKq_;h-EbGf| zNw!I#LbN+6ak88Q1-WpIryGT`V&306H4a)^7*vy^7G2Xc)zN!2RT-wb5yVqBcPCzt z7;>g+9d~Nff&^)8G1*#M6pf$aiu0dCaq=cUu)N8wEn3glQ5Z^(2op6b%#zZ4!duVL7d#!pBfsjc_In$yh{3@o( zGCRF>s_`5G0?%ah02(wsnx=W{6hIs%E)bJtIiDO&5oi=|#i@wasgOgFs;9+&wyK|& zq{OO?J`z7VCWfw!KwGWNOjqK4^C7>UM=&KixT0Oy;4J-%lB zCEmxA zXO*YS56oeBs+i;y_f<=>1?Yo*&|w;yx~jG$C=$~G zrxeVDnPmT$P^Qt~uts|Ha^MSQ7<=)49ZL{TM#cL|GM>OpbyXq~__a#Yz?(E*+Jrr^ z_|q1@ITZs)nOq`e0<|pfh(0vZH#GTitI(UF?Fn_UPJNB&%T#;OgA8?cVH5sQ2nLQN z;=%PkqD4WXj8hbhUcOOv!79GxwW3Pv5+8XTe`MbB9K~96Q_0i~JP>HBKHH4e)Wca` z$!LNp&|{yhHXlVkZs+`;`#dn9S{t-WE2$U}WHnPGXNkU5y&#)LRx3Y8VtA@072+3D zm&7%tRh0_~={3VoIQ}#FUgFJ~Qf_hISqfBR@#kuS6iZPOJv@v0q>}&|&Co$fxxVzU z%{?R|4Oo1s>!-l|xK7xoxIiv6pTz&hob}|tT5g1Ih}d&cQM8CEO3^TgfkZ|+Z-Hd! zq}E~lMNJkevumu)?7*Si>RhW}K&{xIVOHAwIH`3C;t?Lp0ArmEt{qJ|WtJMB459j{ zMY|hmH$;>I#XwS)y&G3J4(Q|%B$UZYrRC;X#(Ht$i&B5{l8~w#>Xb^mSfx2pUXLhu zrHwDBL_txr4r~=To5|78pA60B@dciz0^JshdNhxT3slRZQd*#Ji=OY;%mFP#se_u5 zT^jXlF_^f>rUTkICoOVJ8ylh~iHLfZ+uTGPQbC{sX}rcYETJl`Pw~Q;`cgs#a+=^e z-&6r*lX{*IrJyOGfJzh-JP*iiQm@V_RzTXZXX#Xo-h2f586k4agcQro8a_~fA2o@G z(cc6b8h^Ewptbdri7f!ys8%nIV%CI+ITO4gV%vDYkic{Z%aCueli3ZezMWT<_9Yor zRL;aC zKk4~U{oBND{t;B~3i=09eYIu(FsgS&J|C)|o4yIaBaZ_tET`Y>^Idr=O`SWN&l-Ooa zW^$WQ;%z6$B;kObG{v}sc#K~Z9CUDkPI0YgX%fN1?-cf)!8yfjPyhvTR2>1;>3zTv zMnFXelj4%7r9CDM8w62dE@36!hgEc8Ag!q?depJQ z=_H|e+?aWGm}o#AB?qM&G)&Q!_;L_r#w@@1NikA|=4%_gJHpgUNl|72A;TDyE19f>5ddK0SC!f{F}J*&{|I`rFJquIJ$VL z9g~K(oiPxw0>pG;H5@7h0wC^R<`E%H0Uv=On{bdlz>M0z)KhKY$lj}>>~32o2!`xx z1gg};gmi$dzkl)Abj(MKM~@kJer3empb3oKU%<@44s-J2U+>vBs-9HN(0P~7-WrEA zc~Sd`7jwZt~niR;HI`PD|jaFk~xS2(B9nuBVTl|i;^iu&oooOh{XnKiNIAfC=p94kT3fkbs4ir(ZyXsPAnG~L)icznno>E{*&5`G^c_;bu1QdAXotSJL0T)Iai%eu)bE<-u-aeb!)R1*8f)a#cSC|H$CZYg@Qfy|Y@+~T9fEolpgP`rrO%^P{*S{o=kSTps8iJ@) zNir3meAIUh1y^1w4iPf7OX|sj4*W|{LJxwz61RJ;W7xXA))5_`7(lfKe(EOZEG&kN znze-U0dl1MeLS?E7`jfkQUXhTtGy=C#p5S!UNHw~;d0iEIZ#jlKEk zR^hbRGp1t|^c#j`^uek8cHk8^#^1GPmM=FdXMBGW4ggsEW<{00B#^e!2yl`Rl2^Pf ztX}d3KGP}?ndT9$;RT?eV#DvX@7_n*ur%TUQ|LJ`!LIaSYk%VivMvAd7fuo#> z9}+h}P6!}ce{3j^xrEIZdG(95)W!-?sZao&cnFNWx6O&W1dn!Jc_zNwz&`GRH^_xj zDnw%p8kzl^i6(Lb&0$_fw`V66QK&kd`dC<(#}ctqA}!JPYv3|I(HKGri|-EEDp6hS zPAEkXOYv%QR=2A_l~4kAt6*QH?;hAFqcW3@>qyXe97O1tZu%GiJ?=6Q_GZ=GCDH&R zR1UE_9a*F1u#W2p9@Ubx6J7VABy4$&s+Vh+dPWmk(Xm=RSV|CMGUW(4uJB;Oc1dif zkc+u$8>}XCm(V0`?~v-m#FFamZd38#>OCH=2sYJ`s&bEqNy4#JHI1ptE(`g4t8zt* zt4iRgVZR?PWt&j1q+(T$CA`6}^(aBs&7mhebj~`kz|0}Hoqa$8jf2U)+XL<}3giQH ztgK|PInJ|hD1$-!I~wAVx!d3#kGszsEW(Phgr#?y4wQ*Igd-LHyDw_&y&!(4Xsz^5 z_8DGly9SZ&DBzT>c`0oAXy*Ssx?5pzXj)}V0BF7J=W^aqup_Y2phJktP1p@un>hh1 zHFh@B)TY*w&`p5_=}_YEEv%-|)#j+AI=S8xPGzE7E2el<=vO(k$6_``Ejo>cLwoph zGS^mAj>J+^I-;U;>BS5Qlx*oLG3j8=aCQ#PZfbxQicv}P6c$RAkCtboZ(0k1>il4K zMIF%xr*exgWa1nAekWOVA|IxWG5O_z4b(?>XyKr!tG<$oC5o;#Z6Om)>SC8)CPD*Q z%cG?Ic6RIH$vleMlZx!u1H*IfFk(Y018DMnT-6-)dBQhrpT)oJ7DVh7T#JOK9y4>W_#JnB1xQJ@w% zGxKGV!yu}%4uTb6SbxhvgGF&Ymh6pQbq70V0v@7CN$adg zNdc%VuKm$846rQmH;XV^62hDbJqUb?7Gbs|gb9G$U@)*4&9umHdpehbFeYGaCpC&~ ziLA@omPmfiTLVUi^v9gVmR>Y%`*1=i4AeL{EE;gsr{MRlu0b?}2^ za2-f>j#0un-9Xd`KvZGZutC_s`i}rJ?!e-Bze%t=#2LI|{KOsmkY{P=W`*MR_+E?s zQkur}t0>+-CieULly+Hs!GUMV)U{GwaHc!P|@8? zUrCDJ_wAr$4OdP=g*$1F4508U!0-$h?le>f3<1qdk!oQc*xTXB@|D0Y_H2hI2gU}n zZcUagPZmrdNy8DEW+{uto`$MZ7k%qDU$suF*?_QRUCrqLhcmz8Kl~krsCbj$_3hw-?W?hh|_di1g9D#>`ht%7Rb!RTgz@0 zz>n4;F(RxDk}VNSD5cfE`TXA7?%s3X@BhQwqlg)+pT6>+zWBDs?z;1HSM1qsTbooQ znn_J5F6UeR{VU)6`R)Jmm+z!G|ND=(eD}k*{{H5VlhYl2W&oxu(syW(@qS29B0#$0 z?FKMRyR~2v52lUdBBdpwYX!IhBtz3Hp5-|a88eI@`WmpXv%&JRAC*2TA*e}}KHjH~ zh)XYqe7~=N4n6;ha{S7RKdl$%HN?VTgQpz7RNM*0molkOa9^E?gCB|!`ramxy6J{6 zqm$5OCg9JE{?f=K750jYX*9T#EeBeQzkd}KDZIUOw@O27S0YPhfxv)K zdx%ytEj5~pzhnS(mY(smTUx<<&s>dHcn#G?7y6Tn)kOF;jP9!;gW3U<_KeE4Ek2wP z%lqgkW_+2H^jYA!S}-g#ZulB*N6yNwL}$4d18qki`S3K;-O)RF$L^iX$Q4I67Yhjl z@zs_<1ff}thNVoLJQ^aVs_p^n(Q!_GJeOZ#`$gqolU6s#?T>7VkAa!bCjHV20Qo3#e zUhWn_L{u}?lI@%Ld-#d7M(i|2*{!&EF@q;C#uz*6OjA;P4(W6&Mzxxvm_-rfh5^@Y z>pjv!N`aR2=uiR+0l_Y|<%`-46PzKtZvrhIg_MDP$`M-=rO23fue$CP>qu+59IJp; zqJpw>ZcI5K*ib5K0^8|^75(Q>OHLVZ=wIkPD#!}sD%%1sr;QrR=ibtxM`7Hr5 zZKvY_YG%@6JKdr5e_S@N7y_PFfd?f4f4g7)D?N0RS3>)fP_amg(q~%g;m(+C^c}$gp%ckbh|tNOw`n^wT|En%5*Sqz578@_;W9}?>V?AkhUxvvhe-@6}P|D>}Z!W)#v*!xM z;$z48LdQO@_{ed;&X&B>JxcI1t9CGQ1% zD+QiC({RJM@RV`kX@qBy??_JROY=-8j=Pd7;d`m~h2z3%N8iT2jT?ITGr-^8&3*Pu zUR~61Z|}eck`1iiylq2oTK<1f-n^RfenwbrZ@jSgqP~IX*oaKgynP{W`%3epS6|#a z5DjncTi-WuQFOsR|AKv9%kW6g&`5OQr6avjA64y$_DzfWhNHo4Bhlc-=)%Em0~-i! zOp-UNz~`EHg0knT?190Ns3%%~anFz|zh~&ul~Lb_KJ^U@_igBn_7$Res%1FZmz)1A zwLib&_W3LHzqikC+viOaTv8YqP==$Pk!W9IbRnRi+b-xOP#OcD717AWz0qcXOBeP{ zvK)$W09BkfX{cx5qTa|sI{|b#g|hp$ad60x)2j|s*hM23ldci( zukRhE>Cssi5B2tJh)x|`-?KS76Jqb7Ioo?SZ|jXVY#S0mP<_wFk=~(bL+|j&(6;p> zeS-rlXu-Z&E2!G&ZbP)OZ*#99aHA;0J_q{SHneScWPRJR<=yRFor^nr7jNuYvaF-M zYe~n28!zlxzqn&V*Yd8;4PCt*JxjYg+uJtxT{r~Z+j=%{9$eoxJhZ-T{ov-!j6y0M zZo6<~sJFLmOV8G}-hq*!OBW3d|BowJKRDEDrNeC_Mhxo!eb45$fx&eyJp6p%P;Do~ zqk;zdb}W2#&+wLoTQ_fm87)+JOV1{UjIBexTZaZknuC`3e@Z^%u)C#q%leD|K?RNs zwQcPgVLS{xAFxy!2=>|rwrzPnWu+K&*lIKE{UgfT+A{#5!UCR8VO9JeIFJW=w)b7s zGt&Ef%C7)&8;u>@(A%~Zre)*o`IP#st)`aeQ+B1r>w7nEZW|m}UoEX|L+{q%wu^g) zFTQYS@YMqg+ZQeFT(sDR=TOh9+qMjD_=i;NSl;t2tqLNVwr+dx`Z3LOJ=w5XtaNZa z9a*wyvFk`0r=C=Jfuw|W6aGKu!$eFXNz*tyxNQj0Y@+9geM^q0${*yCiQZvJyWS1a z(BR+*+R9MWyJM@=oDs`3>=lwCj@UQ#2uatAdIx%kAjm|XS#(7)*)#x3ETc37udY>^ z8KuwL=g;>FwSl?o2e)3jZsXA4mURZ|6;V=;(%eC^HhsTY5&;Uu+__ zZ=grgE|K|_#8gtz7YEVhj;Yb_UW9%(ttPyQu=LqfAN&Ae>7a{h;?Ce*c~af?XIami zGA`U&6Mm5I3SYRbZ}SNH*p|NGVW|g4q!C9%Jwq35+k#}S)VqiD_57q_9N#xoiNA_C zoH#uPmJ@c2^rGJ(;45X0|1d$>*O0%1p9B0bT8E(nDburlQ}2f3UC1zBDb_+0lywW` zsGTCE;R+E{;rA83weU;S9lA2yyTde}GmQojS0cq>1jEJ1(j-~6q%}@WB9r7s&*n6| zZMf3V3Tz&wo<@FZ>zIbq?vat9KGe>Np8pZj9mNl^xv_6YCAAAqkbW;|oA@2VuYq5N zpXg#XzeD-0@7dbJ=(sc*+}=C1ar5A-iTiV*sdxocYp=xO5 zq6nl&k>9d)bFT=*5%^H=$hIL3&z^y(cW8*=X%i7?$in8(=1Y|og!cAqi8l0Y&^YnD z*JeuI$&`ORb13!Y1k3r_XhF0hie6j4pnhI`{p(&8y^c?nJ|50{#IKf~-dsl=u9 z|Mn#O%0-JJ{`vnu|D$Ek^I!0+|H5(qGWu6r<3BMc7T#~>SH~|6@8(_jZEtU1+`goJ zX?sWevi8pQuJ-Qs<%`=FFJ8Q4@zTW|i#|d-3um?MoIfS+Zp5l8z65l8&Vv9UaR$Iy<^Lx;vIHYhSi_ z*^*^Tmvt;#wybkm*Rt+q%RAedM=a@F+S$>$th2MTtFyavc~^Va;;toKOS?L{mUVS@ zb#--jG2!T5+`XiGX?I8WvhL3AuI}#c<;!W}a;jfW(aXuUTv3acEbUm<+10%~UVYs0 zC!AOf_guIho`Nd885+Id^sP4WS+Q;4)k8g7=gq$$f=Bm67o0XY(0hTKF?l)cSbiqR z=JP?JP#4tW^wE?l`7Obu?Bv-~{Hek8U}od&?4gCj{W*SrW>YW_d_4SQaC>l1@ZI3P zhVRwgA3PBJz~7(!N$_+37nxrMk3~;po(}$){gdBt#L8n&JN^3izW0B<_Kk0O`#*o` z^H+T`m#^zQ=Gd1#dGB{KQ)YB_z3iMTKJu}Te{tFVNw2@=P4CS#HnmKezqn(?2`8?3 z@oA@T=zYVr*Ij?xm%sAWyS{b*dAEIL*6e(tzBpxi=kgUF`tbK3tn0er#t-G|S01~u z@A@~l46eK5vBzI=;paa8yFWj3=21A>|hy`=W<`@#y2v+<8|fdch0l&Rg2O;^Y^fy7tVoUv};*&O85t z^}QQ64exl}6>oU|M?U_EefNIs;{$_V`_Yfza(?->*)X#(+!*?8i$tgg?>&YIhuIVIbc zsn6HtS4Br;8tOX371=}bnTGt@HC;;^m*y80>T`3$Lkq#^y)P~03U#fE3P(;mWbW)K z)9X&7vf~?P=Ie7O7mldgR$P7Tk-3%G`rH{gKU)g3#g`X?LoYn+b#md%#x?o+mle(}H`V7(tUok7X-!wyM8(S~ zdqJjt^ba%hE5k$2_M4Y9?t0h8ZN<^AzTwpMjl0`R)9SB(SN6)2Zu;DctM0mTdH%@E zdAYguC)Ur)PTu*66{B~(GTX7>l+5yc%c^WKU-$0&=SMxm@B7QnrOp1Kxu#5E=e5^l zHf0;bx_s&S3r7DE?)-FadUH$jNp&MfkN%;4xUhBViLaT`Fs0#~x|yS|-+5BFXLa+` z-D?leKnsykxmxrIZ+Vz@GM7-$;()9$Rt?ZDYwF39H!`MQ?+*~OU+GaE}y4b7QS zIBC-4x@rFO%nW~Kcu3tWe|B&<2NoO^EG#ba+cS%UCH{wlk7RBw{4w}b_OIbHb$fSQ z`o`=2x&54T-+1i}vwzgoe9EbR`tzc;7oGR2bwAyG-J5QF^M^nA`7hjg*EhcT<41n> zjF+(?(Y0dbF>7A@s@-qmY)y-_X1J#&^Bv8{fI7v1R^> zl_#t@=auKL+t7R6jUOh*o!@x)k)J)$*mA;}4ZWkgKXv<;?)csVPyBY*o;SY#gI~Ji z&ad6`gCCy!&M)45*LUt&bK2?Wy!`xi*SzWaPu}*KJHC9^*B)q@HvN_7{o$#_ZbojvF>_gYR?(&a){KpT!G zvTg37wp%{_nLECA&jSzNE7%eU08 zZJv}rJ0E6d*Vl!GFdqg8hlWfxEav>CsoB%=v-0QUgY1lkwVC6>g>bl*Tyw*U%%Lw_ z7j4P(zi{;H*(*O09+JEAFX7Ac)9Pl_iRAhjD2L=;oX_--IYgrhwL9%)D+RT+VO)E~l`mNcv>`LfsM&0O_ z$|DV<_aCyelpXy^-G99|?5f*&-jvbL7Dj(G<>;_J*IhV~qc28^hlQ`myu5C7*NoZq z)9OymjJ_dv>-!s~XBNLNv-1aY^9|YTmH!eQnd8HCnb9wWvqIbecjZgo*~l~Lf6}GW zuymwOe&}dJ3o%M;#{jt@T0gjDYtK;MFuZ@`wgEFsRdsk~b}|@mzO<_8AR$HuM=-4o z4x!O}AuM_8DD%9~^Q23y$d$ZPc%FkUqcFP1qE~ray}Y*&^@DXK-#@w7KDFfc=7Ss_ za+w$Shs}CraYbF7KO^JUA#Acog)8fhoZ&}ZWXKc%bG|+})L)_OnF2}bgG2lvSdLW6 z1fbPFi~}x<3TH{+PYI?Wvq(+Z1wS9w2Z#A9$=g8Ac@$5nVHSkvS+7*7w85tk3(pRg z6H042)IXWST`Alz_-FV*zM*iTAJi4|Yl2yn=lfkvJ~d{G{+v30W5&->R4_Bhge4Au z2!fp7?0cb?nH?S)92TtNwA_Lp6zhD5&EFQ3{p}ndSm)=$AJPC?n^%cJAy*&x?Tdnj zDVg^AOgOi$0axiD*UcxgL03WW&XCKd@~SQj?po#fUupHi>--BMFV`1%8K3>5YXi6u zEuR@={dWY1OltJ!7G@TGe?izz-)JO~`Z!=8a3NL!Uy!8$APBPb{K%l-KPJHWp6``P zC6AJR>c1`Ph4eEsFU4)^pT5?Iizzqn9}~{W`h{az3+|`` zYMH=a7X}#s4LS3hg$xV7Uy`*(pM7q z815Dwfokl_bqRm z(YoTv!&*;S)$zVF59|5KSx@x$oPGM>PtJbN?L9B^?&&$F_q*>q$NNF+x!(Twz5HW8 zZGGjVkNo7kdw<#Us>pln{3rY?)}4$6CNd*ZbikBo7xSSb8LKbkr!|(IVHku>*f;ht zeZ=-Hga5HnJKr9Jx4e?)Ad@2L&jMsPrBnhkv#8Dp1(?-zfPXt zPo57X&)+A{KP1nml4siP3n)|YOvA6>U42W#J-m1GV+8=)f6oBZ$$>uRgUi6jONd`U zqgM0$SQ7p(=t%S0H2mba@Kc1Nn(qpW{|^kZs>9*}(^qDpn#4w%dN1Vz|FkrzfAeOS zTtxCWwX@XGhSP`>yGZ_xNjE+a2=F8yAOAPP&n-P|IoQ8v`E;->e^-2reSFJS+0jM@ z4NZMLoBLkVi%m|JpoM0kX0op}Be%YHVXvn88-@nAZtdORd0%CYe?B#C;wSo_MOgGN zJZmnWhEL*sI=^&0XswlH)=Kz&ylX6M!FX`2#-+Y7PGmp$BtO}|WXq?mYH9xKe6wy+9$ur@q0u8eOaWu#@!g`iYtnV)Qk z8GbL|r@8T&gIjuEf!=cg;hRafi06TQ>D#a)IyyQoUK<~`=B$^e{O`Tsb7H>PyWyhV zMHlz>U3Bq?Krr09d1D$}7%fio-WY0x?uX^ZHr@iz8n?C!p zHNgk#I*@eJ)87chH@8cRDwh+?~zM z&AB<<^Q(vC?wnsPj=z>}n6!@F2=*Dk8wWd*oP?k*PJ zs-jMO_x_y|@49E(w!41%XYPO3j`zO*{=_M*_r7~(?!D`Q zdv@G6{=DtI5AI0P#rOCB%(iVwSI>pze{je4_ul`WkYive_w3lQ{lR;iLfh}X=iPVx z^n1768!F8|aL@Mplf=ru>-`UW@UB?iAQ)ImQ{Iv$MoIVI^S*nNTm`%Dxp&81$`#?X zEC#0)X+3by{qKDjZTz+SrfVe`|Mgm(B+pZqcLwdOoh0ofZ?*C~$&wYB>m==NmbH?8 zH_eiEp69KVY1x9FdDd>XlU|l)X{(cEN!m$TnF?jSUe;<+OOlh4C9czQN$Pq@J57_M zmAKRmda3@)2I*kcIq3wo=2WiQbFkM=x^2KJ&TZvcQMfE=wUa&&CXUkLq?}sSP`j11 zTCGmE+w1VuZ)HWh-OoFKOn2K6KXyVClQwd;fcP+_m$E zlK<27@4D;V_w2amu6yr)cjk(_?z->Zdmgy!r|x<1-Yo51>k{vZH?97NyZY*Df5ZLw zJ1_iu``ljl3HLd7$o;kZ_wIjlFS-8!DPDEocHeRT(f!QMhaUX7-ENP2)cwp~xcT&3 z?z9`Ye*c$#J^!YAi%aLKWHRjzT}8d+N_(i1%6T_^McUP4KJBZklBBrAIm??^o;sbW z+>QAZc~j-4@N`l9h}Bo6v7%v?PF3l3bv}_)`crLHHeITG>dWj>!K4-nJrk5-vFK*iMV-St$#-C)?FM~U^J zYF$CQfN(`hE42d>6|+-3AD=Ih4G9Und*8A3iPi|0i{dYtHl(()Wdj|@vEz0dxKAEmI@VO}iPIDo1Q002JMrZ}$ zLTe5!{NtZm2$b4!-GuW;c=CTJTwOfvpV7nP|7am8ABNy6JnU7OFe{2|Nb}7QCGp*9 zvJa*wQ(95U3Q815U7F6NyZrvIE+l|$fTUA_58V1Bo#Njv2#{PCn2U)ZP-IddySFX<&!+g|Ix|M#YS?}O|zn8lzSIzUf z%ebSUxCOHL15Li9@Q40;%6vQVr?{5?HCVP1#ZRV~oLxRCSJw_k30+9wC;kb5Eu!^& zx<^eM-a`|KKe-TUAd90Tn=_hx#iRt< zB>aO5VS-jQsJi8vs;$=HW{GyMsCq_HM>kUx2*v7D+z5C#(hDv#$a`8@pUUedQtBzD zs}+dwBPqNtyez5$n(s^8T-Bixnnwg+V^iDZ1(v%nE<`NEg^#DMH75kn)6~=Y!?Wsd zi@rI@*ATH8whcSEI#_kKPUHZ9GMcXQ>(#uv+MTJ}H%+7_m^=*Ips58ArWhXi@Sg6r zVV+I-dG6>*Yr5_r3|0PM-JPkr+e3N>VT-Aq8RQ&|Ldl2X<$9R91rlpy{T4|$03-NY z!4Q`j(5ZTdDGlclbO+eNsR1t48Gx=b>H#Y~(>6#dKAysU@N>|Q z5lx^QUQo?90aH@BTObK4IOq)*6$$bl7{7w=*GRPh{9;P!|Kh_7q6`{0lPoRPV8mDc z*Q?>^?lWtkap|t*YY=fj&3DV93}?dmh$MCq&XNuXTPM?6t}XtgB>gm&`JF}P&8barf z%5WdbU`AkVD5Bknd5Z&In-AiJyd~yZ8%ggR}duiro*=prYz;-nzQ79|` zzL=q7Ep@CT`tS6oA9=Mi%dK6f{Uh`FHec?ny25`O^CN|p?f~nkBL6DJKivTm)h!T% zhk*kSJJee25Q1oJb3ys}JRtUU8yw`)W@-5`H|$E;avx9#(-<1tBA&Xlbpn%WlrvJ& z`{W%H$%Uk(vcPwvPW6D3miq_dLq1*9AY?|Bra;w@o4%k9P&P9|yeG`lWo}TnZbjX+ zDimCc_v&viFgYxK(N4KHZe~S_(@sgO00P(Y-#M-1ol^7z54$DkR1+Y~5F$h~LK$ze zSuBArc%EK8%OVTHsRg!eRVvF41`2yCj>lGEN>;f<%CYLrSn**Z*r*iEXbEPyh$C)@rYkQRI;+vm1O=ITima8Eyg1tETd~n8d63r`X|={u z|3pxt__r-H)aKFP!CH%1d2!XEdGDmkxf=}af_#{T!dY0qZQAd3(<=+VpRtEOAH!EA z#cw&+o}-rzAwd$|W$u$Xc6zW4@tPa9p$l4x_Z_4UJKdJHp?WN?E;$U}@q-y?0nO#C zw$Qg`@H0fbU-il#lJN@E#E1$_Y0z7^A2f&wm8u6p!T6$BF#>YOIaauYB?^ABB-?v+%~+xfAyhk1QGnXVZDaR)$#zw%90yL@w~h^~Pi&>Y4~fG{N& z4cvO;pqm-A*b`iWccgTq?AIKw6!h7T{{(HpRqE%{t?GspISK~<9V({~3ghR@g+W$$ zho-8^UY+_%o`nPhBVPU`V_8Y<9<*0VLulqOAl#b{bAUq6?jhZ*o*wq4I7eyOt5P{j zszr}~smfYb*sFt<5$3P~pnvIEuzHby`k&*EUx8ZB0AGIL11^isbt)Yo_+9xwI$BK; zp4vq)c^K3gLPr;x;STMhaK-d{VNfJQ9K1OGF@4ieh!*hx)ZqI^o>t&(qzbUb;t3a= zvH=$cWnFf}`9&>yR)T^aQ%$J;`f(l3q-v&x(pEWE-BNWIT3_S&z8kxq7=G zA6Yj?9h#pP3ohns_+Rgr6!%Gi&Wqv)56YcB=Sehl)ZjI`T(J>Dm)(gnd3EmQbdxw< z>d8FRNiOGXMlZnvP&xK2^v`}nq(ds-7dKAe^ZLh3iRb=>WO}Hv0f3`^7~g^CeAuEP z)lzkEOj_9J)WLck-;n{x{k~XuKi9&yEjvc^wXa$l)ll`OVXu7NVQx%sgH^LWE9m>& zgZP+N0~mxtu+f|*9RuNaWi<%cHp0{l@nqNPRpb$d0&e<#a4X`gg1nnhX~udJGmK=r z62pFZSGB5|Pzlnzs)4iv4$a!Kc5{D#f{Zm|g)sFM=|1so&(xzK`VJn^==+t7Apl<* zj*{^j?rWu9G?cMOUp1a=Ue+N8(AeH-H;K(T~%^J}7HT?NQu9MpSFO zn89Znc$a_RYV`jVSVOv=L@Wt~>S}h~1X@YTf;{Cf8;f?$hJtXkj1&UBltLi1gnhk! zg0ZfB0V@ft7#i%R!m>r+6+mOUEcbI0-HCAcfTti*+$ZD^;_yL^ch5i#-!R zqs)XXpvFjD?HVL(wVw9^ zGxo-rQT!H|F&$&ZwwSR4pQXZI&zNxnN3)(lcwol1F=Nk|QAuLPwwRHQTf4@LRT+`l zG02QrFyN}Y3zMaWs-C$@wT;aD0qUj-VctKkc0x$ihM^tUUECqnfVGD?R?D2>A)CNn zN%#b`Ap;Hni@{-ZakY$qR0O}UFWS3X{-ZIr#{SW0BZp#X{CMI*3_LK`K(wT|6&~~o zX#(&Qm@5zgpKFX=foDUyZnL-y$BDMrYvd+knF0{!segm6f;D|lSZx8uC0OGRiga@szk9G&$s=@LWO}F00Gy z$%}gQrCcgf)-)M42H<&J$XJy(jAbltqIt|6;|OQ&l|26{!7DN4Qp+pA~1C5buL>!fYf{(5d2$Dk$=MC_f783caLNs-&$$v++8fPI& z&|>Il9t2r{S2BSN8jOlk=LmRh)d^08r7FG zYUV(SL(lc`6m^Kw7^3foE#a3CVer6B-sEfx_i(luc{#lHg0hcERWo3tOb0)&#QH@) zIzZ1rqJbXRz>bX%8Yk{lbmD$|#H5S?s2vssuO3!xCUal~6O*l=jk^IOeHW(;^cY$( zzc&l6@g*8xBNZkZ-7_YqaI7^s_3t;N7ZQY*H<--S1NrPUGQ2>SWVi-Q4CG5+!-f7{ zS*@uq5CiyQzX%!MSg$5mU=oQcPntC2k`FcTC2}F6yps#GyNwH~_hv2_!E3n;@Di>_ zZ{T_a@5=$i{yssx^+`IVf7NQgX1ZQEYC;{CJ5a9@HGqOQ^&7YssORdj9A0FCJba@r zUHr~B=$)%p4GX&Bs&j)j7gGEypRg{Jx;K5KE|jXe;KEodbb{DRef48a%d zeZRM_x?kumsr&if=4`&N2pP3L@6@}=^!TXuSDV_)&FRakoEA@wrJamvdjWI=#XJvk zFJ7iHG0rLVjO$CfGJAAVSLPF5jPFmx_bjSU$`Qg&=)Hb zJn9bvlP~{DSp)<$!j%hg5u@@U5e!sF7<3hc)^)kt7hjP+BT5{HhHDv;t;?na6Ynp+ zUXy#WZO{fZ)K!#Yz}OFZ>jb^Sw2U_49Q*S_+H~Ve~B2C?DNJn2%&csq$z*TGS9;W!lFWJ#$)tUd~QMm zeU8Bh=~U>Qf5l>Rh2Q%aWO%y@Jv^My<)Ptuv~ViE{^;oS4w!(~4-8j;o%4bdvce_^ z6-}JD#dHf{D{OYy$n3-1w)*3gE<3v0!Jcuhd-l|r91)CsOa7}_1p zY7c=-6+$j`BG2=vvCX5goBstp<~+(Zn(!ajBe7ZLfc1Dlk6j*HVPIg12V)VrUCQ)?o);VV z^;pGq69o{_t58qIr^Y6M zZmZE8N31IO?`x^2zJ4P0NwkVIg~m2&(o|wi@ryUDRM0Y0Icv|B8=$d6DB}JoPPEV=dMqiDClh&TP@flA)a<^x#IqU8 z6|rB@xJE8DZk3VsL3JY+j_63XLStH$HK z;ku$)SDo+k9oR^kqbUt`Tr`)}#8t^4v|}!FpRC~Ee0ftK0Ek}>hPFDV{7qYqP#$#y zX7>6D9Pa^a?oRaJCUrsj#_z|U9I(jd-+<1Ku-^`%2?%C`8@c;dK znTvy!tjS=g;9j7>rAstPRen{|g5Ra-<+}8m&IBDjs=0_Yk2;4T{r)=leYb$hH;JLi z1${N=2DtYUUdb* zc3ADAE7HTdzA^Yh)f=wBx#gL5pRy}GhtZ?IBjXpmo`0>ShxC>aKq_H@4oH!ijU+Gx zQ!-H&B^=T{=4sW%hSVmVP!rNrw8d34R+}~qIAN_EcGJq4iOtOUj86zf7`4zeeerla znc81eL;sqFO%Dag%>OFFFhxQm1t?%md|v4djmGANQ*n^xa|5{Gx&cYvF%=3^UxpAk7DiBJjclilylqggw-;ATH(VQL)XB$G zDbF{cg1OulrW7>ZASA<>xZHFd)*K^0ttdd3NeUbHm$YM(3>Fn8V}k}@=rB96vb;xPY5dR^5H7@W#TgTQHOQPcECb&bY^1c_Dx!ufGknRC`FR{FEx4Jk&LR4)u>30OU4?lbGF1{@nU_BS_t3hb zOk`gN=HQJr1Tk<6R|i1DWCHv(igp%6>L+kwfIh*56EU+ueUicm7FU@J^{fx|RFNJ7 z+`00<5Z3Pi4lcLTBQDq04iABw8){>qLb3i8%sZjeFlL&bH?YoH9geis18r4n-8Xm8 z=zB-KssO}+dUrV&{E1Bgs!OR9=Xs#0uh4i-`CnG}Jw)JxgU;nPO;O>#4=11vsK31~ zArNRcnktSdl9rd(aHY*Dfsd;pE$jo-w+HWoS)4&|eq8y(s;&#Pq`_+;(nyFz;T z!+6O=8Z;M+I}v&ngkB*+Lz?eeYE>jJfixcDa zl~(~VQxqdwF+l*-%=!glFi|TkB>|m@`jE(fiY&8w#4`9!-}9^lY&Hft2=FIl)-@wH z(3sTLqqoIVqp1ez9*aLJ7O&_n0CrG%z4EQ5@)GR+C%dkv`1CO%odj9Om5Zt zGIt~G(|1}y*F`eu0!zgJ*X9cuSKF#m7Rm__Le5&1kO}7mQMD+AtN;~wz6Dr0YdV_Q z73Qd=ZxYQv5n>;48W@qzvTD|ZYF6`vRA+wmboq}hw0tvw5=g#5OVL@3Qx`zR&6RaW zEZ7nY_F+MY%SxQrukISUui~JF%guF8b$t-T!wq$>VCu~}H_3%L()9tV3V={ESs2fz zJKQ<}&?9)=FOunhS2W~w-l4NS_Wl%I7>VS}c-ffyZgr!G;&fKh9dnZ{qU z9&6Ul$?*xzEeUex-N53#%N@Lthy+Cpa3DJ912tcM#m`A@G zqe{*@v6AL&O4xhnhe@Rz6==8hn*39weH;L+eVwPdI8)np)zE-97LkV zHU;cz+Q_JU>2&!|tj{+D5=5%zqG}teYNvL|e2F3MkZpcIvIC$Nh&!3b!I1SS>s!I@A81`~~pB1hnwB1Uio?!))4Ng&7w zLaAS4qnyRYiZ z(ovWxL7TdvKdh346cojFNs8!$@|O~Jm;=*@#2UZCtWq34+vz*oxb5D!mMJsGf?gJ& zP;Aq*XYGJWsVA_N zicbrF6|q>pq&ZCnHFe<0V;+9?NxG_evK8fTxcZ#(X~n}>?Bg%kS+68kf_ZDhgJk5RKk}+W8rB-l58e>)Sl%Q_j6SVu-0T|h_C6Mp@EGM!bE~!nwMVydD3~G zHyRFr+mzGw1~YL;<5cgCU2cC0)(F&U!+Z-jVRT1!PNmA z+pa|h7757dVFyXw+L&ljP^(&H{cNe91|zt!u7bhJQmxtn79ScsF%n4D-`wU(z<-C~kDjO7)1%V3vB(i#`2%4V2*pltGX3ET}c#qYYb%hVqh zJ_~Y40m0466uc^!pT?0?tp#E63Jw4(k_h^Et&UqU z7$QBuz8xbWMi~*KC#~``4)bi2u3$8?#TBjcS7a&kMR#KjePQYlK@&G4h80m9#vif! zf)Og`#ou93UVt8XGKWkWIdp-=K1|O)F8H;ALw-SCep*XSP{edAku_7Dru;!wtaBaH zr>u9dol$gqY(rC(6uVv6e22!6(Blk#)JZLBI~9gjqJ^LdudDIiv`!4ifdmL`?Dr{? z;m9tgPFW=$F$mxw9G}w^(Cx<3)??{NX+^5AwTP=kYMk30-fc%UvetJ+;`bd~Nc>wE z%`tiNLh2^s(ua$XvxRX87Y<`uE@B&Z^hxeuQ)^?SckcBu?~JSdQNV=sBH$? zxQnbP0XdcMb~?t}_IUQ?FvqkOgqp%I#QNNkjR@S97`G|GxJ_da)(Jo#u4#&E$>sw2 z3A%LInIvk@sE_9OrD!%GipuO77cw=EkySeZZy*DXyKyZZl1467Vwf>Ui2}Bc!AK9K zjj!M%*h_2H#>OfFi2(x&!4rI_|sHGwQ#*nXlX85rUOF{-%DPv z2^p?b;#CQbhQBQMDZG=PLSdOJp4M}LJXOfDPA*V31#_M z5ld6jx7oMZOoUL71#qL-b&inPG9OtL#8|^$i(_>ax@Jru@p{CfLSkfEy%1+qp=&K9 zqA>NXmWQq3#4>&)l!8u`w$);eTfr}PGwehW;#sbWx3n?wbcw-*)zwCZ=PKJt#KIN| zrCzp;_A;ai^Pwq|o#xcF(y0st2p@U@qD`LhIS?(4Y-G04o$O^{T^X?|){z}N%(x77 z2u)ZH#!5nZ8+RWYrHEJA1VSM1&4D!QwdTVJeXU_=MABd{*(VL?(NDd3*o7+jb#)Jd z=#LErl8QVm`jHLQ zu9u&t^-ug%{tti4$UaqBR{YzuW<{m-N*WW!3~|6?8r$H2S>aAttAd?IR%TI;i+$M9 zg!|zE&D_zj)q4GfC`=gRU6>gV;5H^dK<7d3Rml&8=k#mYsQK(Qm zcFjlEPEbJN>cMkHC|M=c#qvmJ%}Nc!x`_aEP}(IAa+d` zq;*tQiweSyny?E~DNA=@e)7Ql4LWE6WKJPUBAcF!rEkD#K2IAFKveXIjulzEoxWw=KR6xc5!XXi16f;_zc;&S1 z{H}6cw9=3nw@F>L{MGON{@0$|_s75aa`I@{W$L#4hkx_&zxw2JFMR1mC6(l&Or^_~ zfBWJu{?#A8^vBP>q&G6<$(C>a?(gmXqo@DmOMj)LWRY_;*viU3m9KRgR$WQv7F)TI zHA|C@3s?_ZCu}#NRG${-WzYsJYoNnrl)@4)mey?${6Cytx#h~A-yk5nDK>V;QTm*RR!Nh@bB6MKno{_vx9XHyG4cFne2uy@PpWcQY4gy(h*Crt6m#r^zl+F*S?6XVQ0i8t^+)l3MHc`c)XmNxNh2TF9q84o%TV7x)ATV~Z-I~$T&z{3}t26N&VI z6j9c)=a#+Y&)I=(IF)b^0LU1XB%=^7t{2)0z%VGGjDX5b%6lAo_u?-{NJG{zYWJhdO$a}0q7ch|24*8Z@kc&(oc z*kMf+k zApWKJK$QSNN2prxXTXoR;e;_kCF9#bPYs&+YWZC^PJlo`PtvUu78eHQ!eGWnYx<#u z9Yk3I^lc2bK~Jqqq347NGSw|Xuc1+EDfH|o2C}nLg&vA3P=Wo?Yz5)(rnNKl$I+ie zSYU{w^}sk@@(68j%(YxF!o->6qFl+vVw(1D6HoXElUxV}tmo3#Afa2WmJRD_rdkm; zZ0e<4a0bt*KwFjK{go_HAFc$Du9g%};C9vL;w4txtwS=oXo<~)tOS#Ja>CiON|)K= zv36Kej;WjQchg#ukiOKXx2zqm0@8_8+liT=6aGw~m~dg-4|Z&M1@9|@01ytw-Mhu@ zeiHO|D%3~;Y%a1TYn0`qusADf<5E;&hDPHxWj4k zQhEk|9@2Bd04=hrp?wpD5F4?8q(fIm(^5h__WfgwZ;bP3sXBMMraD?8PM&KP$Eu(% zZ7g%}5o%an zEB(iJFG!ser?gPY^+`HavA0YYtf#2KMN+E?Ij|6xJH1^VwsbFL8R_2wDn`nV?j*&h z`etU3Yf~+O45J?56kEGH*^XZT#;V9^}*aqeT+9r}Hsu8-;q>7c& zz0ym!B$d07t=dBtwm-cXuwVUzJDsGRtV zXBoL;lexW%z|n1@%OUdFM;4Y4NKOj9jbegq;lv6Y8nd!SyU-r&SrVyHWMFg6eIJRBvm_l zn+l`cFmNRX1vlZT%lRj?$CykT{zsubo(b)R6Wvlw3bTw9kh?0KuR$9R+TDQKh>O=W z9>56LzC+oK6E~)7=W^Sbg+u4C7|!;+B>ox2B)i&hIQ~!L0d0fP{FC$A;X=aG&Yx87 zPWgj+JgD~y@NxfyV_eLFNhF3&`uSfXTUl<|AT z?reih;*Y4bHR2DX)8;i(&wrq5Bw1!Q}%k$J!HtmmnX@SXB z3kT3@^KeKeh;iLf+C3WY1;YaO{!peqWEjEDTGSt%BB53OZD|BsE)9WfsUAkrmP6jkK^qeJL0dQ19rXzah{eqsKz^7!{mVY&qC&Wq*&J@h1P$(RWmkLQB zv%Exo+MhNU6xkMHq#xAO!~?08RAF&Hr`8?DSyGY5Q?_zKI5z&hCJ#Hv>AU(02KeO+ z2Kbno%C~8&8^sQ5jCYJ|h73598MfH}mnlH{F%8pBac{FGR3Vg3rq5&*HTY9tzQZgg zDUaD`Qtxq`dEOtH3U?9k@%JE~+QxkboWm)5?x@3NHZ#p^S|O@sQk7X;akVA0m@sm> z^RGZ36p8TAQwFX3ig?is(|sIju!gfKs*AK_I|S6l683<927BffAPRT{8&dqNm9?Hu zriWz$ZPN6x9U&R(L_h}|+HppMIEEG?rcCG*^k@L3Dr41Wsg4+y2bL%pjIv-jOT;u_ zpN;mDT%K}|(rG}IsB zWX}`aX>H$vt`~34*vH_ib(=F&n~(=ZCIF=xOl|Iq*FCz29N~nSDhjahd`0I^=Vxp<5LlJPuO!bPX}Wrwz6BM12Gf( z+08udPr;E{0e-PX!3rJbV5cCYhC6sDHN;>FLj3UA6s81}g|xhef~MQBeL`NmHq(I8(|PKMoCM{OfM7JN)3unOoUFsELlx4__$V7;G{8z9zgcu9CQQyYzio+K3L z8v8wFzt5-Te^oOcvZazqBpFYpyaK6?u#!l{j^;)xB%8O-F9qv4z%t^Tg^*_utrZDI zt#qa#TEa?M|E!$Gwf;Dlvf}xaZgif=eo65q{uwUz;*a&b#=qP=@0GNz@dvrs`!DMG zLVqki12dsJF_o@jD()8KnP9VZ4+0U1%zxz{E+8bruYU}NvGIM7+QdI04l2qG3YrMF z5nm>B=q`$Z9q9+8iAqq{WZI05QufC*1YcBC5~OCm0y%tMNK9=vD{8j9l$-(i0Pco>YrEDttdIBb4mZ6=Lia3(@I4_AC|7!+gvnS4f!ztTM@LD; z3@1f3${Y@>lk)Gf2F5@-oJJkRGGLtr&6dm=8-#*_$#hBpvf!X*WVb?CW(iwzvKomwE!0D*T{nGh_7)nL6nmJeADflvhrs-y1EW&_g~2%G7U zM(AK-annNPcM#5J3A}Ac;Q%v%t9ke#a>91uRFkN7=BIi6d2P0x*#%NTA(xzio(*#P z=@fWCH$xnm=ff#%snX2;P$>sflkvhpW&g%$7tWkMyOIC&5e*t=?hZ3LQWu$K(M6mXmn?{zg-pcsZtK4_gR zC79CCWuj8fFigSIKw~C0j)m*+w{W10gFFz*YZg;+HN=8wK`yOky^ z9G^9d>6HJc$F*i5*Oh~kO`)B0LML=nCq3E{2wsLjjwg!-+o(gNq?`LF8@BgQ|JVt5VfwX zIXIxaQ@6p%lT8|mZRsG~coc++V~^+xA{XM7LF*$J>W2&+ME`X!;-pjK|3Hhq5psoN zisPwnQQ%eQ+uOw zG&vz4!#blDGc(#`ZBh}8f!Vv5TN2mCw_f)o#sf34_es#e8$`;**(hN+TYCYVf7S*#=VGP%cA7S;---0qVGSl?3V&#}9s5bI znKH2z#)`Ud2us&m#d#-u??>kzbB-df8RxAI>rXW?tdl`k!RZE6*N64bnp0%s*eaO} zaH~`8I0!r=C~tj*gKc3Eh0WgAJtghW$U$x%9Zkuh8EOO#_+h`R@vy8li{NbO2d)U2l!BJqfF~H#zZRT=6~YdFjqP*e zB&a$8WMIGpsFDD+mOW(ZxYnC$m1Dw7FBWh}9*EiNXvRXUnFdA{a?$um z@|zSHOl&*%>q66(PG@BFt8>eHBqo>Fq8~S$0L&gv7!;bZLOQU8?-Qj*&lw$=cV&Xd zM2>~QGW}Yy*AqaMRzJ`RGmjFEWmqb%TVG_VF>a!~u^5vC>ab>%_c`s#v7RyUZGMnc zHvE|f@y0JOu<;WLrbl0MxZcupB(Y#DH&T`KT1*h2Umpy4W#@bgRsQ%h_&aa*^Q<|v zGx?s@^9BAW7d=B2)&WF?k7UDoIzpbIoG_uF%x7e?U)LpchG`%m0~uoP6IT#1OdSj(L@2A(G-MLX$$rFj)gjBtclc>P3yCHjSZa z3yMYtrJu*L*sQtsh&Mr;Y?y;np<$1dEk3SsykuFdT&LxG z^(IWob_`*q z2cS^I49H(Fs-NK36T{|rAEt)!k^MfwZ(6<;_(w9c549(mA%qdRcfW?K$?yUnloq6b zF-L>pqXwE*f#Y)?i4SYcYm_9X+SFrXZqI`a01T|7jG5F3<81j`a$HS?UT}tNHky(5 zCZoI`mr-fX5V#Gq{ApBkqFPjR(f~H#!Z``vstveO35xD*$PjTe{* zs1&q}U&chpX?zyEhT$4l!74llm3pToY_*d_cj>k=mmEa}oY<6RXlI@vY;+$Pz zVyyGsJA-R|Mh94sW*yLKkMoh^k!mc>3nPihVY(>iCEM~jf^(weYdzIk;sfbW{B;g3 zLxhutbPin8^+jmUEH$^{wilGdxj)Bs{h#gC^LiEO?IUvh*X)(>BQ_Vwz1UpWqxQu%p(uk2bz3VQ^3jTiB#cdEA&^s6 zJS4W6saf$rh4Tov`PTp^1jx}hUzB4qYzl=V9qoFdsUR;P2#3fHbJrLNPY??ZWp@32 zNJ@7TI8k#h+7*Gj?Bgc?* z!yD=2TorV#R?n#q;QJVbL@kppuE9BsK1b9ua_4@^+B4VCB0R5g4Ifa3;2N6q{(7z< zt&VvaW~tGH=4F`59QQILvt#v}ERJ~@A=-wavd`}ke>%*Vq<{+^^O(`n7l! zG1V3e#8j3X>*$Vk+Ul@w_i$Y^+LCCkfnzB>#<3h~qCwg?&rWgxZruvJe!>lt(BGF> z{jUDXwWhyX4fERmh5-Tj#`?<<&WN*izPvX7a{Xmqn*J`=GpNkD5X<+B#TuyA@tzs^ zm-b9?Z1eq3gNtFcu3~p^ zafLckFmdq^1)IHU4pG2v6BR_!?6gYZ0}fGOcO)Q7$?=b^vZF&3Q1yyKaG+6hhys&v zN{fdm;IdAVfC61#Wm^u&BaVkCXgh~iOQQ<3x=>5EhJ69dM^u&}!32PCjslSp?J(hY zLQL9Ish$PtmoH@;YCuySJB@o0w#sleV!Qktr(@|1g%;;5+6tUOSb?MM26l#mJrj1+ zsR?T!H&{~WBe@Bd@0#A(^zJ1c4UKh8>eUSy@nI+ICqMWVsQe-74J$;jp`M|6CztRr=Y zPBleNyeP8H-_{w4QZvS6wVPpfBSbudUKZ1h4Sk#LEZqmzRbj=Tv=!`zr8_}A?29_T zkeLPQ(pZ7-D3K_Jwq$r)4}5nge6Vh`>GR*XW@GO7rUrJ^%k^MygErUeun9C}v!MZn z6x=t5*3JBCddw;+g)KuFlpo&{30V@dsBblP32P4+HC1Zy3io&wfXnmF5cx4HpZN1V zhw2Gneu%nnOz8Q>q*wkYN1QPUUa|ZEYBu$a9lYSI1rE5;^4|&}^1ML(+;*)!6IEdZ{(*L+m$M3>6A2y!l<4b@Kd3i-tGEBnR3sg#C# zXW#w7k2>43O zgYA;r46^{6M#W%N-ydOC%qBk-umBy1g|H#u8D9sDiyuKQ;$O7E)PWp#Ch3eg>adXt z-~tk<8|~jJFM`c!7%Vd2nPuj7ks*rnXxoIOmM+(wJ_37zls^dTu}Bd*Xdcv8cdt+Wd+B5(EW z7%m(DBaQ{+5S;;krg|x5JgMR&HdXR+CM6b>?p2T(*wcI_#%P*Rpm0KK6Y9r3^!j9z`g1pEr&@02<(8192^-M8n7hHuMVg7dX&A zB1j82U^2-TD=xtO+P`5#QXSu8L(5>|F(@eDfcp$g9OYYD=nG-ueCv5KikT25_@7*{ zT=MUed}pLJyRidiFn5uxj8qX8Q-qqgNWq<~1?bhhg;)b^E`jxlH2j$(RU_!zs6N=g z7B)v`o2Yu3Wv2Pq%7FpFuq_5Qcpjh^Lu4bKMU)IU>Wu1kT({U_Q=!ZTa zE9OLdir#utsa{oYj&v6m;0Z%Qx3CzgaRVF<>)UJz_2lHyxSbqhQx(*Aw`kZ*w<$~! zMsVb#a~vrsQ$)7gZQezg5l}jDFj1XW^fi%G0U%41C_9oBzC;a>Mt~QwzOq6Ye_h(a zTxbG`MV?H`X`?Obq8ls`Q=lpWmN_(^? zBe!NuDF9&54)La-X6ZbB;p5^@)9|TQ&PkEOLOSozU^v8U zM9pngmQK+M;v>y110Gn;6u4NfF;7Yc=>aU@8D%h5ELJ8Yo%rPiSP;7HD87~9ng#{N zp1E*9fo3)VfO8i(r9dB>4*O7LqZI{#ysKP@e~Qw5XafE4M$@VAuWBuDB%pe){ib!- z&ucR)GQ5DTB zO{z0Iofh1Byymf?(S98Dl;pc%krmzXh^xAuoxxe-SziQ_5tYy+fT)DapJ zjU$}!#7-@tH@_^gd+j*FiISVX=L5zuXO0EvV0z zxm=%S*Rj2o@@tJ=Nalp9;%cY!4tiHRt?;vO=@J}z*gvAq!Wv=8A*fRjD)g291Zv8d z>0k&Omt;vo-5RjM3m-5AG|W=Sd0j2X!ki0;poKkfOIX-LR-R!&>UqWSi9VBLE6YkC zi;)F(l1g(bN1Tjf16b4spH~v6pwp24V6ZrkO>&;j5eY>7asVO97Kf3r8&}`GMITTR zKY4717xrP(IoULZ2_T~Upa_D16hT6uBQgM`Nrei!Pecj~gGc@dZb?n^$|MyjB(FSQ z$)lZo+6|DJ{sJ}BhR|3=fNzK^{*R{TTHuM;Kmu%oyHSDd3%y*2`4T&Fp{Kukn5Q>I z-3dqz;}!T8RBJ+k3_mvO=jS-vgxmH`!K(7S0oPY0;1JGPaNF^YC5pJ{JHcVzlv4K7 zr<*ecu1Yi|#cm-LYxxtSVpbOpkKWm=RV*KJI)Bbi|9~|C-sl01NuYGfL!kDjTN*>* zC~};MB-EAj42CL}-A((4IHrYjhSv@q892L)5vlZhhZo`~I*MqUR-tF9+y^$~(gf*W{QJe>Po zv%papo{V`79^-a#k6|P8EA3M0B>#{VcbL(0b*(62uh>qrp;)y<{{WkwEkTRA1EUreOn6C#ysJv8YN;j+RhHPu%{&-s$F{l%%!PKvoR9mjK0WSFfI9I?yau->_ zat|N?5%Xpiwp zs4VId1{N#fA#6ml1EqwOo!PDuJ7(_-tig=C7y{N{zn1RA8h`_9nAf?4H4sNJ;gaSt zZKLKRYSiFEvk}lN zjO3J3ry&hOrz7CC29nOmYXy9kJ6@*HHqEciLXU;d8L-0(K>MGQ(2l zvTs3YbEi;T7CVWWCUVr&XtIT-8TB$bqn8LNz3eRZ62oD=l&)yV5wx*VAe)hBD5G&K zk_case!)I+Vz@;4(1p&%FA1BnUStkjqjBbtay{`Cs3{BtTv&_&TrwJkqh%f87|FHJ z03xt_OxE##sO91F`F^X(51g30T z?hLg+HpCw?l7BTBlv6IrMLT^y3xD!O?d#VnmGgo~b~I6?F?j2?H$Wl3Qhf9nj_=OK zRgOzk*HA8pY(gf38D$Vju=Vb8U64XfJtN`9I+rv$S_Mv$%=Iu(Q#Lm1(4*&QOrTqL>{X2rO=xSfrhkJIK-Nz+`$+YM|x6Fs}%WFLzE6I`MOuF^^LL z=h7Kpax^ikp#4e8Q2YG+!|)z+y+VO;<5tb@>n%0Xw1n>^CAoj`5=hT}N+T)Wh-kM%Picm-KKVzR`7*?PWcjjBj)uWqVZ* zr{Wu3N7?rL4?MgQ-{?BZwoh;7`4McojkH2|b4@yy02gVsiZ z;u~E@+4kwpJU;?d*HN~|^sqO+(RGyVaXsvhZ*(1HdqNKf;u~E@*`C(J!T3hkQMPCF z@Kk)G>nPi^dN>r{=sL=FL=T7K8(l})p3}oZe530q+tKkiFY;hAWkzh08Qb7Cgn^2X zA^Dv4lL{T2Qb{6II4+6&OIL9qqKppVVDe!v2*w8CUX*ZYPu4_BVWR=%S90Z7*VdnC`TxtPirOo}Qcw-6(|%4u&P(1txDQG-o=Y>B*V zJQ0=`;SV?lYcLl_X;LF#f`l4$5_i#wO}w;fm&lUU5Wq0}2sNrdh@GM+FA6#{g32(e znxKkt0riJ!<$C3fbEoJ}Ob@=aRbX9?f(?CirznGlVsa9ixtcZYN) zqR!ZCecomTbcSW=fAvuiWJ~Sr#7ZXRA#~!O=EPrb(}G$CM4-I*YfUss^`tP5?@5s) z{`XFQNDeLur6Q~|b}B0=7RVra5aJlngRME5dmu0Z#x%nOIi*M}B3>noi|{4wy>ZyE+&p@B!HIR&sP=;$bU1X(6^7p2|8 zF#?c)+^pAWu|arERzPvoK0q4f7DUo71!|}WwH1A68|P6;pjg@nuRPRr)E3H6RSE-A zc2jXpcbk!IcsFkG?mpq>dk9f3W`q1 zu{-Jt8ccPW`qK~CT&}JL{-V&yzUfpXHjC^unZ~y3gk2@mCK|@-K=Ed{fEdj0(A-=7 z{J&fX`)NNOo)@@iKW*{3ZU~E`{~R^hDk5+V9+~M3R6m2@l2Qn+iO>q3q}+%INT$Z* zXA<1h!ygYMn} z{=`@TR-ROsy}B}s#s^}lrF;;FVDu=$&>)8+aWQyCXSHm}#s_LcZW?I4g{XhxH|XT@ z#)DEtLU=rj7$89KG}8kVV|a#nwJ4r(l0mbClU@Uj_&8V)3*#LN?PB_9=FDIjn6WSt zezj}jNX&GZl~58`4uK?L=Gc)C>r)KG`K7lMOzM{L+7gIZvNEhECmsRjNtrL=Ypy_pJ zi6+bxB2RRdPfO)ZRKO;7&|s9M1WIk_%`kM5+kM zTcj%EE&4bFnq*%HUdUcFK_*N{jA2zAM2vzGtPJ&6fc9=qS%wMxIkz?0cj@8^o`>1S;P&^3~HN>3?U+( z6*;#ca0oz!Q=!#M>t19J3qUPKniOjXcUm5lqF$P~6q;vsSX&apTjK#J7F*CY3?}#m zIvRd$@F}qfG*qZG6Q`ksMa_|CTmXylf_XNM3%m9b4hnO6)-RJisAI#Pz|P$b7Z6x$Ruqpe%yCorZ zbux7Wd0{NP>EMOQpW(z9^5k*L#W3oIK{^E%dmQsDi-9;ZQXzq!Yl4zEqqH~s8bn4N z7~}hLkW0>rfs8{!ZjdR6Wa1g;0ON((lr#mN#Kt7?eXz(kyDB&fl`96W+PI@y(P=W3 zvW74ib5=h|h+z{NwGX_F>Yx~$%EzecuEl6Aek)1fQ(_Yc6t7xZl)#!AN)KokXbXQ@ zZX)NN1Rs&XB$dc#Zw2bChyW_CO``ViD7;6fn3uXL;jj*PORqS{HB_OvEr1Uvw7^2s z3uk zXX0R{&*Na*?$~E)&Oc)HMdytN0&5GJd?UkR3iC$L^*PJ z-8LR?`;#5C0?ANtm}FBRmtjNd`TB*-7ZGBH-AV)bM9RLN++Sz9Y||k_tmc^YFOmfN z?6@p>;xkiMC!%sXQ|UV=FOn zh@6$_OnW{LYmLQ)Ij>*+tACT~HKaGT)5S*D=%lZ=?4wv98`ngD^HJIid;sFDYgt%; zDVz2W&oUSuvS>;hOH=!YPO%TE)U5?(XC8HIy=e?`wEQWW zYq=yz|4h;;+{+NUg(pKje`&N56YgMu1VV1(TA0NMT6 z;d@$QalUmR^=peb&*LlN9GPKwB`udqhMnWAURr*PI5Y1H{}zfI!LD)|l$fsKFttI= z0f%Acj-p*LI0-=>sL`^upb}1z8Nla|gIQdeq+~OC>sUtENd`5vMSd_84or{{wMKcXJ{OCh zw=C9EF6dO+h+5UbR~5$Rs$@c)y($@S(LjMj?Fj;W%xd}b%!7jgzUIem`43dYmS_H} zRO?cqD^^PTp%Xv<9XP1FQ*0wi9}dKF8GrPLbILSxj^DzPVQqZt>mKJuji4`kt2X1q(R>`Fhv{N1{l?-O@=Rh4!d`OPOz&OsZp&}DpWc%ajbf` znI9O;KJPWw5Z<(UY10mY4Uzbvqb3|Y4Zm!c18xCC!m+Rw*I-$*8t8XbkD&N2-`3NV z2t7;C8JJ6~iD?BW`5;i}P)9{TL|Jy(`Bw!oCocU)KdcPjgw<>3zI=GD4}G~Z7Y z9!@C1pZGl@148#_PT39d%|9w;`~z}asQ(oAf4VI`%nxS#r+0*hW1r>WXXA?xUpeFd zoJssHM)bA2K;_~c9Y`Q4n%8J7ZK zXrQ{po%wDb=X@qdWPI!LWPFoRU_{0Z_Bc+)jg~S_#!D?_oQ!X=lyNftAi$52@lD?! z882IgjBkRBh3^+NWIm7fusN&gaVm48crV_2;~CiSQZ~DdCKN)1s!iJGy$aeq0}g-0 zKh8y`%)e7#>qF?YmyLH2s4~T`lD;g+2sgaVtC4#Oei2qAG>ex_@_+#u9yfd4f z1q4JCKdt!5I8vrrs>;Ju70aF;WUg+T*0Y*?Y9}^Tu&G+dsE;;p=n>w}Vthd}%!sj_ ziy`Ol#>jc+MdQOUjVXfLrdXKX3njWLKbql^qIHkp0W zHo;3z=u`rF&Zz|QP+6vHJJfSo!T|p2Gqwn$nQ$~r`B~(Yg^gD`jzK4eMJJ&h+%MNx zI93Yv*on-yg-y2D=`Kh#x*>%!b~@x&P-sl=FsSH?s}hUt*%K9L$Es{rm}Io-lj8T) zE0|T2&gu~$K{AEXjPNX;f$(}X!c%GRw6u#h=w=2A8bNZgpdWlQBaN=lnE_6hdoT`> zb!`PAH;<8P+Qv)TM9Bh7c4$bgq&q5xJlg&5Kq6ktR*mqjzjz3SDo`WDD8e%YT7yi8 zBcvC$ss--Te7SmK+{eQ9$bBqe03eWH*D(&U4VxmSP;aYSw=St~2{+4hHQcNYsya+d}@#E|!I9`X{#oVm9@eW40H98wsrqwMZ}4Tn_+ZyXN4c&W!xK@-vWw z%iVgdijp8Sm~2}cS`kgukWa|xBJP$_SqB;X2`0@6rPJ(zucF6b@kA_LJn|ahywj}z zva)#sPviTQ%{zi<-hOt`{1-uzBt~}zDMBv9x=hmIzpa?YD8?28_9f-8*O1-MsBjJ-5n6XNE^jQT0zGEAU0n}T|Y*?^l`!w<%hlU zvtt~AZ*iw2ekn7gQz>HV2XZSS2Uv`n@r?YoMOG+G-WaPa&kAB)*u}IAr-N(B2a^Lc zmA)lP{Rl!z3psosts>viHtaS#ZFGWD;98bRmA~f1>l-roiTR*#{xt*b@qj{Lvas6_ zmA{mi?Q!7uN7g>mAQ(gL*S^Ip$K-P*^t5n`LOO)U&Yc;A&?>l2E{ zm)CL{W-*AlhBLA_p@Z*13mKd?qN7JzAUT{OO($*t8j*a_UY`tS-zP)us<&3Ly0>Hd zlN$&QxY-qH;uuu>5MQ+=;wa0)q6;i?8-ST5F&5j5&UlziGvX;A0LykdaM(|B1%fF0 zk8N9S&IJWli7Hi^@NzgNlk(KBQxTfo&a5n;%P`nw?ioy*BwfmTX}Oae!-?>y+`o0B zh-}UgU zt!P{GOn)m3!;T*LL>%mBcMiZVMeT}ovlT23#&=O)TBdA0Cb%5G#3I{SCSp6t+N2RC#t~DjDF=Wap{Hw zEG-i9XL-6Z#`4VX?17@H=R30;Eg)eqq%cc{sI(zWmoH>zNqw#k{Oj!9()82quFlfZCoPrZvPe|JY+gPc=Ia9iEPqS!WQ}j_NfT@FD}FKA4p;9~ zoV_J@aMY*lBGD?iB!#Mr9YANx8!|A62}bXfAig+S^{QP_{!$=$0PL_O{|t zrvQ~cfJ;D2Ba93vYSSO(e+QUyk~Ge+fdltCiIIB_$khpHsBn!w_yB_@n#yM=upyg7 z$#bb70Xf$VaRJNUQdJrZdV+4~1}^Of#v z1Y>wJRf5SIc>2GJ zI}`Z2$}<0-?e1A__a!Zyd#$B~wn^GFNy}ny*#&9YL6$Z&w~#hX+9buI1yWiRR0PIx zR8$6L9C53t_($BGapC{{J@2_U=>mUe@bmc>Zq9kndDrKC z_V+pO%L?{d@KPUo^YAFal|#6ScUxFDj6k`>u?~2hmMayYDsV`9df52Bq=AJdCTeU#$k6O;NO`*KrC zcF1gl5zv(zkuBLcz*5Ce1zMyrpfjD`4kIIh&8Zo#_{C%N?o=V$g5ae3-Xm)*)*(8+ zoTqpCnZvne0;0p7<$%7{L0hl~)qwp(Xvb;94md*1YfI35vPZ)LmzJ?15;bOX$|IX-rA~@D1t<;!*aH}7+4<{$3>NYF_erd-%PGQ zO|HMTtF{EQgY0!?+e5Y3BgVsCM=`3~=1ri@g5a#pQbC9&h-`7Vc%LLU&u5mi;0QN! zB!on_@U^J&866~WpTWB*#hG}BLh>23X^YH}$PFsC0Wd;CP3GYOQIjRCqI~hQ)Up^~ zx&xt2oHWW}?S?XD?neBw79hu;vIMf)CyQS0Ee8&{yqNP0)DaGpl)@m* zq`5#o%gZ$~k~~i~HbTg{Df|yhzQ~l;rKw4iPvT9uPBdnij34k?NIDzxX=on@S-QO-!Ku$S=vs$%1%o zWT!GR#;!w~G&GkjVhKs4FS+8D`ILr;A;U)?nKfc-~Asbby17;7}uv>rH%F2AY0iH-T?V_?gIJKOHepOMNzOo#hJ*-V9T%#;U(*@%~K7F7^;sFdv z`_!-?=X#Av-s7Uz}H6(Kug+-YOPaByQZFpGo~CEzm)D#PSg0*%orAUYk% zjtJbR(OTAhD$?+@mq+TVoyi!jT2JmZTSrNN#{?%$I5LD;VgV?=LivTSW>RFN9((-Er0v=zF)EK_T1B{iQ4c@ir3{!%)2dm0qfHvs z;l|R$k!g21TFbGq@L}eVM%co$oH<<_RNW@4;zjq(s)&hN*4!C|z0zjxXi35VTHK^a z+G;BFhf$*aR0@T?0c_4zX_Clkq1@+_!NCNCa)D-zd4ty~>W52Y-^(&bfJ;(MUx`%= z%G^bCoQndD9@FRz(i&dn@qLp|IM^@`_V(-bq%m>_D%1?qznrpdl&k3hGS*Evffon6 z!CSN$^kq$nMJ-s3wdOVQ{n|&go@>u)oj}c*4Sd7%B|y2J?cg1rS=d_&rPvs*Kjqh? z0^mv|6#CBx0T)w$j~nd+w)LN1xx1D$Oh9j9R-{ESnF&QPX?~H)QWFP-P>>$Zz{&wk zEe=+DXYv3J&bAq-W$EMhI{AFC!Zc7X4A)oTvkxpR9rHHg~~T;<8fNtleHwn^D9r)Q|PBPw70# z6rPYgoz|{C-NkFBd1<<=NbhC43wr_oCRFnhwQ54ryOLT3_iv{;m!QV-K_~%t;(x~D z@2W#nupLJviPSol$s{)l4HF7=YfIaqgj2T-Wuj?i z=|`1&HLMJt_!uO2Y}b(I(b{lnbe|Eyo5XX*M?|yrqEm(~MReIDFYTlLMS^+sr>*NY1Dr{Ky?NfcmlT=^0IN(`x_C z`_yg18`8in%Qo(+D`8Af4Fi%lOQU-TA3Zp0bHHql&xbThzw+f8fJrfJnkS3(O(uc< z29+`8i4q}6`XQxLh+#<)piZ9R!)_B#)qpQ|wVzD5-PIr9RAEZoyi}5@HPP!@9-lyG zxzzeV8}i9b?5tnwtcGT-jTfd*VhVHwjCFGR*W%UsG)n@{PyjB*vX)T5r}xLgJtYr8 zfe;kH-E|NYaB-R?4?2X#4Fyp9t}TZ`f$$|!Ae6}Dy^uhSQ8;u6Ku|mgATKC$Xa^GH zTv(QAI2K?jJ@_E=4(CAJGVjK&s1wo?ll8`XN zhcf7vf`ynk5xNc*?Ayr_lNM_-2{GVlNr)k!1Y;}gw)1~E#ukFx4=B!mflkMo3yQk| zGm`-GAAK7)>L02cJgb&6D0qq0Cd<#s4ziiO%h2edoz*VgovA~S5}qsNP|n~dM3aKx za9D1`XcLZN`F2uqUXzs9*lKZ@kqA>2Lzto;M{~9EHBwPErg7SOA&r|)7ndh>aXvEk zFm#wdr3DcBl@){>qWq%G(aGRsxiR+mxXIjfYnoL-2yC%M2z#?9NB~<7DMMunH$`dN z>})KS1MTBg@5F7aE5vBgSjxP{zdXA_0~k!Y3-}LICPgoJ+fpDmU*%a%7XMfHyl?R_ zpi_xbwgM}Kvs`mkSf9cQL9MDN*u2=o=%50Sivh~%FmC?@K$3CL9{+`Zp5w)6x@5B_ z->mEdxw{=CV83n=r!uO6XdQA}3pdyg_PMw>#0pH|v9NB@)J3vLC@$N6%_f9=xl28> zBavT1BxXf4MD(w28su4(>TFxH%XQ*p(>=b_H4Cr=TST*(KVEmD@~l!xDg_8ys_M${gY>nSA+zYv%j}D2fmOX_Z6U%+t#4O8ali1Sui?bLTCbS|V}zii zW~VL+*4G!r-2x3*1<0{Xe$41KHgrHYWma%P-*9;&S%8mzfUg!e!9)5fN3c=sMK01U zG22-&7sc9txcW^afp&qa+l7X^1~qA}c}o}>?*kM-L>q}DOTJ4(HMjn2eH>}8E}+t= zQ*cnXoHdnYri!ougFajZLIrY~#A(OfGIEgEW`W{fn#;tI=iGD<1KkqU2+blQ&hgUm zL=HUWEvT#$rlfJe#i=378Psyh^9K&$pR@d171p8^r3QNd`B}zvW%39v#TUG7sy(ci zag{}z3buZWswF=&p2;VPCIE$Pq*}=XMWzO;Krd!%s!(3vSu37LYY<9l<6tc;h`EuJ zJ0qi7Km`>+g2M1pZjoUo6bavvc2;+eX5|lNQe~qQ4&%ft4BhXNu=5WkN)>iBwNa^tfF<4Pz zX*vYyzF%YS7uJ{sHrQ3Ts^XV0#NviiP({s)w%Ni8xK9%qpVP_hd`#mO*6sZ%Eu|Se zf*m!2SE2f-$w57kHwykBnbl~k`>hHWw^#a{LcD~WGvZ7+v=~L*X+UDq4`gg zqjF>A&S&obuV8p{+{jmQdY!0C`9&)(}jqpZc>zhW3m|kn&eDz9xvv>m9)lXtYVJNs1nN3X}M}*KC|%R zX8qyu0vYoQfG;5-Z|p@EAb_W7Z}XtW2^Quy)WVs2vDac3(RNgcxLTlDf9QL32^j z+@bSo(5wW|-c-~x;=E+rw~5{lPRBZN=1>K2nXLdD0BpsRjDF$q_y}>M9q|6decw03 zA!23@JZXp3!~3@x|jAO)yQHFNHZ(rY4#k_ zmwP;8&bJk&jE}9ICbv~GmlqDzv!M8G^@8OK!B>f(f@Ci~?HB$K@F5cK?A$G`Qx($9 z)qN7Hl`+zstObkrD)Ts-)2l2I_mq*j{?Lx?Ign9Mp?jD3oHZN{ z_#q>C;216CWSrt30BT`iTk+Y>9*SHdEb$;2hz)Z_K0bhhW{_G&-n;GGNYsj+WQ=)oXgCznLNT_p*BZ; z`~*!M7APJe%PMqdb;!vz{+7s%6bgW~fYJq>GY42hRz!fLhN2opz*&!^?u!a&KXR+; z7$^W+Y0Zq4aa!3%pXEiE1#X%$c&* z3WTR=sTxoKj8j^Q&uEp+`qw4^D6+PD+uk?tDW|)_HsxX2GwFYvcHJp{hcQV09pzrQ z(!6sK$gJi&9!z0<>Z!vH!c|OuYK?glG#c~A&@ND!P$wM_Bvcc#qjfc)4KG&-x|Y#S zn9tz0YqMvxU)X8~idImPICmNDhH?h(gsZJ5l-sl%`xvsJD}1h{s2V+_{bin-2uJq5 zIga*_A>=3yZPq8Cn0Hy7+|)iX7urFVLh2l%%)=Sn`y+0(FK*mS6!|2? zEM=MvJ#rOxTtNRzWLrLwZTW;xCX5Uor>R4Shv}k>Xg(oCiDkH>d9t7l=*0QMxY{Ay z6hmT*&wyhL6~^uC-H8>^^=4NY6!`L$Ng+H)Z8+jD;0upszGfN10TctF!G6P%GeaAa^WUAbvI@8i}a2iXGmrlc?d#XuG%a&x&@ z3s+>%Z+e%PAwevyFpa8l0ufM~CnNcr1qVH}np7I6sPG$`rt(R(cnYjJhm^RjmZ0+^ z;EQ+;^QPa)Ej+R~n%UGFTlN@h2VnsYGq1((f=aGfp-jt{Z`ylR+=!H6TeH1!NE1=C;H4})y7=ld)JIPeH93}$X=(m z#GI2T#KwOk07#6oUoi1Oa0t4Sb%4?of!xU~xdnkEJY~-<<=*74V&TO zt_2ZT*_;+?vn=;hoU~e=gnc%pz9HHajYm0!i z(Boxlo2S~G;h09=zf5hcjX7j{nc8saDfD%LFoNr%gVk4Hpp$qxuU6lreaBi5KBh5` z6Uik=iy6o{1`h-`nMdtW!c-c!Q7{Y{z{=WoayiwQ*qBaDZV+Sl+3pGVrgB@*v~aD) zCt|D&@KxKW$k$ddk3Fr}C&P>>8M~O&#F{ZglF#hKt)!A9z2p zJz2stgYud#7}Io<3e!~a320(&t^H85+~i=FbjARud=5$UF^#&A01h3`*jwdaB8YPJod@!TaC{R6+;OlLXKG3jiR zAg7$Jv&}Dq(N^EVsv5`Wm#J-kjDG3b>KOepwH=7j2iBKjpv^$Oj?v8%$I2`pfYA+x z(N4onYXJe3(cDl8Dq{vFU^7$yKvYI5s#4)UctCzUI|aRSaQc7&d_6TpD9OwzCB^Y{XK5>{xqK zC8*&jcGend4Kz7bNv9=7@*w0H5KGExSnh6DCl;;rOLPrYEe8-3j@r(rgpMnk@-#t* z!K+iekSPI*Hctr@MVuOih%okW{7JStF5;d~hPUH*Re&Nn#50^VhaPjxu&LgKyKOlc z!Sul9(1rlE$68k9@YR!=b6Op2$Cvj%DMJ}O2QM`xh@pwMPsD=wy<3ygg zw5?vs53gmO;thVVO8#m@yn>@B%idvKP9sZzu8!7v2ELdA;aok!9@?>dC{tkZgnU5X zLHUiYef&=;v^lw!##gLT`YuO$T7rwV+o+(RI#DYCiQSpU2c?32ac}zE7|yWbPSYlN z8`VFQg5i$WV=HNA;Iwu_k7zpi7}?;%@~A~&FlqI{b0_NcZF!XJB%K?`l+`3E_bg8U zlLl-{A7GfiKNe8N&z*E>rd2@(e2J>uRs^>V3%SBfuH$tVj@@^P43utF`tA3Rd6mY> zdv#%YE0Zd>x?8yvE0c>igJurQ906{}F9TbRHU^NzWC~2S(p1f6x)TnQHOC}3px7uv$TR>2+4ZTjZrU&?9lm`}K zML@1x7xa z@F*D9<+db+oD&-q)f7x~ z4%C~px^4UwJeWuSxMwz@MoXxJ=CEN6j@|5AsJ|P1scTjBkf5se5NH4 zt);y@3vqt$>iEs2*@iI$PzH1A(k`mZt4X_v-8;-%RzYEuxz5>GK#z9HVIpf8K40(D zTZkJwgX>IcMe*z|KDou2;Xb1#fZ$Y8e3`~q(+Ye!jtd%Iee(9dbLPH4$!W!ca)mu` zfm&4g71x5f^qFYwLXxZKY*1AY=jd5#b5jubU~-Oqj*%0boS2mSsg-2=Yhb{b4t&Mu z0E|I}AW0k-&h1=OOr;>&36DQ#z^s3enWG?icUbmP$(#* z1@x@K)KPEzkx)($%p{QG>R{nIAcPXGHQMM|wpr6C%?A(E^aQ`KHY1Qr>np4tWDN~x z@Q$sLW(p*+G6FvZYHb3)2(HL zA%G;N##rtS_!pV?=$q|2nl=kfo#mA*8Hrr?MLu{QkYpEOhg=S&QbH-7K(79#1fk5$ zuSmUx!73q?+%WWr%^vCqW&2De%nk18=xG|)gq9lelG~Ze_%>*j52H$GliJEX4<A3RYTKcvz(#tE~;$rXd#!ReG$(P=Xs#e>#)GW0#Y-m#}EV#bS zIz6&(w0H^4f2I{+^PTUrep$ZtLNvT89 zoVA-VmbQQJ-QE7|Z4o72(0EuG+GkrtMJD^NKb zw@K`#l=~8#q2$qDX~qokcS1qz4(Ux09qd_(IgqjFy0SCLWi-ns-fN54F$ev{_M~xWe<17$9f+?>4Klkmu z>#Ns(kE)XH`ftj!bXsI*O;3|7Q4+)?LwAeJBrFDzhDm(zV?3ij#h8u&LaP`yXZf2{~uoE4_u4q zV!Xe$ZVHMXG$WNas~cvU2sm`ckc@Bh8)Js5O~=V-#09Liy||;;!as%3hsmPk2q!z+ zjQdfYZ6-q@SXDd92z$|IK6So4E(Lo2_#>L5J|Gr(-K42%GK#E!uBP%|}t zFc?D7&`&4=7OH!Yx-QU`wr%n&^ZDsuu+SY01_CArgJE5)CcdlaGAkIeexg^3-h#?> zuy#_Ct?Z^8rsHWKYI2{0lw9Y*dItkVUGg;sD7No;UOp$!GkQCDKKP%X=cNyEkOtVB z@J=^g8#eLw5%bT}#fy?UjAfcEHVsOF=)gn9jZeo3Ys5ao<%9O% zS3Y#@7zn~V6u)e?L+wd%3IIZ1(4VPA*`{eK3+R>fcTttlij$HwWQYP}!zCsUtpYYB z;64d$nHy2*57!eGsLF30BWAK&t@6$(0 zoBy#kys`R-$a+sb;m)A44nQ@WL3J#Ci(r-c3XvMZ6eOarOUk^nk`atu7Yd*d*Y^CE z4CezJwB^EppS}tJg{!sCV)Jf^{@QK&DnuQaLCjv6dlvH*c6MDkcNNiU*qJ-ac{yFh z?epjCC1>s`vSiPa4a!H)@pXH4?u=o=L}Mre7IHUMTo}LMvg$xQ+~2pMZ{X5+{g}Ug%$qbk(lay?UvkArHSQbet?r1&rpJB5 z@!K+&ziF@J=m-P&}`g?}1I5O@V(W}0J;lAE# zJeG|YsFmS(EWPk$T7PBLjrl9}zc=Q$j(O7sml_NVD8q5jNIcdUUjir?wo7^ml=gwo z%6R0mYTOTS8N%3PE1@S2pnB#_9_ksmv>H2TCx9;3pzPjhE?p-8HEB+Bd$$h>A!xp5<4AQV z?yU}w3~k>q(lmLWu#x1X?qr)SN7&s4KG2A+0$gmF3_w=_846bwO;a38O z@qU6ls%W5Z$Kp5i3~yPywSPOzXtB~;dN#Ys*g90*Iy5NM9JI{;TlCp)cDGcwY`E+% zRN=@_>(-tT=waZMfTi9+X0LT%`<7QyS8WE}Y_;0#{Uz$#+A{#6!UA4NW#jXIAR!O* zT;6wS&q(!^)L#eWb~-!QTW#G6(=s}HCAGe6uW99#)LrlKhH8I*>)^nK@!DE@t6PU# zFY6h;?2@6uHw-LpThiXSq}|AKsOJr>TLydoA`Ls1_X=wp2N7LccX@TgKHYO8>FpOQ z9o)b`mM&>`1F1dJNb0;mQo@D_|KI1u!I(snr*U|2`w*hpM9*PkOAi~DKgcB$y~C1r zNT;E}!4b5Tp}4wZtJItkD>Uq7lPV4yn|7GgiA$>k)gcfvk!SX}u9$2dU`ni@Gy^YJ z>aA|h+U*Q>F4A;|pNWXf&G&@9leH;5WYRDVW4L4T%`|Ow8x<|uRbQ7{U4WU;5yQy!% zg!lImQLg{fBY@w|BCd41EqwM(m>v@Mo=p~@1`@_{8rl*~0 z6oYUT;S3|jE<^Gs*~V)+*-aURB;|VgYw6k?e44s3Dn0dZpYGDzk&&T36ys_w?^g01 z!4JW{v2RB`w@V%*|1f!*_|4|mz%Syb`JTgXF24;uTYI42E8@Y+t3w<62j9T6*FqC( zlRQlIOO&nLe+!n=7XuK;JA_}4j3lTIjJPx&8S0Utv1Mz2RcPP{aHu-6eMs!1XCST) z4Z%LlP?83(IUDM~LPb$)sy$oc-o9SZvFAM+7QK_G|HiqVp-d=_GL?#EnC*HZ27Xz zWnIguB#-+Oe#oqhonTXGd2@cgKq5ZOhx2FI~QDdB^hQ%R85ME$?2wqO%QC zWohTK&W_IIot>Rso!y-)y4t$hyOwq>>+0xQ-qqRF)z#gFP1D`py|jB-cSrZ~?#}M6 z?(XgtE9l}1nqNWHD=4-?kJ^_m>sa2|)xDy!`q<-+KVdxGbIAsXA8orIG+V!BtJzj7 zw-3BwsAubfh3jL)Ku^5>w84SudS{e*Y2U|_8Kg3qAe+qv`BWijii-ZEU~+28oT>h_ zU`8;jaZYM(_E3MGzbV=r3^mm!PEY8sqY3q@_!utBzQi4 zA^KhL->KjG4Tl|h%xPHFUNsdOgSdGs;o{POW9qNy`GyUscH z>W_Zn&Igu1H~B5szvI@Zv1!ueh3y?Hk3Zq$*PgbfxBAw%-*DqypZ~%`k9_6H3-7wS zWlkoWFHD`$xnkvqKk}__=elmX`NNs~k;iQ8yYa6k4X%6e`JcW1l6&v_)gN9w^Q>Fm zx1{y31!sTYw%h;ygLiy*?|t__lrA((o4fKg$DjV8JO1UXw`FF{F2DMi*F5{(csF25-iMvKFZq2Ib>^qRS0d-I%VUha_85vk*%ro%@+nw}TUixy-%3a5n~4ZFH> z)AQMxM;^Z-+>p(+O=l&tCDk&&J31xR8s#&&%&PdXs3F%Gu1w9&L=Bm>CwDDtT$Wjq z&8O#wbF;zd<7X7p*<5K!_VDSm=g*lsBX=5&9oIN3lTV+NJuJ7qu=<$8(?_QA>C@AG zsu-pU=VybtmmG3ZHb45Ii_6Cs^6ADYE7SS()H#cy8Kd_e)q7^cNxA$9$G4o6J+txT zO#YngdF7^j`h@)4@Whk5!X_GCLEVd@{OE6HWsVHzp6xd;ZQT97joS;O554u&4UK!+ ziqrEqzAtsniMQOl^39K2vm$eNbYXga{)GI3)RbMHTsiv41*wijr$j382>V>+RP^n^TQpE>pa5{pcUVU7tzMXr9!3Vs7N9(ck2Uvsd^1TNj73qhQynLnp0F`MVxJeDuj9 zB7bzO6JR|z`rC!4M)@e%J?Xepj~@N!qtbqKcB-W#*wwTs>TNhDKl+L8xs8jWTqbBr zkKVHT+u@|JF&uq5-4OZB4N*6PUyxlC1x>lbcAb}<8y3PNqeGac(ckY$d48BmrPDzs zoz3JX<>wS;HOy)(HZ?Ry#c=ZEDY@zXjA*7mE1aEc@#h4GvLJj!u(+_qZ;RT4rT#~P zk47KM{&(>E)Sto^b9;AO@wOZOuI=3O-uCvJ=KOn8^C_qP{tru9UvuF_>%O<=hIicj zS0DM*eV={!kw?G$%nyF}q8AxSbgew{=#yW2(Vlnk@-z2+_K`1t?Xe&H(5n;7k)oK3 zd#ihHe%}Wk{n}%VlNPQ#^7xa_z2M??z116T{s<)=e)QQN{P2awNyneuTOHl=>H9zT z;5VOo;pe-recJ~=^tlHg{^Da#|JzCLdEl#$eC@H5Pg`^D`4_Ld{v9`d>aM#V{QM(d zd}`A485dmmo8SKF#nCO>o_V(E(1F1@bJtyY)hF(J=GoVM_WtQJ4mtFM6Hi+sqPgno ze|q>E-~7%CKmYa6@Qowe=Pzl!fP^YyYbMky?EN1*I$syHWv?T zeg0&n+W8(ni;wk1kW+V%0K(T@=_y=zW5 zE9*zCQAd0jce$%wnX_=PHxtSm}vtey? zY`7Q>H!0oRuriwa>UHs!Xw$1lzm&S>li}?2HGd4x&rHwF%n9W-K`67+=Vy*coseI| zRziN*URV^(X0PATkMru(siVKj9u+o+$E3QmN2IQKanj6e>!ihDxvA7V`u6CWTc#JL zUH8|i*3^;U*Ua4L=gK1uqfgG>RZNY3H}{KM!>-(}3#X3WlO6r{sYiwRba(ayc3BxI z91^}hIzKnMd*+<{^xUb@=v&ht`(VS2sQvb6*VFSe4XM;M{}>;h=EV)s=;y+gu({ED zB3<;(MxIG&lQL9GOUdcvw|%%ev>3m~<@h*z;|+sbw)PD54a564ZXYnu*SML0@hD$z z{}toP3KC*ua0Kt&;1Ehl4)tRlbzT^Ho|K{Ma1@`y^<2~urBNT2T;y%_GTve|2-Z}5 z|D;0Ow4z_l1nD5~qgVNdv|LbFnalYzBR_|*NgWX$nLB)@A9qn8$^zz0KA7vTRPl(D zG5mZmn+&Yq=mKkJ|F2bqTKC4P`AWKIrRsL%JintWPJ75sTQ ze`DmQsVbNiL}8I{*McDJH~U`bMRUTr!67Vy`I)RA6mmYu=5G(m{^e|&ne)@(ztI7D zn^BEHHk}Xrwk1Ks)Tk{Vh4XU_#LoulZeCFgyb6N%glu7yQFCGN$STkOLdgqn@Yly) zx-ak|KOYBc1Go`ApB1G1cL%d4H~RClvkJbyC~RYFbP`E@EU*vQ&MWJ;`pW=75TqFS z;X&4aUV!sG-zyf29yNW>e^<(5Z}C*LAdLKfpp8n(3tpE#q0k<^$?t4l$XN4XJN0J# zqr-VAKYI+Tn;kiz76tygFo*z*{{cVDPBZAbnP|#{sbe|pPSfOv5rbt*RVGZY1VJ@r>Fd8FM}eaT2sCLZIn4W@}fTlzf|?W-pg#sc%R@@1tm?S^%tLy+v!29FZoI1fIddfRq*-hJrjat3VYH&4&bD=j|48LPdl1XMTn^bc1TEIj(?($Pm9 zbJWqt9DR7}tLM+(x$|4Ig{hwRbaMSla{X#@eImJjJ-I%{mF|0wC)c}@>!Zo_%gObg zw zwVG=UHji+x^7^MWKm8kPDSDM}-Hj=~k5~Vdt+;L=S!iAZ?Ca^LH5_DeZY(x;F_x_saNJ_sB~{tAy+earw^n;S zZ!*KWn8r5q6Q=wjfzB_WBAMm2^e^{I|CV$-;l0w5ep|4VkI}ff?@OeWpnd3s^oiWh z;8%nEbjFL9s-C``dtu}W_}I=K+1PDHwyg}BpVD}Pr~zLvgkzcl>UsO!M~;EY-Vd>1TK?Y+YzE>WxJtc=rOYU#%ru*O?UPeHvA zP8?Y8!TLQQFMcu>IzOSFka{)#ZzN0X&jP<$q}_4|WW@6C%6!EZPz~=6Y;dc{S-jS+ zN5otD2G+@8br}Tby$>7{cmGt<&Px?PYP@*Qj`8C4@^@2S^e8QIe^;5{u;rq!Jh^D? ze{R@O`OvxBTHbc=JO5*C-;cif-=|&i$A!1N{+pFMla*kW0s1yxVeWDpVSL%(;O1D) z>$oSbF99XoVjfcBRO-HopJ1eaS$=fIq}9Z}%=dQtnI)`fEWW&L$NO)%;Zsdth#QW3 z@!@61_5ASMk2X!;_Nu9$ym{AMXFT-ayGQ=dzyAEz=b!n&+O|3OoSC}%$X)GIc0BjH zyIMc~?Z5fPZ@=`LFVAWY!ZrUo``SzUF8a==PdnouHs5p3$-#$morgcvvhBKcANy>} l+MCm#>${`ps&$`v>NhvO@zk#$+wuL8dw=e~F#G8D{9o~KLI3~& From 98fb881315f773277a68591a8eba1311307239b4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 15:57:34 +0200 Subject: [PATCH 851/927] packaging/nix-make: Compile units with a nushell builder The per-unit rendering of the configuration macros into -D/-U flags was done at evaluation time, and the bash builder looped over arrays derived from the structured attrs. That processing belongs in the builder, but bash cannot read the typed JSON of the structured attributes (e.g. `null` and booleans in `defines`). The compile step is now compile.nu, a nushell script run after the stdenv setup script has been sourced. It reads $NIX_ATTRS_JSON_FILE and does everything: the source tree, -I/-D/-U flags, pkg-config and the compiler invocation. Nix passes the unit's `usedDefines` as is. The objects are byte-identical to the ones built by the bash builder. The link step is still bash; it can follow. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/compile.nu | 57 ++++++++++++++++++ packaging/nix-make/lib.nix | 106 ++++++++++++---------------------- 2 files changed, 95 insertions(+), 68 deletions(-) create mode 100644 packaging/nix-make/compile.nu diff --git a/packaging/nix-make/compile.nu b/packaging/nix-make/compile.nu new file mode 100644 index 0000000000..45d74d919e --- /dev/null +++ b/packaging/nix-make/compile.nu @@ -0,0 +1,57 @@ +# The builder for a compilation unit. It reads the derivation's structured +# attributes (see `compileUnit` in lib.nix) and runs the compiler once. +# +# This is a nushell script rather than bash because the attributes carry +# typed JSON (e.g. `null` and booleans in `defines`), which bash cannot +# represent. The stdenv setup script has been sourced by the bash prelude +# in lib.nix, so the compiler (`$CXX`), the include paths of the build +# inputs and pkg-config are set up. + +def main [] { + let attrs = open $env.NIX_ATTRS_JSON_FILE + + if ($env.CXX? | is-empty) { + error make { msg: "CXX is not set; was the stdenv setup script sourced?" } + } + + # Recreate the source layout: each header and the source file is a + # separate store path. + mkdir tree + cd tree + for entry in ($attrs.includes | transpose path store) { + mkdir ($entry.path | path dirname) + ln -s $entry.store $entry.path + } + mkdir ($attrs.srcPath | path dirname) + ln -s $attrs.src $attrs.srcPath + + let include_flags = $attrs.includeDirs | each {|dir| + $"-I(if ($dir | is-empty) { '.' } else { $dir })" + } + + # The configuration macros this unit is sensitive to: `null` means + # undefined; strings become C string literals. + let define_flags = $attrs.defines | transpose name value | each {|d| + match ($d.value | describe) { + "nothing" => $"-U($d.name)", + "bool" => $"-D($d.name)=(if $d.value { 1 } else { 0 })", + "int" => $"-D($d.name)=($d.value)", + "string" => $"-D($d.name)=($d.value | to json)", + _ => { error make { msg: $"unsupported value for macro ($d.name): ($d.value)" } }, + } + } + + let pkg_config_flags = if ($attrs.pkgConfigDeps | is-empty) { + [] + } else { + ^pkg-config --cflags ...$attrs.pkgConfigDeps | str trim | split row -r '\s+' | where { $in != "" } + } + + (^$env.CXX + ...$attrs.cxxFlags + ...$include_flags + ...$define_flags + ...$pkg_config_flags + -c $attrs.srcPath + -o $env.out) +} diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 9a5e75b6de..db7bd1ea8a 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -88,17 +88,6 @@ let name: builtins.toFile name "#pragma once\n// The macros of this configuration header are passed to the compiler as -D flags.\n"; - # Render a macro value as a `-D` value: integers and booleans become - # bare, strings become C string literals. - renderDefine = - v: - if builtins.isBool v then - (if v then "1" else "0") - else if builtins.isInt v then - toString v - else - builtins.toJSON v; - # Wrap a file in a C++ raw string literal, like the `gen_header` # generator in nix-meson-build-support/generate-header. mkStringHeader = @@ -134,6 +123,16 @@ let // attrs ); + # Like `mkLeanDerivation`, but the build script is a nushell script, + # which (unlike bash) can read the typed JSON of the structured + # attributes (`$NIX_ATTRS_JSON_FILE`). The stdenv setup script is still + # sourced first, for the compiler and the build inputs. + mkNuDerivation = + attrs: nuScript: + mkLeanDerivation ( + attrs // { nativeBuildInputs = attrs.nativeBuildInputs or [ ] ++ [ pkgs.nushell ]; } + ) "exec nu --no-config-file ${nuScript}"; + # The stdenv's default hardening flags, except that `_FORTIFY_SOURCE` # warns on every unit when not optimizing, so it is disabled then (like # `hardeningDisable = [ "fortify" ]` in `stdenv.mkDerivation`). @@ -180,61 +179,31 @@ let let deps = component.externalDepsFor unit.externalIncludes; in - mkLeanDerivation - { - name = "${baseNameOf unit.path}.o"; - # The config macros this unit is sensitive to (see `configHeaders`), - # as reported by the scanner; `null` means undefined. - defines = lib.mapAttrs (_: renderDefine) (lib.filterAttrs (_: v: v != null) unit.usedDefines); - undefines = lib.attrNames (lib.filterAttrs (_: v: v == null) unit.usedDefines); - # Generated files come from the output of their generator derivation. - src = if unit.src == null then component.allGenerated.${unit.path}.path else unit.src; - includes = - unit.includes - // lib.listToAttrs ( - map (g: lib.nameValuePair g component.allGenerated.${g}.path) unit.generatedIncludes - ); - srcPath = unit.path; - includeDirs = component.includeDirs ++ component.depIncludeDirs; - cxxFlags = - commonCxxFlags - ++ component.extraCxxFlags - ++ (component.unitCxxFlags.${unit.path} or [ ]) - ++ lib.concatMap (d: d.cflags or [ ]) deps; - pkgConfigDeps = lib.concatMap (d: d.pkgconfig or [ ]) deps; - buildInputs = depPackages deps; - nativeBuildInputs = [ pkgs.pkg-config ]; - NIX_HARDENING_ENABLE = hardeningFlags; - } - '' - mkdir tree - cd tree - - # Recreate the source layout: each header and the source file is a - # separate store path. - for name in "''${!includes[@]}"; do - mkdir -p "$(dirname "$name")" - ln -s "''${includes[$name]}" "$name" - done - mkdir -p "$(dirname "$srcPath")" - ln -s "$src" "$srcPath" - - flags=() - for dir in "''${includeDirs[@]}"; do - flags+=("-I''${dir:-.}") - done - for name in "''${!defines[@]}"; do - flags+=("-D''${name}=''${defines[$name]}") - done - for name in "''${undefines[@]}"; do - flags+=("-U''${name}") - done - if [[ ''${#pkgConfigDeps[@]} -gt 0 ]]; then - flags+=($(pkg-config --cflags "''${pkgConfigDeps[@]}")) - fi - - $CXX "''${cxxFlags[@]}" "''${flags[@]}" -c "$srcPath" -o "$out" - ''; + mkNuDerivation { + name = "${baseNameOf unit.path}.o"; + # The config macros this unit is sensitive to (see `configHeaders`), + # as reported by the scanner; `null` means undefined. The builder + # turns them into -D/-U flags. + defines = unit.usedDefines; + # Generated files come from the output of their generator derivation. + src = if unit.src == null then component.allGenerated.${unit.path}.path else unit.src; + includes = + unit.includes + // lib.listToAttrs ( + map (g: lib.nameValuePair g component.allGenerated.${g}.path) unit.generatedIncludes + ); + srcPath = unit.path; + includeDirs = component.includeDirs ++ component.depIncludeDirs; + cxxFlags = + commonCxxFlags + ++ component.extraCxxFlags + ++ (component.unitCxxFlags.${unit.path} or [ ]) + ++ lib.concatMap (d: d.cflags or [ ]) deps; + pkgConfigDeps = lib.concatMap (d: d.pkgconfig or [ ]) deps; + buildInputs = depPackages deps; + nativeBuildInputs = [ pkgs.pkg-config ]; + NIX_HARDENING_ENABLE = hardeningFlags; + } ./compile.nu; # Link the objects into a shared library or an executable. link = @@ -336,8 +305,9 @@ let - `files`: extra files (e.g. generated headers) by path in the root namespace. - `configHeaders`: generated config headers, as an attribute set from path in the root namespace to the `#define`s. The header itself is an - empty stub; each unit gets the macros it uses as `-D` flags. Their - macros are also used to evaluate preprocessor conditionals. + empty stub; each unit gets the macros it uses, and the builder + (compile.nu) turns them into `-D` flags. Their macros are also used + to evaluate preprocessor conditionals. - `defines`, `undefines`: further macros known to be defined (with value) or undefined when evaluating preprocessor conditionals, on top of the platform defaults and the config headers. From e98100d4587dfe97f9b935e72162917b119f877d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 16:03:09 +0200 Subject: [PATCH 852/927] packaging/nix-make: Link with a nushell builder as well link.nu replaces the bash link script, so `mkNuDerivation` is the only derivation helper and `mkLeanDerivation` is gone. `postInstall` is now a nushell snippet. The compile derivations are unchanged; only the link steps rebuild. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/lib.nix | 110 ++++++++++++------------------------- packaging/nix-make/link.nu | 53 ++++++++++++++++++ src/nix/make.nix | 5 +- 3 files changed, 91 insertions(+), 77 deletions(-) create mode 100644 packaging/nix-make/link.nu diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index db7bd1ea8a..52f35e95fe 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -97,13 +97,16 @@ let # The `dev` outputs, as `stdenv.mkDerivation` would pick for `buildInputs`. depPackages = deps: map lib.getDev (lib.concatMap (d: lib.toList d.pkg) deps); - # A minimal derivation that sources the stdenv setup script (which - # provides the C++ compiler, and handles `buildInputs` etc.) but avoids - # the evaluation cost of `stdenv.mkDerivation`, which is significant - # for hundreds of derivations. Unlike `runCommand`, it does not set + # A minimal derivation running a nushell build script, which (unlike + # bash) can read the typed JSON of the structured attributes + # (`$NIX_ATTRS_JSON_FILE`). A bash prelude sources the stdenv setup + # script first, which provides the C++ compiler and handles + # `buildInputs` etc. This avoids the evaluation cost of + # `stdenv.mkDerivation`, which is significant for hundreds of + # derivations, and unlike `runCommand` it does not set # `preferLocalBuild`, so units can be built remotely. - mkLeanDerivation = - attrs: script: + mkNuDerivation = + attrs: nuScript: derivation ( { system = stdenv.hostPlatform.system; @@ -114,25 +117,18 @@ let # With structured attrs, attributes are not in the environment. if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; fi source $stdenv/setup - ${script} + exec nu --no-config-file ${nuScript} '') ]; inherit stdenv; __structuredAttrs = true; } // attrs + // { + nativeBuildInputs = attrs.nativeBuildInputs or [ ] ++ [ pkgs.nushell ]; + } ); - # Like `mkLeanDerivation`, but the build script is a nushell script, - # which (unlike bash) can read the typed JSON of the structured - # attributes (`$NIX_ATTRS_JSON_FILE`). The stdenv setup script is still - # sourced first, for the compiler and the build inputs. - mkNuDerivation = - attrs: nuScript: - mkLeanDerivation ( - attrs // { nativeBuildInputs = attrs.nativeBuildInputs or [ ] ++ [ pkgs.nushell ]; } - ) "exec nu --no-config-file ${nuScript}"; - # The stdenv's default hardening flags, except that `_FORTIFY_SOURCE` # warns on every unit when not optimizing, so it is disabled then (like # `hardeningDisable = [ "fortify" ]` in `stdenv.mkDerivation`). @@ -212,63 +208,27 @@ let # Link against every external dependency used by any unit. deps = component.externalDepsFor component.allExternalIncludes; in - mkLeanDerivation - { - name = "${component.name}-${component.version}"; - # Instantiate the objects and the dependencies in parallel. - objects = parallel (map (d: d.drvPath) component.allDeps ++ map (o: o.drvPath) objects) objects; - inherit (component) - type - libName - exeName - binSymlinks - linkFlags - postInstall - ; - linkPkgConfig = lib.concatMap (d: d.pkgconfig or [ ]) deps; - linkLibs = - map (d: "-l${d.libName}") component.allDeps - ++ lib.concatMap (d: d.libs or [ ]) deps - ++ component.extraLinkLibs - ++ commonLinkLibs; - buildInputs = component.allDeps ++ depPackages deps; - nativeBuildInputs = [ pkgs.pkg-config ]; - } - '' - libs=() - if [[ ''${#linkPkgConfig[@]} -gt 0 ]]; then - libs+=($(pkg-config --libs "''${linkPkgConfig[@]}")) - fi - - case "$type" in - library) - mkdir -p $out/lib - $CXX -shared -fPIC \ - -Wl,-soname,lib$libName.so \ - -Wl,--as-needed -Wl,--no-undefined \ - "''${linkFlags[@]}" \ - -o $out/lib/lib$libName.so \ - "''${objects[@]}" \ - "''${libs[@]}" \ - "''${linkLibs[@]}" - ;; - executable) - mkdir -p $out/bin - $CXX \ - -Wl,--as-needed -Wl,--no-undefined \ - "''${linkFlags[@]}" \ - -o $out/bin/$exeName \ - "''${objects[@]}" \ - "''${libs[@]}" \ - "''${linkLibs[@]}" - for name in "''${binSymlinks[@]}"; do - ln -s $exeName $out/bin/$name - done - ;; - esac - - eval "$postInstall" - '' + mkNuDerivation { + name = "${component.name}-${component.version}"; + # Instantiate the objects and the dependencies in parallel. + objects = parallel (map (d: d.drvPath) component.allDeps ++ map (o: o.drvPath) objects) objects; + inherit (component) + type + libName + exeName + binSymlinks + linkFlags + postInstall + ; + linkPkgConfig = lib.concatMap (d: d.pkgconfig or [ ]) deps; + linkLibs = + map (d: "-l${d.libName}") component.allDeps + ++ lib.concatMap (d: d.libs or [ ]) deps + ++ component.extraLinkLibs + ++ commonLinkLibs; + buildInputs = component.allDeps ++ depPackages deps; + nativeBuildInputs = [ pkgs.pkg-config ]; + } ./link.nu // { inherit objects; inherit (component) units; @@ -285,7 +245,7 @@ let - `libName`: library name without `lib` prefix (e.g. `nixutil`). - `exeName`: executable name; defaults to `name`. - `binSymlinks`: names of symlinks to the executable to create in `bin/`. - - `postInstall`: shell snippet run after linking. + - `postInstall`: nushell snippet run after linking, with `$env.out` set. - `deps`: other components this one depends on. Their public headers are made available under `_deps//` and their libraries are linked. Dependencies are transitive. diff --git a/packaging/nix-make/link.nu b/packaging/nix-make/link.nu new file mode 100644 index 0000000000..5a0057f219 --- /dev/null +++ b/packaging/nix-make/link.nu @@ -0,0 +1,53 @@ +# The builder for linking a component's objects into a shared library or +# an executable. It reads the derivation's structured attributes (see +# `link` in lib.nix). The stdenv setup script has been sourced by the bash +# prelude in lib.nix, so the compiler driver (`$CXX`), the library paths +# of the build inputs and pkg-config are set up. + +def main [] { + let attrs = open $env.NIX_ATTRS_JSON_FILE + + if ($env.CXX? | is-empty) { + error make { msg: "CXX is not set; was the stdenv setup script sourced?" } + } + + let pkg_config_libs = if ($attrs.linkPkgConfig | is-empty) { + [] + } else { + ^pkg-config --libs ...$attrs.linkPkgConfig | str trim | split row -r '\s+' | where { $in != "" } + } + + match $attrs.type { + "library" => { + let lib = $"lib($attrs.libName).so" + mkdir $"($env.out)/lib" + (^$env.CXX -shared -fPIC + $"-Wl,-soname,($lib)" + -Wl,--as-needed -Wl,--no-undefined + ...$attrs.linkFlags + -o $"($env.out)/lib/($lib)" + ...$attrs.objects + ...$pkg_config_libs + ...$attrs.linkLibs) + } + "executable" => { + mkdir $"($env.out)/bin" + (^$env.CXX + -Wl,--as-needed -Wl,--no-undefined + ...$attrs.linkFlags + -o $"($env.out)/bin/($attrs.exeName)" + ...$attrs.objects + ...$pkg_config_libs + ...$attrs.linkLibs) + for name in $attrs.binSymlinks { + ln -s $attrs.exeName $"($env.out)/bin/($name)" + } + } + _ => { error make { msg: $"unknown component type: ($attrs.type)" } } + } + + # A nushell snippet from the component description. + if not ($attrs.postInstall | is-empty) { + ^nu --no-config-file -c $attrs.postInstall + } +} diff --git a/src/nix/make.nix b/src/nix/make.nix index 44ddc733fb..ad38c03f23 100644 --- a/src/nix/make.nix +++ b/src/nix/make.nix @@ -82,8 +82,9 @@ nixMake.mkComponent { "nix-store" ]; + # A nushell snippet. postInstall = '' - mkdir -p $out/libexec/nix - ln -s ../../bin/nix $out/libexec/nix/build-remote + mkdir $"($env.out)/libexec/nix" + ln -s ../../bin/nix $"($env.out)/libexec/nix/build-remote" ''; } From 005d39a46ac3d3346e7b31e9d734f15628faf4f1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 16:07:04 +0200 Subject: [PATCH 853/927] packaging/nix-make: Use builtins.genericClosure for deduplication `uniqueBy` is now a `genericClosure` with a no-op operator, and `allDeps` uses the operator to compute a real transitive closure of the dependencies rather than concatenating their precomputed lists. No change to the resulting derivations. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/lib.nix | 44 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 52f35e95fe..3cfba45cb1 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -147,28 +147,15 @@ let # which instantiates them and all their inputs. uniqueBy = key: xs: - (lib.foldl' - ( - acc: x: - let - k = key x; - in - if acc.seen ? ${k} then - acc - else - { - seen = acc.seen // { - ${k} = null; - }; - result = acc.result ++ [ x ]; - } - ) - { - seen = { }; - result = [ ]; + map (x: x.value) ( + builtins.genericClosure { + startSet = map (x: { + key = key x; + value = x; + }) xs; + operator = _: [ ]; } - xs - ).result; + ); compileUnit = component: unit: @@ -324,7 +311,19 @@ let ++ roots; # The transitive closure of the dependencies, direct ones first. - allDeps = uniqueBy (d: d.component.name) (deps ++ lib.concatMap (d: d.component.allDeps) deps); + allDeps = + let + item = d: { + key = d.component.name; + value = d; + }; + in + map (x: x.value) ( + builtins.genericClosure { + startSet = map item deps; + operator = x: map item x.value.component.deps; + } + ); # Dependencies contribute their roots, public include directories and # extra files under `_deps//` (a prefix that cannot collide with @@ -430,6 +429,7 @@ let files = allFiles; inherit generated allGenerated; inherit + deps units allDeps allDefines From 3d357c192d742290c4e10a59c606a7b34799f5c8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 16:37:30 +0200 Subject: [PATCH 854/927] packaging/nix-make: Silence "structuredAttrs is enabled" The stdenv setup script prints this unconditionally on stdout, once per derivation. Source it with stdout redirected to /dev/null; as `source` is a builtin, the environment it sets up is kept. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/lib.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging/nix-make/lib.nix b/packaging/nix-make/lib.nix index 3cfba45cb1..2eab533e93 100644 --- a/packaging/nix-make/lib.nix +++ b/packaging/nix-make/lib.nix @@ -116,7 +116,8 @@ let (builtins.toFile "builder.sh" '' # With structured attrs, attributes are not in the environment. if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; fi - source $stdenv/setup + # The setup script prints "structuredAttrs is enabled" on stdout. + source $stdenv/setup >/dev/null exec nu --no-config-file ${nuScript} '') ]; From 2cc11cbc1ee43e395dd27d651e4de01221dc0d38 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2026 22:48:59 +0200 Subject: [PATCH 855/927] packaging/nix-make: Add the baked-flake.nix string header Needed by `nix flake bake` (flake-bake.cc), merged upstream. Assisted-by: Claude Fable 5.1 --- src/nix/make.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/make.nix b/src/nix/make.nix index ad38c03f23..67edc90957 100644 --- a/src/nix/make.nix +++ b/src/nix/make.nix @@ -49,6 +49,7 @@ nixMake.mkComponent { "profiles.md.gen.hh" = nixMake.mkStringHeader ../../doc/manual/source/command-ref/files/profiles.md; "unpack-channel.nix.gen.hh" = nixMake.mkStringHeader ./nix-channel/unpack-channel.nix; "buildenv.nix.gen.hh" = nixMake.mkStringHeader ./nix-env/buildenv.nix; + "baked-flake.nix.gen.hh" = nixMake.mkStringHeader ./baked-flake.nix; }; configHeaders = { From 8078b3fdc47f929296e18a053e2e4f8988714793 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Sep 2026 11:34:16 +0200 Subject: [PATCH 856/927] packaging/nix-make: Re-lock the nix input for the bdwgc input The root flake gained a `bdwgc` input, but the nested `nix` input's lock predates it, so `inputs.bdwgc` was missing when evaluating packaging/dependencies.nix through the overlay. `nix flake lock` does not re-read the inputs of an already-locked node; `nix flake update nix` does. Only the bdwgc node is added. Assisted-by: Claude Fable 5.1 --- packaging/nix-make/flake.lock | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packaging/nix-make/flake.lock b/packaging/nix-make/flake.lock index 017d68f335..182570e034 100644 --- a/packaging/nix-make/flake.lock +++ b/packaging/nix-make/flake.lock @@ -1,5 +1,22 @@ { "nodes": { + "bdwgc": { + "flake": false, + "locked": { + "lastModified": 1790015987, + "narHash": "sha256-bPVosUgucJR0hGSap3tJW84AR6eGYegVROV+O8VYX5A=", + "owner": "DeterminateSystems", + "repo": "bdwgc", + "rev": "373957929b9ca3b2ae69a3e16b830c44c15b42ca", + "type": "github" + }, + "original": { + "owner": "DeterminateSystems", + "ref": "user-defined-stacks", + "repo": "bdwgc", + "type": "github" + } + }, "flake-compat": { "flake": false, "locked": { @@ -70,6 +87,7 @@ }, "nix": { "inputs": { + "bdwgc": "bdwgc", "flake-parts": "flake-parts", "git-hooks-nix": "git-hooks-nix", "nixpkgs": "nixpkgs", From e47e4bc77b0353125b92c3c223ae83e70815d590 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Sep 2026 12:37:02 +0200 Subject: [PATCH 857/927] tests/functional: Add a Meson-free test runner Add tests/functional/run-tests.py, a stdlib-only Python program that runs a list of functional test scripts in parallel and reports success/failure like `meson test`. It generates common/subst-vars.sh and config.nix from their .in templates, provides the environment the harness in common/ expects, and runs each script as `bash -x -e -u -o pipefail