From a80c97df6f8b6df9c3752398ced2279ded27e6f7 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 31 Aug 2026 11:03:48 -0400 Subject: [PATCH 1/4] libfetchers: apply immutable URLs to file inputs The Lockable HTTP Protocol only worked for tarball inputs. An input with `flake = false` and no tarball extension is a file input, so Nix ignored the `Link: <...>; rel="immutable"` header. Move the handling into CurlInputScheme so that both schemes share it, and give file inputs a fingerprint now that they can lock to an immutable URL. Assisted-by: Claude Opus 5 (1M context) --- src/libfetchers/tarball.cc | 55 ++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 5586229e56cf..09e89d3f970e 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -408,6 +408,33 @@ struct CurlInputScheme : InputScheme { return (bool) input.getNarHash(); } + + /* Apply the "immutable" URL that the server sent in a `Link: + <...>; rel="immutable"` header (see the Lockable HTTP Protocol) + to the input. + + The immutable URL must denote an input of the same type, because + the server cannot change how Nix reads the resource. + + `requireTree` tells the parser whether the immutable URL denotes + a tree. */ + Input applyImmutableUrl( + const Settings & settings, const Input & input, const std::string & immutableUrl, bool requireTree) const + { + // FIXME: would be nice to support arbitrary flakerefs + // here, e.g. git flakes. + auto immutableInput = Input::fromURL(settings, immutableUrl, requireTree); + + if (immutableInput.getType() != schemeName()) + throw Error( + "the immutable URL '%s' of input '%s' is a '%s' input, but a '%s' input is necessary", + immutableUrl, + input.to_string(), + immutableInput.getType(), + schemeName()); + + return immutableInput; + } }; struct FileInputScheme : CurlInputScheme @@ -445,6 +472,10 @@ struct FileInputScheme : CurlInputScheme auto file = downloadFile(store, settings, getStrAttr(input.attrs, "url"), input.getName()); auto narHash = store.queryPathInfo(file.storePath)->narHash; + + if (file.immutableUrl) + input = applyImmutableUrl(settings, input, *file.immutableUrl, false); + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); auto accessor = ref{store.getFSAccessor(file.storePath)}; @@ -453,6 +484,14 @@ struct FileInputScheme : CurlInputScheme return {accessor, input}; } + + std::optional getFingerprint(Store & store, const Input & input) const override + { + if (auto narHash = input.getNarHash()) + return "file:" + narHash->to_string(HashFormat::SRI, true); + else + return std::nullopt; + } }; struct TarballInputScheme : CurlInputScheme @@ -510,21 +549,15 @@ struct TarballInputScheme : CurlInputScheme return std::nullopt; auto & result = *res; - if (result.immutableUrl) { - auto immutableInput = Input::fromURL(settings, *result.immutableUrl); - // FIXME: would be nice to support arbitrary flakerefs - // here, e.g. git flakes. - if (immutableInput.getType() != "tarball") - throw Error("tarball 'Link' headers that redirect to non-tarball URLs are not supported"); - input = immutableInput; - } + auto narHash = settings.getTarballCache()->treeHashToNarHash(settings, result.treeHash); + + if (result.immutableUrl) + input = applyImmutableUrl(settings, input, *result.immutableUrl, true); if (result.lastModified && !input.attrs.contains("lastModified")) input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); - input.attrs.insert_or_assign( - "narHash", - settings.getTarballCache()->treeHashToNarHash(settings, result.treeHash).to_string(HashFormat::SRI, true)); + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); return {{result.accessor, input}}; } From 5da5311bc9b8c31f5f2ab5c6f4df8ed1039a339e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 31 Aug 2026 11:08:45 -0400 Subject: [PATCH 2/4] libfetchers: check the NAR hash in an immutable URL The Lockable HTTP Protocol says that Nix checks the tarball contents against the `narHash` in the `Link` header. Nix did not do this: it replaced the value with the NAR hash of the data that it downloaded. Assisted-by: Claude Opus 5 (1M context) --- src/libfetchers/tarball.cc | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 09e89d3f970e..fe4232704c8e 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -414,12 +414,18 @@ struct CurlInputScheme : InputScheme to the input. The immutable URL must denote an input of the same type, because - the server cannot change how Nix reads the resource. + the server cannot change how Nix reads the resource. If the + immutable URL contains a `narHash`, it must agree with `narHash`, + the NAR hash of the data that Nix downloaded. `requireTree` tells the parser whether the immutable URL denotes a tree. */ Input applyImmutableUrl( - const Settings & settings, const Input & input, const std::string & immutableUrl, bool requireTree) const + const Settings & settings, + const Input & input, + const std::string & immutableUrl, + const Hash & narHash, + bool requireTree) const { // FIXME: would be nice to support arbitrary flakerefs // here, e.g. git flakes. @@ -433,6 +439,15 @@ struct CurlInputScheme : InputScheme immutableInput.getType(), schemeName()); + if (auto expected = immutableInput.getNarHash(); expected && *expected != narHash) + throw Error( + (unsigned int) 102, + "NAR hash mismatch in the immutable URL '%s' of input '%s': expected '%s', but got '%s'", + immutableUrl, + input.to_string(), + expected->to_string(HashFormat::SRI, true), + narHash.to_string(HashFormat::SRI, true)); + return immutableInput; } }; @@ -474,7 +489,7 @@ struct FileInputScheme : CurlInputScheme auto narHash = store.queryPathInfo(file.storePath)->narHash; if (file.immutableUrl) - input = applyImmutableUrl(settings, input, *file.immutableUrl, false); + input = applyImmutableUrl(settings, input, *file.immutableUrl, narHash, false); input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); @@ -552,7 +567,7 @@ struct TarballInputScheme : CurlInputScheme auto narHash = settings.getTarballCache()->treeHashToNarHash(settings, result.treeHash); if (result.immutableUrl) - input = applyImmutableUrl(settings, input, *result.immutableUrl, true); + input = applyImmutableUrl(settings, input, *result.immutableUrl, narHash, true); if (result.lastModified && !input.attrs.contains("lastModified")) input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); From 59aacd0fe769612431320a5678e5557473e85d2c Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 31 Aug 2026 11:10:33 -0400 Subject: [PATCH 3/4] doc: generalize the Lockable HTTP Protocol documentation Assisted-by: Claude Opus 5 (1M context) --- .../rl-next/lockable-http-file-inputs.md | 21 +++++ doc/manual/source/SUMMARY.md.in | 2 +- .../source/protocols/tarball-fetcher.md | 81 ++++++++++++------- 3 files changed, 76 insertions(+), 28 deletions(-) create mode 100644 doc/manual/rl-next/lockable-http-file-inputs.md diff --git a/doc/manual/rl-next/lockable-http-file-inputs.md b/doc/manual/rl-next/lockable-http-file-inputs.md new file mode 100644 index 000000000000..05be9875006c --- /dev/null +++ b/doc/manual/rl-next/lockable-http-file-inputs.md @@ -0,0 +1,21 @@ +--- +synopsis: "The Lockable HTTP Protocol now applies to file inputs" +--- + +The [Lockable HTTP Protocol](@docroot@/protocols/tarball-fetcher.md) only applied to tarball inputs. +Nix ignored the `Link: <...>; rel="immutable"` header for an input with `flake = false` and a URL without a tarball extension, because such an input is a file input. + +Nix now applies the immutable URL to file inputs too: + +```nix +inputs.determinate-pkg = { + url = "https://install.determinate.systems/determinate-pkg/stable/Universal"; + flake = false; +}; +``` + +The immutable URL must have the same input type as the input that Nix fetched. +For a file input whose immutable URL has a tarball extension, the server must put the prefix `file+` in front of the transport scheme. + +Nix also compares the contents that it downloads against the `narHash` in the immutable URL, and gives an error if the two values are different. +Before, Nix ignored that value. diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 224d901c2eb2..9ab32ef8b46d 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -128,7 +128,7 @@ - [Build Trace Entry](protocols/json/build-trace-entry.md) - [Build Result](protocols/json/build-result.md) - [Store](protocols/json/store.md) - - [Serving Tarball Flakes](protocols/tarball-fetcher.md) + - [Lockable HTTP Protocol](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) - [Nix Archive (NAR) Format](protocols/nix-archive/index.md) - [Binary Cache](protocols/binary-cache/index.md) diff --git a/doc/manual/source/protocols/tarball-fetcher.md b/doc/manual/source/protocols/tarball-fetcher.md index e8cda784cc9e..92dfe1cabd87 100644 --- a/doc/manual/source/protocols/tarball-fetcher.md +++ b/doc/manual/source/protocols/tarball-fetcher.md @@ -1,33 +1,27 @@ -# Lockable HTTP Tarball Protocol - -Tarball flakes can be served as regular tarballs via HTTP or the file -system (for `file://` URLs). Unless the server implements the Lockable -HTTP Tarball protocol, it is the responsibility of the user to make sure that -the URL always produces the same tarball contents. - -An HTTP server can return an "immutable" HTTP URL appropriate for lock -files. This allows users to specify a tarball flake input in -`flake.nix` that requests the latest version of a flake -(e.g. `https://example.org/hello/latest.tar.gz`), while `flake.lock` -will record a URL whose contents will not change -(e.g. `https://example.org/hello/.tar.gz`). To do so, the -server must return an [HTTP `Link` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Link) with the `rel` attribute set to -`immutable`, as follows: +# Lockable HTTP Protocol + +Nix can fetch a flake input over HTTP, or from the file system for `file://` URLs. +If the server does not support the Lockable HTTP Protocol, the user must make sure that the URL always gives the same contents. + +An HTTP server can return an "immutable" URL for the lock file. +Then the user can put an input in `flake.nix` that asks for the most recent version of a resource (for example, `https://example.org/hello/latest.tar.gz`), while `flake.lock` records a URL whose contents do not change (for example, `https://example.org/hello/.tar.gz`). +To do this, the server must send an [HTTP `Link` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Link) with the `rel` attribute set to `immutable`, as follows: ``` Link: ; rel="immutable" ``` -(Note the required `<` and `>` characters around *flakeref*.) +(The `<` and `>` characters around *flakeref* are necessary.) + +*flakeref* must have the same input type as the input that Nix fetched. +Nix gives an error if the two types are different. +For the two input types that this protocol applies to, refer to [Tarball inputs](#tarball-inputs) and [File inputs](#file-inputs). -*flakeref* must be a tarball flakeref. It can contain the tarball flake attributes -`narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its -value must be the [NAR hash][Nix Archive] of the unpacked tarball (as computed via -`nix hash path`). Nix checks the contents of the returned tarball -against the `narHash` attribute. The `rev` and `revCount` attributes -are useful when the tarball flake is a mirror of a fetcher type that -has those attributes, such as Git or GitHub. They are not checked by -Nix. +*flakeref* can contain the flake attributes `narHash`, `rev`, `revCount` and `lastModified`. +If *flakeref* contains `narHash`, its value must be the [NAR hash][Nix Archive] of the contents, as computed by `nix hash path`. +Nix compares the contents that it downloaded against `narHash`, and gives an error if the two values are different. +Nix does not check `rev` and `revCount`. +These two attributes are useful when the input is a mirror of a fetcher type that has them, such as Git or GitHub. ``` Link: ; rel="immutable" ``` -(The linebreaks in this example are for clarity and must not be included in the actual response.) +(The line breaks in this example make it easy to read. Do not put them in the actual response.) + +## Tarball inputs + +Nix unpacks a tarball input into a tree. +Nix uses this input type when the URL has a tarball extension, such as `.tar.gz` or `.zip`, or when the input is a flake. + +For a tarball input, *flakeref* must be a tarball flakeref. + +The value of the `lastModified` flake attribute is the timestamp of the most recent file in the tarball. + +## File inputs + +Nix puts a file input into the store as a single file. +Nix uses this input type when the URL has no tarball extension and the input has `flake = false`. +The URL does not have to look like a file, and it can contain query parameters. + +> **Example** +> +> ```nix +> # flake.nix +> { +> inputs.determinate-pkg = { +> url = "https://install.determinate.systems/determinate-pkg/stable/Universal"; +> flake = false; +> }; +> outputs = { foo }: { /* ... */ }; +> } +> ``` + +For a file input, *flakeref* must be a file flakeref. + +``` +Link: ; rel="immutable" +``` -For tarball flakes, the value of the `lastModified` flake attribute is -defined as the timestamp of the newest file inside the tarball. +The `lastModified` attribute does not apply to file inputs. ## Gitea and Forgejo support From 7ed17d46d424081ab9933ce718d270f2e56a2c2b Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 31 Aug 2026 11:12:02 -0400 Subject: [PATCH 4/4] tests: cover the Lockable HTTP Protocol for file inputs Assisted-by: Claude Opus 5 (1M context) --- tests/nixos/tarball-flakes.nix | 85 +++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix index ab9b200db269..073ba2a231fc 100644 --- a/tests/nixos/tarball-flakes.nix +++ b/tests/nixos/tarball-flakes.nix @@ -8,7 +8,35 @@ let pkgs = config.nodes.machine.nixpkgs.pkgs; - root = pkgs.runCommand "nixpkgs-flake" { } '' + # A resource that is not a tarball. An input that points at it is a + # `file` input, not a `tarball` input. + bom = pkgs.writeText "bom.json" '' + {"hello": "world"} + ''; + + # A flake with a `flake = false` input, to show that the Lockable HTTP + # Protocol applies to `file` inputs. + mkFileFlake = + name: url: + pkgs.writeTextFile { + inherit name; + destination = "/flake.nix"; + text = '' + { + inputs.foo = { + url = "${url}"; + flake = false; + }; + outputs = { self, foo }: { }; + } + ''; + }; + + fileFlake = mkFileFlake "file-flake" "http://localhost/file/stable/aarch64-linux?a=1"; + badHashFlake = mkFileFlake "bad-hash-flake" "http://localhost/file-bad-hash/stable/aarch64-linux"; + badTypeFlake = mkFileFlake "bad-type-flake" "http://localhost/file-bad-type/stable/aarch64-linux"; + + root = pkgs.runCommand "nixpkgs-flake" { nativeBuildInputs = [ pkgs.nix ]; } '' mkdir -p $out/{stable,tags} set -x @@ -27,6 +55,32 @@ let Redirect "/tags/latest.tar.gz" "/stable/${nixpkgs.rev}.tar.gz" Header always set Link "; rel=\"immutable\"" EOF + + # A non-tarball resource, served at a URL that has no tarball + # extension. Nix fetches it as a `file` input. + mkdir -p $out/file/{stable,v1} $out/file-bad-hash/stable $out/file-bad-type/stable + for d in file/stable file/v1 file-bad-hash/stable file-bad-type/stable; do + cp ${bom} $out/$d/aarch64-linux + done + + # The NAR hash of the resource, percent-encoded for a URL query. + narHash=$(nix-hash --to-sri --type sha256 "$(nix-hash --type sha256 $out/file/v1/aarch64-linux)" \ + | sed 's|+|%2B|g; s|/|%2F|g; s|=|%3D|g') + + cat >$out/file/stable/.htaccess <; rel=\"immutable\"" + EOF + + # An immutable URL with a NAR hash that does not agree with the + # contents. + cat >$out/file-bad-hash/stable/.htaccess <; rel=\"immutable\"" + EOF + + # An immutable URL that is a tarball flakeref, not a file flakeref. + cat >$out/file-bad-type/stable/.htaccess <; rel=\"immutable\"" + EOF ''; in @@ -58,6 +112,9 @@ in virtualisation.additionalPaths = [ pkgs.hello pkgs.fuse + fileFlake + badHashFlake + badTypeFlake ]; virtualisation.memorySize = 4096; nix.settings.substituters = lib.mkForce [ ]; @@ -99,6 +156,32 @@ in # Check that fetching fails if we provide incorrect attributes. machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?rev=493300eb13ae6fb387fbd47bf54a85915acc31c0") machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?narHash=sha256-tbudgBSg+bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw=") + + # The protocol also applies to `file` inputs, that is, inputs with + # `flake = false` whose URL has no tarball extension. + machine.succeed("cp -rT ${fileFlake} /tmp/file-flake && chmod -R u+w /tmp/file-flake") + machine.succeed("nix flake lock /tmp/file-flake") + node = json.loads(machine.succeed("cat /tmp/file-flake/flake.lock"))["nodes"]["foo"] + print(node) + + # Check that the lock file records the immutable URL, and that the + # input is still a file. + assert node["locked"]["url"] == "http://localhost/file/v1/aarch64-linux" + assert node["locked"]["type"] == "file" + assert node["original"]["url"] == "http://localhost/file/stable/aarch64-linux?a=1" + + # Check that Nix compares the contents against the `narHash` in the + # immutable URL. + machine.succeed("cp -rT ${badHashFlake} /tmp/bad-hash-flake && chmod -R u+w /tmp/bad-hash-flake") + out = machine.fail("nix flake lock /tmp/bad-hash-flake 2>&1") + print(out) + assert "NAR hash mismatch in the immutable URL" in out + + # Check that Nix rejects an immutable URL of a different input type. + machine.succeed("cp -rT ${badTypeFlake} /tmp/bad-type-flake && chmod -R u+w /tmp/bad-type-flake") + out = machine.fail("nix flake lock /tmp/bad-type-flake 2>&1") + print(out) + assert "is a 'tarball' input, but a 'file' input is necessary" in out ''; }