diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index f42699d73..9f5bdf6d0 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -69,6 +69,7 @@ sources = files( 'search-path.cc', 'tecnix-dependency-tracking.cc', 'tecnix-eval-cache-row.cc', + 'tecnix-git-status.cc', 'tecnix-manifest-fingerprint.cc', 'tecnix-worldtree.cc', 'tectonix.cc', diff --git a/src/libexpr-tests/tecnix-git-status.cc b/src/libexpr-tests/tecnix-git-status.cc new file mode 100644 index 000000000..23375e8f9 --- /dev/null +++ b/src/libexpr-tests/tecnix-git-status.cc @@ -0,0 +1,151 @@ +#include +#include +#include + +#include + +#include "nix/expr/tecnix/source-deps.hh" + +namespace nix { + +namespace { + +using namespace std::string_literals; + +constexpr auto headOid = "d00491fd7e5bb6fa28c517a0bb32b8b506539d4d"; +constexpr auto indexOid = "0cfbf08886fca9a91cb753ec8734c84fcbe52c9f"; +constexpr auto zeroOid = "0000000000000000000000000000000000000000"; + +std::string +ordinary(std::string_view xy, std::string_view modes, std::string_view hH, std::string_view hI, std::string_view path) +{ + return "1 "s + std::string(xy) + " N... " + std::string(modes) + " " + std::string(hH) + " " + std::string(hI) + " " + + std::string(path) + '\0'; +} + +GitStatusEntry staged(std::string path, uint32_t mode = 0100644) +{ + return {.path = std::move(path), .index = GitIndexBlob{.oid = indexOid, .mode = mode}}; +} + +GitStatusEntry onDisk(std::string path) +{ + return {.path = std::move(path)}; +} + +} // namespace + +TEST(TecnixGitStatus, StagedOnlyChangeIsServedFromTheIndex) +{ + // What `git status` reports for a cleanly merged, skip-worktree file of an + // uncommitted merge in a sparse checkout: staged, absent from disk. + auto output = ordinary("M.", "100644 100644 100644", headOid, indexOid, "system/tectonix/default.nix"); + EXPECT_EQ(parseGitPorcelainV2ZStatus(output), std::vector{staged("system/tectonix/default.nix")}); +} + +TEST(TecnixGitStatus, WorktreeChangesStayOnDisk) +{ + auto output = ordinary(".M", "100644 100644 100644", headOid, headOid, "a.nix") + + ordinary("MM", "100644 100644 100644", headOid, indexOid, "b.nix") + + ordinary("AD", "000000 100644 000000", zeroOid, indexOid, "c.nix"); + EXPECT_EQ(parseGitPorcelainV2ZStatus(output), (std::vector{onDisk("a.nix"), onDisk("b.nix"), onDisk("c.nix")})); +} + +TEST(TecnixGitStatus, StagedAdditionIsServedFromTheIndex) +{ + auto output = ordinary("A.", "000000 100644 100644", zeroOid, indexOid, "new.nix"); + EXPECT_EQ(parseGitPorcelainV2ZStatus(output), std::vector{staged("new.nix")}); +} + +TEST(TecnixGitStatus, StagedDeletionHasNoIndexBlob) +{ + auto output = ordinary("D.", "100644 000000 000000", headOid, zeroOid, "gone.nix"); + EXPECT_EQ(parseGitPorcelainV2ZStatus(output), std::vector{onDisk("gone.nix")}); +} + +TEST(TecnixGitStatus, IndexModeIsKept) +{ + auto output = ordinary("M.", "100644 100755 100755", headOid, indexOid, "bin/run") + + ordinary("A.", "000000 120000 120000", zeroOid, indexOid, "link"); + EXPECT_EQ(parseGitPorcelainV2ZStatus(output), (std::vector{staged("bin/run", 0100755), staged("link", 0120000)})); +} + +TEST(TecnixGitStatus, SubmoduleStaysOnDisk) +{ + auto output = "1 M. SC.. 160000 160000 160000 "s + headOid + " " + indexOid + " vendor/sub" + '\0'; + EXPECT_EQ(parseGitPorcelainV2ZStatus(output), std::vector{onDisk("vendor/sub")}); +} + +TEST(TecnixGitStatus, RenameKeepsBothNames) +{ + auto output = "2 R. N... 100644 100644 100644 "s + headOid + " " + indexOid + " R100 new name.nix" + '\0' + + "old name.nix" + '\0'; + EXPECT_EQ(parseGitPorcelainV2ZStatus(output), (std::vector{staged("new name.nix"), onDisk("old name.nix")})); +} + +TEST(TecnixGitStatus, CopySourceIsNotDirty) +{ + auto output = + "2 C. N... 100644 100644 100644 "s + headOid + " " + indexOid + " C100 copy.nix" + '\0' + "source.nix" + '\0'; + EXPECT_EQ(parseGitPorcelainV2ZStatus(output), std::vector{staged("copy.nix")}); +} + +TEST(TecnixGitStatus, WorktreeCopySourceIsNotDirty) +{ + auto output = + "2 .C N... 100644 100644 100644 "s + headOid + " " + headOid + " C100 copy.nix" + '\0' + "source.nix" + '\0'; + EXPECT_EQ(parseGitPorcelainV2ZStatus(output), std::vector{onDisk("copy.nix")}); +} + +TEST(TecnixGitStatus, UnmergedAndUntrackedStayOnDisk) +{ + auto output = "u UU N... 100644 100644 100644 100644 "s + headOid + " " + indexOid + " " + headOid + + " conflicted.nix" + '\0' + "? untracked dir/file.nix" + '\0'; + EXPECT_EQ( + parseGitPorcelainV2ZStatus(output), (std::vector{onDisk("conflicted.nix"), onDisk("untracked dir/file.nix")})); +} + +TEST(TecnixGitStatus, PathsWithSpacesAreKeptWhole) +{ + auto output = ordinary("M.", "100644 100644 100644", headOid, indexOid, "dir with spaces/a b.nix"); + EXPECT_EQ(parseGitPorcelainV2ZStatus(output), std::vector{staged("dir with spaces/a b.nix")}); +} + +TEST(TecnixGitStatus, HeadersAreIgnored) +{ + auto output = "# branch.oid "s + headOid + '\0' + "# stash 2" + '\0' + + ordinary("M.", "100644 100644 100644", headOid, indexOid, "a.nix"); + EXPECT_EQ(parseGitPorcelainV2ZStatus(output), std::vector{staged("a.nix")}); +} + +TEST(TecnixGitStatus, TruncatedRecordIsRejected) +{ + auto output = ordinary("M.", "100644 100644 100644", headOid, indexOid, "a.nix") + "1 M. N... 100644"; + EXPECT_THROW(parseGitPorcelainV2ZStatus(output), Error); +} + +TEST(TecnixGitStatus, MalformedRecordIsRejected) +{ + EXPECT_THROW(parseGitPorcelainV2ZStatus("1 M. N... 100644 100644"s + '\0'), Error); + EXPECT_THROW(parseGitPorcelainV2ZStatus("u UU N... 100644"s + '\0'), Error); +} + +TEST(TecnixGitStatus, RenameMissingItsOriginalPathIsRejected) +{ + auto output = "2 R. N... 100644 100644 100644 "s + headOid + " " + indexOid + " R100 new.nix" + '\0'; + EXPECT_THROW(parseGitPorcelainV2ZStatus(output), Error); +} + +TEST(TecnixGitStatus, PorcelainV1IsRejected) +{ + for (auto record : {"M a.nix", " M a.nix", "MM a.nix", "A a.nix", "D a.nix", "UU a.nix", "?? a.nix"}) + EXPECT_THROW(parseGitPorcelainV2ZStatus(std::string(record) + '\0'), Error) << record; + EXPECT_THROW(parseGitPorcelainV2ZStatus("R new.nix"s + '\0' + "old.nix" + '\0'), Error); +} + +TEST(TecnixGitStatus, EmptyStatusHasNoEntries) +{ + EXPECT_EQ(parseGitPorcelainV2ZStatus(""), std::vector{}); +} + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/tecnix/source-deps.hh b/src/libexpr/include/nix/expr/tecnix/source-deps.hh index 4d9da7c3c..a0bb228b8 100644 --- a/src/libexpr/include/nix/expr/tecnix/source-deps.hh +++ b/src/libexpr/include/nix/expr/tecnix/source-deps.hh @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -46,7 +47,43 @@ void mergeUnpublishedTrackedSourceDepsFrame(TrackedSourceDepsFrame & frame); [[gnu::always_inline]] inline void forceValueTracked(EvalState & state, Value & v, PosIdx pos, TrackingContext & trackingCtx); -std::vector parseGitPorcelainZDirtyPaths(std::string_view output); +/** The staged blob of a dirty path, per `git status --porcelain=v2`. */ +struct GitIndexBlob +{ + /** Hex object id of the blob in the index. */ + std::string oid; + /** Git file mode of the index entry: 0100644, 0100755 or 0120000. */ + uint32_t mode; + + bool operator==(const GitIndexBlob &) const = default; +}; + +/** One path `git status` reports as differing from HEAD. */ +struct GitStatusEntry +{ + std::string path; + + /** + * Set when the working tree has no change of its own against the index, + * so the index entry is the authoritative content of `path`. The file + * need not be on disk: git stages, but never writes, the cleanly merged + * files of an uncommitted merge that lie outside a sparse checkout + * (skip-worktree entries). + */ + std::optional index = std::nullopt; + + bool operator==(const GitStatusEntry &) const = default; +}; + +/** + * Parse `git status --porcelain=v2 -z` output. Every changed, renamed (both + * names), copied (the copy only), unmerged or untracked path yields an + * entry. `index` is set only for ordinary, renamed and copied entries whose + * worktree column is `.` and whose index entry is a regular file or symlink. + * Throws on any record it does not recognise, including porcelain v1 output, + * rather than reporting fewer dirty paths. + */ +std::vector parseGitPorcelainV2ZStatus(std::string_view output); /** * A stack-resident accumulator for one bracketed region of evaluation: the diff --git a/src/libexpr/tecnix/repo-accessor.cc b/src/libexpr/tecnix/repo-accessor.cc index 9559a5c55..426dc80f7 100644 --- a/src/libexpr/tecnix/repo-accessor.cc +++ b/src/libexpr/tecnix/repo-accessor.cc @@ -12,6 +12,7 @@ #include "nix/expr/eval-inline.hh" #include "nix/expr/eval-settings.hh" #include "nix/expr/tecnix/source-accessors.hh" +#include "nix/expr/tecnix/source-deps.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/git-utils.hh" #include "nix/store/store-api.hh" @@ -26,6 +27,7 @@ #include #include +#include #include #include @@ -158,8 +160,9 @@ static std::string gitFingerprintWithMode(const Hash & oid, uint32_t mode) } /** - * The dirty (modified, added, deleted, renamed, untracked) repo-relative - * paths of `checkoutPath`, per `git status --porcelain -z`. + * The dirty (modified, added, deleted, renamed, unmerged, untracked) + * repo-relative paths of `checkoutPath`, per `git status --porcelain=v2 -z`, + * with the staged blob of each path whose content the index holds. * * Scrubs the git discovery environment (GIT_DIR, GIT_WORK_TREE, * GIT_COMMON_DIR, GIT_INDEX_FILE) so an ambient git context — a hook, a @@ -168,7 +171,7 @@ static std::string gitFingerprintWithMode(const Hash & oid, uint32_t mode) * is load-bearing for Tecnix source-closure validity, so callers that can * tolerate an unknown dirty state must catch explicitly. */ -static std::vector gitStatusDirtyPaths(const std::string & checkoutPath) +static std::vector gitStatusDirtyEntries(const std::string & checkoutPath) { StringMap gitEnvironment = getEnv(); gitEnvironment.erase("GIT_DIR"); @@ -178,14 +181,129 @@ static std::vector gitStatusDirtyPaths(const std::string & checkout auto [status, output] = runProgram( {.program = "git", - .args = {"-C", checkoutPath, "--no-optional-locks", "status", "--porcelain", "-z", "--untracked-files=all"}, + .args = {"-C", checkoutPath, "--no-optional-locks", "status", "--porcelain=v2", "-z", "--untracked-files=all"}, .environment = gitEnvironment}); if (!statusOk(status)) throw Error("failed to get git status in '%s': program 'git' %s", checkoutPath, statusToString(status)); - return parseGitPorcelainZDirtyPaths(output); + return parseGitPorcelainV2ZStatus(output); } +/** + * Staged content for the dirty paths whose working tree has no change of its + * own against the index (`git status` worktree column `.`), read from the + * index blobs. Blobs are read raw, like the committed tree: no LFS smudging, + * end-of-line conversion or `ident` expansion. So for a materialized file + * without such filters this is the same content as the disk, and a filtered + * one evaluates as git stores it (an LFS file as its pointer), as it would + * once committed. For a skip-worktree entry outside a sparse checkout, which + * git stages but never writes (the cleanly merged files of an uncommitted + * merge), the blob is the only copy. Serves those paths' files and symlinks + * and nothing else. + */ +struct GitIndexSourceAccessor : SourceAccessor +{ + void anchor() override; + + struct Entry + { + Hash oid; + uint32_t mode; + }; + + using Entries = boost::unordered_flat_map>; + + using DirectorySet = boost::unordered_flat_set>; + + ref repo; + Entries entries; + /** Every proper ancestor directory of an entry. */ + DirectorySet directories; + + GitIndexSourceAccessor(ref repo, Entries && entries) + : repo(std::move(repo)) + , entries(std::move(entries)) + { + for (auto & [path, _] : this->entries) + for (auto p = CanonPath(path); !p.isRoot();) { + p.pop(); + if (p.isRoot() || !directories.emplace(p.rel()).second) + break; + } + } + + bool containsDirectory(const CanonPath & path) const + { + return !path.isRoot() && directories.contains(path.rel()); + } + + const Entry * find(const CanonPath & path) const + { + if (path.isRoot()) + return nullptr; + auto i = entries.find(path.rel()); + return i == entries.end() ? nullptr : &i->second; + } + + bool contains(const CanonPath & path) const + { + return find(path); + } + + const Entry & need(const CanonPath & path) + { + if (auto entry = find(path)) + return *entry; + throw Error("'%s' has no staged content in the git index", showPath(path)); + } + + std::string readBlob(const Entry & entry) + { + GitAccessorOptions opts{.exportIgnore = false, .smudgeLfs = false}; + return repo->getAccessor(entry.oid, opts, "index")->readFile(CanonPath::root); + } + + std::optional maybeLstat(const CanonPath & path) override + { + auto entry = find(path); + if (!entry) + return std::nullopt; + if (entry->mode == 0120000) + return Stat{.type = tSymlink}; + return Stat{.type = tRegular, .isExecutable = entry->mode == 0100755}; + } + + void readFile(const CanonPath & path, Sink & sink, fun sizeCallback) override + { + auto & entry = need(path); + if (entry.mode == 0120000) + throw Error("'%s' is not a regular file", showPath(path)); + auto content = readBlob(entry); + sizeCallback(content.size()); + sink(content); + } + + std::string readLink(const CanonPath & path) override + { + auto & entry = need(path); + if (entry.mode != 0120000) + throw Error("'%s' is not a symlink", showPath(path)); + return readBlob(entry); + } + + DirEntries readDirectory(const CanonPath & path) override + { + throw Error("'%s' is not a directory", showPath(path)); + } + + std::optional getPhysicalPath(const CanonPath &) override + { + return std::nullopt; + } +}; + +void GitIndexSourceAccessor::anchor() {} + /** * Read delegation wrapper that preserves the path-fingerprint semantics Tecnix * historically used with libgit2: fingerprints are the git object id at the @@ -227,17 +345,22 @@ struct TecnixSourceAccessor : SourceAccessor std::string repoPrefix; DirtyPathSet dirtyFiles, dirtyDirs; + /** The dirty files whose content comes from the git index, not `disk`. */ + std::shared_ptr index; + TecnixSourceAccessor( ref clean, ref disk, std::optional git, std::string repoPrefix, - DirtyPathSet && dirtyFiles) + DirtyPathSet && dirtyFiles, + std::shared_ptr index = nullptr) : clean(std::move(clean)) , disk(std::move(disk)) , git(std::move(git)) , repoPrefix(std::move(repoPrefix)) , dirtyFiles(std::move(dirtyFiles)) + , index(std::move(index)) { for (auto & f : this->dirtyFiles) { debug("TecnixSourceAccessor: dirty file: '%s'", f); @@ -274,6 +397,33 @@ struct TecnixSourceAccessor : SourceAccessor return dirtyFiles.contains(path.rel()); } + /** Where a dirty file's content lives: the git index or the checkout. */ + ref dirtySource(const CanonPath & path) + { + if (index && index->contains(path)) + return ref(index); + return disk; + } + + /** + * The type of a dirty path. A directory that only the index has (the + * merge added it outside the sparse set, possibly replacing a file of the + * same name in HEAD) is in neither HEAD nor the working tree. + */ + std::optional dirtyLstat(const CanonPath & path) + { + auto s = dirtySource(path)->maybeLstat(path); + if (!s && index && index->containsDirectory(path)) + s = Stat{.type = tDirectory}; + return s; + } + + /** Where any path's content lives. */ + ref source(const CanonPath & path) + { + return isDirty(path) ? dirtySource(path) : clean; + } + bool tracksEvalAccesses(const CanonPath &) override { return true; @@ -311,11 +461,11 @@ struct TecnixSourceAccessor : SourceAccessor if (path.isRoot()) s = clean->maybeLstat(path); else if (isDirty(path)) - s = disk->maybeLstat(path); + s = dirtyLstat(path); else { s = clean->maybeLstat(path); if (!s && dirtyDirs.contains(path.rel())) - s = disk->maybeLstat(path); + s = dirtyLstat(path); } return s; } @@ -324,26 +474,26 @@ struct TecnixSourceAccessor : SourceAccessor { if (dumpPathDepth == 0) trackAccess(path); - return (isDirty(path) ? disk : clean)->readFile(path, sink, sizeCallback); + return source(path)->readFile(path, sink, sizeCallback); } /** `readFile` without recording the access, for a caller that records its own, finer-grained observation of the file instead. */ std::string readFileUnrecorded(const CanonPath & path) { - return (isDirty(path) ? disk : clean)->readFile(path); + return source(path)->readFile(path); } std::string readLink(const CanonPath & path) override { if (dumpPathDepth == 0) trackAccess(path); - return (isDirty(path) ? disk : clean)->readLink(path); + return source(path)->readLink(path); } std::optional getPhysicalPath(const CanonPath & path) override { - return (isDirty(path) ? disk : clean)->getPhysicalPath(path); + return source(path)->getPhysicalPath(path); } /** The fingerprint of `path` in the clean tree, before the dirty overlay. */ @@ -387,7 +537,7 @@ struct TecnixSourceAccessor : SourceAccessor dirtyUnderPath.push_back(f); } - if (!path.isRoot() && dirtyFiles.contains(rel) && !disk->maybeLstat(path)) + if (!path.isRoot() && dirtyFiles.contains(rel) && !dirtyLstat(path)) return {path, "absent"}; std::string fp = *cleanFp; @@ -396,15 +546,22 @@ struct TecnixSourceAccessor : SourceAccessor HashSink hashSink{HashAlgorithm::SHA256}; for (auto & f : dirtyUnderPath) { hashSink << f; - auto st = disk->maybeLstat(CanonPath(f)); + auto dirtyPath = CanonPath(f); + // Staged content is identified by its blob id, so fingerprinting + // a directory never reads the blobs of a large merge. + if (auto staged = index ? index->find(dirtyPath) : nullptr) { + hashSink << "G" << gitFingerprintWithMode(staged->oid, staged->mode); + continue; + } + auto st = dirtyLstat(dirtyPath); if (!st) { hashSink << "D"; } else if (st->type == Type::tRegular) { hashSink << (st->isExecutable ? "X" : "F"); - hashSink << disk->readFile(CanonPath(f)); + hashSink << disk->readFile(dirtyPath); } else if (st->type == Type::tSymlink) { hashSink << "L"; - hashSink << disk->readLink(CanonPath(f)); + hashSink << disk->readLink(dirtyPath); } } fp += ";dirty=" + hashSink.finish().hash.to_string(HashFormat::Base16, false); @@ -419,6 +576,15 @@ struct TecnixSourceAccessor : SourceAccessor trackAccess(path); auto rel = path.isRoot() ? "" : std::string(path.rel()); + + // A dirty path that is no longer a directory (a merge replaced the + // directory with a file) must not list the old directory's children. + if (!path.isRoot() && isDirty(path)) { + auto st = dirtyLstat(path); + if (!st || st->type != tDirectory) + return dirtySource(path)->readDirectory(path); + } + if (!path.isRoot() && !dirtyDirs.contains(rel)) return clean->readDirectory(path); @@ -435,7 +601,8 @@ struct TecnixSourceAccessor : SourceAccessor auto rest = std::string_view(f).substr(dirPrefix.size()); if (rest.find('/') != std::string_view::npos) continue; - auto stat = disk->maybeLstat(path / rest); + auto child = path / rest; + auto stat = dirtyLstat(child); if (stat) entries[std::string(rest)] = stat->type; else @@ -759,15 +926,35 @@ ref getTecnixRepoAccessor(EvalState & state) // Get all dirty files in the repo. This is load-bearing for source // closure validity: if we cannot determine the dirty overlay, do - // not continue with a clean-tree accessor (gitStatusDirtyPaths + // not continue with a clean-tree accessor (gitStatusDirtyEntries // throws). Until the daemon exposes a full repo dirty-path RPC, // materialized checkouts keep using git status at this boundary. TecnixSourceAccessor::DirtyPathSet dirtyFiles; - for (auto & path : gitStatusDirtyPaths(checkoutPath)) - dirtyFiles.insert(std::move(path)); + GitIndexSourceAccessor::Entries indexEntries; + for (auto & entry : gitStatusDirtyEntries(checkoutPath)) { + // The worldtree backend has no libgit2 repository to read + // blobs from, so its dirty files stay on disk. + if (entry.index && gitFingerprints) + indexEntries.insert_or_assign( + entry.path, + GitIndexSourceAccessor::Entry{ + .oid = Hash::parseNonSRIUnprefixed(entry.index->oid, HashAlgorithm::SHA1), + .mode = entry.index->mode, + }); + dirtyFiles.insert(std::move(entry.path)); + } + + std::shared_ptr index; + if (!indexEntries.empty()) + index = std::make_shared(gitFingerprints->repo, std::move(indexEntries)); tecnixData(state)->tecnixRepoAccessor = make_ref( - cleanAccessor, makeFSSourceAccessor(checkoutPath), gitFingerprints, "", std::move(dirtyFiles)); + cleanAccessor, + makeFSSourceAccessor(checkoutPath), + gitFingerprints, + "", + std::move(dirtyFiles), + std::move(index)); } else { tecnixData(state)->tecnixRepoAccessor = make_ref( cleanAccessor, cleanAccessor, gitFingerprints, "", TecnixSourceAccessor::DirtyPathSet{}); diff --git a/src/libexpr/tecnix/source-deps.cc b/src/libexpr/tecnix/source-deps.cc index 3ba129d52..ceabb7239 100644 --- a/src/libexpr/tecnix/source-deps.cc +++ b/src/libexpr/tecnix/source-deps.cc @@ -5,6 +5,8 @@ #include "nix/util/util.hh" #include +#include +#include #include #include @@ -61,38 +63,103 @@ void tecnixValueLabelOutOfRange(const void * value) abort(); } -std::vector parseGitPorcelainZDirtyPaths(std::string_view output) +std::vector parseGitPorcelainV2ZStatus(std::string_view output) { - std::vector paths; + std::vector entries; size_t pos = 0; - while (pos < output.size()) { + + auto malformed = [](std::string_view record) { + return Error("unrecognized 'git status --porcelain=v2 -z' record '%s'", record); + }; + + auto nextRecord = [&]() -> std::optional { + if (pos == output.size()) + return std::nullopt; auto nulPos = output.find('\0', pos); if (nulPos == std::string_view::npos) - break; - - auto entry = output.substr(pos, nulPos - pos); + throw malformed(output.substr(pos)); + auto record = output.substr(pos, nulPos - pos); pos = nulPos + 1; + return record; + }; - // Git porcelain v1 -z format is "XY PATH\0", with an extra - // original-path record only when the X column is R/C. Keep both names - // dirty so source reads of either side see the checkout overlay. - if (entry.size() < 4 || entry[2] != ' ') - continue; - - paths.emplace_back(entry.substr(3)); + // Splits off `count` space-separated header fields and returns the rest of + // the record: the path, which may itself contain spaces. + std::vector fields; + auto splitFields = [&](std::string_view record, size_t count) -> std::optional { + fields.clear(); + for (size_t i = 0; i < count; ++i) { + auto space = record.find(' '); + if (space == std::string_view::npos) + return std::nullopt; + fields.push_back(record.substr(0, space)); + record.remove_prefix(space + 1); + } + if (record.empty()) + return std::nullopt; + return record; + }; - if (entry[0] == 'R' || entry[0] == 'C') { - auto nextNul = output.find('\0', pos); - if (nextNul == std::string_view::npos) - break; + auto parseMode = [](std::string_view field) -> std::optional { + uint32_t mode = 0; + auto [end, ec] = std::from_chars(field.data(), field.data() + field.size(), mode, 8); + if (ec != std::errc{} || end != field.data() + field.size()) + return std::nullopt; + return mode; + }; - auto originalPath = output.substr(pos, nextNul - pos); - pos = nextNul + 1; - if (!originalPath.empty()) - paths.emplace_back(originalPath); + while (auto record = nextRecord()) { + // Headers: `# branch.*` with --branch, `# stash ` with status.showStash. + if (record->starts_with("# ")) + continue; + if (record->size() < 3 || (*record)[1] != ' ') + throw malformed(*record); + + switch ((*record)[0]) { + case '1': + case '2': { + // 1 XY sub mH mI mW hH hI path + // 2 XY sub mH mI mW hH hI Xscore path NUL origPath + bool renamed = (*record)[0] == '2'; + auto path = splitFields(*record, renamed ? 9 : 8); + if (!path || fields[1].size() != 2) + throw malformed(*record); + std::optional originalPath; + if (renamed && !(originalPath = nextRecord())) + throw malformed(*record); + + GitStatusEntry entry{.path = std::string(*path)}; + auto xy = fields[1]; + auto indexMode = parseMode(fields[4]); + if (xy[1] == '.' && indexMode && (*indexMode == 0100644 || *indexMode == 0100755 || *indexMode == 0120000)) + entry.index = GitIndexBlob{.oid = std::string(fields[7]), .mode = *indexMode}; + entries.push_back(std::move(entry)); + + // A rename removes its original path; a copy (status.renames=copies) + // leaves its source unchanged, so the source is not dirty. The + // score field says which it is, for index- and worktree-side + // records alike (`C.` and `.C`). + if (originalPath && !originalPath->empty() && !fields[8].starts_with('C')) + entries.push_back({.path = std::string(*originalPath)}); + break; + } + case 'u': { + // u XY sub m1 m2 m3 mW h1 h2 h3 path + auto path = splitFields(*record, 10); + if (!path) + throw malformed(*record); + entries.push_back({.path = std::string(*path)}); + break; + } + case '?': + case '!': + entries.push_back({.path = std::string(record->substr(2))}); + break; + default: + throw malformed(*record); } } - return paths; + return entries; } static uint64_t hashSourceAccessIds(std::span items) diff --git a/tests/functional/tecnix/builtins.sh b/tests/functional/tecnix/builtins.sh index e33f24aa0..7063d5acb 100755 --- a/tests/functional/tecnix/builtins.sh +++ b/tests/functional/tecnix/builtins.sh @@ -1524,4 +1524,158 @@ assert_jq "$export_ignore_deps" '.reader."secret.txt" | startswith("git:")' \ assert_jq "$export_ignore_deps" '.reader | has(".gitattributes") | not' \ "git attributes are inert in the raw-tree view and should not enter the closure" +# An uncommitted merge in a sparse checkout stages the cleanly merged files +# outside the sparse set but never writes them (skip-worktree). The index holds +# their content, so evaluation must read it from there, not from the disk. +echo "Testing staged files outside a sparse checkout..." +# create_sparse_merge_world DIR [SPARSE-CHECKOUT-SET-FLAGS...] +create_sparse_merge_world() { + local dir="$1" + shift + createGitRepo "$dir" + ( + cd "$dir" + mkdir lib zone + echo '"base"' > lib/marker.nix + echo "file" > lib/swap + mkdir lib/olddir + echo '"old"' > lib/olddir/default.nix + echo "zone file" > zone/README + cat > resolve.nix << 'RESOLVE_EOF' +args: { + allTargetNames = [ "reader" "lister" ]; + resolve = id: if id == "lister" then { + olddir = builtins.readDir ./lib/olddir; + drvPath = "/nix/store/00000000000000000000000000000000-${id}.drv"; + } else rec { + marker = import ./lib/marker.nix; + newdir = if builtins.pathExists ./lib/newdir then import ./lib/newdir else "missing"; + linkType = if builtins.pathExists ./lib/link.nix then builtins.readFileType ./lib/link.nix else "missing"; + viaLink = if linkType == "missing" then "missing" else import ./lib/link.nix; + swap = if builtins.readFileType ./lib/swap == "directory" then import ./lib/swap else "file"; + olddir = if builtins.readFileType ./lib/olddir == "regular" then import ./lib/olddir else "directory"; + lib = builtins.readDir ./lib; + # Copying the directory into the store reads it through the same overlay. + copied = let libCopy = builtins.path { path = ./lib; name = "sparse-lib"; }; in { + marker = import (libCopy + "/marker.nix"); + newdir = import (libCopy + "/newdir"); + swap = import (libCopy + "/swap"); + olddir = import (libCopy + "/olddir"); + link = builtins.readFileType (libCopy + "/link.nix"); + listing = builtins.readDir libCopy; + }; + # Forced through drvPath so the listing is a recorded dependency. + drvPath = "/nix/store/00000000000000000000000000000000-${marker}-${builtins.hashString "sha256" (builtins.toJSON lib)}-${id}.drv"; + }; +} +RESOLVE_EOF + git add -A + git commit -m "sparse merge world" + git checkout -b upstream + echo '"merged"' > lib/marker.nix + mkdir lib/newdir + echo '"newdir"' > lib/newdir/default.nix + ln -s marker.nix lib/link.nix + git rm -q lib/swap + mkdir lib/swap + echo '"swapped"' > lib/swap/default.nix + git rm -q -r lib/olddir + echo '"flattened"' > lib/olddir + git add -A + git commit -m "change and add files outside the sparse set" + git checkout - + echo "zone change" > zone/README + git commit -am "change inside the sparse set" + git sparse-checkout set "$@" zone + git merge --no-commit --no-ff upstream + ) +} + +SPARSE_WORLD="$TEST_ROOT/tecnix-sparse-merge-world" +create_sparse_merge_world "$SPARSE_WORLD" +[[ ! -e "$SPARSE_WORLD/lib/marker.nix" ]] || fail "precondition: lib/marker.nix should not be on disk" +git -C "$SPARSE_WORLD" ls-files -v lib/marker.nix | grepQuiet '^S ' \ + || fail "precondition: lib/marker.nix should be skip-worktree" +SPARSE_HEAD=$(get_head_sha "$SPARSE_WORLD") + +sparse_args="{ gitDir = \"$SPARSE_WORLD/.git\"; resolver = \"resolve.nix\"; args = { }; rev = \"$SPARSE_HEAD\"; checkoutPath = \"$SPARSE_WORLD\"; }" + +sparse_targets=$(tecnix_eval_json_no_cache "builtins.tecnixTargets (($sparse_args) // { targets = [ \"reader\" ]; })") +assert_jq "$sparse_targets" '.reader.marker == "merged"' \ + "a staged file outside the sparse checkout should evaluate with its staged content" +assert_jq "$sparse_targets" '.reader.newdir == "newdir"' \ + "a directory the merge added outside the sparse checkout should exist and import" +assert_jq "$sparse_targets" '.reader.linkType == "symlink" and .reader.viaLink == "merged"' \ + "a symlink the merge added outside the sparse checkout should resolve through the index" +assert_jq "$sparse_targets" '.reader.swap == "swapped"' \ + "a file the merge replaced with a directory outside the sparse checkout should be that directory" +assert_jq "$sparse_targets" '.reader.olddir == "flattened"' \ + "a directory the merge replaced with a file outside the sparse checkout should be that file" +assert_jq "$sparse_targets" '.reader.lib == { "link.nix": "symlink", "marker.nix": "regular", "newdir": "directory", "olddir": "regular", "swap": "directory" }' \ + "listing a directory outside the sparse checkout should combine HEAD and the index" +assert_jq "$sparse_targets" '.reader.copied == { marker: "merged", newdir: "newdir", swap: "swapped", olddir: "flattened", link: "symlink", listing: .reader.lib }' \ + "copying a directory outside the sparse checkout into the store should take its staged content" + +sparse_deps=$(tecnix_eval_json_no_cache "tecnixTargetDependencyPathSet (($sparse_args) // { targets = [ \"reader\" ]; })") +assert_jq "$sparse_deps" '.reader."lib/marker.nix" | contains("dirty=")' \ + "a staged file outside the sparse checkout should carry a dirty fingerprint" +assert_jq "$sparse_deps" '.reader.lib | contains("dirty=")' \ + "a directory holding staged files outside the sparse checkout should carry a dirty fingerprint" + +# Listing a directory the merge replaced with a file must fail, not list nothing. +expect_sparse_listing_fails() { + local args="$1" + local err="$TEST_ROOT/tecnix-sparse-lister.err" + local expr + expr=$(rewrite_tecnix_test_expr "builtins.tecnixTargets (($args) // { targets = [ \"lister\" ]; })") + expect 1 nix eval --json \ + --extra-experimental-features 'nix-command' \ + --option lazy-trees true \ + --option tecnix-eval-cache false \ + --expr "$expr" \ + >/dev/null 2>"$err" + grepQuiet "not a directory" < "$err" || fail "readDir of a file should fail with 'not a directory': $(cat "$err")" +} +expect_sparse_listing_fails "$sparse_args" + +# Materializing everything must not change what the staged files evaluate to +# or how they fingerprint, and a full checkout must fail the same readDir. +git -C "$SPARSE_WORLD" sparse-checkout disable +[[ -e "$SPARSE_WORLD/lib/marker.nix" && -e "$SPARSE_WORLD/lib/swap/default.nix" ]] \ + || fail "precondition: disabling the sparse checkout should materialize the staged files" +full_targets=$(tecnix_eval_json_no_cache "builtins.tecnixTargets (($sparse_args) // { targets = [ \"reader\" ]; })") +assert_json_equal "$full_targets" "$sparse_targets" \ + "staged files should evaluate the same whether or not they are materialized" +full_deps=$(tecnix_eval_json_no_cache "tecnixTargetDependencyPathSet (($sparse_args) // { targets = [ \"reader\" ]; })") +assert_json_equal "$full_deps" "$sparse_deps" \ + "staged files should fingerprint the same whether or not they are materialized" +expect_sparse_listing_fails "$sparse_args" + +# Once the merge is committed the file is clean again, and its content is the same. +git -C "$SPARSE_WORLD" commit -m "merge upstream" +SPARSE_MERGED_HEAD=$(get_head_sha "$SPARSE_WORLD") +merged_args="{ gitDir = \"$SPARSE_WORLD/.git\"; resolver = \"resolve.nix\"; args = { }; rev = \"$SPARSE_MERGED_HEAD\"; checkoutPath = \"$SPARSE_WORLD\"; }" +merged_targets=$(tecnix_eval_json_no_cache "builtins.tecnixTargets (($merged_args) // { targets = [ \"reader\" ]; })") +assert_json_equal "$merged_targets" "$sparse_targets" \ + "committing the merge should not change what the staged files evaluated to" +expect_sparse_listing_fails "$merged_args" + +# World checkouts use a sparse index. git status reports the same entries with +# one, and Tecnix never reads the index through libgit2 (which rejects its +# `sdir` extension), so evaluation must not change. +echo "Testing staged files outside a sparse checkout with a sparse index..." +SPARSE_INDEX_WORLD="$TEST_ROOT/tecnix-sparse-index-merge-world" +create_sparse_merge_world "$SPARSE_INDEX_WORLD" --sparse-index +[[ "$(git -C "$SPARSE_INDEX_WORLD" config index.sparse)" == true ]] \ + || fail "precondition: the checkout should use a sparse index" +[[ ! -e "$SPARSE_INDEX_WORLD/lib/marker.nix" ]] || fail "precondition: lib/marker.nix should not be on disk" +sparse_index_args="{ gitDir = \"$SPARSE_INDEX_WORLD/.git\"; resolver = \"resolve.nix\"; args = { }; rev = \"$(get_head_sha "$SPARSE_INDEX_WORLD")\"; checkoutPath = \"$SPARSE_INDEX_WORLD\"; }" +sparse_index_targets=$(tecnix_eval_json_no_cache "builtins.tecnixTargets (($sparse_index_args) // { targets = [ \"reader\" ]; })") +assert_json_equal "$sparse_index_targets" "$sparse_targets" \ + "a sparse index should not change what staged files evaluate to" +sparse_index_deps=$(tecnix_eval_json_no_cache "tecnixTargetDependencyPathSet (($sparse_index_args) // { targets = [ \"reader\" ]; })") +assert_json_equal "$sparse_index_deps" "$sparse_deps" \ + "a sparse index should not change how staged files fingerprint" +expect_sparse_listing_fails "$sparse_index_args" + echo "Tecnix builtin tests passed!"