From 0ee7dcf9a4d62dc418d5f386aa604b9bf1cddb5a Mon Sep 17 00:00:00 2001 From: Ionite Date: Wed, 30 Sep 2026 17:19:05 -0700 Subject: [PATCH 1/2] refactor(LinkSafeFileSystem): walk the real tree first with one seen set EnumerateFiles needs two rules: remember which physical directories were scanned (links make the tree a graph, and .NET's own recursion follows them forever on a cycle), and follow links only after the real tree is done, so an alias like Swarm's diffusion_models never shadows the real DiffusionModels folder. Replace the three visited maps and two stacks with a recursive walk, one HashSet keyed by real path under the platform comparer, and a queue of links walked afterwards. A single set makes the cross-map check that the previous revision missed impossible to forget. Add an a_alias row to the shadowing test: every existing alias sorted after DiffusionModels, so following links inline still passed all tests. Co-Authored-By: Lykos (Opus 5.5) --- .../Helper/LinkSafeFileSystem.cs | 112 ++++++------------ .../Helper/LinkSafeFileSystemTests.cs | 1 + 2 files changed, 35 insertions(+), 78 deletions(-) diff --git a/StabilityMatrix.Core/Helper/LinkSafeFileSystem.cs b/StabilityMatrix.Core/Helper/LinkSafeFileSystem.cs index 3528ca62a..4bcf3df54 100644 --- a/StabilityMatrix.Core/Helper/LinkSafeFileSystem.cs +++ b/StabilityMatrix.Core/Helper/LinkSafeFileSystem.cs @@ -99,7 +99,8 @@ public static bool WouldLinkCycle(DirectoryPath sourceDir, DirectoryPath linkPat /// /// Recursively enumerates files matching under /// . Symbolic links are followed, but every physical directory is - /// visited at most once, so no file is yielded twice. Directories nested deeper than + /// visited at most once, so no file is yielded twice; a directory reachable both directly and + /// through a link is yielded under its direct path. Directories nested deeper than /// and directories that cannot be read are skipped without aborting /// the enumeration. Yielded paths are rooted at as given, not at its /// resolved target. @@ -110,65 +111,27 @@ public static IEnumerable EnumerateFiles( int maxDepth = DefaultMaxDepth ) { - // A real directory is keyed by its literal path, compared ordinally, so two folders whose - // names differ only in case are both scanned; it is also matched against link targets, so a - // real folder reached through a link is not rescanned. A link is keyed by its resolved - // target, compared with the platform's case sensitivity (PathComparer), because a target is - // stored however the link was created. - var visitedRealDirsExact = new Dictionary(StringComparer.Ordinal); - var visitedRealDirsForLinkTargets = new Dictionary(PathComparer); - var visitedLinkTargets = new Dictionary(PathComparer); - - // Real directories are drained to completion before any link is considered, so a link can - // never take the identity of a real folder and shadow it out of the scan. - var realDirs = new Stack<(string Path, string RealPath, int Depth)>(); - var linkedDirs = new Stack<(string Path, string RealPath, int Depth)>(); - - var rootReal = GetRealPath(rootDir); - realDirs.Push((rootDir, rootReal, 0)); - - while (realDirs.Count > 0 || linkedDirs.Count > 0) - { - // Which stack the entry came from is how the walk knows whether it is a link. - var fromRealDirs = realDirs.Count > 0; - var dir = fromRealDirs ? realDirs.Pop() : linkedDirs.Pop(); + // Physical directories already scanned. Links are what turn the tree into a graph. + var seen = new HashSet(PathComparer); + // Links wait until the real tree is done, so a folder is listed under its own name + // rather than through an alias that points at it + var links = new Queue<(string Path, int Depth)>(); - // Claimed on pop, not on push, so the walk order decides which spelling owns the - // identity instead of the reversed push order. - if (!fromRealDirs) - { - if ( - visitedLinkTargets.TryGetValue(dir.RealPath, out var linkClaimer) - || visitedRealDirsForLinkTargets.TryGetValue(dir.RealPath, out linkClaimer) - ) - { - Logger.Info( - "Skipping {Path}: the same directory was already scanned as {ClaimedBy}", - dir.Path, - linkClaimer - ); - continue; - } + foreach (var file in Walk(rootDir, GetRealPath(rootDir), 0)) + yield return file; - visitedLinkTargets[dir.RealPath] = dir.Path; - } - else - { - if ( - visitedRealDirsExact.TryGetValue(dir.RealPath, out var realClaimer) - || visitedLinkTargets.TryGetValue(dir.RealPath, out realClaimer) - ) - { - Logger.Warn( - "Skipping {Path}: the same directory was already scanned as {ClaimedBy}", - dir.Path, - realClaimer - ); - continue; - } + while (links.TryDequeue(out var link)) + { + foreach (var file in Walk(link.Path, GetRealPath(link.Path), link.Depth)) + yield return file; + } - visitedRealDirsExact[dir.RealPath] = dir.Path; - visitedRealDirsForLinkTargets[dir.RealPath] = dir.Path; + IEnumerable Walk(string dir, string realDir, int depth) + { + if (!seen.Add(realDir)) + { + Logger.Debug("Skipping {Path}: already scanned", dir); + yield break; } List files; @@ -176,48 +139,41 @@ public static IEnumerable EnumerateFiles( try { files = Directory - .EnumerateFiles(dir.Path, searchPattern, EnumerationOptionConstants.TopLevelOnly) + .EnumerateFiles(dir, searchPattern, EnumerationOptionConstants.TopLevelOnly) .ToList(); - subDirs = new DirectoryInfo(dir.Path) + subDirs = new DirectoryInfo(dir) .EnumerateDirectories("*", EnumerationOptionConstants.TopLevelOnly) .ToList(); } catch (Exception e) when (e is IOException or UnauthorizedAccessException) { - Logger.Debug(e, "Skipping unreadable directory {Path}", dir.Path); - continue; + Logger.Debug(e, "Skipping unreadable directory {Path}", dir); + yield break; } foreach (var file in files) - { yield return file; - } - if (dir.Depth >= maxDepth) + if (depth >= maxDepth) { Logger.Warn( "Skipping directories below {Path}: nesting deeper than {MaxDepth}", - dir.Path, + dir, maxDepth ); - continue; + yield break; } - // Pushed in reverse so each stack pops its entries in enumeration order - for (var i = subDirs.Count - 1; i >= 0; i--) + foreach (var subDir in subDirs) { - var subDir = subDirs[i]; - var isLinkDir = subDir.Attributes.HasFlag(FileAttributes.ReparsePoint); - var subReal = isLinkDir ? GetRealPath(subDir.FullName) : Path.Join(dir.RealPath, subDir.Name); - - if (isLinkDir) - { - linkedDirs.Push((subDir.FullName, subReal, dir.Depth + 1)); - } - else + if (subDir.Attributes.HasFlag(FileAttributes.ReparsePoint)) { - realDirs.Push((subDir.FullName, subReal, dir.Depth + 1)); + links.Enqueue((subDir.FullName, depth + 1)); + continue; } + + foreach (var file in Walk(subDir.FullName, Path.Join(realDir, subDir.Name), depth + 1)) + yield return file; } } } diff --git a/StabilityMatrix.Tests/Helper/LinkSafeFileSystemTests.cs b/StabilityMatrix.Tests/Helper/LinkSafeFileSystemTests.cs index 2f86c72ca..fe9c6dce6 100644 --- a/StabilityMatrix.Tests/Helper/LinkSafeFileSystemTests.cs +++ b/StabilityMatrix.Tests/Helper/LinkSafeFileSystemTests.cs @@ -167,6 +167,7 @@ string outerName [DataTestMethod] [DataRow("diffusion_models")] + [DataRow("a_alias")] [DataRow("sub", "alias")] public void EnumerateFiles_RealFolderShadowedByLink_KeepsRealFolderPaths(params string[] linkSegments) { From 9c4c6c64956ca637eca46d359e72bbe717058bc7 Mon Sep 17 00:00:00 2001 From: Ionite Date: Wed, 30 Sep 2026 17:57:38 -0700 Subject: [PATCH 2/2] fix(LinkSafeFileSystem): walk paths in order of how many links they pass Real-folders-first is one case short: when a folder is itself a link (a user moved DiffusionModels to another drive and linked it back), an alias of that link ties with it, and sibling order picks the winner. Swarm's diffusion_models happens to sort after on NTFS; a_alias does not, and on Linux sibling order is unordered. Generalize the rule: links are walked in order of how many links their path goes through, and the first walk to reach a physical directory owns it. Real directories are the zero-link case. GetRealPath now follows one link at a time (returnFinalTarget: false) and counts hops along the way. Following one link at a time also keeps link targets spelled the way they were stored instead of resolving through subst drives, which yielded every file twice when the root was given on a subst letter. Tests: alias of a link (both sort orders), dangling link, root spelled in a different case, a link to an ancestor of the root, a link inside a link target, and two links looping into each other. Plain FIFO link order fails the alias-of-link row. Co-Authored-By: Lykos (Opus 5.5) --- .../Helper/LinkSafeFileSystem.cs | 52 +++++--- .../Helper/LinkSafeFileSystemTests.cs | 121 +++++++++++++++++- 2 files changed, 148 insertions(+), 25 deletions(-) diff --git a/StabilityMatrix.Core/Helper/LinkSafeFileSystem.cs b/StabilityMatrix.Core/Helper/LinkSafeFileSystem.cs index 4bcf3df54..8c251564d 100644 --- a/StabilityMatrix.Core/Helper/LinkSafeFileSystem.cs +++ b/StabilityMatrix.Core/Helper/LinkSafeFileSystem.cs @@ -30,9 +30,19 @@ public static class LinkSafeFileSystem /// ancestors, returning the physical directory path. Segments that cannot be resolved /// (missing, or a reparse point that is not a link) are kept as written. /// - public static string GetRealPath(string path) => GetRealPath(path, 0); + public static string GetRealPath(string path) => GetRealPath(path, out _); - private static string GetRealPath(string path, int hop) + /// + /// receives how many links + /// the path goes through. + /// + private static string GetRealPath(string path, out int hops) + { + hops = 0; + return Resolve(path, ref hops); + } + + private static string Resolve(string path, ref int hops) { var fullPath = Path.TrimEndingDirectorySeparator(Path.GetFullPath(path)); var root = Path.GetPathRoot(fullPath) ?? string.Empty; @@ -52,7 +62,7 @@ var segment in fullPath[root.Length..] if (!info.Exists || !info.Attributes.HasFlag(FileAttributes.ReparsePoint)) continue; - if (hop >= MaxLinkHops) + if (hops >= MaxLinkHops) { Logger.Warn("Gave up resolving links at {Path}: too many nested links", current); continue; @@ -61,7 +71,9 @@ var segment in fullPath[root.Length..] FileSystemInfo? target; try { - target = info.ResolveLinkTarget(returnFinalTarget: true); + // The immediate target, as stored, so it is spelled the way the link's own + // neighbourhood is (subst and mapped drives would otherwise resolve to the volume) + target = info.ResolveLinkTarget(returnFinalTarget: false); } catch (IOException e) { @@ -72,8 +84,10 @@ var segment in fullPath[root.Length..] if (target is null) continue; - // The target may itself sit below other links, so canonicalize it as a whole - current = GetRealPath(target.FullName, hop + 1); + hops++; + + // The target may itself be a link, or sit below other links, so canonicalize it as a whole + current = Resolve(target.FullName, ref hops); } return current; @@ -99,11 +113,11 @@ public static bool WouldLinkCycle(DirectoryPath sourceDir, DirectoryPath linkPat /// /// Recursively enumerates files matching under /// . Symbolic links are followed, but every physical directory is - /// visited at most once, so no file is yielded twice; a directory reachable both directly and - /// through a link is yielded under its direct path. Directories nested deeper than - /// and directories that cannot be read are skipped without aborting - /// the enumeration. Yielded paths are rooted at as given, not at its - /// resolved target. + /// visited at most once, so no file is yielded twice; a directory reachable along several + /// paths is yielded under the one that goes through the fewest links. Directories nested + /// deeper than and directories that cannot be read are skipped + /// without aborting the enumeration. Yielded paths are rooted at + /// as given, not at its resolved target. /// public static IEnumerable EnumerateFiles( string rootDir, @@ -113,16 +127,17 @@ public static IEnumerable EnumerateFiles( { // Physical directories already scanned. Links are what turn the tree into a graph. var seen = new HashSet(PathComparer); - // Links wait until the real tree is done, so a folder is listed under its own name - // rather than through an alias that points at it - var links = new Queue<(string Path, int Depth)>(); + // Links wait until the real tree is done, then go in order of how many links they pass + // through, so a folder is listed under its own name rather than through an alias of it, + // and a linked folder under its link rather than through an alias of the link + var links = new PriorityQueue<(string Path, string RealPath, int Depth), int>(); foreach (var file in Walk(rootDir, GetRealPath(rootDir), 0)) yield return file; - while (links.TryDequeue(out var link)) + while (links.TryDequeue(out var link, out _)) { - foreach (var file in Walk(link.Path, GetRealPath(link.Path), link.Depth)) + foreach (var file in Walk(link.Path, link.RealPath, link.Depth)) yield return file; } @@ -147,7 +162,7 @@ IEnumerable Walk(string dir, string realDir, int depth) } catch (Exception e) when (e is IOException or UnauthorizedAccessException) { - Logger.Debug(e, "Skipping unreadable directory {Path}", dir); + Logger.Debug(e, "Skipping missing or unreadable directory {Path}", dir); yield break; } @@ -168,7 +183,8 @@ IEnumerable Walk(string dir, string realDir, int depth) { if (subDir.Attributes.HasFlag(FileAttributes.ReparsePoint)) { - links.Enqueue((subDir.FullName, depth + 1)); + var realPath = GetRealPath(subDir.FullName, out var hops); + links.Enqueue((subDir.FullName, realPath, depth + 1), hops); continue; } diff --git a/StabilityMatrix.Tests/Helper/LinkSafeFileSystemTests.cs b/StabilityMatrix.Tests/Helper/LinkSafeFileSystemTests.cs index fe9c6dce6..f3c040d4b 100644 --- a/StabilityMatrix.Tests/Helper/LinkSafeFileSystemTests.cs +++ b/StabilityMatrix.Tests/Helper/LinkSafeFileSystemTests.cs @@ -106,10 +106,9 @@ public void EnumerateFiles_TwoLinksToSameDirectory_VisitsItOnce() public void EnumerateFiles_RealDirAlreadyClaimedAsLinkTarget_IsVisitedOnce(bool xyLinkIsDeeper) { // ext/X/Y/y.json is reachable two ways: through a link to ext/X/Y, and as a real subfolder - // of a link to ext/X. Sibling enumeration order is file-system dependent, so nesting the - // links at different depths pins the walk order instead of relying on names: the deeper - // link is always popped first (draining real dirs first pushes it last, and the link stack - // pops last-in first-out). + // of a link to ext/X. Sibling enumeration order is file-system dependent, so the two rows + // nest the links at different depths to swap which one is walked first; whichever it is, + // the other must find the folder already scanned. var root = CreateDir("root"); var sub = CreateDir("root", "sub"); CreateFile("ext", "X", "Y", "y.json"); @@ -117,9 +116,7 @@ public void EnumerateFiles_RealDirAlreadyClaimedAsLinkTarget_IsVisitedOnce(bool var x = Path.Combine(tempDir, "ext", "X"); var xy = Path.Combine(x, "Y"); - // Deep link -> ext/X/Y, shallow link -> ext/X. When the deep link is the one targeting - // ext/X/Y, it is walked first and the shallower ext/X link then reaches that same real - // folder again through its "Y" child. + // One row puts the ext/X/Y link under sub, the other puts the ext/X link there var xyLink = Path.Combine(xyLinkIsDeeper ? sub : root, "inner"); var xLink = Path.Combine(xyLinkIsDeeper ? root : sub, "outer"); TempFiles.CreateDirectoryLink(xyLink, xy); @@ -238,6 +235,116 @@ public void EnumerateFiles_DeeperThanMaxDepth_IsSkipped() ); } + [DataTestMethod] + [DataRow("a_alias")] + [DataRow("z_alias")] + public void EnumerateFiles_LinkedFolderShadowedByAliasOfLink_KeepsLinkFolderPaths(string aliasName) + { + // DiffusionModels is itself a link (user moved it to an external drive), and Swarm's + // alias points at that link. The folder must still be listed under DiffusionModels. + var root = CreateDir("root"); + CreateFile("ext", "diff", "a.json"); + CreateFile("ext", "diff", "b.json"); + var diffusion = Path.Combine(root, "DiffusionModels"); + TempFiles.CreateDirectoryLink(diffusion, Path.Combine(tempDir, "ext", "diff")); + TempFiles.CreateDirectoryLink(Path.Combine(root, aliasName), diffusion); + + var files = LinkSafeFileSystem.EnumerateFiles(root, "*.json").ToList(); + + CollectionAssert.AreEquivalent( + new[] { Path.Combine(diffusion, "a.json"), Path.Combine(diffusion, "b.json") }, + files + ); + } + + [TestMethod] + public void EnumerateFiles_DanglingLink_IsSkipped() + { + var root = CreateDir("root"); + CreateFile("root", "a.json"); + var gone = CreateDir("gone"); + TempFiles.CreateDirectoryLink(Path.Combine(root, "dangling"), gone); + Directory.Delete(gone); + + var files = LinkSafeFileSystem.EnumerateFiles(root, "*.json").ToList(); + + CollectionAssert.AreEquivalent(new[] { Path.Combine(root, "a.json") }, files); + } + + [TestMethod] + public void EnumerateFiles_RootGivenInDifferentCase_LinkToRealChild_IsVisitedOnce() + { + if (!Compat.IsWindows) + { + Assert.Inconclusive("Case-insensitive root spelling is Windows-only."); + return; + } + + var root = CreateDir("root"); + CreateFile("root", "DiffusionModels", "a.json"); + TempFiles.CreateDirectoryLink(Path.Combine(root, "z_alias"), Path.Combine(root, "DiffusionModels")); + + // Caller spells the root differently from how it is stored on disk + var spelledRoot = root.ToUpperInvariant(); + var files = LinkSafeFileSystem.EnumerateFiles(spelledRoot, "*.json").ToList(); + + CollectionAssert.AreEquivalent( + new[] { Path.Combine(spelledRoot, "DiffusionModels", "a.json") }, + files + ); + } + + [TestMethod] + public void EnumerateFiles_LinkToAncestorOfRoot_YieldsEachFileOnce() + { + var root = CreateDir("root"); + CreateFile("root", "a.json"); + CreateFile("other", "o.json"); + TempFiles.CreateDirectoryLink(Path.Combine(root, "up"), tempDir); + + var files = LinkSafeFileSystem.EnumerateFiles(root, "*.json").ToList(); + + CollectionAssert.AreEquivalent( + new[] { Path.Combine(root, "a.json"), Path.Combine(root, "up", "other", "o.json") }, + files + ); + } + + [TestMethod] + public void EnumerateFiles_LinkInsideLinkTarget_IsFollowed() + { + var root = CreateDir("root"); + CreateDir("ext", "A"); + CreateFile("ext", "B", "b.json"); + TempFiles.CreateDirectoryLink(Path.Combine(root, "L1"), Path.Combine(tempDir, "ext", "A")); + TempFiles.CreateDirectoryLink( + Path.Combine(tempDir, "ext", "A", "toB"), + Path.Combine(tempDir, "ext", "B") + ); + + var files = LinkSafeFileSystem.EnumerateFiles(root, "*.json").ToList(); + + CollectionAssert.AreEquivalent(new[] { Path.Combine(root, "L1", "toB", "b.json") }, files); + } + + [TestMethod] + public void EnumerateFiles_TwoLinksLoopingIntoEachOther_AreSkipped() + { + var root = CreateDir("root"); + CreateFile("root", "a.json"); + // A link needs an existing target when created: make b real, link a -> b, then replace b + // with a link back to a + var a = Path.Combine(root, "a"); + var b = CreateDir("root", "b"); + TempFiles.CreateDirectoryLink(a, b); + Directory.Delete(b); + TempFiles.CreateDirectoryLink(b, a); + + var files = LinkSafeFileSystem.EnumerateFiles(root, "*.json").ToList(); + + CollectionAssert.AreEquivalent(new[] { Path.Combine(root, "a.json") }, files); + } + [TestMethod] public void GetRealPath_ResolvesLinkChainsAndLinkedAncestors() {