diff --git a/StabilityMatrix.Core/Helper/LinkSafeFileSystem.cs b/StabilityMatrix.Core/Helper/LinkSafeFileSystem.cs
index 3528ca62a..8c251564d 100644
--- a/StabilityMatrix.Core/Helper/LinkSafeFileSystem.cs
+++ b/StabilityMatrix.Core/Helper/LinkSafeFileSystem.cs
@@ -30,9 +30,19 @@ public static class LinkSafeFileSystem
/// ancestors, returning the physical directory path. Segments that cannot be resolved
/// (missing, or a reparse point that is not a link) are kept as written.
///
- public static string GetRealPath(string path) => GetRealPath(path, 0);
+ public static string GetRealPath(string path) => GetRealPath(path, out _);
- private static string GetRealPath(string path, int hop)
+ ///
+ /// receives how many links
+ /// the path goes through.
+ ///
+ private static string GetRealPath(string path, out int hops)
+ {
+ hops = 0;
+ return Resolve(path, ref hops);
+ }
+
+ private static string Resolve(string path, ref int hops)
{
var fullPath = Path.TrimEndingDirectorySeparator(Path.GetFullPath(path));
var root = Path.GetPathRoot(fullPath) ?? string.Empty;
@@ -52,7 +62,7 @@ var segment in fullPath[root.Length..]
if (!info.Exists || !info.Attributes.HasFlag(FileAttributes.ReparsePoint))
continue;
- if (hop >= MaxLinkHops)
+ if (hops >= MaxLinkHops)
{
Logger.Warn("Gave up resolving links at {Path}: too many nested links", current);
continue;
@@ -61,7 +71,9 @@ var segment in fullPath[root.Length..]
FileSystemInfo? target;
try
{
- target = info.ResolveLinkTarget(returnFinalTarget: true);
+ // The immediate target, as stored, so it is spelled the way the link's own
+ // neighbourhood is (subst and mapped drives would otherwise resolve to the volume)
+ target = info.ResolveLinkTarget(returnFinalTarget: false);
}
catch (IOException e)
{
@@ -72,8 +84,10 @@ var segment in fullPath[root.Length..]
if (target is null)
continue;
- // The target may itself sit below other links, so canonicalize it as a whole
- current = GetRealPath(target.FullName, hop + 1);
+ hops++;
+
+ // The target may itself be a link, or sit below other links, so canonicalize it as a whole
+ current = Resolve(target.FullName, ref hops);
}
return current;
@@ -99,10 +113,11 @@ public static bool WouldLinkCycle(DirectoryPath sourceDir, DirectoryPath linkPat
///
/// Recursively enumerates files matching under
/// . Symbolic links are followed, but every physical directory is
- /// visited at most once, so no file is yielded twice. Directories nested deeper than
- /// and directories that cannot be read are skipped without aborting
- /// the enumeration. Yielded paths are rooted at as given, not at its
- /// resolved target.
+ /// visited at most once, so no file is yielded twice; a directory reachable along several
+ /// paths is yielded under the one that goes through the fewest links. Directories nested
+ /// deeper than and directories that cannot be read are skipped
+ /// without aborting the enumeration. Yielded paths are rooted at
+ /// as given, not at its resolved target.
///
public static IEnumerable EnumerateFiles(
string rootDir,
@@ -110,65 +125,28 @@ public static IEnumerable EnumerateFiles(
int maxDepth = DefaultMaxDepth
)
{
- // A real directory is keyed by its literal path, compared ordinally, so two folders whose
- // names differ only in case are both scanned; it is also matched against link targets, so a
- // real folder reached through a link is not rescanned. A link is keyed by its resolved
- // target, compared with the platform's case sensitivity (PathComparer), because a target is
- // stored however the link was created.
- var visitedRealDirsExact = new Dictionary(StringComparer.Ordinal);
- var visitedRealDirsForLinkTargets = new Dictionary(PathComparer);
- var visitedLinkTargets = new Dictionary(PathComparer);
-
- // Real directories are drained to completion before any link is considered, so a link can
- // never take the identity of a real folder and shadow it out of the scan.
- var realDirs = new Stack<(string Path, string RealPath, int Depth)>();
- var linkedDirs = new Stack<(string Path, string RealPath, int Depth)>();
-
- var rootReal = GetRealPath(rootDir);
- realDirs.Push((rootDir, rootReal, 0));
-
- while (realDirs.Count > 0 || linkedDirs.Count > 0)
- {
- // Which stack the entry came from is how the walk knows whether it is a link.
- var fromRealDirs = realDirs.Count > 0;
- var dir = fromRealDirs ? realDirs.Pop() : linkedDirs.Pop();
+ // Physical directories already scanned. Links are what turn the tree into a graph.
+ var seen = new HashSet(PathComparer);
+ // Links wait until the real tree is done, then go in order of how many links they pass
+ // through, so a folder is listed under its own name rather than through an alias of it,
+ // and a linked folder under its link rather than through an alias of the link
+ var links = new PriorityQueue<(string Path, string RealPath, int Depth), int>();
- // Claimed on pop, not on push, so the walk order decides which spelling owns the
- // identity instead of the reversed push order.
- if (!fromRealDirs)
- {
- if (
- visitedLinkTargets.TryGetValue(dir.RealPath, out var linkClaimer)
- || visitedRealDirsForLinkTargets.TryGetValue(dir.RealPath, out linkClaimer)
- )
- {
- Logger.Info(
- "Skipping {Path}: the same directory was already scanned as {ClaimedBy}",
- dir.Path,
- linkClaimer
- );
- continue;
- }
+ foreach (var file in Walk(rootDir, GetRealPath(rootDir), 0))
+ yield return file;
- visitedLinkTargets[dir.RealPath] = dir.Path;
- }
- else
- {
- if (
- visitedRealDirsExact.TryGetValue(dir.RealPath, out var realClaimer)
- || visitedLinkTargets.TryGetValue(dir.RealPath, out realClaimer)
- )
- {
- Logger.Warn(
- "Skipping {Path}: the same directory was already scanned as {ClaimedBy}",
- dir.Path,
- realClaimer
- );
- continue;
- }
+ while (links.TryDequeue(out var link, out _))
+ {
+ foreach (var file in Walk(link.Path, link.RealPath, link.Depth))
+ yield return file;
+ }
- visitedRealDirsExact[dir.RealPath] = dir.Path;
- visitedRealDirsForLinkTargets[dir.RealPath] = dir.Path;
+ IEnumerable Walk(string dir, string realDir, int depth)
+ {
+ if (!seen.Add(realDir))
+ {
+ Logger.Debug("Skipping {Path}: already scanned", dir);
+ yield break;
}
List files;
@@ -176,48 +154,42 @@ public static IEnumerable EnumerateFiles(
try
{
files = Directory
- .EnumerateFiles(dir.Path, searchPattern, EnumerationOptionConstants.TopLevelOnly)
+ .EnumerateFiles(dir, searchPattern, EnumerationOptionConstants.TopLevelOnly)
.ToList();
- subDirs = new DirectoryInfo(dir.Path)
+ subDirs = new DirectoryInfo(dir)
.EnumerateDirectories("*", EnumerationOptionConstants.TopLevelOnly)
.ToList();
}
catch (Exception e) when (e is IOException or UnauthorizedAccessException)
{
- Logger.Debug(e, "Skipping unreadable directory {Path}", dir.Path);
- continue;
+ Logger.Debug(e, "Skipping missing or unreadable directory {Path}", dir);
+ yield break;
}
foreach (var file in files)
- {
yield return file;
- }
- if (dir.Depth >= maxDepth)
+ if (depth >= maxDepth)
{
Logger.Warn(
"Skipping directories below {Path}: nesting deeper than {MaxDepth}",
- dir.Path,
+ dir,
maxDepth
);
- continue;
+ yield break;
}
- // Pushed in reverse so each stack pops its entries in enumeration order
- for (var i = subDirs.Count - 1; i >= 0; i--)
+ foreach (var subDir in subDirs)
{
- var subDir = subDirs[i];
- var isLinkDir = subDir.Attributes.HasFlag(FileAttributes.ReparsePoint);
- var subReal = isLinkDir ? GetRealPath(subDir.FullName) : Path.Join(dir.RealPath, subDir.Name);
-
- if (isLinkDir)
+ if (subDir.Attributes.HasFlag(FileAttributes.ReparsePoint))
{
- linkedDirs.Push((subDir.FullName, subReal, dir.Depth + 1));
- }
- else
- {
- realDirs.Push((subDir.FullName, subReal, dir.Depth + 1));
+ var realPath = GetRealPath(subDir.FullName, out var hops);
+ links.Enqueue((subDir.FullName, realPath, depth + 1), hops);
+ continue;
}
+
+ foreach (var file in Walk(subDir.FullName, Path.Join(realDir, subDir.Name), depth + 1))
+ yield return file;
}
}
}
diff --git a/StabilityMatrix.Tests/Helper/LinkSafeFileSystemTests.cs b/StabilityMatrix.Tests/Helper/LinkSafeFileSystemTests.cs
index 2f86c72ca..f3c040d4b 100644
--- a/StabilityMatrix.Tests/Helper/LinkSafeFileSystemTests.cs
+++ b/StabilityMatrix.Tests/Helper/LinkSafeFileSystemTests.cs
@@ -106,10 +106,9 @@ public void EnumerateFiles_TwoLinksToSameDirectory_VisitsItOnce()
public void EnumerateFiles_RealDirAlreadyClaimedAsLinkTarget_IsVisitedOnce(bool xyLinkIsDeeper)
{
// ext/X/Y/y.json is reachable two ways: through a link to ext/X/Y, and as a real subfolder
- // of a link to ext/X. Sibling enumeration order is file-system dependent, so nesting the
- // links at different depths pins the walk order instead of relying on names: the deeper
- // link is always popped first (draining real dirs first pushes it last, and the link stack
- // pops last-in first-out).
+ // of a link to ext/X. Sibling enumeration order is file-system dependent, so the two rows
+ // nest the links at different depths to swap which one is walked first; whichever it is,
+ // the other must find the folder already scanned.
var root = CreateDir("root");
var sub = CreateDir("root", "sub");
CreateFile("ext", "X", "Y", "y.json");
@@ -117,9 +116,7 @@ public void EnumerateFiles_RealDirAlreadyClaimedAsLinkTarget_IsVisitedOnce(bool
var x = Path.Combine(tempDir, "ext", "X");
var xy = Path.Combine(x, "Y");
- // Deep link -> ext/X/Y, shallow link -> ext/X. When the deep link is the one targeting
- // ext/X/Y, it is walked first and the shallower ext/X link then reaches that same real
- // folder again through its "Y" child.
+ // One row puts the ext/X/Y link under sub, the other puts the ext/X link there
var xyLink = Path.Combine(xyLinkIsDeeper ? sub : root, "inner");
var xLink = Path.Combine(xyLinkIsDeeper ? root : sub, "outer");
TempFiles.CreateDirectoryLink(xyLink, xy);
@@ -167,6 +164,7 @@ string outerName
[DataTestMethod]
[DataRow("diffusion_models")]
+ [DataRow("a_alias")]
[DataRow("sub", "alias")]
public void EnumerateFiles_RealFolderShadowedByLink_KeepsRealFolderPaths(params string[] linkSegments)
{
@@ -237,6 +235,116 @@ public void EnumerateFiles_DeeperThanMaxDepth_IsSkipped()
);
}
+ [DataTestMethod]
+ [DataRow("a_alias")]
+ [DataRow("z_alias")]
+ public void EnumerateFiles_LinkedFolderShadowedByAliasOfLink_KeepsLinkFolderPaths(string aliasName)
+ {
+ // DiffusionModels is itself a link (user moved it to an external drive), and Swarm's
+ // alias points at that link. The folder must still be listed under DiffusionModels.
+ var root = CreateDir("root");
+ CreateFile("ext", "diff", "a.json");
+ CreateFile("ext", "diff", "b.json");
+ var diffusion = Path.Combine(root, "DiffusionModels");
+ TempFiles.CreateDirectoryLink(diffusion, Path.Combine(tempDir, "ext", "diff"));
+ TempFiles.CreateDirectoryLink(Path.Combine(root, aliasName), diffusion);
+
+ var files = LinkSafeFileSystem.EnumerateFiles(root, "*.json").ToList();
+
+ CollectionAssert.AreEquivalent(
+ new[] { Path.Combine(diffusion, "a.json"), Path.Combine(diffusion, "b.json") },
+ files
+ );
+ }
+
+ [TestMethod]
+ public void EnumerateFiles_DanglingLink_IsSkipped()
+ {
+ var root = CreateDir("root");
+ CreateFile("root", "a.json");
+ var gone = CreateDir("gone");
+ TempFiles.CreateDirectoryLink(Path.Combine(root, "dangling"), gone);
+ Directory.Delete(gone);
+
+ var files = LinkSafeFileSystem.EnumerateFiles(root, "*.json").ToList();
+
+ CollectionAssert.AreEquivalent(new[] { Path.Combine(root, "a.json") }, files);
+ }
+
+ [TestMethod]
+ public void EnumerateFiles_RootGivenInDifferentCase_LinkToRealChild_IsVisitedOnce()
+ {
+ if (!Compat.IsWindows)
+ {
+ Assert.Inconclusive("Case-insensitive root spelling is Windows-only.");
+ return;
+ }
+
+ var root = CreateDir("root");
+ CreateFile("root", "DiffusionModels", "a.json");
+ TempFiles.CreateDirectoryLink(Path.Combine(root, "z_alias"), Path.Combine(root, "DiffusionModels"));
+
+ // Caller spells the root differently from how it is stored on disk
+ var spelledRoot = root.ToUpperInvariant();
+ var files = LinkSafeFileSystem.EnumerateFiles(spelledRoot, "*.json").ToList();
+
+ CollectionAssert.AreEquivalent(
+ new[] { Path.Combine(spelledRoot, "DiffusionModels", "a.json") },
+ files
+ );
+ }
+
+ [TestMethod]
+ public void EnumerateFiles_LinkToAncestorOfRoot_YieldsEachFileOnce()
+ {
+ var root = CreateDir("root");
+ CreateFile("root", "a.json");
+ CreateFile("other", "o.json");
+ TempFiles.CreateDirectoryLink(Path.Combine(root, "up"), tempDir);
+
+ var files = LinkSafeFileSystem.EnumerateFiles(root, "*.json").ToList();
+
+ CollectionAssert.AreEquivalent(
+ new[] { Path.Combine(root, "a.json"), Path.Combine(root, "up", "other", "o.json") },
+ files
+ );
+ }
+
+ [TestMethod]
+ public void EnumerateFiles_LinkInsideLinkTarget_IsFollowed()
+ {
+ var root = CreateDir("root");
+ CreateDir("ext", "A");
+ CreateFile("ext", "B", "b.json");
+ TempFiles.CreateDirectoryLink(Path.Combine(root, "L1"), Path.Combine(tempDir, "ext", "A"));
+ TempFiles.CreateDirectoryLink(
+ Path.Combine(tempDir, "ext", "A", "toB"),
+ Path.Combine(tempDir, "ext", "B")
+ );
+
+ var files = LinkSafeFileSystem.EnumerateFiles(root, "*.json").ToList();
+
+ CollectionAssert.AreEquivalent(new[] { Path.Combine(root, "L1", "toB", "b.json") }, files);
+ }
+
+ [TestMethod]
+ public void EnumerateFiles_TwoLinksLoopingIntoEachOther_AreSkipped()
+ {
+ var root = CreateDir("root");
+ CreateFile("root", "a.json");
+ // A link needs an existing target when created: make b real, link a -> b, then replace b
+ // with a link back to a
+ var a = Path.Combine(root, "a");
+ var b = CreateDir("root", "b");
+ TempFiles.CreateDirectoryLink(a, b);
+ Directory.Delete(b);
+ TempFiles.CreateDirectoryLink(b, a);
+
+ var files = LinkSafeFileSystem.EnumerateFiles(root, "*.json").ToList();
+
+ CollectionAssert.AreEquivalent(new[] { Path.Combine(root, "a.json") }, files);
+ }
+
[TestMethod]
public void GetRealPath_ResolvesLinkChainsAndLinkedAncestors()
{