Improve Perf of MemoryFileSystem.EnumerateItems

src/Zio.Tests/FileSystems/TestAsyncFileSystem.cs

@@ -280,6 +280,33 @@ public async Task TestMemoryFileSystemAsyncSupportsMutationMetadataAndWatch()
         Assert.IsNotNull(watcher);
     }
 
+    [TestMethod]
+    public async Task TestMemoryFileSystemAsyncEnumeratesNormalizedPathsAndTimes()
+    {
+        await using var fs = new MemoryFileSystemAsync();
+        await fs.CreateDirectoryAsync("/a/b");
+        await fs.WriteAllTextAsync("/root.txt", "root");
+        await fs.WriteAllTextAsync("/a/b/c.txt", "c");
+        Assert.AreEqual("c", await fs.ReadAllTextAsync("/a/b/c.txt"));
+
+        var local = new DateTime(2020, 3, 4, 5, 6, 7, DateTimeKind.Local);
+        await fs.SetLastWriteTimeAsync("/root.txt", DateTime.SpecifyKind(local, DateTimeKind.Utc));
+        await fs.SetLastAccessTimeAsync("/root.txt", local);
+
+        var expected = new UPath[] { "/a", "/a/b", "/a/b/c.txt", "/root.txt" };
+        var paths = (await ToListAsync(fs.EnumeratePathsAsync("/", "*", SearchOption.AllDirectories))).OrderBy(path => path.FullName, StringComparer.Ordinal).ToList();
+        CollectionAssert.AreEqual(expected, paths);
+
+        var items = (await ToListAsync(fs.EnumerateItemsAsync("/", SearchOption.AllDirectories))).OrderBy(item => item.Path.FullName, StringComparer.Ordinal).ToList();
+        CollectionAssert.AreEqual(expected, items.Select(item => item.Path).ToList());
+        foreach (var item in items)
+        {
+            Assert.IsTrue(new DateTimeOffset(await fs.GetCreationTimeAsync(item.Path)).EqualsExact(item.CreationTime), $"Creation time of `{item.Path}`");
+            Assert.IsTrue(new DateTimeOffset(await fs.GetLastWriteTimeAsync(item.Path)).EqualsExact(item.LastWriteTime), $"Last write time of `{item.Path}`");
+            Assert.IsTrue(new DateTimeOffset(await fs.GetLastAccessTimeAsync(item.Path)).EqualsExact(item.LastAccessTime), $"Last access time of `{item.Path}`");
+        }
+    }
+
     [TestMethod]
     public async Task TestComposeFileSystemAsyncDelegatesOperationsAndDisposal()
     {

src/Zio.Tests/FileSystems/TestMemoryFileSystemEnumeration.cs

@@ -0,0 +1,94 @@
+// Copyright (c) Alexandre Mutel. All rights reserved.
+// This file is licensed under the BSD-Clause 2 license.
+// See the license.txt file in the project root for more information.
+
+using System.IO;
+
+using Zio.FileSystems;
+
+namespace Zio.Tests.FileSystems;
+
+[TestClass]
+public class TestMemoryFileSystemEnumeration
+{
+    [TestMethod]
+    public void TestEnumeratedPathsAreNormalized()
+    {
+        var fs = new MemoryFileSystem();
+        fs.CreateDirectory("/a/b");
+        fs.WriteAllText("/root.txt", "");
+        fs.WriteAllText("/a/.hidden", "");
+        fs.WriteAllText("/a/b/c d.txt", "");
+        fs.WriteAllText("/a/b/x..y", "");
+        fs.WriteAllText("/a/b/.z.", "");
+
+        var expected = new UPath[] { "/a", "/a/.hidden", "/a/b", "/a/b/.z.", "/a/b/c d.txt", "/a/b/x..y", "/root.txt" };
+
+        var paths = fs.EnumeratePaths("/", "*", SearchOption.AllDirectories).OrderBy(path => path.FullName, StringComparer.Ordinal).ToList();
+        CollectionAssert.AreEqual(expected, paths);
+
+        var items = fs.EnumerateItems("/", SearchOption.AllDirectories).OrderBy(item => item.Path.FullName, StringComparer.Ordinal).ToList();
+        CollectionAssert.AreEqual(expected, items.Select(item => item.Path).ToList());
+        CollectionAssert.AreEqual(expected, items.Select(item => item.AbsolutePath).ToList());
+
+        foreach (var path in paths.Concat(items.Select(item => item.Path)))
+        {
+            // Same path as combining the parent directory and the name, and already normalized
+            Assert.AreEqual(UPath.Combine(path.GetDirectory(), path.GetName()), path);
+            Assert.AreEqual(new UPath(path.FullName).FullName, path.FullName);
+        }
+
+        // Search patterns with a directory, or that don't match the directories that are followed
+        CollectionAssert.AreEqual(new UPath[] { "/a/b/c d.txt" }, fs.EnumerateFiles("/a", "b/*.txt").ToList());
+        CollectionAssert.AreEqual(new UPath[] { "/a/b/c d.txt", "/root.txt" }, fs.EnumerateFiles("/", "*.txt", SearchOption.AllDirectories).OrderBy(path => path.FullName, StringComparer.Ordinal).ToList());
+        CollectionAssert.AreEqual(new UPath[] { "/a/b/x..y" }, fs.EnumerateFiles("/", "x*", SearchOption.AllDirectories).ToList());
+        CollectionAssert.AreEqual(new UPath[] { "/a/b" }, fs.EnumerateDirectories("/", "b", SearchOption.AllDirectories).ToList());
+        CollectionAssert.AreEqual(new UPath[] { "/a", "/a/b" }, fs.EnumerateDirectories("/", "*", SearchOption.AllDirectories).OrderBy(path => path.FullName, StringComparer.Ordinal).ToList());
+        CollectionAssert.AreEqual(new UPath[] { "/a/.hidden", "/a/b" }, fs.EnumeratePaths("/a", "*", SearchOption.TopDirectoryOnly).OrderBy(path => path.FullName, StringComparer.Ordinal).ToList());
+    }
+
+    [TestMethod]
+    public void TestEnumeratedItemTimesMatchTheFileSystemTimes()
+    {
+        var fs = new MemoryFileSystem();
+
+        // All the times are the same
+        fs.CreateDirectory("/directory");
+
+        // The last write and last access times are the same, the creation time is different
+        fs.WriteAllText("/written.txt", "content");
+
+        // All the times are different
+        fs.WriteAllText("/read.txt", "content");
+        Assert.AreEqual("content", fs.ReadAllText("/read.txt"));
+
+        // Same ticks but different kinds
+        var local = new DateTime(2020, 3, 4, 5, 6, 7, DateTimeKind.Local);
+        fs.WriteAllText("/kinds.txt", "content");
+        fs.SetCreationTime("/kinds.txt", local);
+        fs.SetLastWriteTime("/kinds.txt", DateTime.SpecifyKind(local, DateTimeKind.Utc));
+        fs.SetLastAccessTime("/kinds.txt", DateTime.SpecifyKind(local, DateTimeKind.Unspecified));
+
+        // Same ticks and kinds, set explicitly
+        var utc = new DateTime(2021, 6, 7, 8, 9, 10, DateTimeKind.Utc);
+        fs.WriteAllText("/utc.txt", "content");
+        fs.SetCreationTime("/utc.txt", utc);
+        fs.SetLastWriteTime("/utc.txt", local);
+        fs.SetLastAccessTime("/utc.txt", utc);
+
+        var items = fs.EnumerateItems("/", SearchOption.AllDirectories).ToList();
+        Assert.AreEqual(5, items.Count);
+        foreach (var item in items)
+        {
+            AssertSameDateTimeOffset(fs.GetCreationTime(item.Path), item.CreationTime, item.Path);
+            AssertSameDateTimeOffset(fs.GetLastWriteTime(item.Path), item.LastWriteTime, item.Path);
+            AssertSameDateTimeOffset(fs.GetLastAccessTime(item.Path), item.LastAccessTime, item.Path);
+        }
+    }
+
+    private static void AssertSameDateTimeOffset(DateTime expected, DateTimeOffset actual, UPath path)
+    {
+        var expectedOffset = new DateTimeOffset(expected);
+        Assert.IsTrue(expectedOffset.EqualsExact(actual), $"Expecting `{expectedOffset:o}` but got `{actual:o}` for `{path}`");
+    }
+}

src/Zio/FileSystems/MemoryFileSystem.cs

@@ -909,22 +909,23 @@ protected override IEnumerable<FileSystemItem> EnumerateItemsImpl(UPath path, Se
                     {
                         var node = nodePair.Value;
                         var canFollowFolder = searchOption == SearchOption.AllDirectories && nodePair.Value is DirectoryNode;
-                        var fullPath = directoryPath / nodePair.Key;
+                        var fullPath = CombineChildPath(directoryPath, nodePair.Key);
 
                         if (canFollowFolder)
                         {
                             foldersToProcess.Insert(dirIndex++, fullPath);
                         }
 
+                        GetDateTimeOffsets(node, out var creationTime, out var lastWriteTime, out var lastAccessTime);
                         var item = new FileSystemItem
                         {
                             FileSystem = this,
                             AbsolutePath = fullPath,
                             Path = fullPath,
                             Attributes = node.Attributes,
-                            CreationTime = node.CreationTime,
-                            LastWriteTime = node.LastWriteTime,
-                            LastAccessTime = node.LastAccessTime,
+                            CreationTime = creationTime,
+                            LastWriteTime = lastWriteTime,
+                            LastAccessTime = lastAccessTime,
                             Length = node is FileNode fileNode ? fileNode.Content.Length : 0,
                         };
 
@@ -1140,6 +1141,46 @@ private static void ValidateFile([NotNull] FileSystemNode? node, UPath srcPath)
         }
     }
 
+    private static UPath CombineChildPath(UPath directoryPath, string name)
+    {
+        // The name of a child node is always a single part of a validated path (no separator, not `.` or `..`),
+        // so when the directory path is absolute (and normalized, as any UPath), the combined path is already
+        // normalized: this is equivalent to `directoryPath / name` without validating and normalizing again
+        // the name and the combined path.
+        if (!directoryPath.IsAbsolute)
+        {
+            return directoryPath / name;
+        }
+
+        var directory = directoryPath.FullName;
+        return new UPath(directory.Length == 1 ? "/" + name : directory + "/" + name, true);
+    }
+
+    private static void GetDateTimeOffsets(FileSystemNode node, out DateTimeOffset creationTime, out DateTimeOffset lastWriteTime, out DateTimeOffset lastAccessTime)
+    {
+        // Converting a local DateTime to a DateTimeOffset looks up the UTC offset of the local time zone at that
+        // time, so convert only once the times that are identical (e.g. for a directory, for a file that has not
+        // been read or written since it was created, or for a file not read since it was last written).
+        var creation = node.CreationTime;
+        var lastWrite = node.LastWriteTime;
+        var lastAccess = node.LastAccessTime;
+
+        creationTime = creation;
+        lastWriteTime = IsSameDateTime(lastWrite, creation) ? creationTime : lastWrite;
+        lastAccessTime = IsSameDateTime(lastAccess, lastWrite) ? lastWriteTime : IsSameDateTime(lastAccess, creation) ? creationTime : lastAccess;
+    }
+
+    private static bool IsSameDateTime(DateTime left, DateTime right)
+    {
+        // DateTime equality only compares the ticks, while the conversion to a DateTimeOffset also depends on the Kind
+        // (and, for an ambiguous local time, on whether it is a daylight saving time), so compare all the bits.
+        Span<DateTime> values = stackalloc DateTime[2];
+        values[0] = left;
+        values[1] = right;
+        var bits = System.Runtime.InteropServices.MemoryMarshal.Cast<DateTime, ulong>(values);
+        return bits[0] == bits[1];
+    }
+
     private FileSystemNode? TryFindNodeSafe(UPath path)
     {
         EnterFileSystemShared();

src/Zio/generated/FileSystems/MemoryFileSystemAsync.gen.cs

@@ -860,21 +860,22 @@ protected override async ValueTask CreateSymbolicLinkAsyncImpl(UPath path, UPath
                     {
                         var node = nodePair.Value;
                         var canFollowFolder = searchOption == SearchOption.AllDirectories && nodePair.Value is DirectoryNode;
-                        var fullPath = directoryPath / nodePair.Key;
+                        var fullPath = CombineChildPath(directoryPath, nodePair.Key);
                         if (canFollowFolder)
                         {
                             foldersToProcess.Insert(dirIndex++, fullPath);
                         }
 
+                        GetDateTimeOffsets(node, out var creationTime, out var lastWriteTime, out var lastAccessTime);
                         var item = new FileSystemItemAsync
                         {
                             FileSystem = this,
                             AbsolutePath = fullPath,
                             Path = fullPath,
                             Attributes = node.Attributes,
-                            CreationTime = node.CreationTime,
-                            LastWriteTime = node.LastWriteTime,
-                            LastAccessTime = node.LastAccessTime,
+                            CreationTime = creationTime,
+                            LastWriteTime = lastWriteTime,
+                            LastAccessTime = lastAccessTime,
                             Length = node is FileNode fileNode ? fileNode.Content.Length : 0,
                         };
                         if (searchPredicate == null || searchPredicate(ref item))
@@ -1074,6 +1075,45 @@ private static void ValidateFile([NotNull] FileSystemNode? node, UPath srcPath)
         }
     }
 
+    private static UPath CombineChildPath(UPath directoryPath, string name)
+    {
+        // The name of a child node is always a single part of a validated path (no separator, not `.` or `..`),
+        // so when the directory path is absolute (and normalized, as any UPath), the combined path is already
+        // normalized: this is equivalent to `directoryPath / name` without validating and normalizing again
+        // the name and the combined path.
+        if (!directoryPath.IsAbsolute)
+        {
+            return directoryPath / name;
+        }
+
+        var directory = directoryPath.FullName;
+        return new UPath(directory.Length == 1 ? "/" + name : directory + "/" + name, true);
+    }
+
+    private static void GetDateTimeOffsets(FileSystemNode node, out DateTimeOffset creationTime, out DateTimeOffset lastWriteTime, out DateTimeOffset lastAccessTime)
+    {
+        // Converting a local DateTime to a DateTimeOffset looks up the UTC offset of the local time zone at that
+        // time, so convert only once the times that are identical (e.g. for a directory, for a file that has not
+        // been read or written since it was created, or for a file not read since it was last written).
+        var creation = node.CreationTime;
+        var lastWrite = node.LastWriteTime;
+        var lastAccess = node.LastAccessTime;
+        creationTime = creation;
+        lastWriteTime = IsSameDateTime(lastWrite, creation) ? creationTime : lastWrite;
+        lastAccessTime = IsSameDateTime(lastAccess, lastWrite) ? lastWriteTime : IsSameDateTime(lastAccess, creation) ? creationTime : lastAccess;
+    }
+
+    private static bool IsSameDateTime(DateTime left, DateTime right)
+    {
+        // DateTime equality only compares the ticks, while the conversion to a DateTimeOffset also depends on the Kind
+        // (and, for an ambiguous local time, on whether it is a daylight saving time), so compare all the bits.
+        Span<DateTime> values = stackalloc DateTime[2];
+        values[0] = left;
+        values[1] = right;
+        var bits = System.Runtime.InteropServices.MemoryMarshal.Cast<DateTime, ulong>(values);
+        return bits[0] == bits[1];
+    }
+
     private FileSystemNode? TryFindNodeSafe(UPath path)
     {
         EnterFileSystemShared();