From 07526117e16522d3367c5aa435baa2833b295bc0 Mon Sep 17 00:00:00 2001 From: Marko Lahma Date: Sun, 18 Oct 2026 20:00:36 +0300 Subject: [PATCH] Improve Perf of MemoryFileSystem.EnumerateItems EnumerateItemsImpl combined each child with `directoryPath / name`, which validates and normalizes the name and then the whole combined path again, and converted the creation, last write and last access times of every child to DateTimeOffset, each conversion looking up the UTC offset of the local time zone. - Combine a child path by concatenation: the name of a child node is always a single part of a validated path and the directory path is absolute and normalized, so the result is already normalized and is the same path. - Convert identical times only once. The times are compared on their full binary representation, as the conversion also depends on the Kind (and on the daylight saving flag of an ambiguous local time) which DateTime equality ignores, so the enumerated values are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../FileSystems/TestAsyncFileSystem.cs | 27 ++++++ .../TestMemoryFileSystemEnumeration.cs | 94 +++++++++++++++++++ src/Zio/FileSystems/MemoryFileSystem.cs | 49 +++++++++- .../FileSystems/MemoryFileSystemAsync.gen.cs | 48 +++++++++- 4 files changed, 210 insertions(+), 8 deletions(-) create mode 100644 src/Zio.Tests/FileSystems/TestMemoryFileSystemEnumeration.cs diff --git a/src/Zio.Tests/FileSystems/TestAsyncFileSystem.cs b/src/Zio.Tests/FileSystems/TestAsyncFileSystem.cs index 4348332..9c5e191 100644 --- a/src/Zio.Tests/FileSystems/TestAsyncFileSystem.cs +++ b/src/Zio.Tests/FileSystems/TestAsyncFileSystem.cs @@ -280,6 +280,33 @@ public async Task TestMemoryFileSystemAsyncSupportsMutationMetadataAndWatch() Assert.IsNotNull(watcher); } + [TestMethod] + public async Task TestMemoryFileSystemAsyncEnumeratesNormalizedPathsAndTimes() + { + await using var fs = new MemoryFileSystemAsync(); + await fs.CreateDirectoryAsync("/a/b"); + await fs.WriteAllTextAsync("/root.txt", "root"); + await fs.WriteAllTextAsync("/a/b/c.txt", "c"); + Assert.AreEqual("c", await fs.ReadAllTextAsync("/a/b/c.txt")); + + var local = new DateTime(2020, 3, 4, 5, 6, 7, DateTimeKind.Local); + await fs.SetLastWriteTimeAsync("/root.txt", DateTime.SpecifyKind(local, DateTimeKind.Utc)); + await fs.SetLastAccessTimeAsync("/root.txt", local); + + var expected = new UPath[] { "/a", "/a/b", "/a/b/c.txt", "/root.txt" }; + var paths = (await ToListAsync(fs.EnumeratePathsAsync("/", "*", SearchOption.AllDirectories))).OrderBy(path => path.FullName, StringComparer.Ordinal).ToList(); + CollectionAssert.AreEqual(expected, paths); + + var items = (await ToListAsync(fs.EnumerateItemsAsync("/", SearchOption.AllDirectories))).OrderBy(item => item.Path.FullName, StringComparer.Ordinal).ToList(); + CollectionAssert.AreEqual(expected, items.Select(item => item.Path).ToList()); + foreach (var item in items) + { + Assert.IsTrue(new DateTimeOffset(await fs.GetCreationTimeAsync(item.Path)).EqualsExact(item.CreationTime), $"Creation time of `{item.Path}`"); + Assert.IsTrue(new DateTimeOffset(await fs.GetLastWriteTimeAsync(item.Path)).EqualsExact(item.LastWriteTime), $"Last write time of `{item.Path}`"); + Assert.IsTrue(new DateTimeOffset(await fs.GetLastAccessTimeAsync(item.Path)).EqualsExact(item.LastAccessTime), $"Last access time of `{item.Path}`"); + } + } + [TestMethod] public async Task TestComposeFileSystemAsyncDelegatesOperationsAndDisposal() { diff --git a/src/Zio.Tests/FileSystems/TestMemoryFileSystemEnumeration.cs b/src/Zio.Tests/FileSystems/TestMemoryFileSystemEnumeration.cs new file mode 100644 index 0000000..b7d5cb2 --- /dev/null +++ b/src/Zio.Tests/FileSystems/TestMemoryFileSystemEnumeration.cs @@ -0,0 +1,94 @@ +// Copyright (c) Alexandre Mutel. All rights reserved. +// This file is licensed under the BSD-Clause 2 license. +// See the license.txt file in the project root for more information. + +using System.IO; + +using Zio.FileSystems; + +namespace Zio.Tests.FileSystems; + +[TestClass] +public class TestMemoryFileSystemEnumeration +{ + [TestMethod] + public void TestEnumeratedPathsAreNormalized() + { + var fs = new MemoryFileSystem(); + fs.CreateDirectory("/a/b"); + fs.WriteAllText("/root.txt", ""); + fs.WriteAllText("/a/.hidden", ""); + fs.WriteAllText("/a/b/c d.txt", ""); + fs.WriteAllText("/a/b/x..y", ""); + fs.WriteAllText("/a/b/.z.", ""); + + var expected = new UPath[] { "/a", "/a/.hidden", "/a/b", "/a/b/.z.", "/a/b/c d.txt", "/a/b/x..y", "/root.txt" }; + + var paths = fs.EnumeratePaths("/", "*", SearchOption.AllDirectories).OrderBy(path => path.FullName, StringComparer.Ordinal).ToList(); + CollectionAssert.AreEqual(expected, paths); + + var items = fs.EnumerateItems("/", SearchOption.AllDirectories).OrderBy(item => item.Path.FullName, StringComparer.Ordinal).ToList(); + CollectionAssert.AreEqual(expected, items.Select(item => item.Path).ToList()); + CollectionAssert.AreEqual(expected, items.Select(item => item.AbsolutePath).ToList()); + + foreach (var path in paths.Concat(items.Select(item => item.Path))) + { + // Same path as combining the parent directory and the name, and already normalized + Assert.AreEqual(UPath.Combine(path.GetDirectory(), path.GetName()), path); + Assert.AreEqual(new UPath(path.FullName).FullName, path.FullName); + } + + // Search patterns with a directory, or that don't match the directories that are followed + CollectionAssert.AreEqual(new UPath[] { "/a/b/c d.txt" }, fs.EnumerateFiles("/a", "b/*.txt").ToList()); + CollectionAssert.AreEqual(new UPath[] { "/a/b/c d.txt", "/root.txt" }, fs.EnumerateFiles("/", "*.txt", SearchOption.AllDirectories).OrderBy(path => path.FullName, StringComparer.Ordinal).ToList()); + CollectionAssert.AreEqual(new UPath[] { "/a/b/x..y" }, fs.EnumerateFiles("/", "x*", SearchOption.AllDirectories).ToList()); + CollectionAssert.AreEqual(new UPath[] { "/a/b" }, fs.EnumerateDirectories("/", "b", SearchOption.AllDirectories).ToList()); + CollectionAssert.AreEqual(new UPath[] { "/a", "/a/b" }, fs.EnumerateDirectories("/", "*", SearchOption.AllDirectories).OrderBy(path => path.FullName, StringComparer.Ordinal).ToList()); + CollectionAssert.AreEqual(new UPath[] { "/a/.hidden", "/a/b" }, fs.EnumeratePaths("/a", "*", SearchOption.TopDirectoryOnly).OrderBy(path => path.FullName, StringComparer.Ordinal).ToList()); + } + + [TestMethod] + public void TestEnumeratedItemTimesMatchTheFileSystemTimes() + { + var fs = new MemoryFileSystem(); + + // All the times are the same + fs.CreateDirectory("/directory"); + + // The last write and last access times are the same, the creation time is different + fs.WriteAllText("/written.txt", "content"); + + // All the times are different + fs.WriteAllText("/read.txt", "content"); + Assert.AreEqual("content", fs.ReadAllText("/read.txt")); + + // Same ticks but different kinds + var local = new DateTime(2020, 3, 4, 5, 6, 7, DateTimeKind.Local); + fs.WriteAllText("/kinds.txt", "content"); + fs.SetCreationTime("/kinds.txt", local); + fs.SetLastWriteTime("/kinds.txt", DateTime.SpecifyKind(local, DateTimeKind.Utc)); + fs.SetLastAccessTime("/kinds.txt", DateTime.SpecifyKind(local, DateTimeKind.Unspecified)); + + // Same ticks and kinds, set explicitly + var utc = new DateTime(2021, 6, 7, 8, 9, 10, DateTimeKind.Utc); + fs.WriteAllText("/utc.txt", "content"); + fs.SetCreationTime("/utc.txt", utc); + fs.SetLastWriteTime("/utc.txt", local); + fs.SetLastAccessTime("/utc.txt", utc); + + var items = fs.EnumerateItems("/", SearchOption.AllDirectories).ToList(); + Assert.AreEqual(5, items.Count); + foreach (var item in items) + { + AssertSameDateTimeOffset(fs.GetCreationTime(item.Path), item.CreationTime, item.Path); + AssertSameDateTimeOffset(fs.GetLastWriteTime(item.Path), item.LastWriteTime, item.Path); + AssertSameDateTimeOffset(fs.GetLastAccessTime(item.Path), item.LastAccessTime, item.Path); + } + } + + private static void AssertSameDateTimeOffset(DateTime expected, DateTimeOffset actual, UPath path) + { + var expectedOffset = new DateTimeOffset(expected); + Assert.IsTrue(expectedOffset.EqualsExact(actual), $"Expecting `{expectedOffset:o}` but got `{actual:o}` for `{path}`"); + } +} diff --git a/src/Zio/FileSystems/MemoryFileSystem.cs b/src/Zio/FileSystems/MemoryFileSystem.cs index 0279dd2..8b921ca 100644 --- a/src/Zio/FileSystems/MemoryFileSystem.cs +++ b/src/Zio/FileSystems/MemoryFileSystem.cs @@ -909,22 +909,23 @@ protected override IEnumerable EnumerateItemsImpl(UPath path, Se { var node = nodePair.Value; var canFollowFolder = searchOption == SearchOption.AllDirectories && nodePair.Value is DirectoryNode; - var fullPath = directoryPath / nodePair.Key; + var fullPath = CombineChildPath(directoryPath, nodePair.Key); if (canFollowFolder) { foldersToProcess.Insert(dirIndex++, fullPath); } + GetDateTimeOffsets(node, out var creationTime, out var lastWriteTime, out var lastAccessTime); var item = new FileSystemItem { FileSystem = this, AbsolutePath = fullPath, Path = fullPath, Attributes = node.Attributes, - CreationTime = node.CreationTime, - LastWriteTime = node.LastWriteTime, - LastAccessTime = node.LastAccessTime, + CreationTime = creationTime, + LastWriteTime = lastWriteTime, + LastAccessTime = lastAccessTime, Length = node is FileNode fileNode ? fileNode.Content.Length : 0, }; @@ -1140,6 +1141,46 @@ private static void ValidateFile([NotNull] FileSystemNode? node, UPath srcPath) } } + private static UPath CombineChildPath(UPath directoryPath, string name) + { + // The name of a child node is always a single part of a validated path (no separator, not `.` or `..`), + // so when the directory path is absolute (and normalized, as any UPath), the combined path is already + // normalized: this is equivalent to `directoryPath / name` without validating and normalizing again + // the name and the combined path. + if (!directoryPath.IsAbsolute) + { + return directoryPath / name; + } + + var directory = directoryPath.FullName; + return new UPath(directory.Length == 1 ? "/" + name : directory + "/" + name, true); + } + + private static void GetDateTimeOffsets(FileSystemNode node, out DateTimeOffset creationTime, out DateTimeOffset lastWriteTime, out DateTimeOffset lastAccessTime) + { + // Converting a local DateTime to a DateTimeOffset looks up the UTC offset of the local time zone at that + // time, so convert only once the times that are identical (e.g. for a directory, for a file that has not + // been read or written since it was created, or for a file not read since it was last written). + var creation = node.CreationTime; + var lastWrite = node.LastWriteTime; + var lastAccess = node.LastAccessTime; + + creationTime = creation; + lastWriteTime = IsSameDateTime(lastWrite, creation) ? creationTime : lastWrite; + lastAccessTime = IsSameDateTime(lastAccess, lastWrite) ? lastWriteTime : IsSameDateTime(lastAccess, creation) ? creationTime : lastAccess; + } + + private static bool IsSameDateTime(DateTime left, DateTime right) + { + // DateTime equality only compares the ticks, while the conversion to a DateTimeOffset also depends on the Kind + // (and, for an ambiguous local time, on whether it is a daylight saving time), so compare all the bits. + Span values = stackalloc DateTime[2]; + values[0] = left; + values[1] = right; + var bits = System.Runtime.InteropServices.MemoryMarshal.Cast(values); + return bits[0] == bits[1]; + } + private FileSystemNode? TryFindNodeSafe(UPath path) { EnterFileSystemShared(); diff --git a/src/Zio/generated/FileSystems/MemoryFileSystemAsync.gen.cs b/src/Zio/generated/FileSystems/MemoryFileSystemAsync.gen.cs index 86f2b38..8b063bd 100644 --- a/src/Zio/generated/FileSystems/MemoryFileSystemAsync.gen.cs +++ b/src/Zio/generated/FileSystems/MemoryFileSystemAsync.gen.cs @@ -860,21 +860,22 @@ protected override async ValueTask CreateSymbolicLinkAsyncImpl(UPath path, UPath { var node = nodePair.Value; var canFollowFolder = searchOption == SearchOption.AllDirectories && nodePair.Value is DirectoryNode; - var fullPath = directoryPath / nodePair.Key; + var fullPath = CombineChildPath(directoryPath, nodePair.Key); if (canFollowFolder) { foldersToProcess.Insert(dirIndex++, fullPath); } + GetDateTimeOffsets(node, out var creationTime, out var lastWriteTime, out var lastAccessTime); var item = new FileSystemItemAsync { FileSystem = this, AbsolutePath = fullPath, Path = fullPath, Attributes = node.Attributes, - CreationTime = node.CreationTime, - LastWriteTime = node.LastWriteTime, - LastAccessTime = node.LastAccessTime, + CreationTime = creationTime, + LastWriteTime = lastWriteTime, + LastAccessTime = lastAccessTime, Length = node is FileNode fileNode ? fileNode.Content.Length : 0, }; if (searchPredicate == null || searchPredicate(ref item)) @@ -1074,6 +1075,45 @@ private static void ValidateFile([NotNull] FileSystemNode? node, UPath srcPath) } } + private static UPath CombineChildPath(UPath directoryPath, string name) + { + // The name of a child node is always a single part of a validated path (no separator, not `.` or `..`), + // so when the directory path is absolute (and normalized, as any UPath), the combined path is already + // normalized: this is equivalent to `directoryPath / name` without validating and normalizing again + // the name and the combined path. + if (!directoryPath.IsAbsolute) + { + return directoryPath / name; + } + + var directory = directoryPath.FullName; + return new UPath(directory.Length == 1 ? "/" + name : directory + "/" + name, true); + } + + private static void GetDateTimeOffsets(FileSystemNode node, out DateTimeOffset creationTime, out DateTimeOffset lastWriteTime, out DateTimeOffset lastAccessTime) + { + // Converting a local DateTime to a DateTimeOffset looks up the UTC offset of the local time zone at that + // time, so convert only once the times that are identical (e.g. for a directory, for a file that has not + // been read or written since it was created, or for a file not read since it was last written). + var creation = node.CreationTime; + var lastWrite = node.LastWriteTime; + var lastAccess = node.LastAccessTime; + creationTime = creation; + lastWriteTime = IsSameDateTime(lastWrite, creation) ? creationTime : lastWrite; + lastAccessTime = IsSameDateTime(lastAccess, lastWrite) ? lastWriteTime : IsSameDateTime(lastAccess, creation) ? creationTime : lastAccess; + } + + private static bool IsSameDateTime(DateTime left, DateTime right) + { + // DateTime equality only compares the ticks, while the conversion to a DateTimeOffset also depends on the Kind + // (and, for an ambiguous local time, on whether it is a daylight saving time), so compare all the bits. + Span values = stackalloc DateTime[2]; + values[0] = left; + values[1] = right; + var bits = System.Runtime.InteropServices.MemoryMarshal.Cast(values); + return bits[0] == bits[1]; + } + private FileSystemNode? TryFindNodeSafe(UPath path) { EnterFileSystemShared();