Bound memory during ASC MHL rereads and XXH64 Saved Check

Packages/BitMatchEngine/Sources/BitMatchEngine/ASCMHLGenerator.swift

@@ -201,13 +201,23 @@ public enum ASCMHLGenerator: Sendable {
                 var sha = SHA256()
                 var md5 = Insecure.MD5()
                 var count: Int64 = 0
-                while let data = try handle.read(upToCount: 4 * 1_024 * 1_024), !data.isEmpty {
+                while true {
                     try Task.checkCancellation()
-                    sha.update(data: data)
-                    md5.update(data: data)
-                    count += Int64(data.count)
-                    readHooks?.didRead?(path, data.count)
-                    processedBytes += Int64(data.count)
+                    // This is a synchronous, whole-file pass. Drain Foundation's
+                    // temporary read buffer before the next chunk; large exFAT
+                    // media cannot rely on an async suspension to release it.
+                    let chunkCount = try autoreleasepool { () throws -> Int in
+                        let data = try handle.read(upToCount: 4 * 1_024 * 1_024) ?? Data()
+                        if !data.isEmpty {
+                            sha.update(data: data)
+                            md5.update(data: data)
+                        }
+                        return data.count
+                    }
+                    if chunkCount == 0 { break }
+                    count += Int64(chunkCount)
+                    readHooks?.didRead?(path, chunkCount)
+                    processedBytes += Int64(chunkCount)
                     emit()
                 }
                 digestsAndCount = (

Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift

@@ -564,11 +564,18 @@ private enum XXH64 {
         #endif
         var state = State()
         var bytesRead: Int64 = 0
-        while let data = try handle.read(upToCount: 1_048_576), !data.isEmpty {
+        while true {
             try Task.checkCancellation()
             try await PauseGate.waitIfCurrentIsPaused()
-            state.update(data)
-            bytesRead += Int64(data.count)
+            // Check can hash a whole clip without a suspension. Release each
+            // Foundation read buffer before asking for the next chunk.
+            let chunkCount = try autoreleasepool { () throws -> Int in
+                let data = try handle.read(upToCount: 1_048_576) ?? Data()
+                if !data.isEmpty { state.update(data) }
+                return data.count
+            }
+            if chunkCount == 0 { break }
+            bytesRead += Int64(chunkCount)
         }
         guard bytesRead == initial.size, try snapshot(url) == initial else {
             throw NSError(

Packages/BitMatchEngine/Tests/BitMatchEngineTests/ASCMHLMemoryTests.swift

@@ -0,0 +1,84 @@
+import CryptoKit
+import Darwin
+import Foundation
+import XCTest
+@testable import BitMatchEngine
+
+final class ASCMHLMemoryTests: XCTestCase {
+    /// Opt in to keep physical-footprint measurements in an isolated test host.
+    /// Plant: the MHL reread used to retain almost every 4 MiB Foundation
+    /// buffer while hashing one large file, even though verification was bounded.
+    func testLargeDestinationRereadKeepsMemoryBounded() throws {
+        guard ProcessInfo.processInfo.environment["BITMATCH_MHL_MEMORY_TEST"] == "1" else {
+            throw XCTSkip("Set BITMATCH_MHL_MEMORY_TEST=1 for the isolated footprint test")
+        }
+
+        let root = FileManager.default.temporaryDirectory
+            .appendingPathComponent("bitmatch-mhl-memory-\(UUID())", isDirectory: true)
+        try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
+        defer { try? FileManager.default.removeItem(at: root) }
+
+        let media = root.appendingPathComponent("clip.bin")
+        XCTAssertTrue(FileManager.default.createFile(atPath: media.path, contents: nil))
+        let size = UInt64(512 * 1_024 * 1_024)
+        let writer = try FileHandle(forWritingTo: media)
+        try writer.truncate(atOffset: size)
+        try writer.close()
+
+        var sha = SHA256()
+        for _ in 0..<128 {
+            autoreleasepool { sha.update(data: Data(repeating: 0, count: 4 * 1_024 * 1_024)) }
+        }
+        let expected = sha.finalize().map { String(format: "%02x", $0) }.joined()
+        let samples = MHLFootprintSamples()
+
+        let manifest = try ASCMHLGenerator.generateInitialHistory(
+            destinationURL: root,
+            files: [.init(relativePath: "clip.bin", size: Int64(size), expectedSHA256: expected)],
+            startTime: Date(), toolVersion: "memory-test",
+            readHooks: .init(
+                didOpenForRead: { _ in samples.opened() },
+                didRead: { _, count in samples.read(count) },
+                fileSystemType: { "exfat" }
+            )
+        )
+
+        let measured = samples.snapshot()
+        XCTAssertEqual(measured.chunks, 128)
+        XCTAssertEqual(measured.bytes, Int64(size))
+        XCTAssertGreaterThan(measured.initial, 0)
+        XCTAssertLessThan(measured.peak - measured.initial, 128,
+                          "MHL read grew by \(measured.peak - measured.initial) MiB for a 512 MiB file")
+        XCTAssertTrue(FileManager.default.fileExists(atPath: manifest.path))
+        XCTAssertTrue(FileManager.default.fileExists(atPath:
+            manifest.deletingLastPathComponent().appendingPathComponent("ascmhl_chain.xml").path))
+        print("MHL memory initialMiB=\(measured.initial) peakMiB=\(measured.peak) bytes=\(measured.bytes) chunks=\(measured.chunks)")
+    }
+}
+
+private final class MHLFootprintSamples: @unchecked Sendable {
+    private let lock = NSLock()
+    private var initial = 0
+    private var peak = 0
+    private var bytes: Int64 = 0
+    private var chunks = 0
+
+    func opened() {
+        lock.withLock {
+            initial = TransferDiagnosticStore.physicalFootprintMB() ?? 0
+            peak = initial
+        }
+    }
+
+    func read(_ count: Int) {
+        lock.withLock {
+            bytes += Int64(count)
+            chunks += 1
+            peak = max(peak, TransferDiagnosticStore.physicalFootprintMB() ?? 0)
+        }
+    }
+
+    func snapshot() -> (initial: Int, peak: Int, bytes: Int64, chunks: Int) {
+        lock.withLock { (initial, peak, bytes, chunks) }
+    }
+}

Packages/BitMatchEngine/Tests/BitMatchEngineTests/SavedChecksumMemoryTests.swift

@@ -0,0 +1,67 @@
+import Darwin
+import Foundation
+import XCTest
+@testable import BitMatchEngine
+
+final class SavedChecksumMemoryTests: XCTestCase {
+    /// Opt-in footprint test for the XXH64 branch of Check. Other checksum
+    /// algorithms use ChecksumEngine, whose read loops have their own pools.
+    /// Plant: the XXH64 pass retained nearly a whole large file's read buffers.
+    func testXXH64CheckKeepsMemoryBounded() async throws {
+        guard ProcessInfo.processInfo.environment["BITMATCH_CHECK_MEMORY_TEST"] == "1" else {
+            throw XCTSkip("Set BITMATCH_CHECK_MEMORY_TEST=1 for the isolated footprint test")
+        }
+
+        let root = FileManager.default.temporaryDirectory
+            .appendingPathComponent("bitmatch-xxh64-memory-\(UUID())", isDirectory: true)
+        try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
+        defer { try? FileManager.default.removeItem(at: root) }
+
+        let media = root.appendingPathComponent("clip.bin")
+        XCTAssertTrue(FileManager.default.createFile(atPath: media.path, contents: nil))
+        let size = UInt64(512 * 1_024 * 1_024)
+        let writer = try FileHandle(forWritingTo: media)
+        try writer.truncate(atOffset: size)
+        try writer.close()
+
+        // Parsed-evidence construction enters the real Check hashing path;
+        // evidence parsing and XXH64 interoperability are covered elsewhere.
+        let record = SavedChecksumCheck.Record(
+            url: root.appendingPathComponent("ascmhl/reference.mhl"), date: Date(), kind: .ascMHL)
+        let expected = SavedChecksumCheck.ExpectedFile(
+            relativePath: "clip.bin", checksum: "0000000000000000", algorithm: .xxh64, record: record)
+        let discovery = SavedChecksumCheck.Discovery(
+            root: root, expectedFiles: ["clip.bin": expected], records: [record])
+        let sampler = CheckFootprintSamples()
+        let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared)
+        let result = try await checker.check(discovery) { update in
+            sampler.observe(update)
+        }
+
+        XCTAssertEqual(result.changedPaths, ["clip.bin"])
+        let measured = sampler.snapshot()
+        XCTAssertGreaterThan(measured.initial, 0)
+        XCTAssertGreaterThan(measured.completed, 0)
+        XCTAssertLessThan(measured.completed - measured.initial, 128,
+                          "XXH64 Check grew by \(measured.completed - measured.initial) MiB for a 512 MiB file")
+        print("XXH64 Check memory initialMiB=\(measured.initial) completedMiB=\(measured.completed) sizeMiB=512")
+    }
+}
+
+private final class CheckFootprintSamples: @unchecked Sendable {
+    private let lock = NSLock()
+    private var initial = 0
+    private var completed = 0
+
+    func observe(_ update: OperationProgress) {
+        let footprint = TransferDiagnosticStore.physicalFootprintMB() ?? 0
+        lock.withLock {
+            if update.filesProcessed == 0 { initial = footprint }
+            if update.filesProcessed == 1 { completed = footprint }
+        }
+    }
+
+    func snapshot() -> (initial: Int, completed: Int) {
+        lock.withLock { (initial, completed) }
+    }
+}

docs/ISSUE23_MEMORY_AUDIT.md

@@ -0,0 +1,21 @@
+# Large-file memory follow-up
+
+Issue #23 reports BitMatch reaching about 55 GB of application memory during a transfer. The report alone does not identify the app version or stage. The findings below are real local bugs, but they do not establish which path the reporter hit.
+
+## Confirmed paths
+
+- **ASC MHL generation:** on filesystems such as exFAT, BitMatch rereads the verified destination because the filesystem cannot supply a reliable change time for readback reuse. The synchronous 4 MiB read/hash loop retained Foundation buffers across chunks. This loop has shipped since 0.2.1. Its read, SHA-256 and MD5 updates now run inside one autorelease pool per chunk. Cancellation, uncached destination reads, progress, file-stability checks and history publication are unchanged.
+- **Saved Check with XXH64 evidence:** the separate 1 MiB XXH64 read loop had the same retention. Its read and hash update now share a per-chunk pool. MD5 and SHA-256 Check already use pooled `ChecksumEngine` reads. This path does not run during an ordinary copy/verify transfer.
+
+Both regressions use 512 MiB sparse files, verify the actual engine operation, and are opt-in so that physical-footprint assertions run in an isolated test host:
+
+```sh
+BITMATCH_MHL_MEMORY_TEST=1 arch -arm64 swift test --package-path Packages/BitMatchEngine -c release --build-system native --filter ASCMHLMemoryTests --no-parallel
+BITMATCH_CHECK_MEMORY_TEST=1 arch -arm64 swift test --package-path Packages/BitMatchEngine -c release --build-system native --filter SavedChecksumMemoryTests --no-parallel
+```
+
+Before the fixes, the MHL test failed at 10 → 521 MiB (511 MiB growth); after, it passed at 10 → 10 MiB while reading all 128 chunks and publishing the manifest and chain. The XXH64 Check test failed at 5 → 521 MiB (516 MiB retained at completion); its final isolated run passed at 5 → 9 MiB and still reported the deliberately mismatched checksum as changed. The MHL figure is the peak sampled after each chunk; the Check figure is the footprint reported at the end of checking the file.
+
+The existing mounted-exFAT Standard copy → verification → MHL handoff test also passes with the patch. It exercises the real exFAT driver but uses small media. The sparse-file memory tests exercise large logical reads on APFS; the MHL test forces the exFAT reread policy. None of these tests reproduces the reporter's physical drive, footage, or 55 GB workload. The next diagnostic export should identify the version and last stage reached.
+
+Local validation after both fixes: the full serial engine suite passed (126 XCTest cases, 17 intentional opt-in skips, zero failures; 151 Swift Testing cases passed). The explicit mounted-exFAT handoff test passed. The Mac app test build passed (1,099 passed, 13 skipped, zero failed), and the iPad/iPhone simulator build succeeded. The two isolated memory tests passed under their respective flags. Logs and the Mac result summary are in the local ignored `dist/validation/issue23-memory/` directory.