Weight transfer and queue progress by selected bytes

BitMatch-iPadTests/QueueAccessoryPresentationTests.swift

@@ -174,7 +174,8 @@ struct QueueAccessoryPresentationTests {
             id: id, cardName: name, evidence: nil, destinations: "Backup",
             safetyState: state, progressFraction: progress, action: nil, cause: cause,
             copySummary: name, outcome: nil, destinationNames: ["Backup"],
-            verificationModeName: VerificationMode.standard.rawValue
+            verificationModeName: VerificationMode.standard.rawValue,
+            estimatedWorkBytes: 100
         )
     }
 }

BitMatchTests/ByteWeightedProgressPresentationTests.swift

@@ -0,0 +1,76 @@
+import Foundation
+import Testing
+import BitMatchEngine
+@testable import BitMatch
+
+@MainActor
+struct ByteWeightedProgressPresentationTests {
+    @Test func handoffUsesSelectedByteWorkAndNeverResetsOrFinishesEarly() {
+        let pipeline = OperationProgress(overallProgress: 1, currentFile: nil,
+            filesProcessed: 2, totalFiles: 2, currentStage: .completed,
+            speed: nil, elapsedTime: nil, averageSpeed: nil, peakSpeed: nil,
+            bytesProcessed: 200, totalBytes: 100, workBytesProcessed: 500, totalWorkBytes: 500)
+        let withMHL = CopyVerifyExecutor.pipelineProgress(pipeline, destinationCount: 2, includeMHL: true)
+        #expect(abs(withMHL.overallProgress - 5.0 / 7.0) < 0.00001)
+        let beginning = CopyVerifyExecutor.mhlProgress(.init(bytesProcessed: 0, totalBytes: 200,
+            filesProcessed: 0, totalFiles: 2, published: false), precedingWorkBytes: 500)
+        let midway = CopyVerifyExecutor.mhlProgress(.init(bytesProcessed: 100, totalBytes: 200,
+            filesProcessed: 1, totalFiles: 2, published: false), precedingWorkBytes: 500)
+        let published = CopyVerifyExecutor.mhlProgress(.init(bytesProcessed: 200, totalBytes: 200,
+            filesProcessed: 2, totalFiles: 2, published: true), precedingWorkBytes: 500)
+        #expect(beginning.overallProgress == withMHL.overallProgress)
+        #expect(midway.overallProgress > beginning.overallProgress)
+        #expect(published.overallProgress == 0.99)
+        #expect(published.currentStage == .generating)
+        #expect(CopyVerifyExecutor.pipelineProgress(pipeline, destinationCount: 2, includeMHL: false).overallProgress == 0.99)
+        // Mapping the bar cannot change actual copied bytes.
+        #expect(withMHL.bytesProcessed == pipeline.bytesProcessed)
+    }
+
+    @Test func failedBackupKeepsItsUnfinishedBudgetDuringPartialHandoff() {
+        // Two backups reserved 500 pipeline units + 200 handoff units.
+        // One failed early: only 300 pipeline units and one 100-byte job remain.
+        let partial = CopyVerifyExecutor.mhlProgress(.init(bytesProcessed: 0, totalBytes: 100,
+            filesProcessed: 0, totalFiles: 1, published: false), precedingWorkBytes: 500,
+            precedingCompletedWorkBytes: 300, reservedHandoffBytes: 200)
+        let onePublished = CopyVerifyExecutor.mhlProgress(.init(bytesProcessed: 100, totalBytes: 100,
+            filesProcessed: 1, totalFiles: 1, published: true), precedingWorkBytes: 500,
+            precedingCompletedWorkBytes: 300, reservedHandoffBytes: 200)
+        let noJobs = CopyVerifyExecutor.mhlProgress(.init(bytesProcessed: 0, totalBytes: 0,
+            filesProcessed: 0, totalFiles: 0, published: false), precedingWorkBytes: 500,
+            precedingCompletedWorkBytes: 300, reservedHandoffBytes: 200)
+        #expect(partial.overallProgress == 3.0 / 7.0)
+        #expect(onePublished.overallProgress == 4.0 / 7.0)
+        #expect(noJobs.overallProgress == partial.overallProgress)
+    }
+
+    @Test func tinyFinishedCardCannotSpendAHugeWaitingCardsProgress() {
+        let tiny = row(state: .safeToErase, work: 1)
+        let huge = row(state: .copying(progress: 0), fraction: 0, work: 999)
+        let started = QueueAccessoryPresentation.weightedProgress(rows: [tiny, huge])
+        #expect(started == 0.001)
+        var half = huge
+        half = row(state: .verifying(progress: 50), fraction: 0.5, work: 999)
+        #expect(QueueAccessoryPresentation.weightedProgress(rows: [tiny, half]) == 0.5005)
+        #expect(QueueAccessoryPresentation.weightedProgress(rows: [tiny, row(state: .waiting, work: nil)]) == nil)
+        #expect(QueueAccessoryPresentation.weightedProgress(rows: [row(state: .finishing(progress: 100), fraction: 1, work: 100)]) == 0.99)
+    }
+
+    @Test func destinationBarsPreferByteWorkOverEqualFileCounts() throws {
+        let update = OperationProgress(overallProgress: 0.4, currentFile: nil,
+            filesProcessed: 2, totalFiles: 4, currentStage: .copying, speed: nil,
+            elapsedTime: nil, averageSpeed: nil, peakSpeed: nil, bytesProcessed: 2, totalBytes: 1000,
+            perDestinationTotals: [2, 2], perDestinationCompleted: [1, 1], perDestinationProgress: [0.001, 0.7])
+        let rows = TransferProgressPresentation.rows(destinations: [URL(fileURLWithPath: "/one"), URL(fileURLWithPath: "/two")],
+            progress: update, phase: .copying)
+        #expect(rows[0].fraction == 0.001)
+        #expect(rows[1].fraction == 0.7)
+        #expect(rows[0].countText == "1 of 2")
+    }
+
+    private func row(state: CardSafetyState, fraction: Double? = nil, work: Double?) -> QueueSessionRow {
+        QueueSessionRow(id: UUID(), cardName: "Card", evidence: nil, destinations: "Backup",
+            safetyState: state, progressFraction: fraction, action: nil, cause: nil, copySummary: "",
+            outcome: nil, destinationNames: ["Backup"], verificationModeName: "Standard", estimatedWorkBytes: work)
+    }
+}

BitMatchTests/CopyVerifyExecutorIntegrityTests.swift

@@ -326,6 +326,20 @@ final class CopyVerifyExecutorIntegrityTests: XCTestCase {
         XCTAssertEqual(harness.lastState, .cancelled)
     }
 
+    func testImmediateFinalCallbackReachesAuthoritativeResultsWithoutWaitingForUITask() async throws {
+        let final = OperationProgress(overallProgress: 1, currentFile: nil,
+            filesProcessed: 1, totalFiles: 1, currentStage: .completed, speed: nil,
+            elapsedTime: nil, averageSpeed: nil, peakSpeed: nil, bytesProcessed: 10,
+            totalBytes: 10, workBytesProcessed: 30, totalWorkBytes: 30)
+        let harness = ExecutorHarness(returnedResults: [verifiedResult()], emittedResults: [], terminalProgress: final)
+        harness.onAuthoritativeResults = {
+            XCTAssertEqual(harness.observedProgress.last?.totalWorkBytes, 30)
+            XCTAssertEqual(harness.observedProgress.last?.overallProgress, 0.99)
+        }
+        _ = try await harness.execute()
+        XCTAssertEqual(harness.terminalInfo?.success, true)
+    }
+
     func testMHLReadProgressIsNotFinishedBeforePublication() {
         let update = CopyVerifyExecutor.mhlProgress(.init(bytesProcessed: 50, totalBytes: 100,
                                                          filesProcessed: 0, totalFiles: 1, published: false))
@@ -1014,6 +1028,7 @@ private final class ExecutorHarness {
         reportPrefs: ReportPrefs? = nil,
         cameraSettings: CameraLabelSettings = CameraLabelSettings(),
         thrownError: Error? = nil,
+        terminalProgress: OperationProgress? = nil,
         realFileOperations: FileOperationsService? = nil,
         estimatedFiles: Int? = nil,
         estimatedBytes: Int64? = nil,
@@ -1022,7 +1037,8 @@ private final class ExecutorHarness {
         let fileOperations = ExecutorFileOperationsService(
             returnedResults: returnedResults,
             emittedResults: emittedResults,
-            thrownError: thrownError
+            thrownError: thrownError,
+            terminalProgress: terminalProgress
         )
         let platform = ExecutorPlatformManager(fileOperations: realFileOperations ?? fileOperations)
         self.platform = platform
@@ -1089,11 +1105,13 @@ private final class ExecutorFileOperationsService: FileOperationsService {
     private let returnedResults: [FileOperationResult]
     private let emittedResults: [FileOperationResult]
     private let thrownError: Error?
+    private let terminalProgress: OperationProgress?
 
-    init(returnedResults: [FileOperationResult], emittedResults: [FileOperationResult], thrownError: Error? = nil) {
+    init(returnedResults: [FileOperationResult], emittedResults: [FileOperationResult], thrownError: Error? = nil, terminalProgress: OperationProgress? = nil) {
         self.returnedResults = returnedResults
         self.emittedResults = emittedResults
         self.thrownError = thrownError
+        self.terminalProgress = terminalProgress
     }
 
     func performFileOperation(
@@ -1109,6 +1127,7 @@ private final class ExecutorFileOperationsService: FileOperationsService {
             await onFileResult?(result)
         }
         if let thrownError { throw thrownError }
+        if let terminalProgress { progressCallback(terminalProgress) }
         let sourceManifest = returnedResults.reduce(into: [URL]()) { files, result in
             if !files.contains(result.sourceURL) { files.append(result.sourceURL) }
         }

BitMatchTests/QueueSessionPresentationTests.swift

@@ -50,7 +50,7 @@ struct QueueSessionPresentationTests {
             records: [running], sessionIDs: [running.id], progress: verifyingProgress,
             mountedSourceIDs: []
         )
-        #expect(verifying.rows.first?.oneLineStatus().contains("Verifying 40%") == true)
+        #expect(verifying.rows.first?.oneLineStatus().contains("Verifying 70%") == true)
     }
 
     @Test func fileWorkAtOneHundredPercentIsStillFinishingWhileRecordRuns() throws {

Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift

@@ -562,7 +562,7 @@ public final class PinnedDestinationFile: @unchecked Sendable {
 #endif
 
 public final class DestinationWriter {
-    typealias IOByteCallback = @Sendable (Int, TransferIOPhase, Int, TimeInterval, TimeInterval) -> Void
+    typealias IOByteCallback = @Sendable (Int, String, TransferIOPhase, Int, TimeInterval, TimeInterval) -> Void
     struct SourceReadEvidence: Sendable {
         let digests: VerifiedDigests
         let identity: VerifiedFileIdentity
@@ -956,7 +956,10 @@ public final class DestinationWriter {
                         destinationIndex: destination.index,
                         relativePath: relativePath,
                         hooks: hooks,
-                        onIOBytes: onIOBytes
+                        onIOBytes: { index, path, phase, bytes, duration, ended in
+                            let reusePhase: TransferIOPhase = phase == .readingBack ? .validatingExisting : .sourceValidatingExisting
+                            onIOBytes?(index, path, reusePhase, bytes, duration, ended)
+                        }
                     )
                     pendingReuse.append(destination)
                 } else {
@@ -1026,6 +1029,7 @@ public final class DestinationWriter {
                     throw sourceChangedError()
                 }
                 bytesRead += Int64(data.count)
+                onIOBytes?(-1, relativePath, .sourceReading, data.count, 0, ProcessInfo.processInfo.systemUptime)
                 if let sourceDidRead = hooks?.sourceDidRead {
                     let readCount = data.count
                     data = try await sourceDidRead(source, data)
@@ -1058,7 +1062,7 @@ public final class DestinationWriter {
                                     try file.destination.root.requireIncompletePublicationStatus()
                                 }
                                 try writeAll(chunkData, to: file.temporaryFD) { count, seconds, ended in
-                                    onIOBytes?(file.destination.index, .writing, count, seconds, ended)
+                                    onIOBytes?(file.destination.index, relativePath, .writing, count, seconds, ended)
                                 }
                                 return FanOutWriteResult(file: file, error: nil)
                             } catch {
@@ -1109,7 +1113,7 @@ public final class DestinationWriter {
                             source,
                             algorithms: verificationMode.checksumTypes + [.md5],
                             relativePath: relativePath,
-                            hooks: hooks
+                            hooks: hooks, onIOBytes: onIOBytes
                         )
                     )
                 } else if verificationMode != .quick {
@@ -1573,6 +1577,23 @@ public final class DestinationWriter {
     ) async throws -> VerificationResult {
         let startTime = Date()
         var sourceValues: [ChecksumAlgorithm: String] = [:]
+        let sourceSize = try sourceFileSize(source)
+        func sourceChecksum(_ type: ChecksumAlgorithm) async throws -> String {
+            let lastBytes = Mutex<Int64>(0)
+            return try await checksumService.generateChecksum(for: source, type: type) { fraction, _ in
+                guard fraction.isFinite else { return }
+                let current: Int64 = fraction >= 1 ? sourceSize
+                    : max(0, Int64(Double(sourceSize) * max(0, fraction)))
+                let delta = lastBytes.withLock { last -> Int64 in
+                    let delta = max(0, current - last)
+                    last = max(last, current)
+                    return delta
+                }
+                if delta > 0 {
+                    onIOBytes?(destinationIndex, relativePath, .sourceVerifying, Int(delta), 0, ProcessInfo.processInfo.systemUptime)
+                }
+            }
+        }
         if let sourceReadEvidence {
             guard try sourceIdentity(at: source) == sourceReadEvidence.identity else {
                 throw sourceChangedError()
@@ -1581,23 +1602,15 @@ public final class DestinationWriter {
                 if let value = sourceReadEvidence.digests[type] {
                     sourceValues[type] = value
                 } else {
-                    sourceValues[type] = try await checksumService.generateChecksum(
-                        for: source,
-                        type: type,
-                        progressCallback: nil
-                    )
+                    sourceValues[type] = try await sourceChecksum(type)
                 }
             }
             guard try sourceIdentity(at: source) == sourceReadEvidence.identity else {
                 throw sourceChangedError()
             }
         } else {
             for type in requiredTypes {
-                sourceValues[type] = try await checksumService.generateChecksum(
-                    for: source,
-                    type: type,
-                    progressCallback: nil
-                )
+                sourceValues[type] = try await sourceChecksum(type)
             }
         }
         // MD5 is collected in the same pass even when SHA-256 is the only
@@ -1700,7 +1713,8 @@ public final class DestinationWriter {
         _ source: URL,
         algorithms: [ChecksumAlgorithm],
         relativePath: String,
-        hooks: FanOutHooks?
+        hooks: FanOutHooks?,
+        onIOBytes: IOByteCallback? = nil
     ) async throws -> SourceReadEvidence {
         let handle = try uncachedSourceHandle(for: source)
         hooks?.verificationSourceDidOpen?(relativePath)
@@ -1726,6 +1740,7 @@ public final class DestinationWriter {
             }
             guard count > 0 else { throw sourceChangedError() }
             hooks?.verificationSourceDidRead?(relativePath, count)
+            onIOBytes?(-1, relativePath, .sourceVerifying, count, 0, ProcessInfo.processInfo.systemUptime)
             bytesRead += Int64(count)
         }
         let trailingData = try handle.read(upToCount: 1) ?? Data()
@@ -1779,7 +1794,7 @@ public final class DestinationWriter {
             }
             if count == 0 { break }
             hooks?.destinationDidRead?(destinationIndex, relativePath, count)
-            onIOBytes?(destinationIndex, .readingBack, count, readSeconds, readEnded)
+            onIOBytes?(destinationIndex, relativePath, .readingBack, count, readSeconds, readEnded)
             bytesRead += Int64(count)
             if ordinal != nil, bytesRead >= nextBreadcrumb {
                 nextBreadcrumb += verifyReadBreadcrumbInterval
@@ -1844,7 +1859,8 @@ public final class DestinationWriter {
             }
             hooks?.verificationSourceDidRead?(relativePath, sourceCount)
             hooks?.destinationDidRead?(destinationIndex, relativePath, destinationCount)
-            onIOBytes?(destinationIndex, .readingBack, destinationCount, readSeconds, readEnded)
+            onIOBytes?(destinationIndex, relativePath, .readingBack, destinationCount, readSeconds, readEnded)
+            onIOBytes?(destinationIndex, relativePath, .sourceVerifying, sourceCount, 0, readEnded)
             if !same { return false }
             bytesRead += Int64(sourceCount)
         }

Packages/BitMatchEngine/Sources/BitMatchEngine/Models/OperationProgress.swift

@@ -43,6 +43,10 @@ public struct OperationProgress: Codable, Sendable {
     // Per-destination progress (optional)
     public let perDestinationTotals: [Int]?
     public let perDestinationCompleted: [Int]?
+    /// Presentation work only, distinct from actual copied bytes/evidence.
+    public let workBytesProcessed: Double?
+    public let totalWorkBytes: Double?
+    public let perDestinationProgress: [Double]?
 
     // Convenience initializer for backward compatibility
     public init(overallProgress: Double, currentFile: String?, filesProcessed: Int, totalFiles: Int, currentStage: ProgressStage, speed: Double?, reusedCopies: Int? = nil) {
@@ -62,10 +66,13 @@ public struct OperationProgress: Codable, Sendable {
         self.isASCMHL = nil
         self.perDestinationTotals = nil
         self.perDestinationCompleted = nil
+        self.workBytesProcessed = nil
+        self.totalWorkBytes = nil
+        self.perDestinationProgress = nil
     }
 
     // Full initializer with timing information
-    public init(overallProgress: Double, currentFile: String?, filesProcessed: Int, totalFiles: Int, currentStage: ProgressStage, speed: Double?, elapsedTime: TimeInterval?, averageSpeed: Double?, peakSpeed: Double?, bytesProcessed: Int64?, totalBytes: Int64?, stageProgress: Double? = nil, reusedCopies: Int? = nil, perDestinationTotals: [Int]? = nil, perDestinationCompleted: [Int]? = nil, isASCMHL: Bool? = nil) {
+    public init(overallProgress: Double, currentFile: String?, filesProcessed: Int, totalFiles: Int, currentStage: ProgressStage, speed: Double?, elapsedTime: TimeInterval?, averageSpeed: Double?, peakSpeed: Double?, bytesProcessed: Int64?, totalBytes: Int64?, stageProgress: Double? = nil, reusedCopies: Int? = nil, perDestinationTotals: [Int]? = nil, perDestinationCompleted: [Int]? = nil, isASCMHL: Bool? = nil, workBytesProcessed: Double? = nil, totalWorkBytes: Double? = nil, perDestinationProgress: [Double]? = nil) {
         self.overallProgress = overallProgress
         self.currentFile = currentFile
         self.filesProcessed = filesProcessed
@@ -82,5 +89,20 @@ public struct OperationProgress: Codable, Sendable {
         self.isASCMHL = isASCMHL
         self.perDestinationTotals = perDestinationTotals
         self.perDestinationCompleted = perDestinationCompleted
+        self.workBytesProcessed = workBytesProcessed
+        self.totalWorkBytes = totalWorkBytes
+        self.perDestinationProgress = perDestinationProgress
+    }
+    /// Preserve the engine's measurements while mapping the bar across handoff
+    /// and report work. This changes presentation only.
+    public func replacingOverallProgress(_ fraction: Double) -> OperationProgress {
+        OperationProgress(overallProgress: fraction, currentFile: currentFile,
+            filesProcessed: filesProcessed, totalFiles: totalFiles, currentStage: currentStage,
+            speed: speed, elapsedTime: elapsedTime, averageSpeed: averageSpeed, peakSpeed: peakSpeed,
+            bytesProcessed: bytesProcessed, totalBytes: totalBytes, stageProgress: stageProgress,
+            reusedCopies: reusedCopies, perDestinationTotals: perDestinationTotals,
+            perDestinationCompleted: perDestinationCompleted, isASCMHL: isASCMHL,
+            workBytesProcessed: workBytesProcessed, totalWorkBytes: totalWorkBytes,
+            perDestinationProgress: perDestinationProgress)
     }
 }