From a36983208929b14d8caafa3a48da8be0caa74db1 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Sat, 10 Oct 2026 10:39:21 -0400 Subject: [PATCH] Add reviewed AppleDouble exclusions with explicit transfer scope --- .../WorkflowSnapshotTests.swift | 26 +++ BitMatchTests/AppleDoubleWorkflowTests.swift | 183 ++++++++++++++++++ BitMatchTests/WorkflowSnapshotTests.swift | 24 +++ CHANGELOG.md | 3 + .../BitMatchEngine/EngineProtocols.swift | 7 +- .../Evidence/EvidenceWriter.swift | 52 ++++- .../File/AppleDoubleSelection.swift | 101 ++++++++++ .../Models/CameraLabelSettings.swift | 3 + .../BitMatchEngine/Models/Results.swift | 3 + .../Sources/BitMatchEngine/SharedLogger.swift | 1 + .../BitMatchEngine/TransferCompletion.swift | 30 ++- .../BitMatchEngine/TransferPipeline.swift | 34 +++- .../AppleDoubleFilesystemMatrixTests.swift | 158 +++++++++++++++ .../AppleDoubleSelectionTests.swift | 177 +++++++++++++++++ README.md | 2 +- .../CompletionVerdictPresentation.swift | 5 + .../Models/ResultStatusPresentation.swift | 1 + .../Models/TransferOutcomePresentation.swift | 1 + Shared/Core/Services/CopyVerifyExecutor.swift | 11 +- Shared/Core/Services/ReportExporter.swift | 2 +- .../Core/Services/SharedAppCoordinator.swift | 38 +++- .../ViewModels/AppleDoubleReviewModel.swift | 67 +++++++ .../Setup/AppleDoubleSelectionView.swift | 33 ++++ .../Views/Setup/CoordinatorSetupScreen.swift | 8 +- docs/GUIDE.md | 8 + docs/ISSUE16_APPLEDOUBLE.md | 48 +++++ docs/TODO.md | 8 +- 27 files changed, 1006 insertions(+), 28 deletions(-) create mode 100644 BitMatchTests/AppleDoubleWorkflowTests.swift create mode 100644 Packages/BitMatchEngine/Sources/BitMatchEngine/File/AppleDoubleSelection.swift create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/AppleDoubleFilesystemMatrixTests.swift create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/AppleDoubleSelectionTests.swift create mode 100644 Shared/Core/ViewModels/AppleDoubleReviewModel.swift create mode 100644 Shared/Views/Setup/AppleDoubleSelectionView.swift create mode 100644 docs/ISSUE16_APPLEDOUBLE.md diff --git a/BitMatch-iPadTests/WorkflowSnapshotTests.swift b/BitMatch-iPadTests/WorkflowSnapshotTests.swift index 38167ee9..7040e7a1 100644 --- a/BitMatch-iPadTests/WorkflowSnapshotTests.swift +++ b/BitMatch-iPadTests/WorkflowSnapshotTests.swift @@ -102,6 +102,32 @@ final class WorkflowSnapshotTests: XCTestCase { size: CGSize(width: 820, height: 1_100), name: "ipad-history-diagnostics") } + func testCaptureAppleDoubleControls() async throws { + try requireCaptureConfiguration() + let fixture = try SnapshotFixture() + defer { fixture.restoreGlobalPreferences() } + fixture.seedSetup() + let coordinator = fixture.coordinator + let source = try XCTUnwrap(coordinator.sourceURL) + let media = source.appendingPathComponent("Camera clip.bin") + try Data("media".utf8).write(to: media) + var bytes = [UInt8](repeating: 0, count: 42) + func put(_ value: UInt32, _ offset: Int) { + for index in 0..<4 { bytes[offset + index] = UInt8(truncatingIfNeeded: value >> (24 - 8 * index)) } + } + put(0x00051607, 0); put(0x00020000, 4); bytes[25] = 1 + put(2, 26); put(38, 30); put(4, 34) + try Data(bytes).write(to: source.appendingPathComponent("._Camera clip.bin")) + let review = coordinator.appleDoubleSelection + review.enabled = true + let deadline = ContinuousClock.now.advanced(by: .seconds(5)) + while review.paths == nil && ContinuousClock.now < deadline { try await Task.sleep(for: .milliseconds(10)) } + XCTAssertEqual(review.paths, ["._Camera clip.bin"]) + try capture(AppleDoubleSelectionView(review: review).padding(), size: CGSize(width: 320, height: 650), name: "iphone-appledouble-selection") + try capture(AppleDoubleSelectionView(review: review).padding(), size: CGSize(width: 500, height: 650), name: "narrowpad-appledouble-selection") + try capture(AppleDoubleSelectionView(review: review).padding(), size: CGSize(width: 820, height: 650), name: "ipad-appledouble-selection") + } + private func requireCaptureConfiguration() throws { let environment = ProcessInfo.processInfo.environment let enabled = environment["BITMATCH_CAPTURE_WORKFLOW_SNAPSHOTS"] == "1" diff --git a/BitMatchTests/AppleDoubleWorkflowTests.swift b/BitMatchTests/AppleDoubleWorkflowTests.swift new file mode 100644 index 00000000..ab333d7d --- /dev/null +++ b/BitMatchTests/AppleDoubleWorkflowTests.swift @@ -0,0 +1,183 @@ +import XCTest +import Foundation +import PDFKit +import SwiftUI +@testable import BitMatch +import BitMatchEngine + +@MainActor +final class AppleDoubleWorkflowTests: XCTestCase { + private func seed(_ source: URL) throws { + var bytes = [UInt8](repeating: 0, count: 42) + func put(_ value: UInt32, _ offset: Int) { + for index in 0..<4 { bytes[offset + index] = UInt8(truncatingIfNeeded: value >> (24 - 8 * index)) } + } + put(0x00051607, 0); put(0x00020000, 4); bytes[25] = 1 + put(2, 26); put(38, 30); put(4, 34) + try Data(bytes).write(to: source.appendingPathComponent("._A.ARW")) + try Data("legitimate camera file".utf8).write(to: source.appendingPathComponent("._notes.txt")) + } + + // Plant: make the excluded outcome a success; the saved-verdict assertions fail. + func testExecutorReportsMHLAndHistoryRetainExplicitSelection() async throws { + try await FileOperationsTestLock.shared.run { + let folders = try CoordinatorFolders(); defer { folders.cleanup() } + try seed(folders.source) + var settings = CameraLabelSettings() + settings.excludedAppleDoublePaths = try AppleDoubleSelection.review(source: folders.source) + let journal = LocalTransferJournal(fileURL: folders.journalURL) + let id = try journal.enqueue(sourceURL: folders.source, destinationURLs: [folders.primary, folders.secondary], + verificationMode: .standard, cameraSettings: settings, reportSettings: ReportPrefs(), generateASCMHL: true) + try journal.markRunning(id: id) + let executor = CopyVerifyExecutor(platformManager: MacOSPlatformManager.shared, + timingService: OperationTimingService(), errorService: ErrorReportingService(), + stateService: OperationStateService(), backgroundTaskService: IOSBackgroundTaskService.shared) + var terminal: OperationCompletionInfo? + let execution = try await executor.execute(config: CopyVerifyConfig(operationId: id, + sourceURL: folders.source, destinationURLs: [folders.primary, folders.secondary], verificationMode: .standard, + cameraLabelSettings: settings, reportSettings: ReportPrefs(), estimatedFiles: 3, estimatedBytes: 0, + currentMode: .copyAndVerify, generateASCMHL: true), callbacks: CopyVerifyCallbacks( + onProgress: { _ in }, onResult: { _ in }, onStateChange: { state in + if case .completed(let info) = state { terminal = info } + }, onAuthoritativeResults: { _ in })) + let operation = try XCTUnwrap(execution) + XCTAssertEqual(operation.results.count, 6) + XCTAssertEqual(operation.results.filter(\.excludedAppleDouble).count, 2) + XCTAssertEqual(operation.sourceManifest?.count, 3) + let completion = try XCTUnwrap(terminal) + XCTAssertFalse(completion.success) + XCTAssertTrue(completion.message.contains("Selected files verified")) + let presentation = CompletionVerdictPresentation.make(state: .completed(completion), + rows: TransferCompletion.rows(from: operation), hasErrors: false, hasCriticalErrors: false) + XCTAssertEqual(presentation.title, "Selected-file transfer") + XCTAssertTrue(presentation.sourceGuidance.contains("intentionally excluded")) + let rows = TransferCompletion.rows(from: operation) + let requestedUnsafeReport: EnhancedJSONReport = try EvidenceWriter.makeEnhancedJSONReport( + results: rows, jobID: id, started: operation.startTime, finished: Date(), kind: .copyAndVerify, + sourceURL: folders.source, destinationURLs: [folders.primary, folders.secondary], fileCount: 6, + matchCount: 4, totalBytesProcessed: 0, duration: 1, workers: 1, + prefs: ReportPrefs(), project: nil, safeToErase: true) + XCTAssertEqual(requestedUnsafeReport.safeToErase, false) + XCTAssertEqual(requestedUnsafeReport.selection?.countsBasis, "recorded-source-results") + try journal.finish(id: id, results: rows, summary: completion.message, hadIssues: !completion.success, + sourceFingerprint: operation.sourceFingerprint) + let record = try XCTUnwrap(journal.records.first) + XCTAssertEqual(record.state, .issues) + XCTAssertEqual(record.cameraSettings.excludedAppleDoublePaths, ["._A.ARW"]) + XCTAssertEqual(record.results.filter { ResultOutcome(statusText: $0.status) == .excludedAppleDouble }.count, 2) + XCTAssertNil(journal.matchingVerifiedRecord(sourceFingerprint: operation.sourceFingerprint!)) + let persisted = try JSONDecoder().decode([LocalTransferRecord].self, from: Data(contentsOf: folders.journalURL)) + XCTAssertEqual(persisted.first?.results.count, 6) + let reports = try FileManager.default.contentsOfDirectory(at: folders.primary.appendingPathComponent("Reports"), includingPropertiesForKeys: nil) + let json = try XCTUnwrap(reports.first { $0.pathExtension == "json" }) + let report = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(contentsOf: json)) as? [String: Any]) + XCTAssertEqual(report["safeToErase"] as? Bool, false) + let selection = try XCTUnwrap(report["selection"] as? [String: Any]) + XCTAssertEqual(selection["sourceFileCount"] as? Int, 3) + XCTAssertEqual(selection["selectedFileCount"] as? Int, 2) + XCTAssertEqual(selection["excludedFileCount"] as? Int, 1) + let statistics = try XCTUnwrap(report["statistics"] as? [String: Any]) + XCTAssertEqual(statistics["issues"] as? Int, 0) + XCTAssertEqual(statistics["excluded"] as? Int, 2) + let csv = try String(contentsOf: XCTUnwrap(reports.first { $0.pathExtension == "csv" }), encoding: .utf8) + XCTAssertTrue(csv.contains("Excluded Source Files")) + XCTAssertTrue(csv.contains("Whole Card Backed Up")) + let pdf = try XCTUnwrap(PDFDocument(url: XCTUnwrap(reports.first { $0.pathExtension == "pdf" }))) + XCTAssertTrue(try XCTUnwrap(pdf.string).contains("intentionally excluded")) + for destination in [folders.primary, folders.secondary] { + let output = destination.appendingPathComponent(folders.source.lastPathComponent) + XCTAssertFalse(FileManager.default.fileExists(atPath: output.appendingPathComponent("._A.ARW").path)) + let mhlFiles = try FileManager.default.contentsOfDirectory(at: output.appendingPathComponent("ascmhl"), includingPropertiesForKeys: nil) + let text = try String(contentsOf: XCTUnwrap(mhlFiles.first { $0.pathExtension == "mhl" }), encoding: .utf8) + XCTAssertFalse(text.contains("._A.ARW")) + XCTAssertTrue(text.contains("._notes.txt")) + XCTAssertTrue(text.contains("A.ARW")) + } + } + } + + func testQueuedSelectionSurvivesDifferentLivePreferences() async throws { + try await FileOperationsTestLock.shared.run { + let folders = try CoordinatorFolders(); defer { folders.cleanup() } + try seed(folders.source) + var settings = CameraLabelSettings() + settings.excludedAppleDoublePaths = try AppleDoubleSelection.review(source: folders.source) + var reports = ReportPrefs(); reports.makeReport = false + let journal = LocalTransferJournal(fileURL: folders.journalURL) + let id = try journal.enqueue(sourceURL: folders.source, destinationURLs: [folders.primary, folders.secondary], + verificationMode: .standard, cameraSettings: settings, reportSettings: reports, generateASCMHL: false) + let coordinator = SharedAppCoordinator(platformManager: MacOSPlatformManager.shared, + transferJournal: journal, defaults: folders.defaults) + XCTAssertFalse(coordinator.appleDoubleSelection.enabled) + coordinator.startQueue() + let finished = await waitUntil(timeout: .seconds(10)) { + journal.records.first(where: { $0.id == id })?.state == .issues && !coordinator.isOperationInProgress + } + if !finished { coordinator.cancelOperation() } + XCTAssertTrue(finished, coordinator.queueMessage ?? "Queue did not finish") + let record = try XCTUnwrap(journal.records.first { $0.id == id }) + XCTAssertEqual(record.results.count, 6) + XCTAssertEqual(record.results.filter { ResultOutcome(statusText: $0.status) == .excludedAppleDouble }.count, 2) + XCTAssertEqual(record.results.filter(\.isVerifiedStatus).count, 4) + XCTAssertEqual(record.cameraSettings.excludedAppleDoublePaths, ["._A.ARW"]) + XCTAssertFalse(coordinator.appleDoubleSelection.enabled) + XCTAssertFalse(folders.defaults.bool(forKey: "BitMatchExcludeAppleDouble")) + } + } + + func testCancelAfterExclusionRowsKeepsPartialEvidenceAndNeverCompletes() async throws { + try await FileOperationsTestLock.shared.run { + let folders = try CoordinatorFolders(); defer { folders.cleanup() } + try seed(folders.source) + var settings = CameraLabelSettings() + settings.excludedAppleDoublePaths = try AppleDoubleSelection.review(source: folders.source) + let executor = CopyVerifyExecutor(platformManager: MacOSPlatformManager.shared, + timingService: OperationTimingService(), errorService: ErrorReportingService(), + stateService: OperationStateService(), backgroundTaskService: IOSBackgroundTaskService.shared) + var rows: [ResultRow] = [] + var cancelled = false + var completed = false + do { + _ = try await executor.execute(config: CopyVerifyConfig(operationId: UUID(), + sourceURL: folders.source, destinationURLs: [folders.primary], verificationMode: .standard, + cameraLabelSettings: settings, reportSettings: ReportPrefs(), estimatedFiles: 3, estimatedBytes: 0, + currentMode: .copyAndVerify, generateASCMHL: true), callbacks: CopyVerifyCallbacks( + onProgress: { _ in }, onResult: { row in + rows.append(row) + if ResultOutcome(statusText: row.status) == .excludedAppleDouble { executor.cancel() } + }, onStateChange: { state in + if state == .cancelled { cancelled = true } + if case .completed = state { completed = true } + }, onAuthoritativeResults: { _ in })) + XCTFail("Cancelled transfer returned normally") + } catch is CancellationError { + // The executor retains the cancelled UI state and propagates interruption. + } + XCTAssertTrue(cancelled) + XCTAssertFalse(completed) + XCTAssertEqual(rows.filter { ResultOutcome(statusText: $0.status) == .excludedAppleDouble }.count, 1) + XCTAssertFalse(rows.contains(where: \.isVerifiedStatus)) + XCTAssertFalse(FileManager.default.fileExists(atPath: folders.primary.appendingPathComponent("Reports").path)) + } + } + + func testPreviewDefaultPersistenceAndClearAreSafe() async throws { + let folders = try CoordinatorFolders(); defer { folders.cleanup() } + try seed(folders.source) + let review = AppleDoubleReviewModel(defaults: folders.defaults) + XCTAssertFalse(review.enabled) + review.refresh(source: folders.source) + review.enabled = true + XCTAssertNotNil(review.readinessIssue) + let ready = await waitUntil { review.paths != nil } + XCTAssertTrue(ready) + XCTAssertEqual(review.paths, ["._A.ARW"]) + XCTAssertTrue(try XCTUnwrap(review.summaryLine).contains("1 AppleDouble")) + XCTAssertTrue(AppleDoubleReviewModel(defaults: folders.defaults).enabled) + review.refresh(source: nil) + XCTAssertNil(review.paths) + XCTAssertNil(review.readinessIssue) + review.enabled = false + XCTAssertFalse(AppleDoubleReviewModel(defaults: folders.defaults).enabled) + } +} diff --git a/BitMatchTests/WorkflowSnapshotTests.swift b/BitMatchTests/WorkflowSnapshotTests.swift index 5d4c1ad3..0560ca9c 100644 --- a/BitMatchTests/WorkflowSnapshotTests.swift +++ b/BitMatchTests/WorkflowSnapshotTests.swift @@ -41,6 +41,30 @@ final class WorkflowSnapshotTests: XCTestCase { } + func testCaptureAppleDoubleControls() async throws { + try requireCaptureConfiguration() + let fixture = try SnapshotFixture() + defer { fixture.restoreGlobalPreferences() } + fixture.seedSetup() + let coordinator = fixture.sharedCoordinator + let source = try XCTUnwrap(coordinator.sourceURL) + let media = source.appendingPathComponent("Camera clip.bin") + try Data("media".utf8).write(to: media) + var bytes = [UInt8](repeating: 0, count: 42) + func put(_ value: UInt32, _ offset: Int) { + for index in 0..<4 { bytes[offset + index] = UInt8(truncatingIfNeeded: value >> (24 - 8 * index)) } + } + put(0x00051607, 0); put(0x00020000, 4); bytes[25] = 1 + put(2, 26); put(38, 30); put(4, 34) + try Data(bytes).write(to: source.appendingPathComponent("._Camera clip.bin")) + let review = coordinator.appleDoubleSelection + review.enabled = true + let deadline = ContinuousClock.now.advanced(by: .seconds(5)) + while review.paths == nil && ContinuousClock.now < deadline { try await Task.sleep(for: .milliseconds(10)) } + XCTAssertEqual(review.paths, ["._Camera clip.bin"]) + try capture(AppleDoubleSelectionView(review: review).padding(), size: CGSize(width: 580, height: 600), name: "mac-appledouble-selection") + } + private func settle() async { for _ in 0..<8 { await Task.yield() } try? await Task.sleep(nanoseconds: 100_000_000) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40fe1ab7..05d4aba0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +- Added an optional AppleDouble exclusion setting under Advanced on Mac, iPad and iPhone. Everything is still preserved by default. Review the list before starting; legitimate `._` files and other sidecars stay in the transfer. +- Filtered transfers list their exclusions in history and reports. Selected files are still checked, but the whole card is never marked safe to erase when files were skipped. + ## [0.2.5] - 2026-10-05 A reliability update for checking backups and investigating transfer problems. diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/EngineProtocols.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/EngineProtocols.swift index 691af208..f4384d68 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/EngineProtocols.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/EngineProtocols.swift @@ -98,7 +98,10 @@ public struct FileOperationResult: Sendable { /// so this run verified it without writing a new copy. public let wasReused: Bool + public var excludedAppleDouble: Bool = false + public var outcome: ResultOutcome { + if excludedAppleDouble { return .excludedAppleDouble } if let verification = verificationResult { return verification.isValid ? .verified : .checksumMismatch } @@ -120,11 +123,13 @@ public struct FileOperationResult: Sendable { } func preservingReuse(_ reused: Bool) -> Self { - Self( + var result = Self( sourceURL: sourceURL, destinationURL: destinationURL, success: success, error: error, fileSize: fileSize, verificationResult: verificationResult, processingTime: processingTime, clipIntegrity: clipIntegrity, wasReused: success && (wasReused || reused) ) + result.excludedAppleDouble = excludedAppleDouble + return result } } diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/Evidence/EvidenceWriter.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/Evidence/EvidenceWriter.swift index 607a7d5a..c6ea4623 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/Evidence/EvidenceWriter.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/Evidence/EvidenceWriter.swift @@ -18,6 +18,27 @@ public enum ReportExportError: LocalizedError, Sendable { } } +public struct TransferSelectionEvidence: Codable, Sendable { + public let policy: String + /// Partial exports may not contain every source file; counts describe retained rows. + public let countsBasis: String + public let sourceFileCount: Int + public let selectedFileCount: Int + public let excludedFileCount: Int + public let excludedSourcePaths: [String] + + public init(results: [ResultRow]) { + let all = Set(results.map(\.path)) + let excluded = Set(results.filter { ResultOutcome(statusText: $0.status) == .excludedAppleDouble }.map(\.path)) + policy = "exclude-appledouble-companions" + countsBasis = "recorded-source-results" + sourceFileCount = all.count + selectedFileCount = all.subtracting(excluded).count + excludedFileCount = excluded.count + excludedSourcePaths = excluded.sorted() + } +} + // MARK: - Enhanced JSON Report Structures /// Identifies the PDF published last for this report, including its byte digest. public struct ReportPDFEvidence: Codable, Sendable { @@ -52,6 +73,8 @@ public struct EnhancedJSONReport: Codable, Sendable public var notes: String? = nil public let safeToErase: Bool? public let pdfEvidence: ReportPDFEvidence? + /// Explicit scope, present only when source files were deliberately omitted. + public var selection: TransferSelectionEvidence? = nil public struct SourceInfo: Codable, Sendable { public let path: String @@ -95,16 +118,18 @@ public struct EnhancedJSONReport: Codable, Sendable public let totalBytes: Int64 public let matches: Int public let issues: Int + public var excluded: Int? = nil public let successRate: Double public let averageFileSize: Int64 public let largestFile: FileInfo? public let smallestFile: FileInfo? - public init(totalFiles: Int, totalBytes: Int64, matches: Int, issues: Int, successRate: Double, averageFileSize: Int64, largestFile: FileInfo?, smallestFile: FileInfo?) { + public init(totalFiles: Int, totalBytes: Int64, matches: Int, issues: Int, successRate: Double, averageFileSize: Int64, largestFile: FileInfo?, smallestFile: FileInfo?, excluded: Int? = nil) { self.totalFiles = totalFiles self.totalBytes = totalBytes self.matches = matches self.issues = issues + self.excluded = excluded self.successRate = successRate self.averageFileSize = averageFileSize self.largestFile = largestFile @@ -505,13 +530,21 @@ public enum EvidenceWriter: Sendable { // Add summary at the end csvContent += "\n# Summary\n" csvContent += csvRow(["Total Files", String(results.count)]) + if results.contains(where: { ResultOutcome(statusText: $0.status) == .excludedAppleDouble }) { + let scope = TransferSelectionEvidence(results: results) + csvContent += csvRow(["Selection Policy", scope.policy]) + csvContent += csvRow(["Recorded Source Files", String(scope.sourceFileCount)]) + csvContent += csvRow(["Selected Recorded Source Files", String(scope.selectedFileCount)]) + csvContent += csvRow(["Excluded Source Files", String(scope.excludedFileCount)]) + csvContent += csvRow(["Whole Card Backed Up", "No — keep the source"]) + } csvContent += csvRow(["Started", dateFormatter.string(from: started)]) csvContent += csvRow(["Finished", dateFormatter.string(from: started.addingTimeInterval(duration))]) // Verified and copied-but-unverified are counted apart (Promise 2): // a Quick copy is not a match. csvContent += csvRow(["Verified", String(results.filter { $0.isVerifiedStatus }.count)]) csvContent += csvRow(["Copied, not verified", String(results.filter { isMatchStatus($0.status) && !$0.isVerifiedStatus }.count)]) - csvContent += csvRow(["Issues", String(results.filter { !isMatchStatus($0.status) }.count)]) + csvContent += csvRow(["Issues", String(results.filter { !isMatchStatus($0.status) && ResultOutcome(statusText: $0.status) != .excludedAppleDouble }.count)]) csvContent += csvRow(["Duration", "\(String(format: "%.2f", duration)) seconds"]) if copyDurationSeconds != nil || verifyDurationSeconds != nil { csvContent += csvRow(["Copy Duration", copyDurationSeconds.map { "\(String(format: "%.2f", $0)) seconds" } ?? ""]) @@ -612,6 +645,7 @@ public enum EvidenceWriter: Sendable { safeToErase: Bool? = nil, pdfEvidence: ReportPDFEvidence? = nil ) throws -> EnhancedJSONReport { + let safeToErase = results.contains { ResultOutcome(statusText: $0.status) == .excludedAppleDouble } ? false : safeToErase // Calculate file extensions breakdown var extensions: [String: Int] = [:] var largestFile: EnhancedJSONReport.FileInfo? @@ -642,7 +676,7 @@ public enum EvidenceWriter: Sendable { // Calculate issues by type var issuesByType: [String: Int] = [:] - for result in results.filter({ !isMatchStatus($0.status) }) { + for result in results.filter({ !isMatchStatus($0.status) && ResultOutcome(statusText: $0.status) != .excludedAppleDouble }) { let key = normalizedStatus(result.status) issuesByType[key, default: 0] += 1 } @@ -725,7 +759,7 @@ public enum EvidenceWriter: Sendable { } // Create the enhanced report - return EnhancedJSONReport( + var report = EnhancedJSONReport( reportVersion: safeToErase == nil ? "3.0" : "3.1", timestamp: finished, jobId: jobID, @@ -738,11 +772,13 @@ public enum EvidenceWriter: Sendable { matches: matchCount, // Failed rows only: a copied-but-unverified file is neither a // match nor an issue. - issues: results.filter { !isMatchStatus($0.status) }.count, + issues: results.filter { !isMatchStatus($0.status) && ResultOutcome(statusText: $0.status) != .excludedAppleDouble }.count, successRate: fileCount > 0 ? Double(matchCount) / Double(fileCount) * 100 : 100, averageFileSize: averageFileSize, largestFile: largestFile, - smallestFile: smallestFile + smallestFile: smallestFile, + excluded: results.contains { ResultOutcome(statusText: $0.status) == .excludedAppleDouble } + ? results.filter { ResultOutcome(statusText: $0.status) == .excludedAppleDouble }.count : nil ), extensions: extensions, performance: EnhancedJSONReport.Performance( @@ -774,6 +810,10 @@ public enum EvidenceWriter: Sendable { notes: normalizedNotes(prefs.notes), safeToErase: safeToErase, pdfEvidence: pdfEvidence ) + if results.contains(where: { ResultOutcome(statusText: $0.status) == .excludedAppleDouble }) { + report.selection = TransferSelectionEvidence(results: results) + } + return report } /// Automatic reports use the checksums retained by verification. Re-reading diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/AppleDoubleSelection.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/AppleDoubleSelection.swift new file mode 100644 index 00000000..ee0b63f6 --- /dev/null +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/AppleDoubleSelection.swift @@ -0,0 +1,101 @@ +import Foundation +import Darwin + +/// An explicit, reviewed transfer selection. A filename alone never authorizes +/// exclusion: it must describe a bounded AppleDouble v2 container and have a +/// regular-file or directory companion. All other source data is preserved. +public enum AppleDoubleSelection { + // RFC 1740 appendices A–C describe these metadata entries. Fixed-size + // records must not be truncated. Finder info may also carry macOS attrs. + // https://www.rfc-editor.org/rfc/rfc1740.html#appendix-C + private static let minimumEntryLengths: [UInt32: UInt64] = [ + 2: 0, 3: 0, 4: 0, 8: 16, 9: 32, 10: 4, 11: 8, 12: 2, + 13: 0, 14: 4, 15: 4 + ] + + public static func companions(in manifest: [FileEntry]) throws -> [FileEntry] { + try manifest.filter { entry in + try Task.checkCancellation() + guard entry.url.lastPathComponent.hasPrefix("._"), entry.size >= 26 else { return false } + let name = String(entry.url.lastPathComponent.dropFirst(2)) + guard !name.isEmpty else { return false } + let companion = entry.url.deletingLastPathComponent().appendingPathComponent(name) + var companionInfo = stat() + guard lstat(companion.path, &companionInfo) == 0, + (companionInfo.st_mode & S_IFMT) == S_IFREG || (companionInfo.st_mode & S_IFMT) == S_IFDIR else { return false } + let fd = open(entry.url.path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC) + guard fd >= 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + defer { close(fd) } + var before = stat() + guard fstat(fd, &before) == 0, (before.st_mode & S_IFMT) == S_IFREG, + Int64(before.st_size) == entry.size else { + throw FileOperationError.unsafeOperation("Source metadata changed while reviewing exclusions. Review the source again.") + } + func read(_ count: Int, at offset: Int64) throws -> [UInt8] { + var buffer = [UInt8](repeating: 0, count: count) + var done = 0 + while done < count { + try Task.checkCancellation() + let received = buffer.withUnsafeMutableBytes { bytes in + pread(fd, bytes.baseAddress!.advanced(by: done), count - done, offset + Int64(done)) + } + if received < 0 && errno == EINTR { continue } + guard received > 0 else { + throw FileOperationError.unsafeOperation("Unable to read source metadata while reviewing exclusions.") + } + done += received + } + return buffer + } + let header = try read(26, at: 0) + func uint32(_ bytes: [UInt8], _ offset: Int) -> UInt32 { + bytes[offset.. 0, count <= 1024, 26 + count * 12 <= entry.size else { return false } + let records = try read(count * 12, at: 26) + var ids = Set() + var ranges: [(UInt64, UInt64)] = [] + for index in 0..= minimumLength, + ids.insert(id).inserted, + offset >= UInt64(26 + count * 12), offset + length <= UInt64(entry.size) else { return false } + ranges.append((offset, offset + length)) + } + let ordered = ranges.filter { $0.1 > $0.0 }.sorted { $0.0 < $1.0 } + for index in ordered.indices.dropFirst() { + guard ordered[index - 1].1 <= ordered[index].0 else { return false } + } + var after = stat() + guard fstat(fd, &after) == 0, before.st_size == after.st_size, + before.st_mtimespec.tv_sec == after.st_mtimespec.tv_sec, before.st_mtimespec.tv_nsec == after.st_mtimespec.tv_nsec, + before.st_ctimespec.tv_sec == after.st_ctimespec.tv_sec, before.st_ctimespec.tv_nsec == after.st_ctimespec.tv_nsec else { + throw FileOperationError.unsafeOperation("Source metadata changed while reviewing exclusions. Review the source again.") + } + return true + }.sorted { $0.relativePath < $1.relativePath } + } + + public static func review(source: URL) throws -> [String] { + try companions(in: CardSource.enumerateRegularFiles(base: source)).map(\.relativePath) + } + + /// Re-prove the classification at Start. A stale review must never silently + /// omit a new file or a file whose contents are no longer AppleDouble. + public static func excluded(in manifest: [FileEntry], reviewedPaths: [String]?) throws -> [FileEntry] { + guard let reviewedPaths else { return [] } + let entries = try companions(in: manifest) + guard Set(reviewedPaths).count == reviewedPaths.count, + entries.map(\.relativePath) == reviewedPaths.sorted() else { + throw FileOperationError.unsafeOperation("AppleDouble exclusions changed since review. Review the source again; nothing was copied.") + } + return entries + } +} diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/Models/CameraLabelSettings.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/Models/CameraLabelSettings.swift index fe42c5d7..6d6dda90 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/Models/CameraLabelSettings.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/Models/CameraLabelSettings.swift @@ -10,6 +10,9 @@ public struct CameraLabelSettings: Codable, Sendable { public var groupByCamera: Bool = false public var generateUniqueName: Bool = true public var destinationPathComponents: [String]? = nil + /// Reviewed selection carried by this legacy transfer-settings container. + /// nil preserves all files and decodes older queued transfers safely. + public var excludedAppleDoublePaths: [String]? = nil /// Sanitizes a string for safe use as a folder/file name component. /// Prevents path traversal attacks and removes dangerous characters. diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/Models/Results.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/Models/Results.swift index 634e2611..ed23ece0 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/Models/Results.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/Models/Results.swift @@ -9,6 +9,8 @@ public enum ResultOutcome: CaseIterable, Equatable, Sendable { case verified /// Copied without verification (Quick mode). A success, never "verified". case copiedUnverified + /// Intentionally omitted by the reviewed selection; never a success. + case excludedAppleDouble case checksumMismatch case failed @@ -16,6 +18,7 @@ public enum ResultOutcome: CaseIterable, Equatable, Sendable { switch self { case .verified: "✅ Verified" case .copiedUnverified: "✅ Copied" + case .excludedAppleDouble: "⏭ Excluded AppleDouble (selected by user)" case .checksumMismatch: "⚠️ Checksum Mismatch" case .failed: "❌ Failed" } diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift index c8027e53..ac84f932 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift @@ -47,6 +47,7 @@ extension SharedLogger { case verifyStarted, verifyOpenedDestination, verifyRead, verifyDigestFinished, verifyFinished case verifyConfiguration, clipInspectionStarted, clipInspectionFinished case compareStarted, comparePhase, compareProgress, compareFinished + case appleDoubleExcluded } public static func transferEvent(_ event: TransferEvent, run: UUID?, code: Int = 0, taskCancelled: Bool = false, explicit: Bool = false) { diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferCompletion.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferCompletion.swift index 86b95af6..b6976e7b 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferCompletion.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferCompletion.swift @@ -232,6 +232,12 @@ public enum TransferCompletion: Sendable { source: URL, settings: CameraLabelSettings ) -> (jobs: [ASCMHLJob], issues: [String]) { + // A filtered inventory describes only selected, verified destination + // contents. The report/history retain excluded rows and an unsafe + // whole-card verdict; the history is never a whole-source assertion. + let results = results.filter { !$0.excludedAppleDouble } + let excludedPaths = Set(settings.excludedAppleDoublePaths ?? []) + let sourceFiles = sourceFiles.map { files in files.filter { !excludedPaths.contains($0.relativePath(to: source)) } } let entries = results.map { CoverageEntry(source: $0.sourceURL, destination: $0.destinationURL) } let coverage = coverageAnalysis( entries: entries, sourceFiles: sourceFiles, destinations: destinations, @@ -380,7 +386,22 @@ public enum TransferCompletion: Sendable { source: source, settings: settings ) - let incompleteCoverageIssues = coverageIssues(coverage) + let excluded = rows.filter { ResultOutcome(statusText: $0.status) == .excludedAppleDouble } + let excludedCount = Set(excluded.map(\.path)).count + let selectedCoverage: CoverageAnalysis + if excluded.isEmpty { + selectedCoverage = coverage + } else { + let excludedSourcePaths = Set(excluded.map(\.path)) + let selectedSources = sourceFiles.map { files in files.filter { !excludedSourcePaths.contains($0.path) } } + selectedCoverage = coverageAnalysis(entries: coverageRows.map { + CoverageEntry(source: URL(fileURLWithPath: $0.path), + destination: $0.destinationPath.map { URL(fileURLWithPath: $0) }) + }, sourceFiles: selectedSources, destinations: destinations, source: source, settings: settings) + } + let incompleteCoverageIssues = coverageIssues(excludedCount > 0 ? selectedCoverage : coverage) + let excludedDescription = "\(excludedCount) AppleDouble \(excludedCount == 1 ? "file" : "files") intentionally excluded" + let actualIssueCount = rows.filter { !$0.isSuccessStatus && ResultOutcome(statusText: $0.status) != .excludedAppleDouble }.count let issueCount = rows.filter { !$0.isSuccessStatus }.count let fileResultsSucceeded = !rows.isEmpty && issueCount == 0 // Outside Quick, a row that was copied but not verified keeps the @@ -400,6 +421,13 @@ public enum TransferCompletion: Sendable { } else { fileResultsMessage = mode == .quick || !everyRowVerified ? "All files copied" : "All files copied and verified" } + } else if excludedCount > 0 { + let selected = rows.filter { ResultOutcome(statusText: $0.status) != .excludedAppleDouble } + fileResultsMessage = !selected.isEmpty && selectedCoverage.isExact && selected.allSatisfy(\.isVerifiedStatus) + ? "Selected files verified; \(excludedDescription). Keep the source: the whole card is not backed up" + : (mode == .quick && actualIssueCount == 0 && selectedCoverage.isExact + ? "Selected files copied without verification; \(excludedDescription). Keep the source" + : "\(excludedDescription); selected file results need review. Keep the source") } else { fileResultsMessage = issueCount == 1 ? "1 file failed" : "\(issueCount) files failed" } diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift index 5daa455f..2d46fdc5 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift @@ -453,11 +453,20 @@ public final class TransferPipeline: FileOperationsService, Sendable { // Step 2: Build one fail-closed source manifest before safety validation. SharedLogger.debug("Prep: enumerating source manifest at \(operation.sourceURL.path)", category: .transfer) - let sourceManifest = try CardSource.enumerateRegularFiles(base: operation.sourceURL) + let completeSourceManifest = try CardSource.enumerateRegularFiles(base: operation.sourceURL) + let excluded = try AppleDoubleSelection.excluded(in: completeSourceManifest, reviewedPaths: operation.settings.excludedAppleDoublePaths) + if !excluded.isEmpty { + SharedLogger.transferEvent(.appleDoubleExcluded, run: TransferDiagnostics.runID ?? operation.id, code: excluded.count) + } + let excludedPaths = Set(excluded.map(\.relativePath)) + let sourceManifest = excluded.isEmpty ? completeSourceManifest + : completeSourceManifest.filter { !excludedPaths.contains($0.relativePath) } guard !sourceManifest.isEmpty else { - throw FileOperationError.unsafeOperation("Source folder is empty. Choose a source that contains files.") + throw FileOperationError.unsafeOperation(excluded.isEmpty + ? "Source folder is empty. Choose a source that contains files." + : "No files remain after AppleDouble exclusions. Nothing was copied; keep the source.") } - let sourceFingerprint = SourceFingerprint.make(sourceManifest) + let sourceFingerprint = SourceFingerprint.make(completeSourceManifest) let manifestURLByRelativePath = Dictionary( sourceManifest.map { ($0.relativePath, $0.url) }, uniquingKeysWith: { first, _ in first } @@ -489,6 +498,18 @@ public final class TransferPipeline: FileOperationsService, Sendable { let totalStageUnits = operation.verificationMode == .quick ? 1 : 2 // Perf 2: time-based throttle on progress callbacks (500ms) let ledger = RunLedger(destinationCount: destinationCount, filesPerDestination: perSourceFileCount, throttle: 0.5) + // Retain an exclusion row for every source/destination pair before any + // copying, so cancellation and saved history keep the selected scope. + for (index, destination) in operation.destinationURLs.enumerated() { + let root = try SafetyValidator.resolvedDestinationRootChecked(source: operation.sourceURL, destination: destination, settings: operation.settings) + for entry in excluded { + var row = FileOperationResult(sourceURL: entry.url, destinationURL: root.appendingPathComponent(entry.relativePath), + success: false, error: nil, fileSize: entry.size, verificationResult: nil, processingTime: 0) + row.excludedAppleDouble = true + await ledger.record(row, relativePath: entry.relativePath, destination: index) + await onFileResult?(row) + } + } // Free space was checked once, above, by SafetyValidator: the // measured source plus 1 GB, the rule Setup shows. @@ -510,6 +531,9 @@ public final class TransferPipeline: FileOperationsService, Sendable { let elapsed = now.timeIntervalSince(startTime) let speed = elapsed > 0 ? Double(snapshot.bytesCopied) / elapsed : nil let totalBytes: Int64 = { + // The caller's scan includes omitted metadata. For a filtered + // run use the measured selected bytes per backup instead. + if !excludedPaths.isEmpty { return manifestBytes } if let estimate = operation.estimatedTotalBytes, estimate > 0 { return estimate } if snapshot.filesCopied > 0 { return safeMultiply(Int64(totalFiles), snapshot.bytesCopied / Int64(snapshot.filesCopied)) @@ -918,7 +942,7 @@ public final class TransferPipeline: FileOperationsService, Sendable { // change invalidates the whole run, including a file added after the // initial manifest was captured. let finalSourceManifest = try CardSource.enumerateRegularFiles(base: operation.sourceURL) - if let reason = sourceChangeReason(initial: sourceManifest, current: finalSourceManifest) { + if let reason = sourceChangeReason(initial: completeSourceManifest, current: finalSourceManifest) { let error = completionGateError(reason) for (destIndex, destinationURL) in operation.destinationURLs.enumerated() { let root = pinnedDestinations[destIndex]?.logicalRootURL @@ -1061,7 +1085,7 @@ public final class TransferPipeline: FileOperationsService, Sendable { startTime: operation.startTime, endTime: Date(), results: finalResults, - sourceManifest: sourceManifest.map(\.url), + sourceManifest: completeSourceManifest.map(\.url), sourceFingerprint: sourceFingerprint, verificationMode: operation.verificationMode, settings: operation.settings, diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/AppleDoubleFilesystemMatrixTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/AppleDoubleFilesystemMatrixTests.swift new file mode 100644 index 00000000..25b66a88 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/AppleDoubleFilesystemMatrixTests.swift @@ -0,0 +1,158 @@ +#if os(macOS) +import Foundation +import CryptoKit +import Synchronization +import XCTest +@testable import BitMatchEngine + +/// Real drivers and real macOS-created metadata, not filename-only stand-ins. +/// Disk images exercise filesystem semantics, not physical reader/dock faults. +final class AppleDoubleFilesystemMatrixTests: XCTestCase { + func testGeneratedMetadataAcrossAPFSHFSAndExFATInitialRepeatAndFilteredCopies() async throws { + try await FileOperationsTestLock.shared.run { + let fm = FileManager.default + let root = fm.temporaryDirectory.appendingPathComponent("bitmatch-issue16-matrix-\(UUID())") + try fm.createDirectory(at: root, withIntermediateDirectories: true) + var mounted: [URL] = [] + var allDetached = true + func run(_ executable: String, _ args: [String]) throws { + let process = Process(); process.executableURL = URL(fileURLWithPath: executable); process.arguments = args + process.standardOutput = FileHandle.nullDevice + try process.run(); process.waitUntilExit() + guard process.terminationStatus == 0 else { throw NSError(domain: "MatrixSetup", code: Int(process.terminationStatus)) } + } + defer { + for mount in mounted.reversed() { + do { try run("/usr/bin/hdiutil", ["detach", "-quiet", mount.path]) } + catch { allDetached = false; XCTFail("Owned image retained at \(root.path): \(error)") } + } + if allDetached { try? fm.removeItem(at: root) } + } + var sources: [URL] = [] + var backups: [URL] = [] + for (index, fs) in ["APFS", "HFS+", "ExFAT"].enumerated() { + for role in ["source", "backup"] { + let image = root.appendingPathComponent("\(role)-\(index)") + let mount = root.appendingPathComponent("mnt-\(role)-\(index)") + try fm.createDirectory(at: mount, withIntermediateDirectories: true) + try run("/usr/bin/hdiutil", ["create", "-quiet", "-size", "3g", "-type", "SPARSE", "-fs", fs, "-volname", "BM16\(index)\(role)", "-o", image.path]) + try run("/usr/bin/hdiutil", ["attach", "-quiet", "-nobrowse", "-mountpoint", mount.path, image.path + ".sparseimage"]) + mounted.append(mount) + if role == "source" { sources.append(mount) } else { backups.append(mount) } + } + } + // Generate the canonical companion on exFAT through xattr, then use + // those real bytes for APFS/HFS+ sources, where xattrs are native. + let seed = sources[2].appendingPathComponent("seed.bin") + try Data("seed".utf8).write(to: seed) + try run("/usr/bin/xattr", ["-w", "com.apple.quarantine", "0081;00000001;BM16Source;", seed.path]) + let companionBytes = try Data(contentsOf: sources[2].appendingPathComponent("._seed.bin")) + for (sourceIndex, volume) in sources.enumerated() { + let card = volume.appendingPathComponent("Card") + try fm.createDirectory(at: card, withIntermediateDirectories: true) + let media = card.appendingPathComponent("clip.bin") + let bytes = Data(repeating: UInt8(sourceIndex + 17), count: 64 * 1024) + try bytes.write(to: media) + if sourceIndex == 2 { + try run("/usr/bin/xattr", ["-w", "com.apple.quarantine", "0081;00000001;BM16Source;", media.path]) + } else { + try companionBytes.write(to: card.appendingPathComponent("._clip.bin")) + } + try Data("legitimate camera sidecar".utf8).write(to: card.appendingPathComponent("._notes.txt")) + let before = try CardSource.enumerateRegularFiles(base: card) + let sourceHashes = try Dictionary(uniqueKeysWithValues: before.map { ($0.relativePath, SHA256.hash(data: try Data(contentsOf: $0.url))) }) + let excluded = try AppleDoubleSelection.review(source: card) + XCTAssertEqual(excluded, ["._clip.bin"], "source filesystem \(sourceIndex)") + for backupIndex in backups.indices { + let secondary = (backupIndex + 1) % backups.count + let targets = [backups[backupIndex], backups[secondary]].map { $0.appendingPathComponent("pair-\(sourceIndex)-\(backupIndex)") } + for target in targets { try fm.createDirectory(at: target, withIntermediateDirectories: true) } + var settings = CameraLabelSettings() + // Initial and repeat preserve everything by default. + for attempt in 0..<2 { + let trace = Mutex<[String]>([]) + let traceEnabled = sourceIndex == 0 && backupIndex == 2 && attempt == 0 + let hooks: DestinationWriter.FanOutHooks? = traceEnabled ? .init( + beforePublish: { index, root in + guard index == 0 else { return } + let exists = FileManager.default.fileExists(atPath: root.appendingPathComponent("._clip.bin").path) + trace.withLock { $0.append("before publication: companion exists=\(exists)") } + }, afterPublish: { index, root in + guard index == 0 else { return } + let exists = FileManager.default.fileExists(atPath: root.appendingPathComponent("._clip.bin").path) + trace.withLock { $0.append("after publication: companion exists=\(exists)") } + }) : nil + let operation = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared, fanOutHooks: hooks) + .performFileOperation(sourceURL: card, destinationURLs: targets, verificationMode: .standard, + settings: settings, progressCallback: { _ in }, onFileResult: nil) + XCTAssertEqual(operation.results.count, before.count * 2) + if traceEnabled { + let observations = trace.withLock { $0 } + XCTAssertTrue(observations.contains("before publication: companion exists=false")) + XCTAssertTrue(observations.contains("after publication: companion exists=true")) + print("Issue16 fresh exFAT trace: " + observations.joined(separator: "; ")) + } + for result in operation.results { + let relative = result.sourceURL.relativePath(to: card) + if result.success { + XCTAssertEqual(result.verificationResult?.isValid, true) + XCTAssertEqual(SHA256.hash(data: try Data(contentsOf: result.destinationURL)), sourceHashes[relative]) + } else { + // Confirmed fresh-copy behavior: publication on + // exFAT may create a different metadata companion. + // Refusal is required; overwriting it is not a fix. + XCTAssertEqual(relative, "._clip.bin") + XCTAssertTrue(result.destinationURL.path.hasPrefix(backups[2].path)) + XCTAssertTrue(result.error?.localizedDescription.contains("hidden companion") == true) + XCTAssertNotEqual(SHA256.hash(data: try Data(contentsOf: result.destinationURL)), sourceHashes[relative]) + } + } + } + // Deliberately conflicting real metadata must remain intact; + // exclusion avoids comparing/writing it without hiding media failures. + let output = targets[0].appendingPathComponent("Card") + var changed = try Data(contentsOf: output.appendingPathComponent("._clip.bin")) + changed[changed.count - 1] ^= 1 + try changed.write(to: output.appendingPathComponent("._clip.bin")) + let conflict = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared) + .performFileOperation(sourceURL: card, destinationURLs: targets, verificationMode: .standard, + settings: settings, progressCallback: { _ in }, onFileResult: nil) + XCTAssertTrue(conflict.results.contains { $0.sourceURL.lastPathComponent == "._clip.bin" && !$0.success }) + settings.excludedAppleDoublePaths = excluded + let filtered = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared) + .performFileOperation(sourceURL: card, destinationURLs: targets, verificationMode: .standard, + settings: settings, progressCallback: { _ in }, onFileResult: nil) + XCTAssertEqual(filtered.results.filter(\.excludedAppleDouble).count, 2) + XCTAssertTrue(filtered.results.filter { !$0.excludedAppleDouble }.allSatisfy { $0.verificationResult?.isValid == true }) + XCTAssertEqual(try Data(contentsOf: output.appendingPathComponent("._clip.bin")), changed) + XCTAssertEqual(filtered.sourceManifest?.count, before.count) + // First filtered copy into empty targets, then a repeat. + let freshTargets = [backups[backupIndex], backups[secondary]].map { + $0.appendingPathComponent("filtered-\(sourceIndex)-\(backupIndex)") + } + for target in freshTargets { try fm.createDirectory(at: target, withIntermediateDirectories: true) } + for _ in 0..<2 { + let fresh = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared) + .performFileOperation(sourceURL: card, destinationURLs: freshTargets, verificationMode: .standard, + settings: settings, progressCallback: { _ in }, onFileResult: nil) + XCTAssertEqual(fresh.results.filter(\.excludedAppleDouble).count, 2) + for result in fresh.results where !result.excludedAppleDouble { + XCTAssertEqual(result.verificationResult?.isValid, true) + XCTAssertEqual(SHA256.hash(data: try Data(contentsOf: result.destinationURL)), + sourceHashes[result.sourceURL.relativePath(to: card)]) + } + } + } + let after = try CardSource.enumerateRegularFiles(base: card) + XCTAssertEqual(after.map(\.relativePath).sorted(), before.map(\.relativePath).sorted()) + XCTAssertEqual(Dictionary(uniqueKeysWithValues: after.map { ($0.relativePath, $0.modificationDate) }), + Dictionary(uniqueKeysWithValues: before.map { ($0.relativePath, $0.modificationDate) })) + for entry in after { + XCTAssertEqual(SHA256.hash(data: try Data(contentsOf: entry.url)), sourceHashes[entry.relativePath]) + } + } + print("Issue16 matrix: 3 source × 3 primary backup filesystems, 2 backups, initial/repeat/conflict/filtered/fresh-filtered/repeated-filtered = 54 transfers; fresh exFAT metadata conflicts retained, selected-file output and unchanged-source hashes checked.") + } + } +} +#endif diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/AppleDoubleSelectionTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/AppleDoubleSelectionTests.swift new file mode 100644 index 00000000..9173c5dc --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/AppleDoubleSelectionTests.swift @@ -0,0 +1,177 @@ +import Foundation +import XCTest +import CryptoKit +import Synchronization +@testable import BitMatchEngine + +final class AppleDoubleSelectionTests: XCTestCase { + private var root: URL! + override func setUpWithError() throws { + root = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-selection-\(UUID())") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + } + override func tearDownWithError() throws { try FileManager.default.removeItem(at: root) } + + private func container(id: UInt32 = 2) -> Data { + var bytes = [UInt8](repeating: 0, count: 42) + func put(_ value: UInt32, _ offset: Int) { + for index in 0..<4 { bytes[offset + index] = UInt8(truncatingIfNeeded: value >> (24 - 8 * index)) } + } + put(0x00051607, 0); put(0x00020000, 4) + bytes[25] = 1 + put(id, 26); put(38, 30); put(4, 34) + return Data(bytes) + } + private func seed() throws { + try Data("media".utf8).write(to: root.appendingPathComponent("clip.bin")) + try container().write(to: root.appendingPathComponent("._clip.bin")) + } + + // Plant: classify every ._* filename as metadata; the legitimate-file assertions fail. + func testOnlyRecognizedPairedMetadataIsEligible() throws { + try seed() + try Data("legitimate sidecar".utf8).write(to: root.appendingPathComponent("._notes.txt")) + try Data("paired".utf8).write(to: root.appendingPathComponent("notes.txt")) + try container(id: 1).write(to: root.appendingPathComponent("._data.bin")) + try Data("paired".utf8).write(to: root.appendingPathComponent("data.bin")) + try container().write(to: root.appendingPathComponent("._orphan.bin")) + for (name, bytes) in [("unknown.bin", container(id: 99)), ("short.bin", Data(container().prefix(27))), + ("short-finder.bin", container(id: 9)), ("short-dates.bin", container(id: 8))] { + try Data("paired".utf8).write(to: root.appendingPathComponent(name)) + try bytes.write(to: root.appendingPathComponent("._" + name)) + } + var outOfBounds = container() + outOfBounds[34] = 0xff + try Data("paired".utf8).write(to: root.appendingPathComponent("bounds.bin")) + try outOfBounds.write(to: root.appendingPathComponent("._bounds.bin")) + try FileManager.default.createSymbolicLink(at: root.appendingPathComponent("alias.bin"), + withDestinationURL: root.appendingPathComponent("clip.bin")) + try container().write(to: root.appendingPathComponent("._alias.bin")) + XCTAssertEqual(try AppleDoubleSelection.review(source: root), ["._clip.bin"]) + let manifest = try CardSource.enumerateRegularFiles(base: root) + XCTAssertTrue(try AppleDoubleSelection.excluded(in: manifest, reviewedPaths: nil).isEmpty) + XCTAssertEqual(try AppleDoubleSelection.excluded(in: manifest, reviewedPaths: ["._clip.bin"]).count, 1) + } + + // Plant: remove reviewed-path equality; a newly eligible file is silently dropped. + func testChangedReviewRefusesBeforeAnyDestinationWrite() async throws { + try seed() + let reviewed = try AppleDoubleSelection.review(source: root) + try Data("more media".utf8).write(to: root.appendingPathComponent("other.bin")) + try container().write(to: root.appendingPathComponent("._other.bin")) + var settings = CameraLabelSettings(); settings.excludedAppleDoublePaths = reviewed + let backup = root.deletingLastPathComponent().appendingPathComponent("bitmatch-selection-backup-\(UUID())") + try FileManager.default.createDirectory(at: backup, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: backup) } + do { + _ = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared) + .performFileOperation(sourceURL: root, destinationURLs: [backup], verificationMode: .standard, + settings: settings, progressCallback: { _ in }, onFileResult: nil) + XCTFail("Stale review started copying") + } catch let error as FileOperationError { + XCTAssertTrue(error.localizedDescription.contains("exclusions changed")) + } + XCTAssertTrue(try FileManager.default.contentsOfDirectory(atPath: backup.path).isEmpty) + } + + func testChangedContainerAndDuplicatePathsCannotAuthorizeExclusion() throws { + try seed() + let reviewed = try AppleDoubleSelection.review(source: root) + try Data("now legitimate data".utf8).write(to: root.appendingPathComponent("._clip.bin")) + XCTAssertThrowsError(try AppleDoubleSelection.excluded(in: CardSource.enumerateRegularFiles(base: root), reviewedPaths: reviewed)) + try container().write(to: root.appendingPathComponent("._clip.bin")) + XCTAssertThrowsError(try AppleDoubleSelection.excluded(in: CardSource.enumerateRegularFiles(base: root), reviewedPaths: reviewed + reviewed)) + } + + func testOlderSettingsPreserveAllFilesAndNewSelectionRoundTrips() throws { + let old = try JSONEncoder().encode(CameraLabelSettings()) + let decoded = try JSONDecoder().decode(CameraLabelSettings.self, from: old) + XCTAssertNil(decoded.excludedAppleDoublePaths) + var selection = decoded; selection.excludedAppleDoublePaths = ["._clip.bin"] + let restored = try JSONDecoder().decode(CameraLabelSettings.self, from: JSONEncoder().encode(selection)) + XCTAssertEqual(restored.excludedAppleDoublePaths, ["._clip.bin"]) + XCTAssertFalse(ResultOutcome.excludedAppleDouble.isSuccess) + XCTAssertFalse(ResultRow.isVerifiedStatus(ResultOutcome.excludedAppleDouble.statusText)) + var excluded = FileOperationResult(sourceURL: root, destinationURL: root, success: false, error: nil, + fileSize: 42, verificationResult: nil, processingTime: 0) + excluded.excludedAppleDouble = true + XCTAssertEqual(excluded.preservingReuse(false).outcome, .excludedAppleDouble) + } + + func testEveryVerificationModeHonorsSelectionAndMissingMediaStillFailsClosed() async throws { + try seed() + var settings = CameraLabelSettings() + settings.excludedAppleDoublePaths = try AppleDoubleSelection.review(source: root) + let targetRoot = root.deletingLastPathComponent().appendingPathComponent("bitmatch-selection-modes-\(UUID())") + try FileManager.default.createDirectory(at: targetRoot, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: targetRoot) } + for mode in [VerificationMode.quick, .standard, .thorough, .paranoid] { + let backup = targetRoot.appendingPathComponent(mode.rawValue) + try FileManager.default.createDirectory(at: backup, withIntermediateDirectories: true) + let progress = Mutex<[OperationProgress]>([]) + let operation = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared) + .performFileOperation(sourceURL: root, destinationURLs: [backup], verificationMode: mode, + settings: settings, estimatedTotalBytes: 9_000_000, + progressCallback: { update in progress.withLock { $0.append(update) } }, onFileResult: nil) + let copyProgress = progress.withLock { $0.filter { $0.currentStage == .copying || $0.currentStage == .verifying } } + XCTAssertFalse(copyProgress.isEmpty) + XCTAssertTrue(copyProgress.allSatisfy { $0.totalBytes == 5 && $0.totalFiles == 1 }) + XCTAssertEqual(operation.results.filter(\.excludedAppleDouble).count, 1) + XCTAssertTrue(operation.results.filter { !$0.excludedAppleDouble }.allSatisfy(\.success)) + let verdict = TransferCompletion.verdict(rows: TransferCompletion.rows(from: operation), + sourceFiles: operation.sourceManifest, destinations: [backup], source: root, settings: settings, + mode: mode, generateASCMHL: false, handoffIssues: [], reportIssue: nil, + project: .init(didPersist: true, locallySafe: nil)) + XCTAssertFalse(verdict.success) + XCTAssertFalse(verdict.copiedNotVerified) + XCTAssertTrue(verdict.message.contains(mode == .quick ? "without verification" : "Selected files verified")) + let missing = TransferCompletion.verdict(rows: TransferCompletion.rows(from: operation).filter { + ResultOutcome(statusText: $0.status) == .excludedAppleDouble + }, sourceFiles: operation.sourceManifest, destinations: [backup], source: root, settings: settings, + mode: mode, generateASCMHL: false, handoffIssues: [], reportIssue: nil, + project: .init(didPersist: true, locallySafe: nil)) + XCTAssertFalse(missing.success) + XCTAssertFalse(missing.message.contains("Selected files verified")) + XCTAssertTrue(missing.message.contains("no result"), missing.message) + } + } + + func testMultiDestinationSelectionRetainsFullInventoryAndNeverMakesWholeCardSafe() async throws { + try seed() + try Data("keep legitimate file".utf8).write(to: root.appendingPathComponent("._notes.txt")) + let parent = root.deletingLastPathComponent().appendingPathComponent("bitmatch-selection-targets-\(UUID())") + let destinations = [parent.appendingPathComponent("A"), parent.appendingPathComponent("B")] + for url in destinations { try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) } + defer { try? FileManager.default.removeItem(at: parent) } + var settings = CameraLabelSettings(); settings.excludedAppleDoublePaths = try AppleDoubleSelection.review(source: root) + for pipelined in [true, false] { + let operation = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared, pipelinedVerification: pipelined) + .performFileOperation(sourceURL: root, destinationURLs: destinations, verificationMode: .standard, + settings: settings, progressCallback: { _ in }, onFileResult: nil) + XCTAssertEqual(operation.sourceManifest?.count, 3) + XCTAssertEqual(operation.results.count, 6) + XCTAssertEqual(operation.results.filter(\.excludedAppleDouble).count, 2) + XCTAssertTrue(operation.results.filter { !$0.excludedAppleDouble }.allSatisfy { $0.verificationResult?.isValid == true }) + let rows = TransferCompletion.rows(from: operation) + let verdict = TransferCompletion.verdict(rows: rows, sourceFiles: operation.sourceManifest, + destinations: destinations, source: root, settings: settings, mode: .standard, + generateASCMHL: false, handoffIssues: [], reportIssue: nil, project: .init(didPersist: true, locallySafe: nil)) + XCTAssertFalse(verdict.success) + XCTAssertTrue(verdict.message.contains("Selected files verified")) + XCTAssertTrue(verdict.message.contains("Keep the source")) + let scope = TransferSelectionEvidence(results: rows) + XCTAssertEqual(scope.sourceFileCount, 3) + XCTAssertEqual(scope.selectedFileCount, 2) + XCTAssertEqual(scope.excludedFileCount, 1) + let plan = TransferCompletion.ascmhlPlan(results: operation.results, sourceFiles: operation.sourceManifest, + destinations: destinations, source: root, settings: settings) + XCTAssertEqual(plan.jobs.count, 2) + XCTAssertTrue(plan.issues.isEmpty, "\(plan.issues)") + XCTAssertTrue(plan.jobs.allSatisfy { $0.files.count == 2 }) + for job in plan.jobs { + XCTAssertFalse(FileManager.default.fileExists(atPath: job.root.appendingPathComponent("._clip.bin").path)) + XCTAssertEqual(try Data(contentsOf: job.root.appendingPathComponent("._notes.txt")), Data("keep legitimate file".utf8)) + } + } + } +} diff --git a/README.md b/README.md index 1a0020c0..f4c3e22d 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Plug in your card and drives, allow BitMatch to use them once, and it picks up t ## What It Does - **Multi destination copy**: the card is read once and written to every drive at the same time, then each drive is read back and checked with SHA-256. Each destination gets its own results. -- **Careful about "safe to erase"**: a card or folder that changes during the copy, online-only iCloud/Dropbox files, or two "backups" on the same physical drive never count as safe. Hidden `._` files are copied and verified too. +- **Careful about "safe to erase"**: a card or folder that changes during the copy, online-only iCloud/Dropbox files, or two "backups" on the same physical drive never count as safe. Hidden `._` files are copied and verified by default. You can exclude recognized AppleDouble companions under **Advanced**, with a list of what will be skipped. That copy can verify the selected files, but it won't mark the whole card safe to erase. - **Photographer jobs** for shoots with several photographers, cameras, and cards. Save the setup instead of rebuilding it every time. - **Camera detection** for Sony, Canon, ARRI, RED, Blackmagic, Panasonic, Fujifilm, GoPro, DJI, Insta360, and generic DCIM. - **Check** for stuff you already copied: compare two folders, or check a backup against its saved checksums. diff --git a/Shared/Core/Models/CompletionVerdictPresentation.swift b/Shared/Core/Models/CompletionVerdictPresentation.swift index 8f6314b5..b17aace3 100644 --- a/Shared/Core/Models/CompletionVerdictPresentation.swift +++ b/Shared/Core/Models/CompletionVerdictPresentation.swift @@ -81,6 +81,11 @@ struct CompletionVerdictPresentation: Equatable, Sendable { sourceGuidance: "Do not erase \(card)." ) } + if case .completed(let info) = state, + rows.contains(where: { ResultOutcome(statusText: $0.status) == .excludedAppleDouble }) { + return Self(title: "Selected-file transfer", detail: info.message, symbol: "minus.circle", + sourceGuidance: "Keep \(card). AppleDouble companions were intentionally excluded, so the whole card is not backed up.") + } let verdict = CompletionVerdict.resolve( state: state, rows: rows, diff --git a/Shared/Core/Models/ResultStatusPresentation.swift b/Shared/Core/Models/ResultStatusPresentation.swift index 74fbf6d2..9f1c94e9 100644 --- a/Shared/Core/Models/ResultStatusPresentation.swift +++ b/Shared/Core/Models/ResultStatusPresentation.swift @@ -47,6 +47,7 @@ struct ResultStatusPresentation: Equatable, Sendable { // error, and must not share the orange "missing"/warning // triangle. A distinct red glyph tells the two apart. case .checksumMismatch: return Self(tone: .failure, symbol: "xmark.octagon.fill") + case .excludedAppleDouble: return Self(tone: .warning, symbol: "minus.circle") case .failed: return Self(tone: .failure, symbol: "xmark.circle") } } diff --git a/Shared/Core/Models/TransferOutcomePresentation.swift b/Shared/Core/Models/TransferOutcomePresentation.swift index 3c51e301..4b4ea458 100644 --- a/Shared/Core/Models/TransferOutcomePresentation.swift +++ b/Shared/Core/Models/TransferOutcomePresentation.swift @@ -301,6 +301,7 @@ struct TransferOutcomePresentation: Equatable, Sendable { switch outcome { case .verified: return "Verified" case .copiedUnverified: return "Copied, not verified" + case .excludedAppleDouble: return "Excluded AppleDouble" case .checksumMismatch: return "Checksum mismatch" case .failed: return "Failed" } diff --git a/Shared/Core/Services/CopyVerifyExecutor.swift b/Shared/Core/Services/CopyVerifyExecutor.swift index 21d8a536..1f74a59f 100644 --- a/Shared/Core/Services/CopyVerifyExecutor.swift +++ b/Shared/Core/Services/CopyVerifyExecutor.swift @@ -308,7 +308,9 @@ final class CopyVerifyExecutor { loggedFileError = true SharedLogger.transferError(error, run: diagnosticRun) } - timingService.recordFileResult(fileResult, verificationMode: verificationMode) + if !fileResult.excludedAppleDouble { + timingService.recordFileResult(fileResult, verificationMode: verificationMode) + } let resultRow = TransferCompletion.row(from: fileResult, destinationRoots: destinationRoots) callbacks.onResult(resultRow) @@ -534,7 +536,7 @@ final class CopyVerifyExecutor { // Evidence (Promise 3): bytes actually copied, one row per file per // backup. config.estimatedBytes is a progress estimate that falls // back to a placeholder when the source was not measured. - let totalBytesProcessed = results.reduce(Int64(0)) { $0 + $1.size } + let totalBytesProcessed = results.filter { ResultOutcome(statusText: $0.status) != .excludedAppleDouble }.reduce(Int64(0)) { $0 + $1.size } let fileCount = results.count let workers = max(1, ProcessInfo.processInfo.activeProcessorCount) @@ -543,6 +545,11 @@ final class CopyVerifyExecutor { let reportMode = config.currentMode var reportSettings = config.reportSettings reportSettings.verificationMode = config.verificationMode + let excluded = results.filter { ResultOutcome(statusText: $0.status) == .excludedAppleDouble } + if !excluded.isEmpty { + let count = Set(excluded.map(\.path)).count + reportSettings.notes += "\nSelection: \(count) AppleDouble source files intentionally excluded (listed in results). Only selected files were copied. Keep the source; the whole card is not backed up." + } if let handoffSummary { reportSettings.notes += "\nASC MHL: \(handoffSummary)" } let reportResults = results let reportOperation = operation diff --git a/Shared/Core/Services/ReportExporter.swift b/Shared/Core/Services/ReportExporter.swift index 5b39436d..ef17015b 100644 --- a/Shared/Core/Services/ReportExporter.swift +++ b/Shared/Core/Services/ReportExporter.swift @@ -40,7 +40,7 @@ enum ReportExporter { let method = verification.label let destinationPaths = destinationURLs.map { $0.path } - let issues = results.filter { !$0.isSuccessStatus } + let issues = results.filter { !$0.isSuccessStatus && ResultOutcome(statusText: $0.status) != .excludedAppleDouble } // Calculate performance metrics let duration = finished.timeIntervalSince(started) diff --git a/Shared/Core/Services/SharedAppCoordinator.swift b/Shared/Core/Services/SharedAppCoordinator.swift index 91456a10..a8e19a61 100644 --- a/Shared/Core/Services/SharedAppCoordinator.swift +++ b/Shared/Core/Services/SharedAppCoordinator.swift @@ -126,9 +126,21 @@ class SharedAppCoordinator: ObservableObject { /// The camera label for the next transfer: suggested from the card, /// remembered per camera and saved across launches (thesis decision). let cameraLabels: CameraLabelModel + let appleDoubleSelection: AppleDoubleReviewModel var cameraLabelSettings: CameraLabelSettings { - get { cameraLabels.settings } - set { cameraLabels.settings = newValue } + get { + var settings = cameraLabels.settings + settings.excludedAppleDoublePaths = appleDoubleSelection.enabled ? (appleDoubleSelection.paths ?? []) : nil + return settings + } + set { + var labelOnly = newValue + labelOnly.excludedAppleDoublePaths = nil + cameraLabels.settings = labelOnly + appleDoubleSelection.replaying = isReplayingQueuedTransfer + appleDoubleSelection.enabled = newValue.excludedAppleDoublePaths != nil + appleDoubleSelection.replaying = false + } } /// Camera settings for the next run only, used instead of /// `cameraLabelSettings` and cleared when that run starts. A prepared @@ -462,6 +474,7 @@ class SharedAppCoordinator: ObservableObject { self.sourceEjectability = sourceEjectability self.physicalDiskIdentityProvider = physicalDiskIdentityProvider self.defaults = defaults + self.appleDoubleSelection = AppleDoubleReviewModel(defaults: defaults) let environment = ProcessInfo.processInfo.environment let isTesting = environment["XCTestConfigurationFilePath"] != nil || environment["XCTestBundlePath"] != nil let testJournalURL = isTesting ? FileManager.default.temporaryDirectory @@ -553,6 +566,7 @@ class SharedAppCoordinator: ObservableObject { // coordinator must still refresh when it changes. stateService.objectWillChange.sink { [weak self] _ in self?.objectWillChange.send() } .store(in: &cancellables) + appleDoubleSelection.objectWillChange.sink { [weak self] _ in self?.objectWillChange.send() }.store(in: &cancellables) cameraLabels.objectWillChange.sink { [weak self] _ in self?.objectWillChange.send() } .store(in: &cancellables) generalSettings.objectWillChange.sink { [weak self] _ in self?.objectWillChange.send() } @@ -605,6 +619,7 @@ class SharedAppCoordinator: ObservableObject { // Detection no longer delays the scan. $sourceURL .sink { [weak self] url in + self?.appleDoubleSelection.refresh(source: url) Task { @MainActor [weak self] in await self?.folderInfoService.updateSource(url) } @@ -761,8 +776,16 @@ class SharedAppCoordinator: ObservableObject { // one backup, and is a 1 GB guess when the source was not scanned. let plannedTotalBytes: Int64? if let context = activeRunContext { - plannedTotalBytes = context.plannedTotalBytes - ?? prog.totalBytes.map { $0 * Int64(context.destinationURLs.count) } + if context.cameraLabelSettings.excludedAppleDoublePaths?.isEmpty == false, + prog.currentStage == .copying || prog.currentStage == .verifying { + plannedTotalBytes = prog.totalBytes.flatMap { total in + let (bytes, overflow) = total.multipliedReportingOverflow(by: Int64(context.destinationURLs.count)) + return overflow ? nil : bytes + } + } else { + plannedTotalBytes = context.plannedTotalBytes + ?? prog.totalBytes.map { $0 * Int64(context.destinationURLs.count) } + } } else { plannedTotalBytes = sourceFolderInfo.map { $0.totalSize * Int64(destinationURLs.count) } } @@ -1635,7 +1658,10 @@ class SharedAppCoordinator: ObservableObject { preResolvedIndependence: BackupIndependenceAssessment? = nil ) async { // Capture every mutable composer input before execution can suspend. - let runCameraSettings = projectRunCameraSettings ?? cameraLabelSettings + var runCameraSettings = projectRunCameraSettings ?? cameraLabelSettings + if !isReplayingQueuedTransfer { + runCameraSettings.excludedAppleDoublePaths = cameraLabelSettings.excludedAppleDoublePaths + } let projectID = photographerReportFinalizer == nil ? nil : photographerJobViewModel.activeJob?.id let projectCardID = photographerReportFinalizer == nil ? nil : activeProjectCardID let projectIdentity = projectID.flatMap { jobID in @@ -2805,7 +2831,7 @@ class SharedAppCoordinator: ObservableObject { destinations: destinationURLs, settings: cameraLabelSettings, verificationMode: verificationMode, - sourceIssue: folderInfoService.sourceScanError, + sourceIssue: folderInfoService.sourceScanError ?? appleDoubleSelection.readinessIssue, destinationWarnings: independence.warnings, availableBytes: { self.getDriveCapacity(for: $0) }, isWritable: TransferReadiness.isWritableFolder diff --git a/Shared/Core/ViewModels/AppleDoubleReviewModel.swift b/Shared/Core/ViewModels/AppleDoubleReviewModel.swift new file mode 100644 index 00000000..e74c2f1a --- /dev/null +++ b/Shared/Core/ViewModels/AppleDoubleReviewModel.swift @@ -0,0 +1,67 @@ +import Foundation +import Combine +import BitMatchEngine + +/// Owns the preview scan. Source changes invalidate the old review at once; +/// Start independently re-proves these paths in the engine before any writes. +@MainActor +final class AppleDoubleReviewModel: ObservableObject { + @Published var enabled: Bool { + didSet { + if !replaying { defaults.set(enabled, forKey: "BitMatchExcludeAppleDouble") } + if oldValue != enabled { refresh(source: source) } + } + } + @Published private(set) var paths: [String]? + @Published private(set) var error: String? + var replaying = false + private let defaults: UserDefaults + private var source: URL? + private var generation = 0 + private var task: Task? + + init(defaults: UserDefaults) { + self.defaults = defaults + enabled = defaults.bool(forKey: "BitMatchExcludeAppleDouble") + } + deinit { task?.cancel() } + + var summaryLine: String? { + guard enabled, let paths else { return nil } + return paths.isEmpty ? "No AppleDouble companions will be excluded" : "\(paths.count) AppleDouble source \(paths.count == 1 ? "file" : "files") will be excluded. Keep the source." + } + + var readinessIssue: String? { + guard enabled, source != nil else { return nil } + return error ?? (paths == nil ? "Reviewing AppleDouble exclusions…" : nil) + } + + func refresh(source: URL?) { + task?.cancel() + generation += 1 + let token = generation + self.source = source + paths = nil + error = nil + guard enabled, let source else { return } + task = Task { [weak self] in + let scanner = Task.detached { + let access = source.startAccessingSecurityScopedResource() + defer { if access { source.stopAccessingSecurityScopedResource() } } + return try AppleDoubleSelection.review(source: source) + } + do { + let found = try await withTaskCancellationHandler { + try await scanner.value + } onCancel: { scanner.cancel() } + guard !Task.isCancelled, let self, self.generation == token else { return } + paths = found + } catch is CancellationError { + // A superseded preview must not publish either paths or an error. + } catch { + guard !Task.isCancelled, let self, self.generation == token else { return } + self.error = "Could not review AppleDouble exclusions: \(error.localizedDescription)" + } + } + } +} diff --git a/Shared/Views/Setup/AppleDoubleSelectionView.swift b/Shared/Views/Setup/AppleDoubleSelectionView.swift new file mode 100644 index 00000000..f0d454d8 --- /dev/null +++ b/Shared/Views/Setup/AppleDoubleSelectionView.swift @@ -0,0 +1,33 @@ +import SwiftUI + +struct AppleDoubleSelectionView: View { + @ObservedObject var review: AppleDoubleReviewModel + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + Toggle("Exclude AppleDouble companion files", isOn: $review.enabled) + Text("Preserve all files by default. Only recognized AppleDouble metadata companions can be excluded; other sidecars stay in the copy.") + .font(.footnote).foregroundStyle(.secondary) + if review.enabled { + if let paths = review.paths { + Text("\(paths.count) source \(paths.count == 1 ? "file" : "files") will be excluded") + .font(.subheadline.weight(.semibold)) + if !paths.isEmpty { + DisclosureGroup("Review excluded files") { + ScrollView { + VStack(alignment: .leading, spacing: 4) { + ForEach(paths, id: \.self) { Text($0).font(.caption).textSelection(.enabled) } + }.frame(maxWidth: .infinity, alignment: .leading) + }.frame(maxHeight: 160) + } + Text("These companions won't be copied or checked. Keep the source: this cannot mark the whole card safe to erase.") + .font(.footnote).foregroundStyle(.orange) + } + } else { + Text(review.readinessIssue ?? "Choose a source to review exclusions.") + .font(.footnote).foregroundStyle(.secondary) + } + } + } + } +} diff --git a/Shared/Views/Setup/CoordinatorSetupScreen.swift b/Shared/Views/Setup/CoordinatorSetupScreen.swift index 446b3afa..3ca71a07 100644 --- a/Shared/Views/Setup/CoordinatorSetupScreen.swift +++ b/Shared/Views/Setup/CoordinatorSetupScreen.swift @@ -72,7 +72,7 @@ extension SetupPresentation { sourceBytes: coordinator.sourceFolderInfo?.totalSize, destinationCount: independence.independentCopyCount, hasProjectEvidence: !(jobs.dashboardJob?.cardIngests.isEmpty ?? true), - informationalLines: coordinator.alreadyBackedUpLine.map { [$0] } ?? [] + informationalLines: [coordinator.alreadyBackedUpLine, coordinator.appleDoubleSelection.summaryLine].compactMap { $0 } ) } } @@ -124,7 +124,11 @@ struct CoordinatorSetupScreen