diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift index a01de988..45cc461a 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift @@ -100,7 +100,7 @@ public struct SavedChecksumCheck: Sendable { switch self { case .notFound: "No saved checksums found here." case .unreadableEvidence: "Some saved checksums are damaged or inconsistent. Compare with another folder instead." - case .unsupportedEvidence: "These saved checksums include a format or algorithm BitMatch cannot fully read yet. Compare with another folder instead." + case .unsupportedEvidence: "Some saved checksums use a format or history detail BitMatch can't check yet. Compare with another folder instead." } } } @@ -441,6 +441,14 @@ public struct SavedChecksumCheck: Sendable { reject(parser, .unsupportedEvidence) return } + // A later ASC generation can move a file or whole directory and + // record the old location in previousPath. Until Check interprets + // that history, treating each generation as unrelated file paths + // would incorrectly call valid renamed media "missing". + if name == "previouspath" { + reject(parser, .unsupportedEvidence) + return + } stack.append(name) text = "" if stack == ["hashlist", "hashes", "hash"] { diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/EvidenceCompatibilityTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/EvidenceCompatibilityTests.swift index 9d314de9..78692c99 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/EvidenceCompatibilityTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/EvidenceCompatibilityTests.swift @@ -91,6 +91,42 @@ struct EvidenceCompatibilityTests { #expect(result.newPaths == ["unrecorded.txt"]) } + /// The reference tool records both a file rename and a directory move in + /// generation two. Check must not ignore previousPath and imply it has + /// interpreted that history when it only knows how to check file bytes. + @Test func referenceRenameHistoryRequiresStructuralReader() async throws { + let container = try copyReference("md5-renames") + defer { try? FileManager.default.removeItem(at: container) } + let root = container.appendingPathComponent("media") + let before = try contents(container) + await #expect(throws: SavedChecksumCheck.CheckError.unsupportedEvidence) { + _ = try await checker.discover(under: root) + } + #expect(try contents(container) == before) + } + + /// Derived disposable variant: remove only the two file lineage markers + /// from the reference record. The remaining directory marker alone must + /// still keep Check from treating this as ordinary file-only evidence. + @Test func directoryRenameMarkerAloneIsUnsupported() async throws { + let container = try copyReference("md5-renames") + defer { try? FileManager.default.removeItem(at: container) } + let root = container.appendingPathComponent("media") + let history = root.appendingPathComponent("ascmhl") + let manifest = try #require(FileManager.default.contentsOfDirectory(at: history, includingPropertiesForKeys: nil) + .first { $0.lastPathComponent.hasPrefix("0002_") && $0.pathExtension == "mhl" }) + let original = try String(contentsOf: manifest, encoding: .utf8) + let modified = original + .replacingOccurrences(of: "Reel A/clip one.mov", with: "") + .replacingOccurrences(of: "Reel B/clip two.mov", with: "") + #expect(modified != original) + #expect(modified.contains("Reel B")) + try modified.write(to: manifest, atomically: true, encoding: .utf8) + await #expect(throws: SavedChecksumCheck.CheckError.unsupportedEvidence) { + _ = try await checker.discover(under: root) + } + } + /// Plant: silently compactMap an unsupported/invalid entry or swallow a bad record. @Test(arguments: ["unsupported", "invalid-digest", "duplicate", "traversal", "failed-action", "namespace", "version", "truncated"]) func incompleteRecordsCannotBecomeAPartialGreen(kind: String) async throws { @@ -205,6 +241,21 @@ struct EvidenceCompatibilityTests { } } + /// A supported neighbour cannot turn a legacy standalone-format record + /// into an apparently complete ASC check. + @Test func classicMHLBesideReferenceHistoryCannotBecomePartialGreen() async throws { + let container = try copyReference("md5") + defer { try? FileManager.default.removeItem(at: container) } + let root = container.appendingPathComponent("media") + try FileManager.default.copyItem(at: fixture("classic-v1.mhl"), + to: root.appendingPathComponent("ascmhl/classic-v1.mhl")) + let before = try contents(container) + await #expect(throws: SavedChecksumCheck.CheckError.unsupportedEvidence) { + _ = try await checker.discover(under: root) + } + #expect(try contents(container) == before) + } + /// Plant: resolve entity declarations or trust a nested metadata hash as a file digest. @Test(arguments: ["entity", "metadata"]) func xmlExtensionsCannotInventFileEvidence(kind: String) async throws { diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/README.md b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/README.md index ab1a3fb7..a8e94da6 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/README.md +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/README.md @@ -2,7 +2,7 @@ All media bytes are tiny, original synthetic payloads, generated by `Scripts/ascmhl/create_compatibility_fixtures.py`. No camera footage, personal hostname or production metadata is included. The fixtures and handcrafted reports/classic-MHL example are offered under BitMatch's MIT license. -`reference/` was generated with the official [ASC MHL reference implementation](https://github.com/ascmitc/mhl) at commit `0fb61f1e4c7c1c3ff422449aa6f091ce0d3b7687` on October 10, 2026. Its MIT notice is retained in `ASC-LICENSE.md`. The manifests are output from that tool, not BitMatch's writer. MD5 contains two generations; XXH64 and C4 contain one. Each inventory and chain passed the official XSDs, `ascmhl-debug verify` and `ascmhl diff`. The CLI's own timestamp/version fields are retained; the hostname is replaced before generation. +`reference/` was generated with the official [ASC MHL reference implementation](https://github.com/ascmitc/mhl) at commit `0fb61f1e4c7c1c3ff422449aa6f091ce0d3b7687`, with additions on October 14 and 17, 2026. Its MIT notice is retained in `ASC-LICENSE.md`. The manifests are output from that tool, not BitMatch's writer. MD5 and MD5-renames contain two generations; the other datasets contain one. Each inventory and chain passed the official XSDs, `ascmhl-debug verify` and `ascmhl diff`. The CLI's own timestamp/version fields are retained; the hostname is replaced before generation. Regenerate into a new, empty directory using a Python 3.11+ environment with that pinned checkout installed: @@ -16,6 +16,8 @@ The tests copy fixtures into disposable roots. MD5/XXH64, zero-byte files, Unico October 14 additions use the same pinned reference commit and license: `sha1/` and `xxh128/` are valid external inventories that BitMatch must explicitly reject as unsupported, both alone and beside a readable history. `xxh64-boundaries/` contains original byte sequences (`byte[index] = index % 251`) of lengths 0, 1, 31, 32, 33, 255, 256, 257 and 1025, covering XXH64's lane and tail boundaries. These fixtures passed the independent checks below; the new Swift consumer regressions still require an Apple test run. +October 17 addition: `md5-renames/` is another output of the same pinned reference CLI, generated with `create -h md5` and then `create -h md5 -dr`. Between those generations, one synthetic file is renamed and `Reel B` is renamed to `Reel C`. The second manifest records two file `` values and one directory ``; the reference verifier and diff accept the final folder, while a further unrecorded directory move fails. BitMatch Check deliberately rejects this history as unsupported until it can interpret rename lineage and structure. It does not call the old paths missing or claim to have verified the chain. The media payloads and fixture script are original/MIT; the reference-tool output retains its MIT notice in `ASC-LICENSE.md`. + Run the corpus checks on Linux or macOS without Swift, using the reference venv: ``` @@ -24,6 +26,6 @@ python Scripts/ascmhl/validate_corpus.py /path/to/pinned/ascmitc/mhl The runner requires the pinned, unmodified checkout, uses its code and schemas, suppresses the CLI's online update check, and checks inventory/chain schemas, generation counts, C4 chain digests, bytes and paths. It checks changed, missing, renamed, extra, modified-manifest and missing-manifest cases only in disposable copies, requiring specific failure codes. A rename must report both the new and missing path. SHA-1/XXH128/C4 are verified by the reference tool here, not claimed as supported by BitMatch. Originals must remain byte-for-byte unchanged. `validate_reference.sh` runs this subset as well as the Apple-only freshly generated BitMatch workflow. The corpus-only runner is also a Linux CI job. -Generate just new datasets without replacing existing originals using `--datasets sha1 xxh128 xxh64-boundaries`. Default generation creates all six datasets. Generation suppresses the reference CLI's online update check as well. +Generate just new datasets without replacing existing originals using `--datasets sha1 xxh128 xxh64-boundaries md5-renames`. Default generation creates all seven datasets. Generation suppresses the reference CLI's online update check as well. -`classic-v1.mhl` documents the current unsupported format. BitMatch still discovers ASC inventories under `ascmhl/`; this batch does not add standalone classic-MHL discovery or claim full chain, rename or directory-hash verification. ASC checks explicitly describe their file-content scope. Histories are never rewritten by these checks. +`classic-v1.mhl` is a handcrafted boundary fixture, not external-tool output. It is rejected inside an ASC history both alone and beside a valid reference history; standalone classic-MHL discovery is still unsupported. This batch does not claim full chain, rename or directory-hash verification. ASC checks explicitly describe their file-content scope. Histories are never rewritten by these checks. diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/Reel A/clip renamed.mov b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/Reel A/clip renamed.mov new file mode 100644 index 00000000..9e8a1b2a --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/Reel A/clip renamed.mov @@ -0,0 +1 @@ +Synthetic clip one diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/Reel A/sidecar.xml b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/Reel A/sidecar.xml new file mode 100644 index 00000000..8ec1f576 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/Reel A/sidecar.xml @@ -0,0 +1 @@ + diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/Reel C/clip two.mov b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/Reel C/clip two.mov new file mode 100644 index 00000000..3df88abf --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/Reel C/clip two.mov @@ -0,0 +1 @@ +Synthetic clip two diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/ascmhl/0001_md5-renames_2026-10-17_043513Z.mhl b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/ascmhl/0001_md5-renames_2026-10-17_043513Z.mhl new file mode 100644 index 00000000..fe92cb3a --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/ascmhl/0001_md5-renames_2026-10-17_043513Z.mhl @@ -0,0 +1,56 @@ + + + + 2026-10-17T00:35:13-04:00 + bitmatch-fixture-host + ascmhl + + + in-place + + + 5bcd50fab8aa16f017f7a6e755556d03 + + + 2e811871b223a14d3cec169a67a2506f + + + + .DS_Store + ascmhl + ascmhl/ + + + + + Reel A/clip one.mov + bbb3e9a25ab3b29f321946e0ced197e5 + + + Reel A/sidecar.xml + 90dc46f5178d03fba3d712bac451d6e8 + + + Reel A + + 8c63cc817a52bc5235cf3e8a35bc74f3 + + + e7355991ecfb7df08e89868c8c9c028c + + + + Reel B/clip two.mov + d597faaafb635c7ab9b94dce00f2caf3 + + + Reel B + + 1599790324681fcf0afe1efd7e5c1597 + + + fdd92faea2dfe67959219e35d00dcfda + + + + diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/ascmhl/0002_md5-renames_2026-10-17_043513Z.mhl b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/ascmhl/0002_md5-renames_2026-10-17_043513Z.mhl new file mode 100644 index 00000000..02f446de --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/ascmhl/0002_md5-renames_2026-10-17_043513Z.mhl @@ -0,0 +1,59 @@ + + + + 2026-10-17T00:35:13-04:00 + bitmatch-fixture-host + ascmhl + + + in-place + + + 5bcd50fab8aa16f017f7a6e755556d03 + + + 79ad990a57773810cd5d5c9654456bde + + + + .DS_Store + ascmhl + ascmhl/ + + + + + Reel A/clip renamed.mov + bbb3e9a25ab3b29f321946e0ced197e5 + Reel A/clip one.mov + + + Reel A/sidecar.xml + 90dc46f5178d03fba3d712bac451d6e8 + + + Reel A + + 8c63cc817a52bc5235cf3e8a35bc74f3 + + + d9f8b0c2af5056964a0ca7d4eb314a42 + + + + Reel C/clip two.mov + d597faaafb635c7ab9b94dce00f2caf3 + Reel B/clip two.mov + + + Reel C + + 1599790324681fcf0afe1efd7e5c1597 + + + fdd92faea2dfe67959219e35d00dcfda + + Reel B + + + diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/ascmhl/ascmhl_chain.xml b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/ascmhl/ascmhl_chain.xml new file mode 100644 index 00000000..c5bb2cdc --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/reference/md5-renames/ascmhl/ascmhl_chain.xml @@ -0,0 +1,11 @@ + + + + 0001_md5-renames_2026-10-17_043513Z.mhl + c45p2RxQ2T7Fo6ZSkRiiFN7TJv12g8nKE9iu6ApwEvJoyEzUPskAo9LwaJzZtoP67cLHFVqtwj8NAMbL8AbveNiJkz + + + 0002_md5-renames_2026-10-17_043513Z.mhl + c41XRWFSyjnPhJF5mjhn3GGCe9qcXS12nDh7iAG4GFaBXPgQnSXgxYgx9Hqf4eKq58yVss1MjS6qujMnhZQGv8Lz4s + + diff --git a/Scripts/ascmhl/create_compatibility_fixtures.py b/Scripts/ascmhl/create_compatibility_fixtures.py index 4b09e51f..251c93a9 100644 --- a/Scripts/ascmhl/create_compatibility_fixtures.py +++ b/Scripts/ascmhl/create_compatibility_fixtures.py @@ -14,7 +14,7 @@ with patch.object(Updater, '_get_latest_version', return_value=None): from ascmhl.cli.ascmhl import mhltool_cli -datasets = ['md5', 'xxh64', 'c4', 'sha1', 'xxh128', 'xxh64-boundaries'] +datasets = ['md5', 'xxh64', 'c4', 'sha1', 'xxh128', 'xxh64-boundaries', 'md5-renames'] parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('output', type=pathlib.Path) parser.add_argument('--datasets', nargs='+', choices=datasets, default=datasets) @@ -23,15 +23,32 @@ output.mkdir(parents=True, exist_ok=False) runner = CliRunner() -def create(root, algorithm): +def create(root, algorithm, detect_renaming=False): + arguments = ['create', '-h', algorithm] + if detect_renaming: + arguments.append('-dr') + arguments.append(str(root)) with patch('platform.node', return_value='bitmatch-fixture-host'): - result = runner.invoke(mhltool_cli, ['create', '-h', algorithm, str(root)]) + result = runner.invoke(mhltool_cli, arguments) if result.exit_code: raise RuntimeError(result.output) from result.exception for dataset in args.datasets: - algorithm = 'xxh64' if dataset == 'xxh64-boundaries' else dataset + algorithm = {'xxh64-boundaries': 'xxh64', 'md5-renames': 'md5'}.get(dataset, dataset) root = output / dataset + if dataset == 'md5-renames': + # Only the reference tool creates the two manifests and chain. Both + # generations describe original, tiny synthetic media bytes. + (root / 'Reel A').mkdir(parents=True) + (root / 'Reel B').mkdir() + (root / 'Reel A' / 'clip one.mov').write_bytes(b'Synthetic clip one\n') + (root / 'Reel A' / 'sidecar.xml').write_bytes(b'\n') + (root / 'Reel B' / 'clip two.mov').write_bytes(b'Synthetic clip two\n') + create(root, algorithm) + (root / 'Reel A' / 'clip one.mov').rename(root / 'Reel A' / 'clip renamed.mov') + (root / 'Reel B').rename(root / 'Reel C') + create(root, algorithm, detect_renaming=True) + continue if dataset == 'xxh64-boundaries': root.mkdir() for size in [0, 1, 31, 32, 33, 255, 256, 257, 1025]: diff --git a/Scripts/ascmhl/validate_corpus.py b/Scripts/ascmhl/validate_corpus.py index f72934ee..1543e01d 100644 --- a/Scripts/ascmhl/validate_corpus.py +++ b/Scripts/ascmhl/validate_corpus.py @@ -16,7 +16,7 @@ from unittest.mock import patch REFERENCE_COMMIT = '0fb61f1e4c7c1c3ff422449aa6f091ce0d3b7687' -DATASETS = ('md5', 'xxh64', 'c4', 'sha1', 'xxh128', 'xxh64-boundaries') +DATASETS = ('md5', 'xxh64', 'c4', 'sha1', 'xxh128', 'xxh64-boundaries', 'md5-renames') def snapshot(root): @@ -75,7 +75,7 @@ def invoke(label, cli, arguments, expected=0, required_output=()): chain_schema.assertValid(chain) checks.append(f'{dataset}: chain XSD') entries = chain.findall('{urn:ASC:MHL:DIRECTORY:v2.0}hashlist') - if len(entries) != (2 if dataset == 'md5' else 1): + if len(entries) != (2 if dataset in ('md5', 'md5-renames') else 1): raise RuntimeError(f'{dataset}: unexpected generation count') for entry in entries: name = entry.findtext('{urn:ASC:MHL:DIRECTORY:v2.0}path') @@ -84,6 +84,25 @@ def invoke(label, cli, arguments, expected=0, required_output=()): if hasher.string_digest() != entry.findtext('{urn:ASC:MHL:DIRECTORY:v2.0}c4'): raise RuntimeError(f'{dataset}: invalid chain C4 for {name}') checks.append(f'{dataset}: generation count and chain C4') + if dataset == 'md5-renames': + namespace = '{urn:ASC:MHL:v2.0}' + latest = etree.parse(str(sorted(history.glob('*.mhl'))[-1])) + moved_files = { + (entry.findtext(namespace + 'path'), entry.findtext(namespace + 'previousPath')) + for entry in latest.findall('.//' + namespace + 'hash') + if entry.find(namespace + 'previousPath') is not None + } + moved_directories = { + (entry.findtext(namespace + 'path'), entry.findtext(namespace + 'previousPath')) + for entry in latest.findall('.//' + namespace + 'directoryhash') + if entry.find(namespace + 'previousPath') is not None + } + if moved_files != { + ('Reel A/clip renamed.mov', 'Reel A/clip one.mov'), + ('Reel C/clip two.mov', 'Reel B/clip two.mov'), + } or moved_directories != {('Reel C', 'Reel B')}: + raise RuntimeError('md5-renames: missing file or directory rename lineage') + checks.append('md5-renames: file and directory lineage') invoke(f'{dataset}: original bytes', mhldebugtool_cli, ['verify', '-v', str(root)]) invoke(f'{dataset}: original paths', mhltool_cli, ['diff', str(root)]) # Mutate only disposable copies, restoring a pristine case each time. @@ -114,6 +133,12 @@ def invoke(label, cli, arguments, expected=0, required_output=()): manifest.write_bytes(manifest.read_bytes() + b'\n') required = ('found new file', 'missing file(s)') if fault == 'renamed' else () invoke(f'{dataset}: rejects {fault}', cli, [command, str(damaged)], expected, required) + if dataset == 'md5-renames': + damaged = work / 'md5-renames-directory-moved-again' + shutil.copytree(root, damaged) + (damaged / 'Reel C').rename(damaged / 'Reel D') + invoke('md5-renames: rejects unrecorded directory move', mhltool_cli, + ['diff', str(damaged)], 10, ('found new file', 'missing file(s)')) if snapshot(corpus) != before: raise RuntimeError('Validation modified the permanent corpus') diff --git a/docs/EVIDENCE_COMPATIBILITY_VALIDATION.md b/docs/EVIDENCE_COMPATIBILITY_VALIDATION.md index 6082d6e7..212c2549 100644 --- a/docs/EVIDENCE_COMPATIBILITY_VALIDATION.md +++ b/docs/EVIDENCE_COMPATIBILITY_VALIDATION.md @@ -53,3 +53,11 @@ The first signed 0.2.8 candidate completed a four-file Standard transfer to APFS Discovery now excludes an MHL input only when the existing bounded AppleDouble validator proves it is a metadata-only container with a real companion. This does not remove any file or change transfer filtering. A real mounted-exFAT regression uses `setxattr` to make macOS create the companion. A five-case synthetic test covers recognized metadata, a damaged lookalike, an orphan, an AppleDouble data-fork record and legitimate XML with a `._` name. Both metadata regressions failed before the fix and passed afterward; the other four controls remain correct. The first test draft incorrectly expected two records for duplicate file evidence; it was corrected to use distinct files before the retained red run. The official verifier initially rejected the signed exFAT folder for newly generated metadata files absent from the MHL. It verified all four recorded media files when run with its explicit `._*` ignore pattern. That reference invocation checks this known synthetic media set; BitMatch's production record filter is narrower and validates the container and companion. + +## October 17 rename and directory-move corpus + +The pinned ASC reference CLI generated a new, redistributable `md5-renames/` fixture from three tiny synthetic files. Its first generation records `Reel A/clip one.mov` and a file under `Reel B`. After renaming the first file and moving the entire `Reel B` directory to `Reel C`, its second generation records both file lineages and the directory lineage with ``. The official schemas, chain C4 values, byte verifier and diff accept the resulting history. Moving `Reel C` again without recording a generation makes the reference diff report both new and missing paths and fail. The generator and fixture README document exact provenance and licensing. + +This fixture exposed a reader gap: BitMatch Check ignored `` and interpreted a valid recorded move as missing old files. A new Swift regression failed before the guard because discovery silently accepted the history; it passed after Check began reporting `unsupportedEvidence` for rename lineage. That is an honest boundary until structural/history interpretation exists. A classic-v1 record placed beside a valid reference history is also tested as unsupported rather than letting the valid neighbour produce a partial green. The classic file remains handcrafted and standalone discovery remains open. + +The updated corpus runner passed **82 independent checks** against the pinned reference commit `0fb61f1e4c7c1c3ff422449aa6f091ce0d3b7687`, including schema, chain C4, original-media, rename/directory-move and disposable corruption controls. The focused Swift `EvidenceCompatibilityTests` suite passed 18 tests after the guard. These checks do not validate BitMatch's ASC chain/directory hashes, interpret rename history, or establish physical-drive reliability. diff --git a/docs/TODO.md b/docs/TODO.md index 955925d4..c8e87562 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -116,11 +116,11 @@ The priority buckets are not a strict implementation sequence: investigate P0 re ## P2 — Evidence and interoperability -October 14 cloud progress: the independent ASC corpus now includes SHA-1/XXH128 unsupported-format fixtures and XXH64 lane/tail boundaries. The reference runner passed 68 checks on Linux and macOS, including negative controls and unchanged originals. Swift consumer tests, fresh BitMatch output against the pinned reference, Mac tests and iPhone/iPad simulator suites also passed; physical-media and signed-app checks still remain before a release. Interoperability and structural/history work below remain open. +October 14 cloud progress: the independent ASC corpus added SHA-1/XXH128 unsupported-format fixtures and XXH64 lane/tail boundaries. Its reference runner passed 68 checks on Linux and macOS, including negative controls and unchanged originals. Swift consumer tests, fresh BitMatch output against the pinned reference, Mac tests and iPhone/iPad simulator suites also passed. October 17 adds an official reference-tool two-generation file-rename/directory-move fixture; the expanded runner passes 82 checks. Physical-media and signed-app checks still remain before a release. Interoperability and structural/history work below remain open. - [ ] **Upstream MHL / source provenance.** Reuse `SavedChecksumCheck` where appropriate to offer source verification against accompanying classic MHL or ASC MHL evidence. Record “source arrived intact” separately from “BitMatch created verified backups.” Preserve original manifests. Current Check discovers ASC inventories under `ascmhl/`; standalone classic-MHL discovery and an upstream transfer verdict still need work. **Acceptance:** incomplete, mismatched, malformed and unsupported evidence gets an honest separate verdict; a failed provenance check never becomes success and never prevents preservation of damaged-but-irreplaceable media. -- [ ] **MHL interoperability corpus.** The first permanent corpus now covers official ASC reference-tool MD5/XXH64 output, two generations, unsupported C4, Unicode/whitespace paths, nested inventories, damaged records and report compatibility. See [the validation record](EVIDENCE_COMPATIBILITY_VALIDATION.md). Expand the existing generated-fixture/reference checks and multi-file saved-record regressions with permanent, redistributable classic/ASC manifests and representative external-tool output. Cover multiple generations, relative paths, renames, directory changes, unsupported algorithms, corrupt/malformed records, Unicode and unusual filenames. **Acceptance:** validate with independent reference implementations, record fixture provenance/licensing and expected outcomes, and never silently treat unsupported evidence as fully verified. +- [ ] **MHL interoperability corpus.** The permanent corpus covers official ASC reference-tool MD5/XXH64 output, two generations, unsupported C4/SHA-1/XXH128, Unicode/whitespace paths, nested inventories, damaged records, a recorded file rename and directory move, and report compatibility. The classic-v1 boundary is handcrafted and explicitly identified as such. See [the validation record](EVIDENCE_COMPATIBILITY_VALIDATION.md). Continue with redistributable, independently attributable classic MHL and representative output from another implementation before claiming broad interoperability. Keep multi-file saved-record and relative-path regressions current. **Acceptance:** validate with independent reference implementations, record fixture provenance/licensing and expected outcomes, and never silently treat unsupported evidence as fully verified. - [ ] **ASC MHL structural verification.** After the corpus establishes expected semantics, extend Check to validate supported root/directory hashes and detect renamed, moved, missing or structurally altered content. **Acceptance:** distinguish file-content verification from structural verification; older manifests without structural evidence remain usable with an explicit scope. Do not claim full history verification before the chain semantics are implemented and independently tested.