Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ public struct SavedChecksumCheck: Sendable {
switch self {
case .notFound: "No saved checksums found here."
case .unreadableEvidence: "Some saved checksums are damaged or inconsistent. Compare with another folder instead."
case .unsupportedEvidence: "These saved checksums include a format or algorithm BitMatch cannot fully read yet. Compare with another folder instead."
case .unsupportedEvidence: "Some saved checksums use a format or history detail BitMatch can't check yet. Compare with another folder instead."
}
}
}
Expand Down Expand Up @@ -441,6 +441,14 @@ public struct SavedChecksumCheck: Sendable {
reject(parser, .unsupportedEvidence)
return
}
// A later ASC generation can move a file or whole directory and
// record the old location in previousPath. Until Check interprets
// that history, treating each generation as unrelated file paths
// would incorrectly call valid renamed media "missing".
if name == "previouspath" {
reject(parser, .unsupportedEvidence)
return
}
stack.append(name)
text = ""
if stack == ["hashlist", "hashes", "hash"] {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,42 @@ struct EvidenceCompatibilityTests {
#expect(result.newPaths == ["unrecorded.txt"])
}

/// The reference tool records both a file rename and a directory move in
/// generation two. Check must not ignore previousPath and imply it has
/// interpreted that history when it only knows how to check file bytes.
@Test func referenceRenameHistoryRequiresStructuralReader() async throws {
let container = try copyReference("md5-renames")
defer { try? FileManager.default.removeItem(at: container) }
let root = container.appendingPathComponent("media")
let before = try contents(container)
await #expect(throws: SavedChecksumCheck.CheckError.unsupportedEvidence) {
_ = try await checker.discover(under: root)
}
#expect(try contents(container) == before)
}

/// Derived disposable variant: remove only the two file lineage markers
/// from the reference record. The remaining directory marker alone must
/// still keep Check from treating this as ordinary file-only evidence.
@Test func directoryRenameMarkerAloneIsUnsupported() async throws {
let container = try copyReference("md5-renames")
defer { try? FileManager.default.removeItem(at: container) }
let root = container.appendingPathComponent("media")
let history = root.appendingPathComponent("ascmhl")
let manifest = try #require(FileManager.default.contentsOfDirectory(at: history, includingPropertiesForKeys: nil)
.first { $0.lastPathComponent.hasPrefix("0002_") && $0.pathExtension == "mhl" })
let original = try String(contentsOf: manifest, encoding: .utf8)
let modified = original
.replacingOccurrences(of: "<previousPath>Reel A/clip one.mov</previousPath>", with: "")
.replacingOccurrences(of: "<previousPath>Reel B/clip two.mov</previousPath>", with: "")
#expect(modified != original)
#expect(modified.contains("<previousPath>Reel B</previousPath>"))
try modified.write(to: manifest, atomically: true, encoding: .utf8)
await #expect(throws: SavedChecksumCheck.CheckError.unsupportedEvidence) {
_ = try await checker.discover(under: root)
}
}

/// Plant: silently compactMap an unsupported/invalid entry or swallow a bad record.
@Test(arguments: ["unsupported", "invalid-digest", "duplicate", "traversal", "failed-action", "namespace", "version", "truncated"])
func incompleteRecordsCannotBecomeAPartialGreen(kind: String) async throws {
Expand Down Expand Up @@ -205,6 +241,21 @@ struct EvidenceCompatibilityTests {
}
}

/// A supported neighbour cannot turn a legacy standalone-format record
/// into an apparently complete ASC check.
@Test func classicMHLBesideReferenceHistoryCannotBecomePartialGreen() async throws {
let container = try copyReference("md5")
defer { try? FileManager.default.removeItem(at: container) }
let root = container.appendingPathComponent("media")
try FileManager.default.copyItem(at: fixture("classic-v1.mhl"),
to: root.appendingPathComponent("ascmhl/classic-v1.mhl"))
let before = try contents(container)
await #expect(throws: SavedChecksumCheck.CheckError.unsupportedEvidence) {
_ = try await checker.discover(under: root)
}
#expect(try contents(container) == before)
}

/// Plant: resolve entity declarations or trust a nested metadata hash as a file digest.
@Test(arguments: ["entity", "metadata"])
func xmlExtensionsCannotInventFileEvidence(kind: String) async throws {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

All media bytes are tiny, original synthetic payloads, generated by `Scripts/ascmhl/create_compatibility_fixtures.py`. No camera footage, personal hostname or production metadata is included. The fixtures and handcrafted reports/classic-MHL example are offered under BitMatch's MIT license.

`reference/` was generated with the official [ASC MHL reference implementation](https://github.com/ascmitc/mhl) at commit `0fb61f1e4c7c1c3ff422449aa6f091ce0d3b7687` on October 10, 2026. Its MIT notice is retained in `ASC-LICENSE.md`. The manifests are output from that tool, not BitMatch's writer. MD5 contains two generations; XXH64 and C4 contain one. Each inventory and chain passed the official XSDs, `ascmhl-debug verify` and `ascmhl diff`. The CLI's own timestamp/version fields are retained; the hostname is replaced before generation.
`reference/` was generated with the official [ASC MHL reference implementation](https://github.com/ascmitc/mhl) at commit `0fb61f1e4c7c1c3ff422449aa6f091ce0d3b7687`, with additions on October 14 and 17, 2026. Its MIT notice is retained in `ASC-LICENSE.md`. The manifests are output from that tool, not BitMatch's writer. MD5 and MD5-renames contain two generations; the other datasets contain one. Each inventory and chain passed the official XSDs, `ascmhl-debug verify` and `ascmhl diff`. The CLI's own timestamp/version fields are retained; the hostname is replaced before generation.

Regenerate into a new, empty directory using a Python 3.11+ environment with that pinned checkout installed:

Expand All @@ -16,6 +16,8 @@ The tests copy fixtures into disposable roots. MD5/XXH64, zero-byte files, Unico

October 14 additions use the same pinned reference commit and license: `sha1/` and `xxh128/` are valid external inventories that BitMatch must explicitly reject as unsupported, both alone and beside a readable history. `xxh64-boundaries/` contains original byte sequences (`byte[index] = index % 251`) of lengths 0, 1, 31, 32, 33, 255, 256, 257 and 1025, covering XXH64's lane and tail boundaries. These fixtures passed the independent checks below; the new Swift consumer regressions still require an Apple test run.

October 17 addition: `md5-renames/` is another output of the same pinned reference CLI, generated with `create -h md5` and then `create -h md5 -dr`. Between those generations, one synthetic file is renamed and `Reel B` is renamed to `Reel C`. The second manifest records two file `<previousPath>` values and one directory `<previousPath>`; the reference verifier and diff accept the final folder, while a further unrecorded directory move fails. BitMatch Check deliberately rejects this history as unsupported until it can interpret rename lineage and structure. It does not call the old paths missing or claim to have verified the chain. The media payloads and fixture script are original/MIT; the reference-tool output retains its MIT notice in `ASC-LICENSE.md`.

Run the corpus checks on Linux or macOS without Swift, using the reference venv:

```
Expand All @@ -24,6 +26,6 @@ python Scripts/ascmhl/validate_corpus.py /path/to/pinned/ascmitc/mhl

The runner requires the pinned, unmodified checkout, uses its code and schemas, suppresses the CLI's online update check, and checks inventory/chain schemas, generation counts, C4 chain digests, bytes and paths. It checks changed, missing, renamed, extra, modified-manifest and missing-manifest cases only in disposable copies, requiring specific failure codes. A rename must report both the new and missing path. SHA-1/XXH128/C4 are verified by the reference tool here, not claimed as supported by BitMatch. Originals must remain byte-for-byte unchanged. `validate_reference.sh` runs this subset as well as the Apple-only freshly generated BitMatch workflow. The corpus-only runner is also a Linux CI job.

Generate just new datasets without replacing existing originals using `--datasets sha1 xxh128 xxh64-boundaries`. Default generation creates all six datasets. Generation suppresses the reference CLI's online update check as well.
Generate just new datasets without replacing existing originals using `--datasets sha1 xxh128 xxh64-boundaries md5-renames`. Default generation creates all seven datasets. Generation suppresses the reference CLI's online update check as well.

`classic-v1.mhl` documents the current unsupported format. BitMatch still discovers ASC inventories under `ascmhl/`; this batch does not add standalone classic-MHL discovery or claim full chain, rename or directory-hash verification. ASC checks explicitly describe their file-content scope. Histories are never rewritten by these checks.
`classic-v1.mhl` is a handcrafted boundary fixture, not external-tool output. It is rejected inside an ASC history both alone and beside a valid reference history; standalone classic-MHL discovery is still unsupported. This batch does not claim full chain, rename or directory-hash verification. ASC checks explicitly describe their file-content scope. Histories are never rewritten by these checks.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Synthetic clip one
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<sidecar/>
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Synthetic clip two
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<hashlist version="2.0" xmlns="urn:ASC:MHL:v2.0">
<creatorinfo>
<creationdate>2026-10-17T00:35:13-04:00</creationdate>
<hostname>bitmatch-fixture-host</hostname>
<tool version="1.2">ascmhl</tool>
</creatorinfo>
<processinfo>
<process>in-place</process>
<roothash>
<content>
<md5 hashdate="2026-10-17T00:35:13.420594-04:00">5bcd50fab8aa16f017f7a6e755556d03</md5>
</content>
<structure>
<md5 hashdate="2026-10-17T00:35:13.420594-04:00">2e811871b223a14d3cec169a67a2506f</md5>
</structure>
</roothash>
<ignore>
<pattern>.DS_Store</pattern>
<pattern>ascmhl</pattern>
<pattern>ascmhl/</pattern>
</ignore>
</processinfo>
<hashes>
<hash>
<path size="19" lastmodificationdate="2026-10-17T00:35:13-04:00">Reel A/clip one.mov</path>
<md5 action="original" hashdate="2026-10-17T00:35:13.417666-04:00">bbb3e9a25ab3b29f321946e0ced197e5</md5>
</hash>
<hash>
<path size="11" lastmodificationdate="2026-10-17T00:35:13-04:00">Reel A/sidecar.xml</path>
<md5 action="original" hashdate="2026-10-17T00:35:13.418321-04:00">90dc46f5178d03fba3d712bac451d6e8</md5>
</hash>
<directoryhash>
<path lastmodificationdate="2026-10-17T00:35:13-04:00">Reel A</path>
<content>
<md5 hashdate="2026-10-17T00:35:13.418893-04:00">8c63cc817a52bc5235cf3e8a35bc74f3</md5>
</content>
<structure>
<md5 hashdate="2026-10-17T00:35:13.418893-04:00">e7355991ecfb7df08e89868c8c9c028c</md5>
</structure>
</directoryhash>
<hash>
<path size="19" lastmodificationdate="2026-10-17T00:35:13-04:00">Reel B/clip two.mov</path>
<md5 action="original" hashdate="2026-10-17T00:35:13.419556-04:00">d597faaafb635c7ab9b94dce00f2caf3</md5>
</hash>
<directoryhash>
<path lastmodificationdate="2026-10-17T00:35:13-04:00">Reel B</path>
<content>
<md5 hashdate="2026-10-17T00:35:13.420085-04:00">1599790324681fcf0afe1efd7e5c1597</md5>
</content>
<structure>
<md5 hashdate="2026-10-17T00:35:13.420085-04:00">fdd92faea2dfe67959219e35d00dcfda</md5>
</structure>
</directoryhash>
</hashes>
</hashlist>
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
<?xml version="1.0" encoding="UTF-8"?>
<hashlist version="2.0" xmlns="urn:ASC:MHL:v2.0">
<creatorinfo>
<creationdate>2026-10-17T00:35:13-04:00</creationdate>
<hostname>bitmatch-fixture-host</hostname>
<tool version="1.2">ascmhl</tool>
</creatorinfo>
<processinfo>
<process>in-place</process>
<roothash>
<content>
<md5 hashdate="2026-10-17T00:35:13.436542-04:00">5bcd50fab8aa16f017f7a6e755556d03</md5>
</content>
<structure>
<md5 hashdate="2026-10-17T00:35:13.436542-04:00">79ad990a57773810cd5d5c9654456bde</md5>
</structure>
</roothash>
<ignore>
<pattern>.DS_Store</pattern>
<pattern>ascmhl</pattern>
<pattern>ascmhl/</pattern>
</ignore>
</processinfo>
<hashes>
<hash>
<path size="19" lastmodificationdate="2026-10-17T00:35:13-04:00">Reel A/clip renamed.mov</path>
<md5 action="original" hashdate="2026-10-17T00:35:13.433173-04:00">bbb3e9a25ab3b29f321946e0ced197e5</md5>
<previousPath>Reel A/clip one.mov</previousPath>
</hash>
<hash>
<path size="11" lastmodificationdate="2026-10-17T00:35:13-04:00">Reel A/sidecar.xml</path>
<md5 action="verified" hashdate="2026-10-17T00:35:13.433954-04:00">90dc46f5178d03fba3d712bac451d6e8</md5>
</hash>
<directoryhash>
<path lastmodificationdate="2026-10-17T00:35:13-04:00">Reel A</path>
<content>
<md5 hashdate="2026-10-17T00:35:13.434580-04:00">8c63cc817a52bc5235cf3e8a35bc74f3</md5>
</content>
<structure>
<md5 hashdate="2026-10-17T00:35:13.434580-04:00">d9f8b0c2af5056964a0ca7d4eb314a42</md5>
</structure>
</directoryhash>
<hash>
<path size="19" lastmodificationdate="2026-10-17T00:35:13-04:00">Reel C/clip two.mov</path>
<md5 action="original" hashdate="2026-10-17T00:35:13.435402-04:00">d597faaafb635c7ab9b94dce00f2caf3</md5>
<previousPath>Reel B/clip two.mov</previousPath>
</hash>
<directoryhash>
<path lastmodificationdate="2026-10-17T00:35:13-04:00">Reel C</path>
<content>
<md5 hashdate="2026-10-17T00:35:13.435945-04:00">1599790324681fcf0afe1efd7e5c1597</md5>
</content>
<structure>
<md5 hashdate="2026-10-17T00:35:13.435945-04:00">fdd92faea2dfe67959219e35d00dcfda</md5>
</structure>
<previousPath>Reel B</previousPath>
</directoryhash>
</hashes>
</hashlist>
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<ascmhldirectory xmlns="urn:ASC:MHL:DIRECTORY:v2.0">
<hashlist sequencenr="1">
<path>0001_md5-renames_2026-10-17_043513Z.mhl</path>
<c4>c45p2RxQ2T7Fo6ZSkRiiFN7TJv12g8nKE9iu6ApwEvJoyEzUPskAo9LwaJzZtoP67cLHFVqtwj8NAMbL8AbveNiJkz</c4>
</hashlist>
<hashlist sequencenr="2">
<path>0002_md5-renames_2026-10-17_043513Z.mhl</path>
<c4>c41XRWFSyjnPhJF5mjhn3GGCe9qcXS12nDh7iAG4GFaBXPgQnSXgxYgx9Hqf4eKq58yVss1MjS6qujMnhZQGv8Lz4s</c4>
</hashlist>
</ascmhldirectory>
25 changes: 21 additions & 4 deletions Scripts/ascmhl/create_compatibility_fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
with patch.object(Updater, '_get_latest_version', return_value=None):
from ascmhl.cli.ascmhl import mhltool_cli

datasets = ['md5', 'xxh64', 'c4', 'sha1', 'xxh128', 'xxh64-boundaries']
datasets = ['md5', 'xxh64', 'c4', 'sha1', 'xxh128', 'xxh64-boundaries', 'md5-renames']
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('output', type=pathlib.Path)
parser.add_argument('--datasets', nargs='+', choices=datasets, default=datasets)
Expand All @@ -23,15 +23,32 @@
output.mkdir(parents=True, exist_ok=False)
runner = CliRunner()

def create(root, algorithm):
def create(root, algorithm, detect_renaming=False):
arguments = ['create', '-h', algorithm]
if detect_renaming:
arguments.append('-dr')
arguments.append(str(root))
with patch('platform.node', return_value='bitmatch-fixture-host'):
result = runner.invoke(mhltool_cli, ['create', '-h', algorithm, str(root)])
result = runner.invoke(mhltool_cli, arguments)
if result.exit_code:
raise RuntimeError(result.output) from result.exception

for dataset in args.datasets:
algorithm = 'xxh64' if dataset == 'xxh64-boundaries' else dataset
algorithm = {'xxh64-boundaries': 'xxh64', 'md5-renames': 'md5'}.get(dataset, dataset)
root = output / dataset
if dataset == 'md5-renames':
# Only the reference tool creates the two manifests and chain. Both
# generations describe original, tiny synthetic media bytes.
(root / 'Reel A').mkdir(parents=True)
(root / 'Reel B').mkdir()
(root / 'Reel A' / 'clip one.mov').write_bytes(b'Synthetic clip one\n')
(root / 'Reel A' / 'sidecar.xml').write_bytes(b'<sidecar/>\n')
(root / 'Reel B' / 'clip two.mov').write_bytes(b'Synthetic clip two\n')
create(root, algorithm)
(root / 'Reel A' / 'clip one.mov').rename(root / 'Reel A' / 'clip renamed.mov')
(root / 'Reel B').rename(root / 'Reel C')
create(root, algorithm, detect_renaming=True)
continue
if dataset == 'xxh64-boundaries':
root.mkdir()
for size in [0, 1, 31, 32, 33, 255, 256, 257, 1025]:
Expand Down
Loading
Loading