diff --git a/docs/pipeline-setup.md b/docs/pipeline-setup.md index c0b9aae..2248c8b 100644 --- a/docs/pipeline-setup.md +++ b/docs/pipeline-setup.md @@ -174,6 +174,16 @@ the cleaning volume's `setup-receipts/`. Legacy attempt and step receipt paths a An incomplete attempt requires inspection and reconciliation. A complete attempt returns its saved report without downloading again, though a new CLI invocation can still launch a container. +LHM prior staging excludes entries with an exact `.git` path component. This skips the Git LFS +object stored alongside the runtime Sapiens `.pt2` checkpoint and preserves the checkpoint at +its original path. A fresh `lhm` run still reads and hashes the entire source archive. `lhm-resume` +validates the declared source length and pins the ETag on every range. It reads the headers for +skipped entries and seeks past their file payloads. `skippedGitEntries` counts excluded members +and `skippedGitBytes` sums their declared sizes. `rangeBytesReceived` reports the bytes actually +fetched during resume, including headers. Any partial `.git` files left by an older attempt stay +on the volume and are omitted from the final runtime inventory. Existing receipt and recovery +checks still apply. + Staging is outside the paid-stage ledger. The function requests one CPU and 1 GiB of memory, with limits of two CPUs and 4 GiB, one container per function configuration, zero function retries and a two-second scaledown window. Modal's execution timeout is 1,800 seconds. On function entry, diff --git a/scripts/tests/test_stage_weights.py b/scripts/tests/test_stage_weights.py index 7fcdbd6..70573b6 100644 --- a/scripts/tests/test_stage_weights.py +++ b/scripts/tests/test_stage_weights.py @@ -221,6 +221,30 @@ def test_clean_only_refuses_an_incomplete_attempt_before_downloading(tmp_path): assert receipt.read_text() == previous +@pytest.mark.parametrize("status", ["failed", "complete"]) +def test_lhm_resume_preserves_legacy_attempt_receipt(tmp_path, status): + receipt = tmp_path / "lhm/setup-receipts/lhm-resume-attempt.json" + receipt.parent.mkdir(parents=True) + legacy = {"phase": "lhm-resume", "status": status, "steps": []} + original = json.dumps(legacy) + receipt.write_text(original) + raw_stage = synchronizer._translate_in(staging.stage)._raw_f + with ( + patch.object(staging, "Path", side_effect=lambda path: tmp_path / path.lstrip("/")), + patch.object(staging.modal, "current_function_call_id", return_value="fc-offline"), + patch.dict(sys.modules, {"huggingface_hub": SimpleNamespace(snapshot_download=Mock())}), + patch.object(staging, "extract_missing_priors") as resume, + ): + deadline = time.time() + 1200 + if status == "failed": + with pytest.raises(RuntimeError, match="prior incomplete attempt"): + raw_stage("lhm-resume", deadline) + else: + assert raw_stage("lhm-resume", deadline) == legacy + resume.assert_not_called() + assert receipt.read_text() == original + + class StageWeightsTest(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() @@ -232,13 +256,25 @@ def setUp(self): } self.tar = archive_bytes(self.files) - def server(self, *, bad_range=False, bad_etag=False, status=206, truncated=False): + def server( + self, + *, + bad_range=False, + bad_etag=False, + status=206, + truncated=False, + head_etag='"fixed"', + head_length_delta=0, + ): requests = [] def serve(request, timeout): self.assertIn(timeout, (30, 45)) if request.get_method() == "HEAD": - return Response(b"", {"ETag": '"fixed"', "Content-Length": str(len(self.tar))}) + return Response( + b"", + {"ETag": head_etag, "Content-Length": str(len(self.tar) + head_length_delta)}, + ) headers = dict(request.header_items()) self.assertEqual(headers["If-match"], '"fixed"') first, last = map(int, headers["Range"].removeprefix("bytes=").split("-")) @@ -276,6 +312,30 @@ def test_official_auxiliary_root_is_extracted(self): (self.root / "gfpgan/weights/parsing_parsenet.pth").read_bytes(), b"public model" ) + def test_stream_skips_exact_git_components_but_hashes_the_full_archive(self): + import hashlib + + runtime = { + "pretrained_models/sapiens/model.pt2": b"sapiens", + "pretrained_models/.gitkeep": b"keep", + "pretrained_models/runtime.git/config": b"runtime", + "gfpgan/.github/config": b"config", + } + metadata = { + "./pretrained_models/sapiens/.git/lfs/objects/blob": b"metadata blob", + "gfpgan/.git": b"git worktree pointer", + } + data = archive_bytes(metadata | runtime) + report = extract_priors(io.BytesIO(data), self.root, len(data)) + self.assertEqual(report["archiveBytes"], len(data)) + self.assertEqual(report["archiveSha256"], hashlib.sha256(data).hexdigest()) + self.assertEqual(report["archiveEntries"], len(runtime)) + self.assertEqual(report["skippedGitEntries"], len(metadata)) + self.assertEqual(report["skippedGitBytes"], sum(map(len, metadata.values()))) + for name, contents in runtime.items(): + self.assertEqual((self.root / name).read_bytes(), contents) + self.assertEqual(list(self.root.rglob(".git")), []) + def test_stream_rejects_wrong_length(self): with self.assertRaisesRegex(ValueError, "content length mismatch"): extract_priors(io.BytesIO(self.tar), self.root, len(self.tar) + 1) @@ -286,6 +346,7 @@ def test_stream_rejects_unsafe_member_names(self): "/pretrained_models/escape", "other/root", "pretrained_models/../escape", + "pretrained_models/.git/../../escape", ): with self.subTest(name=name), self.assertRaises(ValueError): data = archive_bytes({name: b"unsafe"}) @@ -323,6 +384,61 @@ def test_range_reuses_complete_files_and_repairs_partial_files(self): for name, contents in self.files.items(): self.assertEqual((self.root / name).read_bytes(), contents) + def test_range_never_requests_git_payload_and_extracts_the_runtime_checkpoint(self): + model = "pretrained_models/sapiens/checkpoints/sapiens_1b_epoch_173_torchscript.pt2" + metadata = "pretrained_models/sapiens/checkpoints/sapiens-pretrain-1b-torchscript/.git/lfs/objects/b5/09/excluded-blob" + weights = b"a" * (2 * 1024 * 1024) + runtime = {model: weights, "pretrained_models/.gitkeep": b"keep after skipped blob"} + self.tar = archive_bytes({model: weights, metadata: weights} | runtime) + with tarfile.open(fileobj=io.BytesIO(self.tar), mode="r:") as archive: + skipped = archive.getmember(metadata) + partial = self.root / metadata + partial.parent.mkdir(parents=True) + partial.write_bytes(b"previous partial data") + + serve, requests = self.server() + with patch("urllib.request.urlopen", side_effect=serve): + report = extract_missing_priors( + "https://example.invalid/models.tar", self.root, len(self.tar) + ) + + self.assertFalse( + any( + first < skipped.offset_data + skipped.size and last >= skipped.offset_data + for first, last in requests + ) + ) + self.assertEqual( + report["rangeBytesReceived"], sum(last - first + 1 for first, last in requests) + ) + self.assertEqual(report["rangeRequests"], len(requests)) + self.assertEqual(report["archiveBytes"], len(self.tar)) + self.assertIsNone(report["archiveSha256"]) + self.assertEqual(report["sourceETag"], '"fixed"') + self.assertEqual(report["skippedGitEntries"], 1) + self.assertEqual(report["skippedGitBytes"], skipped.size) + self.assertEqual(report["extractedMembers"], len(runtime)) + self.assertEqual(report["reusedMembers"], 0) + for name, contents in runtime.items(): + self.assertEqual((self.root / name).read_bytes(), contents) + self.assertEqual(partial.read_bytes(), b"previous partial data") + (partial.parent / "broken").symlink_to("missing") + inventory = staging.inventory(self.root, skip_git_metadata=True) + self.assertEqual({entry["path"] for entry in inventory["files"]}, set(runtime)) + + def test_range_requires_source_etag_and_exact_archive_length(self): + for options in ({"head_etag": ""}, {"head_length_delta": 1}): + with self.subTest(options=options): + serve, requests = self.server(**options) + with ( + patch("urllib.request.urlopen", side_effect=serve), + self.assertRaisesRegex(ValueError, "known-size object and ETag"), + ): + extract_missing_priors( + "https://example.invalid/models.tar", self.root, len(self.tar) + ) + self.assertEqual(requests, []) + def test_range_fails_closed_on_unpinned_or_truncated_responses(self): for options in ( {"bad_range": True}, @@ -342,10 +458,17 @@ def test_range_fails_closed_on_unpinned_or_truncated_responses(self): self.assertEqual(len(requests), 1, "transport failure must not retry") def test_range_rejects_unsafe_member(self): - self.tar = archive_bytes({"pretrained_models/../escape": b"unsafe"}) - serve, _ = self.server() - with patch("urllib.request.urlopen", side_effect=serve), self.assertRaises(ValueError): - extract_missing_priors("https://example.invalid/models.tar", self.root, len(self.tar)) + for name in ("pretrained_models/../escape", "pretrained_models/.git/../../escape"): + with self.subTest(name=name): + self.tar = archive_bytes({name: b"unsafe"}) + serve, _ = self.server() + with ( + patch("urllib.request.urlopen", side_effect=serve), + self.assertRaises(ValueError), + ): + extract_missing_priors( + "https://example.invalid/models.tar", self.root, len(self.tar) + ) if __name__ == "__main__": diff --git a/worker/modal_stage_weights.py b/worker/modal_stage_weights.py index 9c6047b..0ae1c1d 100644 --- a/worker/modal_stage_weights.py +++ b/worker/modal_stage_weights.py @@ -49,11 +49,14 @@ def digest_file(path): return digest.hexdigest() -def inventory(root): +def inventory(root, *, skip_git_metadata=False): """Hash real files and preserve/validate links without duplicating blob bytes in totals.""" records = [] for path in sorted(Path(root).rglob("*")): - name = str(path.relative_to(root)) + relative = path.relative_to(root) + if skip_git_metadata and ".git" in relative.parts: + continue + name = str(relative) if path.is_symlink(): if not path.exists() or not path.resolve().is_relative_to(Path(root).resolve()): raise ValueError("cache has a broken or escaping symlink") @@ -99,6 +102,7 @@ def read(self, size=-1): reader = Reader() count = 0 + skipped_git_entries = skipped_git_bytes = 0 with tarfile.open(fileobj=reader, mode="r|") as archive: for member in archive: normalized = Path(member.name) @@ -107,6 +111,10 @@ def read(self, size=-1): continue if not parts or parts[0] not in ("pretrained_models", "gfpgan") or ".." in parts: raise ValueError("unexpected prior archive root") + if ".git" in parts: + skipped_git_entries += 1 + skipped_git_bytes += member.size + continue archive.extract(member, path=destination, filter="data") count += 1 if count % 25 == 0: @@ -121,17 +129,20 @@ def read(self, size=-1): "archiveBytes": reader.bytes, "archiveSha256": reader.digest.hexdigest(), "archiveEntries": count, + "skippedGitEntries": skipped_git_entries, + "skippedGitBytes": skipped_git_bytes, } def extract_missing_priors(url, destination, expected_bytes): - """Continue an interrupted uncompressed tar, using pinned ranges to skip intact members. + """Continue an interrupted uncompressed tar using pinned ranges for missing runtime files. - The resumed receipt has no whole-archive digest. Inventory hashes every final file instead. + The resumed receipt has no whole-archive digest. Inventory hashes final runtime files instead. """ import tarfile import urllib.request + range_chunk_bytes = 16 * 1024 * 1024 with urllib.request.urlopen(urllib.request.Request(url, method="HEAD"), timeout=30) as head: etag = head.headers.get("ETag") if not etag or int(head.headers["Content-Length"]) != expected_bytes: @@ -140,8 +151,6 @@ def extract_missing_priors(url, destination, expected_bytes): class RangeReader: def __init__(self): self.position = 0 - self.buffer_start = 0 - self.buffer = b"" self.bytes = 0 self.requests = 0 @@ -162,44 +171,47 @@ def read(self, size): chunks = [] remaining = min(size, expected_bytes - self.position) while remaining: - offset = self.position - self.buffer_start - if not 0 <= offset < len(self.buffer): - count = 65536 if remaining <= 65536 else 16 * 1024 * 1024 - end = min(expected_bytes, self.position + count) - 1 - request = urllib.request.Request( - url, headers={"Range": f"bytes={self.position}-{end}", "If-Match": etag} - ) - with urllib.request.urlopen(request, timeout=45) as response: - expected_range = f"bytes {self.position}-{end}/{expected_bytes}" - if ( - response.status != 206 - or response.headers.get("Content-Range") != expected_range - or response.headers.get("ETag") != etag - ): - raise ValueError("upstream did not honor pinned byte range") - self.buffer = response.read(end - self.position + 2) - if len(self.buffer) != end - self.position + 1: - raise ValueError("range response length mismatch") - self.buffer_start = self.position - self.bytes += len(self.buffer) - self.requests += 1 - offset = 0 - take = min(remaining, len(self.buffer) - offset) - chunks.append(self.buffer[offset : offset + take]) - self.position += take - remaining -= take + # A header read must stop before the member payload, which may be skipped. + count = min(remaining, range_chunk_bytes) + end = self.position + count - 1 + request = urllib.request.Request( + url, headers={"Range": f"bytes={self.position}-{end}", "If-Match": etag} + ) + with urllib.request.urlopen(request, timeout=45) as response: + expected_range = f"bytes {self.position}-{end}/{expected_bytes}" + if ( + response.status != 206 + or response.headers.get("Content-Range") != expected_range + or response.headers.get("ETag") != etag + ): + raise ValueError("upstream did not honor pinned byte range") + chunk = response.read(count + 1) + if len(chunk) != count: + raise ValueError("range response length mismatch") + chunks.append(chunk) + self.position += count + self.bytes += count + self.requests += 1 + remaining -= count return b"".join(chunks) reader = RangeReader() reused = extracted = 0 + skipped_git_entries = skipped_git_bytes = 0 with tarfile.open(fileobj=reader, mode="r:") as archive: - archive.copybufsize = 4 * 1024 * 1024 + archive.copybufsize = range_chunk_bytes for member in archive: parts = Path(member.name).parts if not parts and member.isdir(): continue if not parts or parts[0] not in ("pretrained_models", "gfpgan") or ".." in parts: raise ValueError("unexpected prior archive root during recovery") + if ".git" in parts: + skipped_git_entries += 1 + skipped_git_bytes += member.size + # TarFile.next otherwise probes the last byte of the skipped member's block. + reader.seek(archive.offset) + continue path = Path(destination) / member.name if ( member.isfile() @@ -225,6 +237,8 @@ def read(self, size): "rangeRequests": reader.requests, "reusedMembers": reused, "extractedMembers": extracted, + "skippedGitEntries": skipped_git_entries, + "skippedGitBytes": skipped_git_bytes, } @@ -457,7 +471,7 @@ def step_done(name, document): for name in ("human_model_files", "gagatracker", "dense_sample_points", "voxel_grid"): if not (pending / "pretrained_models" / name).exists(): raise ValueError("required prior subtree missing") - prior_inventory = inventory(pending) + prior_inventory = inventory(pending, skip_git_metadata=True) pending.rename(destination) step_done( "lhm-priors",