Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions docs/pipeline-setup.md
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,16 @@ the cleaning volume's `setup-receipts/`. Legacy attempt and step receipt paths a
An incomplete attempt requires inspection and reconciliation. A complete attempt returns its
saved report without downloading again, though a new CLI invocation can still launch a container.

LHM prior staging excludes entries with an exact `.git` path component. This skips the Git LFS
object stored alongside the runtime Sapiens `.pt2` checkpoint and preserves the checkpoint at
its original path. A fresh `lhm` run still reads and hashes the entire source archive. `lhm-resume`
validates the declared source length and pins the ETag on every range. It reads the headers for
skipped entries and seeks past their file payloads. `skippedGitEntries` counts excluded members
and `skippedGitBytes` sums their declared sizes. `rangeBytesReceived` reports the bytes actually
fetched during resume, including headers. Any partial `.git` files left by an older attempt stay
on the volume and are omitted from the final runtime inventory. Existing receipt and recovery
checks still apply.

Staging is outside the paid-stage ledger. The function requests one CPU and 1 GiB of memory,
with limits of two CPUs and 4 GiB, one container per function configuration, zero function retries
and a two-second scaledown window. Modal's execution timeout is 1,800 seconds. On function entry,
Expand Down
135 changes: 129 additions & 6 deletions scripts/tests/test_stage_weights.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,30 @@ def test_clean_only_refuses_an_incomplete_attempt_before_downloading(tmp_path):
assert receipt.read_text() == previous


@pytest.mark.parametrize("status", ["failed", "complete"])
def test_lhm_resume_preserves_legacy_attempt_receipt(tmp_path, status):
receipt = tmp_path / "lhm/setup-receipts/lhm-resume-attempt.json"
receipt.parent.mkdir(parents=True)
legacy = {"phase": "lhm-resume", "status": status, "steps": []}
original = json.dumps(legacy)
receipt.write_text(original)
raw_stage = synchronizer._translate_in(staging.stage)._raw_f
with (
patch.object(staging, "Path", side_effect=lambda path: tmp_path / path.lstrip("/")),
patch.object(staging.modal, "current_function_call_id", return_value="fc-offline"),
patch.dict(sys.modules, {"huggingface_hub": SimpleNamespace(snapshot_download=Mock())}),
patch.object(staging, "extract_missing_priors") as resume,
):
deadline = time.time() + 1200
if status == "failed":
with pytest.raises(RuntimeError, match="prior incomplete attempt"):
raw_stage("lhm-resume", deadline)
else:
assert raw_stage("lhm-resume", deadline) == legacy
resume.assert_not_called()
assert receipt.read_text() == original


class StageWeightsTest(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
Expand All @@ -232,13 +256,25 @@ def setUp(self):
}
self.tar = archive_bytes(self.files)

def server(self, *, bad_range=False, bad_etag=False, status=206, truncated=False):
def server(
self,
*,
bad_range=False,
bad_etag=False,
status=206,
truncated=False,
head_etag='"fixed"',
head_length_delta=0,
):
requests = []

def serve(request, timeout):
self.assertIn(timeout, (30, 45))
if request.get_method() == "HEAD":
return Response(b"", {"ETag": '"fixed"', "Content-Length": str(len(self.tar))})
return Response(
b"",
{"ETag": head_etag, "Content-Length": str(len(self.tar) + head_length_delta)},
)
headers = dict(request.header_items())
self.assertEqual(headers["If-match"], '"fixed"')
first, last = map(int, headers["Range"].removeprefix("bytes=").split("-"))
Expand Down Expand Up @@ -276,6 +312,30 @@ def test_official_auxiliary_root_is_extracted(self):
(self.root / "gfpgan/weights/parsing_parsenet.pth").read_bytes(), b"public model"
)

def test_stream_skips_exact_git_components_but_hashes_the_full_archive(self):
import hashlib

runtime = {
"pretrained_models/sapiens/model.pt2": b"sapiens",
"pretrained_models/.gitkeep": b"keep",
"pretrained_models/runtime.git/config": b"runtime",
"gfpgan/.github/config": b"config",
}
metadata = {
"./pretrained_models/sapiens/.git/lfs/objects/blob": b"metadata blob",
"gfpgan/.git": b"git worktree pointer",
}
data = archive_bytes(metadata | runtime)
report = extract_priors(io.BytesIO(data), self.root, len(data))
self.assertEqual(report["archiveBytes"], len(data))
self.assertEqual(report["archiveSha256"], hashlib.sha256(data).hexdigest())
self.assertEqual(report["archiveEntries"], len(runtime))
self.assertEqual(report["skippedGitEntries"], len(metadata))
self.assertEqual(report["skippedGitBytes"], sum(map(len, metadata.values())))
for name, contents in runtime.items():
self.assertEqual((self.root / name).read_bytes(), contents)
self.assertEqual(list(self.root.rglob(".git")), [])

def test_stream_rejects_wrong_length(self):
with self.assertRaisesRegex(ValueError, "content length mismatch"):
extract_priors(io.BytesIO(self.tar), self.root, len(self.tar) + 1)
Expand All @@ -286,6 +346,7 @@ def test_stream_rejects_unsafe_member_names(self):
"/pretrained_models/escape",
"other/root",
"pretrained_models/../escape",
"pretrained_models/.git/../../escape",
):
with self.subTest(name=name), self.assertRaises(ValueError):
data = archive_bytes({name: b"unsafe"})
Expand Down Expand Up @@ -323,6 +384,61 @@ def test_range_reuses_complete_files_and_repairs_partial_files(self):
for name, contents in self.files.items():
self.assertEqual((self.root / name).read_bytes(), contents)

def test_range_never_requests_git_payload_and_extracts_the_runtime_checkpoint(self):
model = "pretrained_models/sapiens/checkpoints/sapiens_1b_epoch_173_torchscript.pt2"
metadata = "pretrained_models/sapiens/checkpoints/sapiens-pretrain-1b-torchscript/.git/lfs/objects/b5/09/excluded-blob"
weights = b"a" * (2 * 1024 * 1024)
runtime = {model: weights, "pretrained_models/.gitkeep": b"keep after skipped blob"}
self.tar = archive_bytes({model: weights, metadata: weights} | runtime)
with tarfile.open(fileobj=io.BytesIO(self.tar), mode="r:") as archive:
skipped = archive.getmember(metadata)
partial = self.root / metadata
partial.parent.mkdir(parents=True)
partial.write_bytes(b"previous partial data")

serve, requests = self.server()
with patch("urllib.request.urlopen", side_effect=serve):
report = extract_missing_priors(
"https://example.invalid/models.tar", self.root, len(self.tar)
)

self.assertFalse(
any(
first < skipped.offset_data + skipped.size and last >= skipped.offset_data
for first, last in requests
)
)
self.assertEqual(
report["rangeBytesReceived"], sum(last - first + 1 for first, last in requests)
)
self.assertEqual(report["rangeRequests"], len(requests))
self.assertEqual(report["archiveBytes"], len(self.tar))
self.assertIsNone(report["archiveSha256"])
self.assertEqual(report["sourceETag"], '"fixed"')
self.assertEqual(report["skippedGitEntries"], 1)
self.assertEqual(report["skippedGitBytes"], skipped.size)
self.assertEqual(report["extractedMembers"], len(runtime))
self.assertEqual(report["reusedMembers"], 0)
for name, contents in runtime.items():
self.assertEqual((self.root / name).read_bytes(), contents)
self.assertEqual(partial.read_bytes(), b"previous partial data")
(partial.parent / "broken").symlink_to("missing")
inventory = staging.inventory(self.root, skip_git_metadata=True)
self.assertEqual({entry["path"] for entry in inventory["files"]}, set(runtime))

def test_range_requires_source_etag_and_exact_archive_length(self):
for options in ({"head_etag": ""}, {"head_length_delta": 1}):
with self.subTest(options=options):
serve, requests = self.server(**options)
with (
patch("urllib.request.urlopen", side_effect=serve),
self.assertRaisesRegex(ValueError, "known-size object and ETag"),
):
extract_missing_priors(
"https://example.invalid/models.tar", self.root, len(self.tar)
)
self.assertEqual(requests, [])

def test_range_fails_closed_on_unpinned_or_truncated_responses(self):
for options in (
{"bad_range": True},
Expand All @@ -342,10 +458,17 @@ def test_range_fails_closed_on_unpinned_or_truncated_responses(self):
self.assertEqual(len(requests), 1, "transport failure must not retry")

def test_range_rejects_unsafe_member(self):
self.tar = archive_bytes({"pretrained_models/../escape": b"unsafe"})
serve, _ = self.server()
with patch("urllib.request.urlopen", side_effect=serve), self.assertRaises(ValueError):
extract_missing_priors("https://example.invalid/models.tar", self.root, len(self.tar))
for name in ("pretrained_models/../escape", "pretrained_models/.git/../../escape"):
with self.subTest(name=name):
self.tar = archive_bytes({name: b"unsafe"})
serve, _ = self.server()
with (
patch("urllib.request.urlopen", side_effect=serve),
self.assertRaises(ValueError),
):
extract_missing_priors(
"https://example.invalid/models.tar", self.root, len(self.tar)
)


if __name__ == "__main__":
Expand Down
82 changes: 48 additions & 34 deletions worker/modal_stage_weights.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,14 @@ def digest_file(path):
return digest.hexdigest()


def inventory(root):
def inventory(root, *, skip_git_metadata=False):
"""Hash real files and preserve/validate links without duplicating blob bytes in totals."""
records = []
for path in sorted(Path(root).rglob("*")):
name = str(path.relative_to(root))
relative = path.relative_to(root)
if skip_git_metadata and ".git" in relative.parts:
continue
name = str(relative)
if path.is_symlink():
if not path.exists() or not path.resolve().is_relative_to(Path(root).resolve()):
raise ValueError("cache has a broken or escaping symlink")
Expand Down Expand Up @@ -99,6 +102,7 @@ def read(self, size=-1):

reader = Reader()
count = 0
skipped_git_entries = skipped_git_bytes = 0
with tarfile.open(fileobj=reader, mode="r|") as archive:
for member in archive:
normalized = Path(member.name)
Expand All @@ -107,6 +111,10 @@ def read(self, size=-1):
continue
if not parts or parts[0] not in ("pretrained_models", "gfpgan") or ".." in parts:
raise ValueError("unexpected prior archive root")
if ".git" in parts:
skipped_git_entries += 1
skipped_git_bytes += member.size
continue
archive.extract(member, path=destination, filter="data")
count += 1
if count % 25 == 0:
Expand All @@ -121,17 +129,20 @@ def read(self, size=-1):
"archiveBytes": reader.bytes,
"archiveSha256": reader.digest.hexdigest(),
"archiveEntries": count,
"skippedGitEntries": skipped_git_entries,
"skippedGitBytes": skipped_git_bytes,
}


def extract_missing_priors(url, destination, expected_bytes):
"""Continue an interrupted uncompressed tar, using pinned ranges to skip intact members.
"""Continue an interrupted uncompressed tar using pinned ranges for missing runtime files.

The resumed receipt has no whole-archive digest. Inventory hashes every final file instead.
The resumed receipt has no whole-archive digest. Inventory hashes final runtime files instead.
"""
import tarfile
import urllib.request

range_chunk_bytes = 16 * 1024 * 1024
with urllib.request.urlopen(urllib.request.Request(url, method="HEAD"), timeout=30) as head:
etag = head.headers.get("ETag")
if not etag or int(head.headers["Content-Length"]) != expected_bytes:
Expand All @@ -140,8 +151,6 @@ def extract_missing_priors(url, destination, expected_bytes):
class RangeReader:
def __init__(self):
self.position = 0
self.buffer_start = 0
self.buffer = b""
self.bytes = 0
self.requests = 0

Expand All @@ -162,44 +171,47 @@ def read(self, size):
chunks = []
remaining = min(size, expected_bytes - self.position)
while remaining:
offset = self.position - self.buffer_start
if not 0 <= offset < len(self.buffer):
count = 65536 if remaining <= 65536 else 16 * 1024 * 1024
end = min(expected_bytes, self.position + count) - 1
request = urllib.request.Request(
url, headers={"Range": f"bytes={self.position}-{end}", "If-Match": etag}
)
with urllib.request.urlopen(request, timeout=45) as response:
expected_range = f"bytes {self.position}-{end}/{expected_bytes}"
if (
response.status != 206
or response.headers.get("Content-Range") != expected_range
or response.headers.get("ETag") != etag
):
raise ValueError("upstream did not honor pinned byte range")
self.buffer = response.read(end - self.position + 2)
if len(self.buffer) != end - self.position + 1:
raise ValueError("range response length mismatch")
self.buffer_start = self.position
self.bytes += len(self.buffer)
self.requests += 1
offset = 0
take = min(remaining, len(self.buffer) - offset)
chunks.append(self.buffer[offset : offset + take])
self.position += take
remaining -= take
# A header read must stop before the member payload, which may be skipped.
count = min(remaining, range_chunk_bytes)
end = self.position + count - 1
request = urllib.request.Request(
url, headers={"Range": f"bytes={self.position}-{end}", "If-Match": etag}
)
with urllib.request.urlopen(request, timeout=45) as response:
expected_range = f"bytes {self.position}-{end}/{expected_bytes}"
if (
response.status != 206
or response.headers.get("Content-Range") != expected_range
or response.headers.get("ETag") != etag
):
raise ValueError("upstream did not honor pinned byte range")
chunk = response.read(count + 1)
if len(chunk) != count:
raise ValueError("range response length mismatch")
chunks.append(chunk)
self.position += count
self.bytes += count
self.requests += 1
remaining -= count
return b"".join(chunks)

reader = RangeReader()
reused = extracted = 0
skipped_git_entries = skipped_git_bytes = 0
with tarfile.open(fileobj=reader, mode="r:") as archive:
archive.copybufsize = 4 * 1024 * 1024
archive.copybufsize = range_chunk_bytes
for member in archive:
parts = Path(member.name).parts
if not parts and member.isdir():
continue
if not parts or parts[0] not in ("pretrained_models", "gfpgan") or ".." in parts:
raise ValueError("unexpected prior archive root during recovery")
if ".git" in parts:
skipped_git_entries += 1
skipped_git_bytes += member.size
# TarFile.next otherwise probes the last byte of the skipped member's block.
reader.seek(archive.offset)
continue
path = Path(destination) / member.name
if (
member.isfile()
Expand All @@ -225,6 +237,8 @@ def read(self, size):
"rangeRequests": reader.requests,
"reusedMembers": reused,
"extractedMembers": extracted,
"skippedGitEntries": skipped_git_entries,
"skippedGitBytes": skipped_git_bytes,
}


Expand Down Expand Up @@ -457,7 +471,7 @@ def step_done(name, document):
for name in ("human_model_files", "gagatracker", "dense_sample_points", "voxel_grid"):
if not (pending / "pretrained_models" / name).exists():
raise ValueError("required prior subtree missing")
prior_inventory = inventory(pending)
prior_inventory = inventory(pending, skip_git_metadata=True)
pending.rename(destination)
step_done(
"lhm-priors",
Expand Down
Loading