From 58d193a93c5bd6832bf2b3101e0641b00d6a96ba Mon Sep 17 00:00:00 2001 From: DeadlyVirusIn Date: Fri, 9 Oct 2026 05:57:43 -0400 Subject: [PATCH 1/6] feat(scene): bounded read-only scene contract (scene/1) for project inspection Adds a backend scene contract and job API that describe a 3D project's real geometry without modifying it: units, file-qualified node identities, repeated instances, nested transforms, plate membership, printable parts vs modifier/negative volumes, per-node findings and explicit limitations. - scene.py/scene_limits.py/units.py: strict scene-only archive and XML readers (lxml iterparse, element/triangle/vertex budgets, DTD and external relationship refusal, cancel/deadline hooks); fail-closed when a role, plate or unit cannot be proven; shared 3MF unit handling now also used by geometry, intelligence and placement (millimeter projects unchanged). - scene_jobs.py + server routes: single-worker job state machine, idempotent start, 60 s deadline, 8 MiB response cap, per-process snapshot directories with safe startup sweep. - stl_io.py: additive bounded streaming STL decoder. - plate_placement.py: translation rewrite now converts to the model's unit. - Schema scene-1.schema.json, golden fixture, benchmark and docs under docs/testing/scene-contract/. Verified: backend 3091 passed/49 skipped (py -3.13); 233 scene/unit tests; millimeter-unchanged differential over 50 fixtures identical to b01f7d0 (sha256 73a018e50d8397d4715d5183e4db8d372db34cb64e8a3a00e087b9f7c34a69ef). Reviewed on the final state by Opus, Sol and Astra: all APPROVE. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 3 +- backend/pyproject.toml | 2 + backend/snapstudio_api/scene_jobs.py | 634 +++++++ backend/snapstudio_api/server.py | 22 +- .../snapstudio_core/data/scene-1.schema.json | 224 +++ backend/snapstudio_core/geometry.py | 23 +- backend/snapstudio_core/intelligence.py | 9 +- backend/snapstudio_core/placement.py | 20 +- backend/snapstudio_core/plate_placement.py | 18 +- backend/snapstudio_core/scene.py | 1459 +++++++++++++++++ backend/snapstudio_core/scene_limits.py | 64 + backend/snapstudio_core/stl_io.py | 130 ++ backend/snapstudio_core/units.py | 102 ++ .../tests/fixtures/scene/golden-scene-1.json | 166 ++ backend/tests/fixtures/scene/make_golden.py | 53 + backend/tests/scene_fixtures.py | 187 +++ backend/tests/test_scene_api.py | 168 ++ backend/tests/test_scene_build.py | 310 ++++ backend/tests/test_scene_contract.py | 150 ++ backend/tests/test_scene_hostile.py | 355 ++++ backend/tests/test_scene_jobs.py | 746 +++++++++ backend/tests/test_scene_repair.py | 703 ++++++++ backend/tests/test_units_consumers.py | 213 +++ backend/tools/scene_benchmark.py | 303 ++++ docs/testing/scene-contract/README.md | 291 ++++ docs/testing/scene-contract/results.json | 915 +++++++++++ 26 files changed, 7250 insertions(+), 20 deletions(-) create mode 100644 backend/snapstudio_api/scene_jobs.py create mode 100644 backend/snapstudio_core/data/scene-1.schema.json create mode 100644 backend/snapstudio_core/scene.py create mode 100644 backend/snapstudio_core/scene_limits.py create mode 100644 backend/snapstudio_core/units.py create mode 100644 backend/tests/fixtures/scene/golden-scene-1.json create mode 100644 backend/tests/fixtures/scene/make_golden.py create mode 100644 backend/tests/scene_fixtures.py create mode 100644 backend/tests/test_scene_api.py create mode 100644 backend/tests/test_scene_build.py create mode 100644 backend/tests/test_scene_contract.py create mode 100644 backend/tests/test_scene_hostile.py create mode 100644 backend/tests/test_scene_jobs.py create mode 100644 backend/tests/test_scene_repair.py create mode 100644 backend/tests/test_units_consumers.py create mode 100644 backend/tools/scene_benchmark.py create mode 100644 docs/testing/scene-contract/README.md create mode 100644 docs/testing/scene-contract/results.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ecef129..27d075a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,8 +37,7 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -e . - pip install pytest + pip install -e ".[test]" - name: Test run: python -m pytest -q diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 304a6bcc..d42ea6b8 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -22,6 +22,8 @@ classifiers = [ [project.optional-dependencies] dev = ["pytest>=8"] +# Test-only: validates the committed scene/1 JSON Schema with a real validator (never imported by the engine). +test = ["pytest>=8", "jsonschema>=4.18,<5"] [project.scripts] u1convert = "u1convert.cli:cli" diff --git a/backend/snapstudio_api/scene_jobs.py b/backend/snapstudio_api/scene_jobs.py new file mode 100644 index 00000000..72368b1b --- /dev/null +++ b/backend/snapstudio_api/scene_jobs.py @@ -0,0 +1,634 @@ +"""Scene jobs: the bounded, cancellable background build behind /scene/start|status|result|cancel. + +Model (frozen by plan v3/v4/v5/v6): + +* **One worker thread, at most one queued job.** A new start cancels and replaces the client's own + earlier work (the newest queued job wins; a running job is marked cancelled at once). The worker + slot is released only when the thread has actually exited; a replacement therefore waits, queued, + until the cancelled thread observes the flag (bounded by the cooperative-interruption units in + ``snapstudio_core.scene_limits``). +* **States** ``queued | running | succeeded | failed | cancelled``. Terminal states are immutable and + the FIRST terminal wins: a worker finishing after a cancel discards its result. Cancel is legal from + queued|running and a late cancel on a terminal job is a no-op. There is no stored "expired" state: + an evicted or unknown id answers 404 EXPIRED. +* **Retention.** A terminal job is kept for ``RESULT_TTL_SECONDS`` or until more than + ``MAX_TERMINAL_JOBS`` terminal jobs exist (oldest first). Only jobs whose worker has exited are evicted. +* **Deadline.** 60 s wall clock from the moment the job becomes ``running``, observed cooperatively. + Accepted limitation: a single stuck native/filesystem call cannot be interrupted in-process (there is + no multiprocessing in the frozen sidecar). The backstop is fail-closed: if the worker is still alive + ``WEDGE_GRACE_SECONDS`` after a cancel or after the deadline, the job is reported failed with TIMEOUT + (when it was the deadline) and every new start answers 503 WORKER_WEDGED until the thread exits. +* **Snapshot.** The source is copied once into a private file under + ``/scene-tmp/-/`` (streamed 1 MiB chunks, hard ceiling), hashed (the revision is the + SHA-256 of those bytes), and ONLY the copy is parsed. A scene is a consistent snapshot of the file as + copied. ``SOURCE_CHANGED`` therefore applies only to (a) a file that changed while it was being + copied, (b) a reused request_id+path whose file now hashes differently from that job's revision, and + (c) an ``expected_revision`` that differs from the job's revision. It does NOT apply to edits made + after the snapshot was taken. +* **Snapshot lifetime.** The WORKER deletes its own snapshot in its ``finally`` BEFORE setting + ``worker_done``. Eviction removes only the snapshot of a terminal job whose worker is done, with a + missing-file-tolerant unlink. Cancel never deletes a file. A startup sweep removes only + ``scene-tmp/-`` directories whose pid is dead (or older than 24 h) and tolerates sharing + violations; it never touches a live process's directory. +* **Nothing here touches the original file except to read it.** +""" +from __future__ import annotations + +import errno +import hashlib +import os +import re +import secrets +import shutil +import sys +import threading +import time +import uuid + +from snapstudio_core import paths as _paths +from snapstudio_core import scene +from snapstudio_core import scene_limits as L +from . import request_validation as rv + +ROUTES = ("/scene/start", "/scene/status", "/scene/result", "/scene/cancel") +TERMINAL = ("succeeded", "failed", "cancelled") +_REQUEST_ID = re.compile(r"^[A-Za-z0-9_-]{8,%d}$" % L.MAX_REQUEST_ID_LENGTH) +_REVISION = re.compile(r"^[0-9a-f]{64}$") + + +class JobError(Exception): + """An answer the API gives instead of a result: HTTP status, stable code, safe message.""" + + def __init__(self, http: int, code: str, message: str, **extra) -> None: + self.http, self.code, self.message, self.extra = http, code, message, extra + super().__init__(message) + + def body(self) -> dict: + return {"error": self.code, "message": self.message, **self.extra} + + +class Job: + def __init__(self, job_id: str, request_id: str, path: str, now: float) -> None: + self.job_id, self.request_id, self.path = job_id, request_id, path + self.source = path # the path as the client gave it (for reading) + self.state = "queued" + self.stage: str | None = None + self.completed: int | None = None + self.total: int | None = None + self.error: dict | None = None + self.revision: str | None = None + self.result: bytes | None = None + self.snapshot: str | None = None + self.created = now + self.deadline: float | None = None + self.terminal_at: float | None = None + self.cancelled_at: float | None = None + self.worker_done = threading.Event() + self.cancel_flag = threading.Event() + + +class _Control(scene.Control): + def __init__(self, registry: "SceneJobs", job: Job) -> None: + self.registry, self.job = registry, job + + def check(self) -> None: + job = self.job + if job.cancel_flag.is_set() or job.state != "running": + raise scene.SceneCancelled() + if self.registry._clock() > job.deadline: + raise scene.SceneTimeout() + + def progress(self, stage, completed, total) -> None: + self.job.stage, self.job.completed, self.job.total = stage, completed, total + + +def pid_alive(pid: int) -> bool: + """Whether a process with this pid exists. Errs towards 'alive' when it cannot tell.""" + if pid == os.getpid(): + return True + if pid <= 0: + return False + if sys.platform == "win32": + import ctypes + from ctypes import wintypes + k32 = ctypes.WinDLL("kernel32", use_last_error=True) # type: ignore[attr-defined] + k32.OpenProcess.restype = wintypes.HANDLE + k32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + k32.GetExitCodeProcess.argtypes = [wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD)] + k32.CloseHandle.argtypes = [wintypes.HANDLE] + handle = k32.OpenProcess(0x1000, False, pid) # PROCESS_QUERY_LIMITED_INFORMATION + if not handle: + return ctypes.get_last_error() == 5 # access denied: the process exists + try: + code = wintypes.DWORD() + if not k32.GetExitCodeProcess(handle, ctypes.byref(code)): + return True + return code.value == 259 # STILL_ACTIVE + finally: + k32.CloseHandle(handle) + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return True + return True + + +def sweep_stale(root: str | None = None, *, now: float | None = None) -> list[str]: + """Remove scene-tmp directories left by engines that are gone. Returns what it removed. + + A ``-`` folder is removed only when its owner pid is NOT running AND the folder is older + than ``SNAPSHOT_DIR_MIN_AGE_SECONDS``. A folder owned by a live process is never touched, however old + (pid reuse is an accepted cost: a stale folder next to a recycled pid waits until that pid exits). + Sharing violations are tolerated and retried at the next start.""" + base = root or os.path.join(_paths.data_dir(create=False), "scene-tmp") + removed: list[str] = [] + try: + names = os.listdir(base) + except OSError: + return removed + now = time.time() if now is None else now + for name in names: + m = re.match(r"^(\d+)-([0-9a-f]+)$", name) + full = os.path.join(base, name) + if not m or not os.path.isdir(full): + continue + pid = int(m.group(1)) + if pid == os.getpid(): + continue # our own directory, whatever else is true + try: + age = now - os.stat(full).st_mtime + except OSError: + continue + if pid_alive(pid): + continue # NEVER a live process's folder, whatever its age + if age < L.SNAPSHOT_DIR_MIN_AGE_SECONDS: + continue # a just-exited engine's folder is left alone a while + try: + shutil.rmtree(full) + removed.append(name) + except OSError: + pass # sharing violation / in use: leave it for next time + return removed + + +class SceneJobs: + def __init__(self, *, clock=time.monotonic, builder=None, snapshot_root: str | None = None, + deadline: float = L.JOB_DEADLINE_SECONDS, wedge_grace: float = L.WEDGE_GRACE_SECONDS, + ttl: float = L.RESULT_TTL_SECONDS, max_terminal: int = L.MAX_TERMINAL_JOBS) -> None: + self._clock = clock + self._builder = builder or scene.build_scene + self._root = snapshot_root + self._deadline, self._grace, self._ttl, self._max_terminal = deadline, wedge_grace, ttl, max_terminal + self._lock = threading.RLock() + self._jobs: dict[str, Job] = {} + self._by_request: dict[str, Job] = {} + self._queued: Job | None = None + self._running: Job | None = None + self._worker: threading.Thread | None = None + self._dir: str | None = None + self._pending: list[list] = [] # [path, attempts, next_try] snapshot files that resisted deletion + + # ------------------------------------------------------------------ public API + def start(self, path: str, request_id: str) -> dict: + key = os.path.normcase(os.path.abspath(path)) + ext = os.path.splitext(key)[1].lower() + if ext not in (".3mf", ".stl"): + raise JobError(422, "UNSUPPORTED_FORMAT", "Only .3mf and .stl files can be shown.") + if not os.path.isfile(path): + raise JobError(400, "INVALID_REQUEST", "That file could not be found.") + for _attempt in range(2): + with self._lock: + self._housekeeping() + existing = self._by_request.get(request_id) + if existing is None: + return self._create_locked(path, key, request_id) + self._same_path_or_raise(existing, key) + revision = existing.revision + # Hash OUTSIDE the lock (it can be slow), then re-check under it: the retained job may have + # been evicted meanwhile, and an evicted job must never be handed back. + if revision is not None and self._current_hash(path) != revision: + raise JobError(409, "SOURCE_CHANGED", "The file changed since that scene was built. Start a new one.") + with self._lock: + self._housekeeping() + if self._jobs.get(existing.job_id) is existing and self._by_request.get(request_id) is existing: + return self._start_body(existing, None) + # evicted while hashing: go round once more and start a fresh job for the same request + with self._lock: + self._housekeeping() + existing = self._by_request.get(request_id) + if existing is not None: + self._same_path_or_raise(existing, key) + return self._start_body(existing, None) + return self._create_locked(path, key, request_id) + + def _create_locked(self, path: str, key: str, request_id: str) -> dict: + if self._wedged(): + raise JobError(503, "WORKER_WEDGED", "The scene worker is stuck. Try again after it recovers.") + replaced = None + if self._queued is not None: + replaced = self._queued.job_id + self._cancel_locked(self._queued) + self._queued = None + if self._running is not None and self._running.state == "running": + replaced = self._running.job_id + self._cancel_locked(self._running) + job = Job(uuid.uuid4().hex, request_id, key, self._clock()) + job.source = path + self._jobs[job.job_id] = job + self._by_request[request_id] = job + self._queued = job + if self._worker is None: + self._worker = threading.Thread(target=self._worker_main, name="scene-worker", daemon=True) + self._worker.start() + return self._start_body(job, replaced) + + def status(self, job_id: str) -> dict: + with self._lock: + self._housekeeping() + return self._status_body(self._get(job_id)) + + def result(self, job_id: str, expected_revision: str | None = None) -> tuple[int, bytes | dict]: + with self._lock: + self._housekeeping() + job = self._get(job_id) + if expected_revision is not None and job.revision is not None and job.revision != expected_revision: + raise JobError(409, "SOURCE_CHANGED", "That scene was built from a different version of the file.") + if job.state == "succeeded": + return 200, job.result + if job.state == "failed": + return 422, {"error": job.error["code"], "message": job.error["message"]} + if job.state == "cancelled": + return 409, {"error": "CANCELLED", "message": "That scene was cancelled."} + return 409, {"error": "NOT_READY", "message": "That scene is not ready yet.", "state": job.state} + + def cancel(self, job_id: str) -> dict: + with self._lock: + self._housekeeping() + job = self._get(job_id) + if job.state in ("queued", "running"): + if job is self._queued: + self._queued = None + self._cancel_locked(job) + body = self._status_body(job) + self._housekeeping() # a cancelled-while-queued job can push past the cap + return body + + # ------------------------------------------------------------------ introspection (tests, benchmark) + def snapshot_dir(self) -> str | None: + return self._dir + + def retained(self) -> int: + with self._lock: + return len(self._jobs) + + def wedged(self) -> bool: + with self._lock: + return self._wedged() + + def worker_alive(self) -> bool: + with self._lock: + return self._worker is not None and self._worker.is_alive() + + def close(self, timeout: float = 5.0) -> None: + """Cancel everything and wait for the worker to exit (tests / orderly shutdown).""" + with self._lock: + for job in list(self._jobs.values()): + if job.state in ("queued", "running"): + if job is self._queued: + self._queued = None + self._cancel_locked(job) + worker = self._worker + if worker is not None: + worker.join(timeout) + if worker.is_alive(): + return + with self._lock: + for entry in list(self._pending): + self._try_unlink(entry) + if self._dir and not self._pending: + shutil.rmtree(self._dir, ignore_errors=True) # best effort; a later start sweeps what is left + + # ------------------------------------------------------------------ internals (lock held) + def _same_path_or_raise(self, job: Job, key: str) -> None: + if job.path != key: + raise JobError(409, "INVALID_REQUEST", "That request_id is already used for another file.", + reason="request_id reused for another path") + + def _get(self, job_id: str) -> Job: + job = self._jobs.get(job_id) + if job is None: + raise JobError(404, "EXPIRED", "That scene is no longer available.") + return job + + def _cancel_locked(self, job: Job) -> None: + if job.state not in ("queued", "running"): + return + was_queued = job.state == "queued" + now = self._clock() + job.state = "cancelled" + job.error = {"code": "CANCELLED", "message": "That scene was cancelled."} + job.terminal_at = job.cancelled_at = now + job.cancel_flag.set() + if was_queued: + job.worker_done.set() # it never had a worker or a snapshot + + def _wedged(self) -> bool: + job = self._running + if job is None or job.worker_done.is_set(): + return False + now = self._clock() + if job.state == "cancelled" and job.cancelled_at is not None and now >= job.cancelled_at + self._grace: + return True + return job.deadline is not None and now >= job.deadline + self._grace + + def _housekeeping(self) -> None: + now = self._clock() + for entry in list(self._pending): + if now >= entry[2]: + self._try_unlink(entry) + job = self._running + if (job is not None and job.state == "running" and job.deadline is not None + and now >= job.deadline + self._grace and not job.worker_done.is_set()): + self._fail_locked(job, "TIMEOUT", "Building the scene took too long.") # backstop; the thread is wedged + evictable = [j for j in self._jobs.values() if j.state in TERMINAL and j.worker_done.is_set()] + for j in evictable: + if now - j.terminal_at > self._ttl: + self._drop(j) + terminal = [j for j in self._jobs.values() if j.state in TERMINAL] + excess = len(terminal) - self._max_terminal + if excess > 0: + for j in sorted((j for j in terminal if j.worker_done.is_set()), key=lambda j: j.terminal_at)[:excess]: + self._drop(j) + + def _drop(self, job: Job) -> None: + if job.snapshot: + self._defer_unlink(job.snapshot) + job.snapshot = None + job.result = None + self._jobs.pop(job.job_id, None) + if self._by_request.get(job.request_id) is job: + del self._by_request[job.request_id] + + def _defer_unlink(self, path: str) -> None: + """Delete a snapshot file now; if Windows (or anything) still holds it, KEEP the path on a retry list + with back-off. It is forgotten only when it is gone, or when the process exits (the startup sweep + of a later engine then removes the folder).""" + entry = [path, 0, self._clock()] + if not self._try_unlink(entry) and len(self._pending) < L.MAX_PENDING_UNLINKS: + self._pending.append(entry) + + def _try_unlink(self, entry: list) -> bool: + try: + os.unlink(entry[0]) + except FileNotFoundError: + pass + except OSError: + entry[1] += 1 + entry[2] = self._clock() + min(2.0 ** entry[1], 30.0) # 2 s, 4 s, ... capped at 30 s + return False + if entry in self._pending: + self._pending.remove(entry) + return True + + def _fail_locked(self, job: Job, code: str, message: str) -> None: + if job.state in TERMINAL: + return # the first terminal state wins + job.state = "failed" + job.error = {"code": code, "message": message} + job.terminal_at = self._clock() + job.cancel_flag.set() + + def _status_body(self, job: Job) -> dict: + running = job.state == "running" + return {"job_id": job.job_id, "request_id": job.request_id, "state": job.state, + "stage": job.stage if running else None, + "completed": job.completed if running else None, + "total": job.total if running else None, + "error": job.error, "revision": job.revision} + + def _start_body(self, job: Job, replaced: str | None) -> dict: + return {"job_id": job.job_id, "request_id": job.request_id, "state": job.state, + "revision": job.revision, "replaced_job_id": replaced} + + # ------------------------------------------------------------------ worker + def _worker_main(self) -> None: + try: + while True: + with self._lock: + job = self._queued + if job is None: + self._worker = None # the slot is released only here, as the thread exits + self._housekeeping() + return + self._queued = None + now = self._clock() + job.state = "running" + job.deadline = now + self._deadline # the clock starts when the job starts running + self._running = job + try: + self._execute(job) + finally: + with self._lock: + if self._running is job: + self._running = None + self._housekeeping() # retention does not depend on a client polling + except BaseException: # noqa: BLE001 - never leave the slot taken + with self._lock: + self._worker = None + raise + + def _execute(self, job: Job) -> None: + ctl = _Control(self, job) + try: + job.stage, job.completed, job.total = "reading", 0, None + snap = self._snapshot(job, ctl) + body = self._builder(snap, job.revision, ctl) + with self._lock: + if job.state == "running": + job.state = "succeeded" + job.result = body + job.terminal_at = self._clock() + # else: cancelled / failed first -- the late result is discarded + except scene.SceneCancelled: + with self._lock: + if job.state == "running": + self._cancel_locked(job) + except scene.SceneTimeout: + with self._lock: + self._fail_locked(job, "TIMEOUT", "Building the scene took too long.") + except scene.SceneError as exc: + with self._lock: + self._fail_locked(job, exc.code, exc.message) + except JobError as exc: + with self._lock: + self._fail_locked(job, exc.code, exc.message) + except BaseException as exc: # noqa: BLE001 + try: + from . import server + server._log_unexpected(exc) + except Exception: # noqa: BLE001 + pass + with self._lock: + self._fail_locked(job, "INTERNAL", "The scene could not be built.") + finally: + snap_path = job.snapshot + if snap_path: + try: + os.unlink(snap_path) # the worker deletes its own file ... + job.snapshot = None + except FileNotFoundError: + job.snapshot = None + except OSError: + pass # ... and eviction retries if Windows still holds it + job.worker_done.set() # ... BEFORE it signals that it is done + + # ------------------------------------------------------------------ snapshot + def _snapshot_directory(self) -> str: + with self._lock: + if self._dir is None: + base = self._root or os.path.join(_paths.data_dir(), "scene-tmp") + self._dir = os.path.join(base, f"{os.getpid()}-{secrets.token_hex(4)}") + os.makedirs(self._dir, exist_ok=True) # on EVERY job: something may have removed it + return self._dir + + def _snapshot(self, job: Job, ctl: _Control) -> str: + src = job.source + directory = self._snapshot_directory() + ext = os.path.splitext(src)[1].lower() + dest = os.path.join(directory, job.job_id + ext) + try: + fh = open(src, "rb") + except OSError as exc: + raise JobError(400, "INVALID_REQUEST", "That file could not be read.") from exc + try: + st0 = os.fstat(fh.fileno()) + size0 = st0.st_size + if size0 > L.SNAPSHOT_MAX_BYTES: + raise scene.SceneError("LIMIT_EXCEEDED", "The file is larger than Studio will open here.") + try: + free = shutil.disk_usage(directory).free + except OSError: + free = None + if free is not None and free < size0 + L.DISK_HEADROOM_BYTES: + raise scene.SceneError("INTERNAL", "There is not enough free disk space to read this file.") + digest = hashlib.sha256() + count = 0 + job.snapshot = dest + try: + with open(dest, "wb") as out: + while True: + ctl.check() + chunk = fh.read(L.SNAPSHOT_CHUNK_BYTES) + if not chunk: + break + count += len(chunk) + if count > L.SNAPSHOT_MAX_BYTES or count > size0: + break + digest.update(chunk) + out.write(chunk) + ctl.progress("reading", count, size0) + except OSError as exc: + if exc.errno == errno.ENOSPC: + raise scene.SceneError("INTERNAL", "There is not enough free disk space to read this file.") from exc + raise scene.SceneError("INTERNAL", "Studio could not write its temporary copy of this file.") from exc + st1 = os.fstat(fh.fileno()) + finally: + fh.close() + try: + st2 = os.stat(src) + except OSError: + st2 = None + changed = (count != size0 + or (st1.st_size, st1.st_mtime_ns) != (size0, st0.st_mtime_ns) + or st2 is None or (st2.st_size, st2.st_mtime_ns) != (size0, st0.st_mtime_ns) + or (st0.st_ino and st2 is not None and st2.st_ino and st0.st_ino != st2.st_ino)) + if changed: + raise scene.SceneError("SOURCE_CHANGED", "The file changed while Studio was reading it.") + job.revision = digest.hexdigest() + return dest + + @staticmethod + def _current_hash(path: str) -> str | None: + """SHA-256 of the file as it is now (None if it cannot be read or is over the ceiling).""" + digest = hashlib.sha256() + total = 0 + try: + with open(path, "rb") as fh: + while True: + chunk = fh.read(L.SNAPSHOT_CHUNK_BYTES) + if not chunk: + break + total += len(chunk) + if total > L.SNAPSHOT_MAX_BYTES: + return None + digest.update(chunk) + except OSError: + return None + return digest.hexdigest() + + +# ---------------------------------------------------------------------------------------------- +# HTTP glue (kept out of server.py so that file only gains a route branch) +# ---------------------------------------------------------------------------------------------- + +_default: SceneJobs | None = None +_default_lock = threading.Lock() + + +def registry() -> SceneJobs: + global _default + with _default_lock: + if _default is None: + _default = SceneJobs() + return _default + + +def reset_default_for_tests() -> None: + global _default + with _default_lock: + if _default is not None: + _default.close() + _default = None + + +def _request_id(data: dict) -> str: + value = rv.require_str(data, "request_id") + if not _REQUEST_ID.match(value): + raise rv.ValidationError("Invalid 'request_id'") + return value + + +def _job_id(data: dict) -> str: + value = rv.require_str(data, "job_id") + if len(value) > 128: + raise rv.ValidationError("Invalid 'job_id'") + return value + + +def handle(route: str, data: dict, jobs: SceneJobs | None = None) -> tuple[int, bytes | dict]: + """Run one /scene/* request. Returns (HTTP status, dict body or pre-serialized bytes).""" + jobs = jobs or registry() + if not isinstance(data, dict): + return 400, {"error": "INVALID_REQUEST", "message": "The request body must be a JSON object."} + try: + if route == "/scene/start": + path = rv.require_bounded_str(data, "path", 4096) + return 200, jobs.start(path, _request_id(data)) + if route == "/scene/status": + return 200, jobs.status(_job_id(data)) + if route == "/scene/result": + expected = data.get("expected_revision") + if expected is not None and (not isinstance(expected, str) or not _REVISION.match(expected)): + raise rv.ValidationError("Invalid 'expected_revision'") + return jobs.result(_job_id(data), expected) + if route == "/scene/cancel": + return 200, jobs.cancel(_job_id(data)) + except rv.ValidationError as exc: + return 400, {"error": "INVALID_REQUEST", "message": str(exc)} + except JobError as exc: + return exc.http, exc.body() + return 404, {"error": "not found"} diff --git a/backend/snapstudio_api/server.py b/backend/snapstudio_api/server.py index 2797f104..2c01acea 100644 --- a/backend/snapstudio_api/server.py +++ b/backend/snapstudio_api/server.py @@ -20,6 +20,7 @@ from snapstudio_core.material_mapping import MappingFileUnavailable from . import _lifeline from . import service +from . import scene_jobs from . import request_validation as rv from .request_validation import ValidationError @@ -185,7 +186,11 @@ def _cors(self): self.send_header("Access-Control-Allow-Headers", "Content-Type, X-Auth-Token") def _send(self, code: int, obj: dict): - body = json.dumps(obj).encode("utf-8") + self._send_bytes(code, json.dumps(obj).encode("utf-8")) + + def _send_bytes(self, code: int, body: bytes): + """Send an already-serialized JSON body. The scene routes use this so a body that was + size-checked once (<= 8 MiB) is not serialized a second time.""" self.send_response(code) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) @@ -1320,6 +1325,17 @@ def _or_none(v): self._send(404, {"error": "unknown job"}) else: self._send(200, status) + elif self.path in scene_jobs.ROUTES: + # scene/1: bounded, cancellable, authenticated (the token was checked above). + try: + status, body = scene_jobs.handle(self.path, data) + if isinstance(body, bytes): + self._send_bytes(status, body) + else: + self._send(status, body) + except Exception as exc: + _log_unexpected(exc) + self._send(500, {"error": "INTERNAL", "message": "The scene could not be built."}) elif self.path == "/shutdown": # Graceful half of the L4 zero-orphan sequence (see # sidecar::shutdown_sidecar): reply first, THEN stop the @@ -1384,6 +1400,10 @@ def serve(host: str = "127.0.0.1", port: int = 0) -> None: _watch_parent_then_exit() _lifeline.start_parent_lifeline() httpd, token = build_server(host, port) + try: + scene_jobs.sweep_stale() # leftovers of engines that are gone; never a live engine's folder + except Exception: # noqa: BLE001 - housekeeping must not stop the engine starting + pass actual_port = httpd.server_address[1] print(json.dumps({"port": actual_port, "token": token}), flush=True) # handshake line try: diff --git a/backend/snapstudio_core/data/scene-1.schema.json b/backend/snapstudio_core/data/scene-1.schema.json new file mode 100644 index 00000000..d96dcb13 --- /dev/null +++ b/backend/snapstudio_core/data/scene-1.schema.json @@ -0,0 +1,224 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://snapmaker-studio.invalid/schema/scene-1.schema.json", + "title": "Snapmaker Studio scene/1", + "description": "Read-only description of a project's geometry, placement and engine findings. FROZEN: every scene object is closed (additionalProperties false), every field is required, and unknown is null. Millimetres, right-handed, Z up. A scene is a consistent snapshot of the file as it was copied for the job. Matrices (local_to_parent_mm, world_mm) are 16 numbers in COLUMN-MAJOR order acting on COLUMN vectors (x' = M * x; the translation is elements 12..14). Mesh coordinates are LOCAL to the mesh resource (before any node matrix), in millimetres. A node's bounds_mm and printable concern its role=part volumes only. An instance's effective volume role is node.role_context when that is not null (Bambu/Orca part records are per use), otherwise the roles of the mesh volumes. PLACEMENT_OUTSIDE_BED.overhang_mm measures how far the printable bounds intrude past the bed polygon shrunk inward by edge_margin_mm, in X and Y only (never height).", + "type": "object", + "additionalProperties": false, + "required": ["schema", "revision", "status", "units", "axes", "sources", "bed", "meshes", "nodes", "plates", "findings", "limitations", "counts", "limits"], + "properties": { + "schema": {"const": "scene/1"}, + "revision": {"$ref": "#/$defs/sha256"}, + "status": {"enum": ["complete", "partial"]}, + "units": {"const": "mm"}, + "axes": {"const": "right-handed-z-up"}, + "sources": {"type": "array", "maxItems": 256, "items": {"$ref": "#/$defs/source"}}, + "bed": {"$ref": "#/$defs/bed"}, + "meshes": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/mesh"}}, + "nodes": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/node"}}, + "plates": {"type": "array", "maxItems": 256, "items": {"$ref": "#/$defs/plate"}}, + "findings": {"type": "array", "maxItems": 500, "items": {"$ref": "#/$defs/finding"}}, + "limitations": {"type": "array", "maxItems": 50, "items": {"$ref": "#/$defs/limitation"}}, + "counts": {"$ref": "#/$defs/counts"}, + "limits": {"$ref": "#/$defs/limits"} + }, + "$defs": { + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "id": {"type": "string", "minLength": 1, "maxLength": 128}, + "uint": {"type": "integer", "minimum": 0, "maximum": 4294967295}, + "vec3": {"type": "array", "minItems": 3, "maxItems": 3, "items": {"type": "number"}}, + "matrix16": {"type": "array", "minItems": 16, "maxItems": 16, "items": {"type": "number"}}, + "resource_key": { + "type": "object", "additionalProperties": false, "required": ["part", "object_id"], + "properties": {"part": {"type": "string", "minLength": 1, "maxLength": 256}, "object_id": {"$ref": "#/$defs/id"}} + }, + "unit_name": {"enum": ["micron", "millimeter", "centimeter", "inch", "foot", "meter"]}, + "source": { + "description": "One model part that was read. When the part declares a unit outside the six defined ones, unit and mm_per_unit are the FALLBACK millimetre values, and the UNSUPPORTED_UNIT limitation (whose target_ids are the nodes living in that part) says so; no fit statement is made for such a scene.", + "type": "object", "additionalProperties": false, "required": ["part", "unit", "mm_per_unit"], + "properties": { + "part": {"type": "string", "minLength": 1, "maxLength": 256}, + "unit": {"$ref": "#/$defs/unit_name"}, + "mm_per_unit": {"type": "number", "exclusiveMinimum": 0} + } + }, + "bed": { + "type": "object", "additionalProperties": false, "required": ["polygon_mm", "height_mm", "edge_margin_mm", "policy"], + "properties": { + "polygon_mm": {"type": "array", "minItems": 3, "maxItems": 16, + "items": {"type": "array", "minItems": 2, "maxItems": 2, "items": {"type": "number"}}}, + "height_mm": {"type": ["number", "null"]}, + "edge_margin_mm": {"type": "number", "minimum": 0}, + "policy": {"enum": ["u1_template", "fallback"]} + } + }, + "volume_role": {"enum": ["part", "modifier", "negative", "support_enforcer", "support_blocker", "unknown"]}, + "volume": { + "type": "object", "additionalProperties": false, "required": ["id", "triangle_start", "triangle_count", "role", "source"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "triangle_start": {"$ref": "#/$defs/uint"}, + "triangle_count": {"$ref": "#/$defs/uint"}, + "role": {"$ref": "#/$defs/volume_role"}, + "source": {"enum": ["prusa_range", "bambu_part", "object", null]} + } + }, + "mesh": { + "type": "object", "additionalProperties": false, + "required": ["key", "vertex_count", "triangle_count", "positions_f32le_base64", "indices_u32le_base64", "volumes"], + "properties": { + "key": {"$ref": "#/$defs/resource_key"}, + "vertex_count": {"$ref": "#/$defs/uint"}, + "triangle_count": {"$ref": "#/$defs/uint"}, + "positions_f32le_base64": {"type": "string", "pattern": "^[A-Za-z0-9+/]*={0,2}$"}, + "indices_u32le_base64": {"type": "string", "pattern": "^[A-Za-z0-9+/]*={0,2}$"}, + "volumes": {"type": "array", "minItems": 1, "maxItems": 4096, "items": {"$ref": "#/$defs/volume"}} + } + }, + "plate_ref": { + "type": "object", "additionalProperties": false, "required": ["state", "plate_id", "source"], + "properties": { + "state": {"enum": ["known", "ambiguous", "unknown"]}, + "plate_id": {"type": ["string", "null"], "maxLength": 128}, + "source": {"enum": ["plate_config", null]} + }, + "if": {"properties": {"state": {"const": "known"}}}, + "then": {"properties": {"plate_id": {"type": "string"}, "source": {"const": "plate_config"}}} + }, + "instance_ref": { + "type": "object", "additionalProperties": false, "required": ["build_index", "component_path"], + "properties": { + "build_index": {"type": ["integer", "null"], "minimum": 0}, + "component_path": {"type": "array", "maxItems": 64, "items": {"type": "integer", "minimum": 0}} + } + }, + "bounds": { + "type": ["object", "null"], "additionalProperties": false, "required": ["min", "max"], + "properties": {"min": {"$ref": "#/$defs/vec3"}, "max": {"$ref": "#/$defs/vec3"}} + }, + "node": { + "type": "object", "additionalProperties": false, + "required": ["id", "parent_id", "resource", "mesh_key", "local_to_parent_mm", "world_mm", "mirrored", "role_context", "build_index", "instance_ref", + "plate", "placement_state", "printable", "has_non_part_volumes", "bounds_mm", "finding_ids"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "parent_id": {"type": ["string", "null"], "maxLength": 128}, + "resource": {"$ref": "#/$defs/resource_key"}, + "mesh_key": {"oneOf": [{"$ref": "#/$defs/resource_key"}, {"type": "null"}]}, + "local_to_parent_mm": {"$ref": "#/$defs/matrix16"}, + "world_mm": {"$ref": "#/$defs/matrix16"}, + "mirrored": {"type": "boolean"}, + "role_context": {"enum": ["part", "modifier", "negative", "support_enforcer", "support_blocker", "unknown", null]}, + "build_index": {"type": ["integer", "null"], "minimum": 0}, + "instance_ref": {"$ref": "#/$defs/instance_ref"}, + "plate": {"$ref": "#/$defs/plate_ref"}, + "placement_state": {"enum": ["known", "unknown"]}, + "printable": {"type": ["boolean", "null"]}, + "has_non_part_volumes": {"type": "boolean"}, + "bounds_mm": {"$ref": "#/$defs/bounds"}, + "finding_ids": {"type": "array", "maxItems": 500, "items": {"$ref": "#/$defs/id"}} + } + }, + "plate": { + "type": "object", "additionalProperties": false, "required": ["id", "ui_number", "origin_mm"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "ui_number": {"type": ["integer", "null"], "minimum": 0}, + "origin_mm": {"oneOf": [{"$ref": "#/$defs/vec3"}, {"type": "null"}]} + } + }, + "finding": { + "type": "object", "additionalProperties": false, "required": ["id", "engine", "schema", "pointer", "scope", "kind", "target_ids", "value"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "engine": {"type": "string", "minLength": 1, "maxLength": 64}, + "schema": {"type": "string", "minLength": 1, "maxLength": 64}, + "pointer": {"type": "string", "maxLength": 128}, + "scope": {"enum": ["project", "plate", "instance"]}, + "kind": {"enum": ["placement", "size", "note"]}, + "target_ids": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/id"}}, + "value": { + "type": "object", "additionalProperties": false, "required": ["code", "overhang_mm"], + "properties": { + "code": {"enum": ["PLACEMENT_OUTSIDE_BED", "SIZE_EXCEEDS_BED"]}, + "overhang_mm": { + "type": ["object", "null"], "additionalProperties": false, "required": ["left", "right", "front", "back"], + "properties": {"left": {"type": "number"}, "right": {"type": "number"}, "front": {"type": "number"}, "back": {"type": "number"}} + } + } + } + } + }, + "limitation_code": {"enum": [ + "UNKNOWN_VOLUME_ROLE", "UNKNOWN_PLACEMENT", "PLATE_MEMBERSHIP_AMBIGUOUS", "PLATE_MEMBERSHIP_UNKNOWN", + "PLATES_UNAVAILABLE", "BED_TEMPLATE_UNAVAILABLE", "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED", + "NON_MM_SOURCE_UNIT", "UNSUPPORTED_UNIT", "MULTI_PLATE_PLACEMENT_UNCHECKED", "NO_BUILD_ITEMS"]}, + "limitation": { + "type": "object", "additionalProperties": false, "required": ["code", "target_ids"], + "properties": { + "code": {"$ref": "#/$defs/limitation_code"}, + "target_ids": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/id"}} + } + }, + "counts": { + "type": "object", "additionalProperties": false, + "required": ["nodes", "meshes", "vertices", "triangles", "rendered_triangles", "plates", "findings", "limitations"], + "properties": { + "nodes": {"$ref": "#/$defs/uint"}, "meshes": {"$ref": "#/$defs/uint"}, "vertices": {"$ref": "#/$defs/uint"}, + "triangles": {"$ref": "#/$defs/uint"}, "rendered_triangles": {"$ref": "#/$defs/uint"}, + "plates": {"$ref": "#/$defs/uint"}, "findings": {"$ref": "#/$defs/uint"}, "limitations": {"$ref": "#/$defs/uint"} + } + }, + "limits": { + "type": "object", "additionalProperties": false, + "required": ["max_archive_bytes", "max_model_xml_bytes", "max_rendered_triangles", "max_vertices", "max_nodes", "max_depth", + "max_response_bytes", "max_findings", "max_limitations", "max_id_length"], + "properties": { + "max_archive_bytes": {"$ref": "#/$defs/uint"}, "max_model_xml_bytes": {"$ref": "#/$defs/uint"}, + "max_rendered_triangles": {"$ref": "#/$defs/uint"}, "max_vertices": {"$ref": "#/$defs/uint"}, + "max_nodes": {"$ref": "#/$defs/uint"}, "max_depth": {"$ref": "#/$defs/uint"}, + "max_response_bytes": {"$ref": "#/$defs/uint"}, "max_findings": {"$ref": "#/$defs/uint"}, + "max_limitations": {"$ref": "#/$defs/uint"}, "max_id_length": {"$ref": "#/$defs/uint"} + } + }, + "error_code": {"enum": [ + "INVALID_REQUEST", "UNSUPPORTED_FORMAT", "INVALID_ARCHIVE", "INVALID_GEOMETRY", "UNRESOLVED_REFERENCE", + "LIMIT_EXCEEDED", "SOURCE_CHANGED", "CANCELLED", "TIMEOUT", "EXPIRED", "NOT_READY", "WORKER_WEDGED", "INTERNAL"]}, + "job_state": {"enum": ["queued", "running", "succeeded", "failed", "cancelled"]}, + "error_response": { + "type": "object", "additionalProperties": false, "required": ["error"], + "properties": { + "error": {"$ref": "#/$defs/error_code"}, + "message": {"type": "string", "maxLength": 512}, + "state": {"$ref": "#/$defs/job_state"}, + "reason": {"type": "string", "maxLength": 256} + } + }, + "job_status": { + "type": "object", "additionalProperties": false, + "required": ["job_id", "request_id", "state", "stage", "completed", "total", "error", "revision"], + "properties": { + "job_id": {"$ref": "#/$defs/id"}, + "request_id": {"$ref": "#/$defs/id"}, + "state": {"$ref": "#/$defs/job_state"}, + "stage": {"enum": ["reading", "parsing", "encoding", null]}, + "completed": {"type": ["integer", "null"], "minimum": 0}, + "total": {"type": ["integer", "null"], "minimum": 0}, + "error": {"type": ["object", "null"], "additionalProperties": false, "required": ["code", "message"], + "properties": {"code": {"$ref": "#/$defs/error_code"}, "message": {"type": "string", "maxLength": 512}}}, + "revision": {"oneOf": [{"$ref": "#/$defs/sha256"}, {"type": "null"}]} + } + }, + "job_start": { + "type": "object", "additionalProperties": false, + "required": ["job_id", "request_id", "state", "revision", "replaced_job_id"], + "properties": { + "job_id": {"$ref": "#/$defs/id"}, + "request_id": {"$ref": "#/$defs/id"}, + "state": {"$ref": "#/$defs/job_state"}, + "revision": {"oneOf": [{"$ref": "#/$defs/sha256"}, {"type": "null"}]}, + "replaced_job_id": {"type": ["string", "null"], "maxLength": 128} + } + } + } +} diff --git a/backend/snapstudio_core/geometry.py b/backend/snapstudio_core/geometry.py index ea5eb845..94861085 100644 --- a/backend/snapstudio_core/geometry.py +++ b/backend/snapstudio_core/geometry.py @@ -14,6 +14,7 @@ from .container import ThreeMF from .config_io import load_model_settings +from . import units as _units # Mirror intelligence.py's byte guard; plus a triangle cap for analysis cost. _MAX_BYTES = 80 * 1024 * 1024 @@ -44,6 +45,7 @@ def _parse_3mf_model(raw: bytes, verts: list, faces: list) -> None: hardened XML parser (no XXE/DTD/network). Triangle indices are local to each , so we offset by the vertex count at the start of that mesh.""" root = load_model_settings(raw) + scale = _units.mm_per_unit(raw) # the part's declared unit -> millimetres (1.0 for mm) for mesh in root.iter(f"{_3MF_CORE_NS}mesh"): vstart = len(verts) vnode = mesh.find(f"{_3MF_CORE_NS}vertices") @@ -51,7 +53,11 @@ def _parse_3mf_model(raw: bytes, verts: list, faces: list) -> None: if vnode is None or tnode is None: continue for v in vnode.iterfind(f"{_3MF_CORE_NS}vertex"): - verts.append((float(v.get("x", 0)), float(v.get("y", 0)), float(v.get("z", 0)))) + if scale == 1.0: + verts.append((float(v.get("x", 0)), float(v.get("y", 0)), float(v.get("z", 0)))) + else: + verts.append((float(v.get("x", 0)) * scale, float(v.get("y", 0)) * scale, + float(v.get("z", 0)) * scale)) for t in tnode.iterfind(f"{_3MF_CORE_NS}triangle"): faces.append((vstart + int(t.get("v1")), vstart + int(t.get("v2")), vstart + int(t.get("v3")))) if len(faces) > _MAX_TRIANGLES: @@ -113,8 +119,11 @@ def build_item_dims(path: str) -> list[dict]: # object_id -> (verts, [(component_objectid, component_path, component_xform)]), per file parsed: dict[str, dict[str, tuple]] = {} + unit_scale: dict[str, float] = {} for fname, raw in model_files.items(): root = load_model_settings(raw) + scale = _units.mm_per_unit(raw) # this part's declared unit -> millimetres + unit_scale[fname] = scale objs: dict[str, tuple] = {} for ob in root.iter(f"{_3MF_CORE_NS}object"): oid = ob.get("id") @@ -124,14 +133,18 @@ def build_item_dims(path: str) -> list[dict]: vn = mesh.find(f"{_3MF_CORE_NS}vertices") if vn is not None: for v in vn.iterfind(f"{_3MF_CORE_NS}vertex"): - verts.append((float(v.get("x", 0)), float(v.get("y", 0)), float(v.get("z", 0)))) + if scale == 1.0: + verts.append((float(v.get("x", 0)), float(v.get("y", 0)), float(v.get("z", 0)))) + else: + verts.append((float(v.get("x", 0)) * scale, float(v.get("y", 0)) * scale, + float(v.get("z", 0)) * scale)) comps = [] cn = ob.find(f"{_3MF_CORE_NS}components") if cn is not None: for c in cn.iterfind(f"{_3MF_CORE_NS}component"): comps.append((c.get("objectid"), c.get(f"{_3MF_PROD_NS}path"), - _xform(c.get("transform")))) + _units.scale_translation(_xform(c.get("transform")), scale))) objs[oid] = (verts, comps) parsed[fname] = objs @@ -170,7 +183,9 @@ def collect(objid, prefer_file, xform, acc, seen): for it in load_model_settings(model_files[root_file]).iter(f"{_3MF_CORE_NS}item"): oid = it.get("objectid") acc: list = [] - collect(oid, root_file, _xform(it.get("transform")), acc, frozenset()) + collect(oid, root_file, + _units.scale_translation(_xform(it.get("transform")), unit_scale[root_file]), + acc, frozenset()) if not acc: continue xs = [p[0] for p in acc]; ys = [p[1] for p in acc]; zs = [p[2] for p in acc] diff --git a/backend/snapstudio_core/intelligence.py b/backend/snapstudio_core/intelligence.py index 8f6a70c2..0dd30bec 100644 --- a/backend/snapstudio_core/intelligence.py +++ b/backend/snapstudio_core/intelligence.py @@ -10,6 +10,7 @@ from .doctor import diagnose_path, READY from .container import ThreeMF from .config_io import load_project_settings +from . import units as _units SCHEMA_VERSION = "insights/1" SETTINGS = "Metadata/project_settings.config" @@ -36,10 +37,14 @@ def _bbox_and_triangles(tm: ThreeMF): hi = [float("-inf")] * 3 seen = False for p in model_parts: - for m in _VERT_RE.finditer(tm.read_part(p)): + raw = tm.read_part(p) + # Each part's coordinates are in the unit its own header declares; the result is + # millimetres. Millimetre parts (the default) are multiplied by exactly 1.0. + scale = _units.mm_per_unit(raw) + for m in _VERT_RE.finditer(raw): seen = True for i in range(3): - v = float(m.group(i + 1)) + v = float(m.group(i + 1)) * scale if v < lo[i]: lo[i] = v if v > hi[i]: hi[i] = v if not seen: diff --git a/backend/snapstudio_core/placement.py b/backend/snapstudio_core/placement.py index 8f2915c2..9a5d3b99 100644 --- a/backend/snapstudio_core/placement.py +++ b/backend/snapstudio_core/placement.py @@ -25,6 +25,8 @@ import re import zipfile +from . import units as _units + SCHEMA_VERSION = "placement/1" #: The states an object's footprint can be in against the target polygon. They are @@ -226,6 +228,7 @@ def read_objects(path: str) -> dict: with zipfile.ZipFile(path) as archive: names = archive.namelist() root = _read(archive, ROOT_MODEL) + root_scale = _units.mm_per_unit(root) # the unit the root model declares -> mm settings_text = _read(archive, MODEL_SETTINGS) try: project = json.loads(_read(archive, PROJECT_SETTINGS) or "{}") @@ -234,15 +237,16 @@ def read_objects(path: str) -> dict: meshes: dict[str, list[tuple]] = {} for name in sorted(n for n in names if n.startswith(OBJECTS_DIR)): body = _read(archive, name) + part_scale = _units.mm_per_unit(body) # each part's own declared unit for mesh_id in re.findall(r'', body, re.S) if block: - meshes[mesh_id] = _points(block.group(0)) + meshes[mesh_id] = _scaled(_points(block.group(0)), part_scale) if not meshes: # A project whose geometry is still in the root model. for mesh_id, block in zip(_OBJECT_WITH_ID.findall(root), _OBJECT_BLOCK.findall(root)): - meshes[mesh_id] = _points(block) + meshes[mesh_id] = _scaled(_points(block), root_scale) polygon = polygon_of(project) parts_by_object = multipart._parts_by_object(settings_text) if settings_text else {} @@ -251,7 +255,7 @@ def read_objects(path: str) -> dict: objects = [] for object_id, transform_text in placements.items(): - item = parse_transform(transform_text) + item = _units.scale_translation(parse_transform(transform_text), root_scale) part_ids = [part_id for part_id, _subtype in parts_by_object.get(object_id, [])] roles = dict(parts_by_object.get(object_id, [])) own_components = components.get(object_id) or [ @@ -262,7 +266,8 @@ def read_objects(path: str) -> dict: points = meshes.get(mesh_id) if points is None: continue - whole = compose(item, parse_transform(component_transform)) + whole = compose(item, _units.scale_translation(parse_transform(component_transform), + root_scale)) moved = [apply(whole, point) for point in points] every.extend(moved) if roles.get(mesh_id, "normal_part") == "normal_part": @@ -282,6 +287,13 @@ def read_objects(path: str) -> dict: "polygon_source": PROJECT_SETTINGS if polygon else None} +def _scaled(points: list[tuple], factor: float) -> list[tuple]: + """Mesh points in millimetres. A millimetre part is returned untouched.""" + if factor == 1.0: + return points + return [(x * factor, y * factor, z * factor) for x, y, z in points] + + def _components_by_object(root: str) -> dict: out: dict[str, list] = {} for match in re.finditer( diff --git a/backend/snapstudio_core/plate_placement.py b/backend/snapstudio_core/plate_placement.py index 574fe055..e007f9cd 100644 --- a/backend/snapstudio_core/plate_placement.py +++ b/backend/snapstudio_core/plate_placement.py @@ -32,6 +32,7 @@ from importlib.resources import files from pathlib import Path +from . import units as _units from .container import ThreeMF SCHEMA_VERSION = "placement/1" @@ -606,7 +607,11 @@ def prepare_placed_copy(path: str, out_dir: str | None = None, "before": before} offset = before["suggested_offset"] - offset_for = _uniform_offset(offset["x"], offset["y"]) + # The geometry above is measured in millimetres, but a build item's translation is written in the + # ROOT model's own unit. Convert the offset back before it is added, or an inch project would be + # shifted by 25.4 times too little (and the move would silently not land). + root_scale = _units.mm_per_unit(tm.read_part(ROOT_MODEL)) + offset_for = _uniform_offset(offset["x"] / root_scale, offset["y"] / root_scale) try: rewritten, moved = _rewrite_items(tm.read_part(ROOT_MODEL), offset_for) @@ -643,14 +648,13 @@ def prepare_placed_copy(path: str, out_dir: str | None = None, } ok = bool(after.get("available")) and not after.get("off_plate") if not ok: - # The copy is left in place so a user can inspect it, but the result says - # plainly that the move did not achieve what it was supposed to. + # A copy that did not achieve what it was supposed to is not kept on disk: the + # result says plainly that the move did not work and names no file. + out.unlink(missing_ok=True) return { "schema_version": SCHEMA_VERSION, "ok": False, - "reason": ("Studio moved the arrangement but the copy still has objects " - "off the plate, so it is not safe to rely on. Open the original " - "in Snapmaker Orca and use Arrange."), - "output_path": str(out), "output_name": out.name, + "reason": ("Studio could not move the objects onto the plate, so it did not " + "keep a copy. Open the original in Snapmaker Orca and use Arrange."), "before": before, "after": after, } diff --git a/backend/snapstudio_core/scene.py b/backend/snapstudio_core/scene.py new file mode 100644 index 00000000..d8240c11 --- /dev/null +++ b/backend/snapstudio_core/scene.py @@ -0,0 +1,1459 @@ +"""scene/1 -- a read-only, bounded description of a project's geometry, placement and findings. + +The contract is the committed JSON Schema ``data/scene-1.schema.json`` (frozen: closed objects, +every field required, unknown is null). This module is the only producer of it. + +What a scene IS: a consistent snapshot of the file as it was copied for the job. It is not a live +view. ``SOURCE_CHANGED`` therefore only ever applies to a reused request_id+path or an +``expected_revision`` mismatch (see ``snapstudio_api.scene_jobs``); it does not apply to edits made +after the snapshot was taken. + +Design rules, in order of importance: + +* **Untrusted input.** The archive is opened by a SELECTIVE reader that owns its own checks: + colliding or traversal-looking entry names, external relationships, DTD/ENTITY declarations and + any text encoding other than UTF-8 are rejected BEFORE parsing; only the ``.model``, ``.rels`` and + the two metadata configs the scene needs are ever read (never thumbnails or embedded G-code); + every read is metered against the budgets in ``scene_limits`` while it happens. + ``ThreeMF.open()`` and ``config_io`` are deliberately untouched. +* **Cooperative interruption.** Archive reads are <= 1 MiB, lxml ``iterparse`` checks the cancel / + deadline hook every <= 1,000 events (and on every read), geometry encode/transform runs in + batches of <= 5,000 triangles. The worst uninterruptible call is one such unit. Accepted + limitation: a single stuck native/filesystem call cannot be interrupted in-process; the backstop + is the job registry's fail-closed WORKER_WEDGED state (no multiprocessing in v1). +* **Honest unknowns.** Placement, plate membership and volume roles that the file does not prove + are reported as ``unknown``/``ambiguous`` with a limitation, never guessed. Bed fit is only + claimed when the bed is Studio's U1 template, placement is known and the instance is not one of + several build items of the same object id (the engine's own placement answer for those is + unverified, see ``REPEATED_INSTANCE_PLACEMENT_UNVERIFIED``). +* **Never silent truncation.** Over any budget the build stops with ``LIMIT_EXCEEDED``. +""" +from __future__ import annotations + +import base64 +import hashlib +import io +import json +import math +import posixpath +import re +import sys +import zipfile +import zlib +from array import array +from importlib.resources import files + +from lxml import etree + +from . import scene_limits as L +from . import units as _units +from .errors import SnapStudioError + +SCHEMA_ID = "scene/1" +SCHEMA_RESOURCE = "scene-1.schema.json" + +CORE_NS = "http://schemas.microsoft.com/3dmanufacturing/core/2015/02" +PROD_NS = "http://schemas.microsoft.com/3dmanufacturing/production/2015/06" +_T = {n: f"{{{CORE_NS}}}{n}" for n in ( + "model", "resources", "object", "mesh", "vertices", "vertex", "triangles", "triangle", + "components", "component", "build", "item")} +_PROD_PATH = f"{{{PROD_NS}}}path" + +ROOT_MODEL_DEFAULT = "3D/3dmodel.model" +MODEL_SETTINGS = "Metadata/model_settings.config" +PRUSA_CONFIG = "Metadata/Slic3r_PE_model.config" +_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" +_MODEL_REL_TYPE = "http://schemas.microsoft.com/3dmanufacturing/2013/01/3dmodel" + +# limitation codes that make a scene "partial" rather than "complete" +PARTIAL_LIMITATIONS = frozenset({ + "UNKNOWN_VOLUME_ROLE", "UNKNOWN_PLACEMENT", "PLATE_MEMBERSHIP_AMBIGUOUS", "PLATE_MEMBERSHIP_UNKNOWN", "UNSUPPORTED_UNIT", + "BED_TEMPLATE_UNAVAILABLE", "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED", "NON_MM_SOURCE_UNIT", + "NO_BUILD_ITEMS"}) +#: Information only: the scene is still as complete as the file allows. +INFORMATIONAL_LIMITATIONS = frozenset({"PLATES_UNAVAILABLE", "MULTI_PLATE_PLACEMENT_UNCHECKED"}) + +ERROR_CODES = ( + "INVALID_REQUEST", "UNSUPPORTED_FORMAT", "INVALID_ARCHIVE", "INVALID_GEOMETRY", + "UNRESOLVED_REFERENCE", "LIMIT_EXCEEDED", "SOURCE_CHANGED", "CANCELLED", "TIMEOUT", + "EXPIRED", "NOT_READY", "WORKER_WEDGED", "INTERNAL") + + +class SceneError(SnapStudioError): + """A refusal with a stable machine code (one of ERROR_CODES) and a message safe to show.""" + + def __init__(self, code: str, message: str) -> None: + assert code in ERROR_CODES, code + self.code = code + self.message = message + super().__init__(message) + + +class SceneCancelled(Exception): + """Raised by the job control hook when the job was cancelled.""" + + +class SceneTimeout(Exception): + """Raised by the job control hook when the job's deadline passed.""" + + +class Control: + """The cancel/deadline/progress hook. The default does nothing.""" + + def check(self) -> None: # pragma: no cover - trivial + return None + + def progress(self, stage: str, completed: int | None, total: int | None) -> None: # pragma: no cover + return None + + +def load_schema() -> dict: + return json.loads((files("snapstudio_core.data") / SCHEMA_RESOURCE).read_text("utf-8")) + + +# -------------------------------------------------------------------------------------------- +# Selective archive reader +# -------------------------------------------------------------------------------------------- + +_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]") +_DRIVE = re.compile(r"^[A-Za-z]:") +_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*:") # http:, file:, C: ... never a package-relative path +_XML_DECL_ENCODING = re.compile(rb"""^\s*<\?xml[^>]*?encoding\s*=\s*["']([^"']+)["']""", re.I) +_BOMS_REJECTED = (b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff", b"\xff\xfe", b"\xfe\xff") +_READ_ERRORS = (zipfile.BadZipFile, NotImplementedError, zlib.error, EOFError, RuntimeError, OSError) + + +def normalize_entry_name(name: str) -> tuple[str, bool]: + """A zip entry name as a normalized archive-relative path, and whether it is a directory. + + Rejected: control characters, rooted names, drive letters, any ``..`` segment. Backslashes are + read as separators, so ``a\\b`` and ``a/b`` collide (and are rejected as a pair by the caller). + """ + n = name.replace("\\", "/") + if _CONTROL_CHARS.search(n): + raise SceneError("INVALID_ARCHIVE", "The archive has an entry name with control characters.") + if n.startswith("/") or _DRIVE.match(n): + raise SceneError("INVALID_ARCHIVE", "The archive has an entry with an absolute path.") + segments = n.split("/") + if ".." in segments: + raise SceneError("INVALID_ARCHIVE", "The archive has an entry that escapes its own folder.") + clean = [s for s in segments if s not in ("", ".")] + if not clean: + raise SceneError("INVALID_ARCHIVE", "The archive has an entry with an empty name.") + return "/".join(clean), n.endswith("/") + + +def resolve_reference(target: str, base_dir: str = "") -> str: + """Resolve a relationship Target / component path to a normalized part name. + + A leading ``/`` means "from the package root", which is how every slicer writes them, so it is + accepted. A reference that climbs above the package root is rejected. + """ + t = target.replace("\\", "/") + if _CONTROL_CHARS.search(t) or _SCHEME.match(t): + raise SceneError("INVALID_ARCHIVE", "The project has a malformed internal reference.") + parts = [] if t.startswith("/") else [s for s in base_dir.split("/") if s] + for seg in t.split("/"): + if seg in ("", "."): + continue + if seg == "..": + if not parts: + raise SceneError("INVALID_ARCHIVE", "A reference in the project escapes the archive.") + parts.pop() + else: + parts.append(seg) + if not parts: + raise SceneError("INVALID_ARCHIVE", "The project has an empty internal reference.") + return "/".join(parts) + + +class GuardedStream: + """A file-like over one archive member that meters, scans and honours cancellation. + + Reads at most ``CHUNK`` bytes per call (bounded uninterruptible work). The DTD/ENTITY scan and + the encoding check run on the raw bytes BEFORE the parser sees them. + """ + + def __init__(self, archive: "Archive", info: zipfile.ZipInfo, kind: str, cap: int, scan: bool = True): + self.archive, self.kind, self.cap, self.scan = archive, kind, cap, scan + self.bytes_read = 0 + self._carry = b"" + self._first = True + try: + self._fh = archive.zf.open(info, "r") + except _READ_ERRORS as exc: + raise SceneError("INVALID_ARCHIVE", "A part of the archive could not be opened.") from exc + + def close(self) -> None: + try: + self._fh.close() + except Exception: # noqa: BLE001 + pass + + def read(self, size: int = -1) -> bytes: + self.archive.ctl.check() + n = L.SNAPSHOT_CHUNK_BYTES if size is None or size < 0 else min(size, L.SNAPSHOT_CHUNK_BYTES) + try: + chunk = self._fh.read(n) + if self._first and chunk and len(chunk) < 4: + chunk += self._fh.read(4 - len(chunk)) + except _READ_ERRORS as exc: + raise SceneError("INVALID_ARCHIVE", "A part of the archive is damaged.") from exc + if not chunk: + return chunk + self.bytes_read += len(chunk) + self.archive.account(len(chunk), self.kind) + if self.bytes_read > self.cap: + raise SceneError("LIMIT_EXCEEDED", "A part of the archive expands past what Studio will read.") + if self.scan: + if self._first: + self._check_encoding(chunk) + hay = (self._carry + chunk).lower() + if b" None: + if head.startswith(b"\xef\xbb\xbf"): + head = head[3:] + elif head.startswith(_BOMS_REJECTED) or b"\x00" in head[:4]: + raise SceneError("INVALID_ARCHIVE", "The project is not UTF-8 text, which is the only encoding Studio accepts.") + found = _XML_DECL_ENCODING.match(head) + if found and found.group(1).strip().lower() not in (b"utf-8", b"utf8"): + raise SceneError("INVALID_ARCHIVE", "The project is not UTF-8 text, which is the only encoding Studio accepts.") + + +class Archive: + """Read-only, selective, metered access to one 3MF.""" + + def __init__(self, path: str, ctl: Control): + self.ctl = ctl + self.total_read = 0 + self.model_read = 0 + self.model_declared = 0 + try: + self.zf = zipfile.ZipFile(path) + except (zipfile.BadZipFile, OSError, EOFError) as exc: + raise SceneError("INVALID_ARCHIVE", "This file is not a readable 3MF archive.") from exc + infos = self.zf.infolist() + if len(infos) > L.ARCHIVE_MAX_ENTRIES: + raise SceneError("LIMIT_EXCEEDED", "The archive has more entries than Studio will read.") + self.index: dict[str, tuple[str, zipfile.ZipInfo]] = {} + for info in infos: + norm, is_dir = normalize_entry_name(info.filename) + key = norm.casefold() + if key in self.index: + raise SceneError("INVALID_ARCHIVE", "The archive has duplicate or colliding entry names.") + if info.flag_bits & 0x1: + raise SceneError("INVALID_ARCHIVE", "The archive is encrypted.") + self.index[key] = (norm, info) + if is_dir: + self.index[key] = (norm + "/", info) + + def close(self) -> None: + self.zf.close() + + def account(self, n: int, kind: str) -> None: + self.total_read += n + if self.total_read > L.ARCHIVE_EXPANSION_BYTES: + raise SceneError("LIMIT_EXCEEDED", "The archive expands past what Studio will read.") + if kind == "model": + self.model_read += n + if self.model_read > L.MODEL_XML_BYTES: + raise SceneError("LIMIT_EXCEEDED", "The model data is larger than Studio will read.") + + def has(self, name: str) -> bool: + entry = self.index.get(name.casefold()) + return entry is not None and not entry[0].endswith("/") + + def names(self) -> list[str]: + return [norm for norm, _ in self.index.values() if not norm.endswith("/")] + + def open(self, name: str, kind: str, scan: bool = True) -> GuardedStream: + entry = self.index.get(name.casefold()) + if entry is None or entry[0].endswith("/"): + raise SceneError("UNRESOLVED_REFERENCE", "The project refers to a part that is not in the archive.") + info = entry[1] + if len(entry[0]) > L.MAX_PART_NAME_LENGTH: + # part names are carried on nodes, meshes and sources: bound them before they are placed + raise SceneError("LIMIT_EXCEEDED", f"A part name is longer than the {L.MAX_PART_NAME_LENGTH} characters Studio carries.") + cap = L.MODEL_XML_BYTES if kind == "model" else L.CONFIG_PART_BYTES + if info.file_size > cap: + raise SceneError("LIMIT_EXCEEDED", "A part of the archive is larger than Studio will read.") + if kind == "model": + self.model_declared += info.file_size + return GuardedStream(self, info, kind, cap, scan) + + def read_small(self, name: str) -> bytes: + stream = self.open(name, "config") + try: + out = [] + while True: + chunk = stream.read(L.SNAPSHOT_CHUNK_BYTES) + if not chunk: + break + out.append(chunk) + return b"".join(out) + finally: + stream.close() + + +def iter_xml(stream: GuardedStream, ctl: Control, known=None, consume=frozenset(), local=False): + """Yield lxml ``(event, element)`` pairs with the hardened options. + + The parser is also asked, on its first event, whether it saw a doctype / internal DTD or a + non-UTF-8 declaration, so rejection does not depend on the raw byte scan alone. + + Every completed element is COUNTED against ``MAX_XML_ELEMENTS`` (so a part under its byte cap that is + made of millions of tiny elements is refused early). When ``known`` (the local names the caller reads) + is given, any completed element outside it is released as soon as the caller has seen it, so unknown + junk never accumulates (an element in the namespace the caller does not read is unknown too), and so is + any element named in ``consume`` (self-contained records the caller has read at their own end event); + the caller releases the recognized parents it consumed. ``local=True`` matches by local name. + """ + try: + it = etree.iterparse(stream, events=("start", "end"), resolve_entities=False, + load_dtd=False, no_network=True, huge_tree=False) + first, count, elements = True, 0, 0 + for event, elem in it: + if first: + first = False + info = elem.getroottree().docinfo + if info.doctype or info.internalDTD is not None or info.system_url or info.public_id: + raise SceneError("INVALID_ARCHIVE", "The project contains a DTD, which Studio does not accept.") + if (info.encoding or "UTF-8").upper() not in ("UTF-8", "UTF8"): + raise SceneError("INVALID_ARCHIVE", "The project is not UTF-8 text, which is the only encoding Studio accepts.") + count += 1 + if count % L.XML_EVENTS_PER_CHECK == 0: + ctl.check() + if event == "end": + elements += 1 + if elements > L.MAX_XML_ELEMENTS: + raise SceneError("LIMIT_EXCEEDED", f"A part of the project has more than {L.MAX_XML_ELEMENTS:,} elements.") + yield event, elem + if known is not None and event == "end": + tag = etree.QName(elem).localname if local else elem.tag + if tag not in known or tag in consume: + _free(elem) + except etree.LxmlError as exc: + raise SceneError("INVALID_ARCHIVE", "A part of the project is not well-formed XML.") from exc + finally: + stream.close() + + +def _free(elem) -> None: + """Release ONE completed element: clear it and detach it from its parent. + + Never its siblings. Records that a parent consumes later (the ````/````/```` + children of a settings ````/````) are siblings of whatever unknown element happens to follow + them, and deleting them would silently turn a negative part into a printable one. The caller must have + read everything it needs from ``elem`` (as plain data) BEFORE calling this. + """ + elem.clear() + parent = elem.getparent() + if parent is not None: + parent.remove(elem) + + +# -------------------------------------------------------------------------------------------- +# Matrices (column-major 4x4 as 16 floats; a 3MF transform is 12 numbers: 3 columns + translation) +# -------------------------------------------------------------------------------------------- + +IDENTITY16 = [1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0] + + +def parse_transform(text: str | None, mm_per_unit: float) -> list[float] | None: + """A 3MF ``transform`` attribute as a column-major 16-float matrix in millimetres.""" + if text is None: + return None + parts = text.split() + if len(parts) != 12: + raise SceneError("INVALID_GEOMETRY", "A transform does not have twelve numbers.") + try: + v = [float(p) for p in parts] + except ValueError as exc: + raise SceneError("INVALID_GEOMETRY", "A transform has a value that is not a number.") from exc + if not all(math.isfinite(x) for x in v): + raise SceneError("INVALID_GEOMETRY", "A transform has a value that is not finite.") + return [v[0], v[1], v[2], 0.0, v[3], v[4], v[5], 0.0, v[6], v[7], v[8], 0.0, + v[9] * mm_per_unit, v[10] * mm_per_unit, v[11] * mm_per_unit, 1.0] + + +def mat_mul(a: list[float], b: list[float]) -> list[float]: + out = [0.0] * 16 + for c in range(4): + for r in range(4): + out[c * 4 + r] = (a[r] * b[c * 4] + a[4 + r] * b[c * 4 + 1] + + a[8 + r] * b[c * 4 + 2] + a[12 + r] * b[c * 4 + 3]) + return out + + +def mat_det3(m: list[float]) -> float: + return (m[0] * (m[5] * m[10] - m[9] * m[6]) - m[4] * (m[1] * m[10] - m[9] * m[2]) + + m[8] * (m[1] * m[6] - m[5] * m[2])) + + +def _round_matrix(m: list[float]) -> list[float]: + return [float(f"{x:.9g}") + 0.0 for x in m] # + 0.0 turns -0.0 into 0.0 + + +# -------------------------------------------------------------------------------------------- +# Model parts +# -------------------------------------------------------------------------------------------- + +class Budget: + """Counters shared by every model part of one scene, enforced WHILE parsing (never after).""" + + def __init__(self) -> None: + self.vertices = 0 + self.triangles = 0 + self.elements = 0 + self.objects = 0 + + +class MeshData: + __slots__ = ("pos", "idx", "nverts", "ntris", "max_index") + + def __init__(self) -> None: + self.pos = array("d") + self.idx = array("I") + self.nverts = 0 + self.ntris = 0 + self.max_index = -1 + + +class Component: + __slots__ = ("object_id", "part", "transform") + + def __init__(self, object_id: str, part: str, transform): + self.object_id, self.part, self.transform = object_id, part, transform + + +class ObjectDef: + __slots__ = ("id", "mesh", "components") + + def __init__(self, oid: str): + self.id = oid + self.mesh: MeshData | None = None + self.components: list[Component] = [] + + +class BuildItem: + """A build item. ``part`` is the model FILE its object lives in: the root unless the item carries a + production-extension ``p:path`` (so identity everywhere is the pair part + object id).""" + __slots__ = ("object_id", "part", "transform") + + def __init__(self, object_id: str, part: str, transform): + self.object_id, self.part, self.transform = object_id, part, transform + + +class ModelFile: + def __init__(self, part: str): + self.part = part + self.unit = _units.DEFAULT_UNIT + self.mm_per_unit = 1.0 + self.unit_recognized = True + self.objects: dict[str, ObjectDef] = {} + self.build: list[BuildItem] = [] + + +def _check_id(value: str | None, what: str) -> str: + if value is None or not value.strip(): + raise SceneError("INVALID_ARCHIVE", f"A {what} has no id.") + if len(value) > L.MAX_ID_LENGTH: + raise SceneError("LIMIT_EXCEEDED", f"A {what} id is longer than Studio will carry.") + return value + + +def _package_path(ref: str) -> str: + """A production-extension ``p:path``: an ABSOLUTE package path (leading ``/``); relative ones are refused.""" + if not ref.startswith("/"): + raise SceneError("UNRESOLVED_REFERENCE", "A p:path reference must be an absolute package path.") + return resolve_reference(ref) + + +def parse_model(arc: Archive, part: str, budget: Budget, ctl: Control, is_root: bool) -> ModelFile: + """Stream one ``.model`` part into objects, meshes, components and (root only) build items.""" + mf = ModelFile(part) + stream = arc.open(part, "model") + cur: ObjectDef | None = None + mesh: MeshData | None = None + saw_root = False + t_vertex, t_triangle, t_component, t_item = _T["vertex"], _T["triangle"], _T["component"], _T["item"] + t_object, t_mesh, t_model = _T["object"], _T["mesh"], _T["model"] + limit_v, limit_t, max_coord = L.MAX_VERTICES, L.MAX_PARSED_TRIANGLES, L.MAX_COORDINATE_MM + for event, elem in iter_xml(stream, ctl): + tag = elem.tag + if event == "start": + if not saw_root: + saw_root = True + if tag != t_model: + raise SceneError("INVALID_ARCHIVE", "A model part does not start with a model element.") + # The SAME function the legacy readers use, on the real root element. + info = _units.resolve_unit(elem.get("unit")) + mf.unit_recognized = info.recognized + mf.unit = info.name if info.recognized else _units.DEFAULT_UNIT # documented fallback + mf.mm_per_unit = info.factor + elif tag == t_object: + budget.objects += 1 + if budget.objects > L.MAX_OBJECT_DEFINITIONS: + raise SceneError("LIMIT_EXCEEDED", f"The project defines more than {L.MAX_OBJECT_DEFINITIONS:,} objects.") + oid = _check_id(elem.get("id"), "object") + if oid in mf.objects: + raise SceneError("INVALID_ARCHIVE", "A model part defines the same object id twice.") + cur = ObjectDef(oid) + mf.objects[oid] = cur + elif tag == t_mesh and cur is not None: + mesh = MeshData() + continue + # --- end events + budget.elements += 1 + if budget.elements > L.MAX_XML_ELEMENTS: + raise SceneError("LIMIT_EXCEEDED", f"The model data has more than {L.MAX_XML_ELEMENTS:,} elements.") + if tag == t_vertex: + if mesh is None: + continue + try: + x, y, z = float(elem.get("x")), float(elem.get("y")), float(elem.get("z")) + except (TypeError, ValueError) as exc: + raise SceneError("INVALID_GEOMETRY", "A vertex has a coordinate that is missing or not a number.") from exc + scale = mf.mm_per_unit + x, y, z = x * scale, y * scale, z * scale + if not (math.isfinite(x) and math.isfinite(y) and math.isfinite(z)): + raise SceneError("INVALID_GEOMETRY", "A vertex has a coordinate that is not finite.") + if abs(x) > max_coord or abs(y) > max_coord or abs(z) > max_coord: + raise SceneError("INVALID_GEOMETRY", "A vertex coordinate is outside the range Studio can represent.") + budget.vertices += 1 + if budget.vertices > limit_v: + raise SceneError("LIMIT_EXCEEDED", "The project has more vertices than Studio will draw.") + mesh.pos.append(x) + mesh.pos.append(y) + mesh.pos.append(z) + mesh.nverts += 1 + elif tag == t_triangle: + if mesh is None: + continue + try: + a, b, c = int(elem.get("v1")), int(elem.get("v2")), int(elem.get("v3")) + except (TypeError, ValueError) as exc: + raise SceneError("INVALID_GEOMETRY", "A triangle has an index that is missing or not a whole number.") from exc + if a < 0 or b < 0 or c < 0 or a > 0xFFFFFFFF or b > 0xFFFFFFFF or c > 0xFFFFFFFF: + raise SceneError("INVALID_GEOMETRY", "A triangle has a vertex index outside the supported range.") + budget.triangles += 1 + if budget.triangles > limit_t: + raise SceneError("LIMIT_EXCEEDED", f"The project has at least {budget.triangles:,} triangles; Studio draws at most {limit_t:,}.") + top = max(a, b, c) + if top > mesh.max_index: + mesh.max_index = top + mesh.idx.append(a) + mesh.idx.append(b) + mesh.idx.append(c) + mesh.ntris += 1 + elif tag == t_mesh: + if mesh is not None and cur is not None: + if mesh.nverts == 0 or mesh.ntris == 0: + raise SceneError("INVALID_GEOMETRY", "A mesh has no vertices or no triangles.") + if mesh.max_index >= mesh.nverts: + raise SceneError("INVALID_GEOMETRY", "A triangle refers to a vertex that does not exist.") + cur.mesh = mesh + mesh = None + elif tag == t_component: + if cur is None: + continue + oid = _check_id(elem.get("objectid"), "component") + ref = elem.get(_PROD_PATH) + if ref is not None and not is_root: + raise SceneError("UNRESOLVED_REFERENCE", "A component path is only honoured in the root model part.") + target = _package_path(ref) if ref is not None else part + cur.components.append(Component(oid, target, parse_transform(elem.get("transform"), mf.mm_per_unit))) + if len(cur.components) > L.MAX_NODES: + raise SceneError("LIMIT_EXCEEDED", "An object has more components than Studio will draw.") + elif tag == t_object: + if cur is not None and cur.mesh is not None and cur.components: + raise SceneError("INVALID_ARCHIVE", "An object has both a mesh and components.") + cur = None + elif tag == t_item and is_root: + oid = _check_id(elem.get("objectid"), "build item") + ref = elem.get(_PROD_PATH) + item_part = _package_path(ref) if ref is not None else part + mf.build.append(BuildItem(oid, item_part, parse_transform(elem.get("transform"), mf.mm_per_unit))) + if len(mf.build) > L.MAX_NODES: + raise SceneError("LIMIT_EXCEEDED", "The build has more items than Studio will draw.") + # Every completed element is released, whatever it is (unknown extensions, junk, containers): + # memory stays flat however many elements a model part holds, and the counters above stop it. + _free(elem) + if not saw_root: + raise SceneError("INVALID_ARCHIVE", "A model part is empty.") + return mf + + +_RELS_KNOWN = frozenset({"Relationships", "Relationship"}) +_BAMBU_KNOWN = frozenset({"config", "object", "part", "metadata", "plate", "model_instance", "mesh_stat"}) +_PRUSA_KNOWN = frozenset({"config", "object", "volume", "metadata", "mesh"}) + + +def find_root_model(arc: Archive) -> str: + """The root model part named by ``_rels/.rels`` (default ``3D/3dmodel.model``). + + Every ``.rels`` part is read and an external relationship in any of them is a rejection. + """ + rels = [n for n in arc.names() if n.lower().endswith(".rels")] + if len(rels) > L.MAX_RELS_PARTS: + raise SceneError("LIMIT_EXCEEDED", "The archive has more relationship parts than Studio will read.") + root = None + for name in rels: + stream = arc.open(name, "config") + for event, elem in iter_xml(stream, arc.ctl, known=_RELS_KNOWN, consume=frozenset({"Relationship"}), local=True): + if event != "end" or etree.QName(elem).localname != "Relationship": + continue + if (elem.get("TargetMode") or "").lower() == "external": + raise SceneError("INVALID_ARCHIVE", "The project has an external relationship, which Studio does not follow.") + target = elem.get("Target") + if not target: + raise SceneError("INVALID_ARCHIVE", "A relationship has no target.") + # a relationship Target is relative to the folder that holds the PART the .rels describes + parent = posixpath.dirname(name) + base = posixpath.dirname(parent) if posixpath.basename(parent).casefold() == "_rels" else parent + resolved = resolve_reference(target, base) + if name.casefold() == "_rels/.rels" and elem.get("Type") == _MODEL_REL_TYPE and root is None: + root = resolved + root = root or ROOT_MODEL_DEFAULT + if not arc.has(root): + raise SceneError("UNRESOLVED_REFERENCE", "The project has no root model part.") + return arc.index[root.casefold()][0] + + +#: Stands in for a subtype when two records for one (object, part) disagree; ``role_of`` reads it as unknown. +CONFLICTING_SUBTYPE = "conflicting_records" + + +class SettingsInfo: + """What model_settings.config / Slic3r_PE_model.config say, reduced to what the scene uses.""" + + def __init__(self) -> None: + self.dialect: str | None = None # "bambu" | "prusa" | None + self.part_subtypes: dict[str, dict[str, str | None]] = {} # object id -> {part id: subtype} + self.metadata_objects: set[str] = set() # object ids with ANY part record (even one that is malformed) + self.plates: list[dict] = [] # {"id","ui_number","members":[(oid, inst|None)]} + self.prusa_volumes: dict[str, list[dict]] = {} + + +def parse_settings(arc: Archive, ctl: Control) -> SettingsInfo: + from .assignments import role_of # imported lazily: assignments is a heavy module + + info = SettingsInfo() + if arc.has(MODEL_SETTINGS): + info.dialect = "bambu" + stream = arc.open(arc.index[MODEL_SETTINGS.casefold()][0], "config") + objects = 0 + for event, elem in iter_xml(stream, ctl, known=_BAMBU_KNOWN): + if event != "end": + continue + if elem.tag == "object": + objects += 1 + if objects > 5 * L.MAX_NODES: + raise SceneError("LIMIT_EXCEEDED", "The settings describe more objects than Studio will read.") + oid = elem.get("id") or "" + # a duplicated record may not silently overwrite another: two records for the same + # (object id, part id) that disagree make the role unknown (identical ones are fine) + parts = info.part_subtypes.setdefault(oid, {}) + for p in elem.iterchildren("part"): + info.metadata_objects.add(oid) # the object HAS metadata, whatever this record says + pid = p.get("id") + if pid is None: + continue + subtype = p.get("subtype") + if pid in parts and parts[pid] != subtype: + subtype = CONFLICTING_SUBTYPE + parts[pid] = subtype + _free(elem) + elif elem.tag == "plate": + if len(info.plates) >= 256: + raise SceneError("LIMIT_EXCEEDED", "The settings describe more plates than Studio will read.") + meta = {m.get("key"): m.get("value") for m in elem.iterchildren("metadata")} + number = meta.get("plater_id") + members = [] + for mi in elem.iterchildren("model_instance"): + mm = {m.get("key"): m.get("value") for m in mi.iterchildren("metadata")} + if mm.get("object_id") is not None: + members.append((mm["object_id"], mm.get("instance_id"))) + if len(members) > 10 * L.MAX_NODES: + raise SceneError("LIMIT_EXCEEDED", "A plate lists more objects than Studio will read.") + ui = (int(number) if number is not None and number.isascii() and number.isdigit() + and len(number) <= 6 else None) + info.plates.append({"ui_number": ui, "members": members}) + _free(elem) + elif arc.has(PRUSA_CONFIG): + info.dialect = "prusa" + stream = arc.open(arc.index[PRUSA_CONFIG.casefold()][0], "config") + count = 0 + for event, elem in iter_xml(stream, ctl, known=_PRUSA_KNOWN): + if event != "end" or elem.tag != "object": + continue + count += 1 + if count > L.MAX_OBJECT_DEFINITIONS: + raise SceneError("LIMIT_EXCEEDED", "The settings describe more objects than Studio will read.") + entries = [] + for vol in elem.iterchildren("volume"): + # attributes are read BY NAME: their order in the tag does not matter + first, last = vol.get("firstid"), vol.get("lastid") + valid = (first is not None and last is not None and first.isascii() and last.isascii() + and first.isdigit() and last.isdigit() and len(first) <= 9 and len(last) <= 9) + kind = None + for m in vol.iterchildren("metadata"): + if m.get("key") == "volume_type" and m.get("type") in (None, "volume"): + kind = m.get("value") + entries.append({"range": (int(first), int(last)) if valid else None, "role": role_of(kind)}) + if len(entries) > L.MAX_VOLUMES_PER_MESH: + raise SceneError("LIMIT_EXCEEDED", "An object has more volumes than Studio will read.") + if entries: + info.prusa_volumes.setdefault(elem.get("id") or "", []).extend(entries) + _free(elem) + return info + + +# -------------------------------------------------------------------------------------------- +# Bed +# -------------------------------------------------------------------------------------------- + +def _bed(limitations: list[dict]) -> tuple[dict, dict]: + """(bed object for the scene, rectangle used for findings). Surfaces a fallback, never hides it.""" + from . import plate_placement as pp + + rect = None + try: + rect = pp.parse_printable_area(pp._u1_printable_area()) + except Exception: # noqa: BLE001 - a missing/corrupt template is a limitation, not a crash + rect = None + policy = "u1_template" + if rect is None: + policy = "fallback" + rect = {"min_x": 0.0, "min_y": 0.0, "max_x": 270.0, "max_y": 270.0} + limitations.append({"code": "BED_TEMPLATE_UNAVAILABLE", "target_ids": []}) + height = None + try: + data = json.loads((files("snapstudio_core.data") / "templates" / "u1_base_project_settings.json").read_text("utf-8")) + height = float(data.get("printable_height")) + except Exception: # noqa: BLE001 + height = None + poly = [[rect["min_x"], rect["min_y"]], [rect["max_x"], rect["min_y"]], + [rect["max_x"], rect["max_y"]], [rect["min_x"], rect["max_y"]]] + return ({"polygon_mm": poly, "height_mm": height, "edge_margin_mm": pp.EDGE_MARGIN_MM, "policy": policy}, rect) + + + + +# -------------------------------------------------------------------------------------------- +# Traversal +# -------------------------------------------------------------------------------------------- + +class Node: + __slots__ = ("id", "parent", "resource", "mesh_key", "local", "world", "mirrored", "build_index", + "path", "plate", "placement_known", "printable", "has_non_part", "bounds", + "finding_ids", "children", "top", "repeated", "role_ctx") + + def __init__(self) -> None: + self.parent: Node | None = None + self.children: list[Node] = [] + self.finding_ids: list[str] = [] + self.bounds = None + self.printable = None + self.has_non_part = False + self.mesh_key = None + self.role_ctx = None + + +class TopContext: + """What every node under one build item inherits.""" + + def __init__(self, ordinal: int, build_index, plate: dict, placement_known: bool, repeated: bool): + self.ordinal, self.build_index, self.plate = ordinal, build_index, plate + self.placement_known, self.repeated = placement_known, repeated + self.node: Node | None = None + self.top_part: str | None = None + + +class MeshInfo: + """Volumes and printable-geometry facts for one emitted mesh.""" + + def __init__(self, key: tuple[str, str], data: MeshData): + self.key, self.data = key, data + self.volumes: list[dict] = [] + self.part_verts: list[int] = [] + self.lo = self.hi = None + self.has_part = False + self.has_non_part = False + self.has_unknown = False + + +class Traversal: + def __init__(self, files_by_part: dict[str, ModelFile], settings: SettingsInfo, root_part: str, ctl: Control): + self.files, self.settings, self.root_part, self.ctl = files_by_part, settings, root_part, ctl + self.nodes: list[Node] = [] + self.rendered = 0 + self.mesh_order: list[tuple[str, str]] = [] + self._ids: set[str] = set() + self.mesh_users: dict[tuple[str, str], list[str]] = {} + + def add_tree(self, key: tuple[str, str], transform, ctx: TopContext) -> None: + ctx.top_part = key[0] + self._expand(key, transform, None, [], ctx, ()) + + def _context_role(self, parent: Node | None, via: str | None, inherited: str | None, + ctx: "TopContext | None" = None) -> str | None: + """The role this USE imposes (Bambu/Orca ````), following the traversal context. + + A part record belongs to one (object, component) pair, so a mesh shared by two objects can be a + negative part under one and an ordinary part under the other, and a role given to a component + ASSEMBLY applies to everything beneath it (``inherited`` wins unless it is a plain part). An object + that HAS part records but none for this component gives ``unknown``, never a printable part. + """ + if ctx is not None and self.settings.dialect is not None and ctx.top_part != self.root_part: + # Part records are matched against ROOT-file objects. A build item whose object lives in another + # file (p:path) cannot be associated provably with any record, so every part beneath it is + # unknown (fail closed): never printable, never judged against the bed. + return "unknown" + own = None + if parent is not None and via is not None and self.settings.dialect == "bambu" and parent.resource[0] == self.root_part: + subtypes = self.settings.part_subtypes.get(parent.resource[1]) + if parent.resource[1] in self.settings.metadata_objects and subtypes is not None: + from .assignments import role_of + own = role_of(subtypes[via]) if via in subtypes else "unknown" + if inherited not in (None, "part"): + return inherited + return own if own is not None else inherited + + def _expand(self, key, local, parent: Node | None, path: list[int], ctx: TopContext, + stack: tuple, via: str | None = None, inherited: str | None = None) -> Node: + self.ctl.check() + mf = self.files.get(key[0]) + obj = mf.objects.get(key[1]) if mf else None + if obj is None: + raise SceneError("UNRESOLVED_REFERENCE", "The project refers to an object that is not defined.") + if key in stack: + raise SceneError("INVALID_ARCHIVE", "The project's components refer to themselves.") + if len(path) >= L.MAX_DEPTH: + raise SceneError("LIMIT_EXCEEDED", "The project nests objects deeper than Studio will follow.") + if len(self.nodes) >= L.MAX_NODES: + raise SceneError("LIMIT_EXCEEDED", "The project has more objects than Studio will draw.") + node = Node() + node.parent = parent + node.resource = key + node.path = list(path) + node.local = local if local is not None else IDENTITY16 + node.world = node.local if parent is None else mat_mul(parent.world, node.local) + if not all(math.isfinite(x) for x in node.world): + raise SceneError("INVALID_GEOMETRY", "A combined transform is not finite.") + node.mirrored = mat_det3(node.world) < 0 + node.id = self._node_id(ctx, parent, path) + if node.id in self._ids: # digested ids give 128-bit practical uniqueness; a collision is detected + raise SceneError("INTERNAL", "Two nodes would share an id.") + self._ids.add(node.id) + node.role_ctx = self._context_role(parent, via, inherited, ctx) + node.build_index, node.plate = ctx.build_index, ctx.plate + node.placement_known, node.repeated, node.top = ctx.placement_known, ctx.repeated, ctx + self.nodes.append(node) + if parent is None: + ctx.node = node + else: + parent.children.append(node) + if obj.mesh is not None: + node.mesh_key = key + self.rendered += obj.mesh.ntris + if self.rendered > L.MAX_RENDERED_TRIANGLES: + raise SceneError("LIMIT_EXCEEDED", f"The project has at least {self.rendered:,} triangles counting repeated objects; Studio draws at most {L.MAX_RENDERED_TRIANGLES:,}.") + if key not in self.mesh_users: + self.mesh_users[key] = [] + self.mesh_order.append(key) + self.mesh_users[key].append(node.id) + elif obj.components: + for i, comp in enumerate(obj.components): + self._expand((comp.part, comp.object_id), comp.transform, node, path + [i], ctx, + stack + (key,), via=comp.object_id, inherited=node.role_ctx) + else: + raise SceneError("INVALID_GEOMETRY", "An object has neither a mesh nor components.") + return node + + @staticmethod + def _node_id(ctx: TopContext, parent: Node | None, path: list[int]) -> str: + """``b`` plus ``.c`` per component level. When that would not fit in + ``MAX_ID_LENGTH`` the ancestry is replaced by a deterministic digest of it (``b3.~<32 hex>``), + which is unique per ancestry, so 128 characters always suffice at the deepest allowed nesting.""" + natural = f"b{ctx.ordinal}" + "".join(f".c{i}" for i in path) + if len(natural) <= L.MAX_ID_LENGTH: + return natural + return f"b{ctx.ordinal}.~" + hashlib.sha256(natural.encode("ascii")).hexdigest()[:32] + + # ---- volumes ------------------------------------------------------------------------- + def mesh_data(self, key) -> MeshData: + return self.files[key[0]].objects[key[1]].mesh + + def build_mesh_info(self, key) -> MeshInfo: + data = self.mesh_data(key) + info = MeshInfo(key, data) + total = data.ntris + if (self.settings.dialect == "prusa" and key[0] == self.root_part + and key[1] in self.settings.prusa_volumes): + info.volumes = self._prusa_ranges(self.settings.prusa_volumes[key[1]], total) + else: + info.volumes = [{"id": "v0", "triangle_start": 0, "triangle_count": total, "role": "part", "source": "object"}] + if len(info.volumes) > L.MAX_VOLUMES_PER_MESH: + raise SceneError("LIMIT_EXCEEDED", "A mesh has more volumes than Studio will carry.") + used: set[int] = set() + for vol in info.volumes: + if vol["role"] == "part": + info.has_part = True + s, c = vol["triangle_start"], vol["triangle_count"] + used.update(data.idx[3 * s:3 * (s + c)]) + else: + info.has_non_part = True + if vol["role"] == "unknown": + info.has_unknown = True + if used: + pos = data.pos + lo = [math.inf] * 3 + hi = [-math.inf] * 3 + info.part_verts = sorted(used) + for n, vi in enumerate(info.part_verts): + if n % L.VERTEX_BATCH == 0: + self.ctl.check() + b = 3 * vi + for ax in range(3): + v = pos[b + ax] + if v < lo[ax]: + lo[ax] = v + if v > hi[ax]: + hi[ax] = v + info.lo, info.hi = lo, hi + return info + + @staticmethod + def _prusa_ranges(entries: list[dict], total: int) -> list[dict]: + """Partition ALL triangles into volumes. An unmapped gap is an ``unknown`` volume; ranges that + overlap or leave the mesh make the whole mesh one ``unknown`` volume (never a guess).""" + whole = [{"id": "v0", "triangle_start": 0, "triangle_count": total, "role": "unknown", "source": "prusa_range"}] + if any(e["range"] is None for e in entries): + return whole # a range Studio cannot read: never a printable guess + spans = sorted((e["range"][0], e["range"][1], e["role"]) for e in entries) + out: list[dict] = [] + cursor = 0 + for first, last, role in spans: + if first < cursor or last < first or last >= total: + return whole + if first > cursor: + out.append({"id": f"v{len(out)}", "triangle_start": cursor, "triangle_count": first - cursor, + "role": "unknown", "source": "prusa_range"}) + out.append({"id": f"v{len(out)}", "triangle_start": first, "triangle_count": last - first + 1, + "role": role, "source": "prusa_range"}) + cursor = last + 1 + if cursor < total: + out.append({"id": f"v{len(out)}", "triangle_start": cursor, "triangle_count": total - cursor, + "role": "unknown", "source": "prusa_range"}) + return out or whole + + +def _world_bounds(info: MeshInfo, world: list[float], ctl: Control): + """Exact world-space bounds of the printable (role=part) vertices of one instance.""" + if not info.part_verts: + return None + m = world + # Rows with at most one non-zero entry map each output axis from a single input axis, so the + # transformed local box is exact (no rotation). Anything else transforms the vertices. + rows = [] + simple = True + for i in range(3): + nz = [(j, m[j * 4 + i]) for j in range(3) if abs(m[j * 4 + i]) > 1e-12] + if len(nz) > 1: + simple = False + break + rows.append(nz[0] if nz else None) + if simple: + lo_out = [0.0] * 3 + hi_out = [0.0] * 3 + for i in range(3): + t = m[12 + i] + if rows[i] is None: + lo_out[i] = hi_out[i] = t + else: + j, c = rows[i] + a, b = c * info.lo[j] + t, c * info.hi[j] + t + lo_out[i], hi_out[i] = (a, b) if a <= b else (b, a) + return lo_out, hi_out + pos = info.data.pos + lo_out = [math.inf] * 3 + hi_out = [-math.inf] * 3 + m0, m1, m2, m4, m5, m6, m8, m9, m10, t0, t1, t2 = (m[0], m[1], m[2], m[4], m[5], m[6], m[8], m[9], m[10], m[12], m[13], m[14]) + for n, vi in enumerate(info.part_verts): + if n % L.VERTEX_BATCH == 0: + ctl.check() + b = 3 * vi + x, y, z = pos[b], pos[b + 1], pos[b + 2] + px = m0 * x + m4 * y + m8 * z + t0 + py = m1 * x + m5 * y + m9 * z + t1 + pz = m2 * x + m6 * y + m10 * z + t2 + if px < lo_out[0]: lo_out[0] = px + if px > hi_out[0]: hi_out[0] = px + if py < lo_out[1]: lo_out[1] = py + if py > hi_out[1]: hi_out[1] = py + if pz < lo_out[2]: lo_out[2] = pz + if pz > hi_out[2]: hi_out[2] = pz + return lo_out, hi_out + + +# -------------------------------------------------------------------------------------------- +# Plates +# -------------------------------------------------------------------------------------------- + +def _plate_assignment(settings: SettingsInfo, oid: str, ordinal: int, instances_of_object: int) -> tuple[dict, bool]: + """(plate ref, ambiguous?) for the ``ordinal``-th build item of object ``oid``.""" + if not settings.plates: + return {"state": "unknown", "plate_id": None, "source": None}, False + found: set[str] = set() + for idx, plate in enumerate(settings.plates): + pid = f"p{plate['ui_number']}" if plate["ui_number"] is not None else f"plate-{idx}" + for moid, inst in plate["members"]: + if moid != oid: + continue + if inst is None: + # a record that does not say which instance covers all of them only if there is one + if instances_of_object == 1: + found.add(pid) + else: + return {"state": "ambiguous", "plate_id": None, "source": "plate_config"}, True + elif inst.isascii() and inst.isdigit() and int(inst) == ordinal: + found.add(pid) + if len(found) == 1: + return {"state": "known", "plate_id": next(iter(found)), "source": "plate_config"}, False + if len(found) > 1: + return {"state": "ambiguous", "plate_id": None, "source": "plate_config"}, True + return {"state": "unknown", "plate_id": None, "source": None}, False + + +# -------------------------------------------------------------------------------------------- +# Assembly and encoding +# -------------------------------------------------------------------------------------------- + +class _Limitations: + ORDER = ("UNKNOWN_VOLUME_ROLE", "UNKNOWN_PLACEMENT", "PLATE_MEMBERSHIP_AMBIGUOUS", "PLATE_MEMBERSHIP_UNKNOWN", + "PLATES_UNAVAILABLE", "BED_TEMPLATE_UNAVAILABLE", "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED", + "NON_MM_SOURCE_UNIT", "UNSUPPORTED_UNIT", "MULTI_PLATE_PLACEMENT_UNCHECKED", "NO_BUILD_ITEMS") + + def __init__(self) -> None: + self._by: dict[str, list[str]] = {} + + def has(self, code: str) -> bool: + return code in self._by + + def add(self, code: str, target_ids=()) -> None: + bucket = self._by.setdefault(code, []) + for t in target_ids: + if t not in bucket: + bucket.append(t) + + def render(self) -> list[dict]: + out = [{"code": c, "target_ids": self._by[c][:2000]} for c in self.ORDER if c in self._by] + if len(out) > L.MAX_LIMITATIONS: + raise SceneError("LIMIT_EXCEEDED", "The scene has more limitations than Studio will carry.") + return out + + +def _b64_len(n: int) -> int: + return 4 * ((n + 2) // 3) + + +def _encode_f32(pos: array, ctl: Control) -> str: + step = 3 * L.GEOMETRY_BATCH_TRIANGLES + out = [] + for s in range(0, len(pos), step): + ctl.check() + chunk = array("f", pos[s:s + step]) + if sys.byteorder == "big": + chunk.byteswap() + out.append(base64.b64encode(chunk.tobytes()).decode("ascii")) + return "".join(out) + + +def _encode_u32(idx: array, ctl: Control) -> str: + assert idx.itemsize == 4 + step = 3 * L.GEOMETRY_BATCH_TRIANGLES + out = [] + for s in range(0, len(idx), step): + ctl.check() + chunk = idx[s:s + step] + if sys.byteorder == "big": + chunk = array("I", chunk) + chunk.byteswap() + out.append(base64.b64encode(chunk.tobytes()).decode("ascii")) + return "".join(out) + + +def _rounded(v) -> list[float]: + return [round(float(x), 6) + 0.0 for x in v] + + +def _finish(revision: str, sources: list[dict], bed: dict, rect: dict, trav: Traversal, tops: list[TopContext], + plates: list[dict], plate_count: int, lims: _Limitations, ctl: Control, *, + repeated_tops: list[str]) -> dict: + from . import plate_placement as pp + + ctl.progress("encoding", 0, trav.rendered) + infos = {key: trav.build_mesh_info(key) for key in trav.mesh_order} + + # per-node printable facts, children before parents + for node in reversed(trav.nodes): + if node.mesh_key is not None: + info = infos[node.mesh_key] + if node.role_ctx not in (None, "part"): + # this USE is a modifier / negative part / support helper (or unknown): nothing prints + node.printable = None if node.role_ctx == "unknown" else False + node.has_non_part = True + node.bounds = None + else: + node.printable = True if info.has_part else (None if info.has_unknown else False) + node.has_non_part = info.has_non_part + node.bounds = _world_bounds(info, node.world, ctl) if info.has_part else None + else: + kids = node.children + node.has_non_part = any(k.has_non_part for k in kids) + node.printable = True if any(k.printable is True for k in kids) else ( + None if any(k.printable is None for k in kids) else False) + boxes = [k.bounds for k in kids if k.bounds is not None] + node.bounds = ([min(b[0][i] for b in boxes) for i in range(3)], + [max(b[1][i] for b in boxes) for i in range(3)]) if boxes else None + for node in trav.nodes: + if node.mesh_key is not None and node.role_ctx == "unknown": + lims.add("UNKNOWN_VOLUME_ROLE", [node.id]) + elif node.mesh_key is not None and node.role_ctx in (None, "part") and infos[node.mesh_key].has_unknown: + lims.add("UNKNOWN_VOLUME_ROLE", [node.id]) + + # findings -- only what the geometry plus the file prove + findings: list[dict] = [] + all_mm = all(s["unit"] == "millimeter" for s in sources) and not lims.has("UNSUPPORTED_UNIT") + if not all(s["unit"] == "millimeter" for s in sources): + lims.add("NON_MM_SOURCE_UNIT") + if plate_count > 1: + lims.add("MULTI_PLATE_PLACEMENT_UNCHECKED") + usable_x = rect["max_x"] - rect["min_x"] - 2 * pp.EDGE_MARGIN_MM + usable_y = rect["max_y"] - rect["min_y"] - 2 * pp.EDGE_MARGIN_MM + enabled = bed["policy"] == "u1_template" and plate_count <= 1 + all_mm_or_known = not lims.has("UNSUPPORTED_UNIT") + index_of = {id(n): i for i, n in enumerate(trav.nodes)} + for ctx in tops: + node = ctx.node + if node is None or not enabled or node.bounds is None or node.printable is not True: + continue + if plate_count > 0 and node.plate["state"] != "known": + continue # the project has plates but this instance is provably on none / on two + if ctx.repeated or not ctx.placement_known: + # REPEATED_INSTANCE_PLACEMENT_UNVERIFIED / unknown placement: geometry stays visible, + # but no bed-fit statement of any kind is attached. + continue + lo, hi = node.bounds + pointer = f"/nodes/{index_of[id(node)]}" + if all_mm: + over = pp._overhang({"min": lo, "max": hi}, rect) + if any(v > 0 for v in over.values()): + fid = f"f{len(findings)}" + findings.append({"id": fid, "engine": "scene", "schema": SCHEMA_ID, "pointer": pointer, + "scope": "instance", "kind": "placement", "target_ids": [node.id], + "value": {"code": "PLACEMENT_OUTSIDE_BED", "overhang_mm": over}}) + node.finding_ids.append(fid) + if all_mm_or_known and ((hi[0] - lo[0]) > usable_x + 1e-9 or (hi[1] - lo[1]) > usable_y + 1e-9): + fid = f"f{len(findings)}" + findings.append({"id": fid, "engine": "scene", "schema": SCHEMA_ID, "pointer": pointer, + "scope": "instance", "kind": "size", "target_ids": [node.id], + "value": {"code": "SIZE_EXCEEDS_BED", "overhang_mm": None}}) + node.finding_ids.append(fid) + if len(findings) > L.MAX_FINDINGS: + raise SceneError("LIMIT_EXCEEDED", "The scene has more findings than Studio will carry.") + if repeated_tops: + lims.add("REPEATED_INSTANCE_PLACEMENT_UNVERIFIED", repeated_tops) + + # response preflight: the geometry alone decides this exactly, before any encoding work + geom = 0 + for info in infos.values(): + geom += _b64_len(info.data.nverts * 12) + _b64_len(info.data.ntris * 12) + if geom > L.MAX_RESPONSE_BYTES: + raise SceneError("LIMIT_EXCEEDED", "The geometry alone is larger than the response Studio will send.") + + meshes = [] + done = 0 + for key in trav.mesh_order: + info = infos[key] + d = info.data + meshes.append({ + "key": {"part": key[0], "object_id": key[1]}, + "vertex_count": d.nverts, "triangle_count": d.ntris, + "positions_f32le_base64": _encode_f32(d.pos, ctl), + "indices_u32le_base64": _encode_u32(d.idx, ctl), + "volumes": info.volumes}) + done += d.ntris + ctl.progress("encoding", done, trav.rendered) + + nodes = [] + for node in trav.nodes: + nodes.append({ + "id": node.id, + "parent_id": node.parent.id if node.parent else None, + "resource": {"part": node.resource[0], "object_id": node.resource[1]}, + "mesh_key": ({"part": node.mesh_key[0], "object_id": node.mesh_key[1]} if node.mesh_key else None), + "local_to_parent_mm": _round_matrix(node.local), + "world_mm": _round_matrix(node.world), + "mirrored": node.mirrored, + "role_context": node.role_ctx, + "build_index": node.build_index, + "instance_ref": {"build_index": node.build_index, "component_path": node.path}, + "plate": node.plate, + "placement_state": "known" if node.placement_known else "unknown", + "printable": node.printable, + "has_non_part_volumes": node.has_non_part, + "bounds_mm": ({"min": _rounded(node.bounds[0]), "max": _rounded(node.bounds[1])} if node.bounds else None), + "finding_ids": node.finding_ids}) + limitations = lims.render() + status = "partial" if any(x["code"] in PARTIAL_LIMITATIONS for x in limitations) else "complete" + return { + "schema": SCHEMA_ID, "revision": revision, "status": status, "units": "mm", "axes": "right-handed-z-up", + "sources": sources, "bed": bed, "meshes": meshes, "nodes": nodes, "plates": plates, + "findings": findings, "limitations": limitations, + "counts": {"nodes": len(nodes), "meshes": len(meshes), + "vertices": sum(i.data.nverts for i in infos.values()), + "triangles": sum(i.data.ntris for i in infos.values()), + "rendered_triangles": trav.rendered, "plates": len(plates), + "findings": len(findings), "limitations": len(limitations)}, + "limits": dict(L.LIMITS_ECHO)} + + +def serialize(scene: dict) -> bytes: + """The whole response body, once, compact ASCII JSON, with the cap enforced AS IT IS BUILT. + + Each top-level member (and each element of a list member) is serialized separately and the running + length is checked before it is kept, so an over-cap scene is refused after at most one element past + the cap: the full body (or a UTF-8 copy of it) is never built first. + """ + cap = L.MAX_RESPONSE_BYTES + + def dump(value) -> str: + try: + return json.dumps(value, separators=(",", ":"), allow_nan=False) + except ValueError as exc: + raise SceneError("INVALID_GEOMETRY", "The scene contains a value that is not finite.") from exc + + def over() -> SceneError: + return SceneError("LIMIT_EXCEEDED", "The scene is larger than the response Studio will send.") + + total = 2 # the braces; every other byte is added exactly, before it is kept + members = [] + for key, value in scene.items(): + head = dump(key) + ":" + total += len(head) + (1 if members else 0) # the comma between members + if total > cap: + raise over() + if isinstance(value, list) and value and isinstance(value[0], dict): + items = [] + total += 2 # [ ] + for item in value: + text = dump(item) + total += len(text) + (1 if items else 0) + if total > cap: + raise over() + items.append(text) + members.append(head + "[" + ",".join(items) + "]") + else: + text = dump(value) + total += len(text) + if total > cap: + raise over() + members.append(head + text) + return ("{" + ",".join(members) + "}").encode("ascii") + + +def _sha256_file(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(L.SNAPSHOT_CHUNK_BYTES), b""): + h.update(chunk) + return h.hexdigest() + + +# -------------------------------------------------------------------------------------------- +# Front ends +# -------------------------------------------------------------------------------------------- + +def _build_3mf(path: str, revision: str, ctl: Control) -> dict: + lims = _Limitations() + bed, rect = _bed([]) + if bed["policy"] == "fallback": + lims.add("BED_TEMPLATE_UNAVAILABLE") + arc = Archive(path, ctl) + try: + ctl.progress("parsing", 0, None) + root_part = find_root_model(arc) + budget = Budget() + files_by_part: dict[str, ModelFile] = {} + queue = [root_part] + while queue: + part = queue.pop(0) + if part in files_by_part: + continue + if len(files_by_part) >= L.MAX_MODEL_PARTS: + raise SceneError("LIMIT_EXCEEDED", "The project uses more model parts than Studio will read.") + if not arc.has(part): + raise SceneError("UNRESOLVED_REFERENCE", "The project refers to a model part that is not in the archive.") + canon = arc.index[part.casefold()][0] + mf = parse_model(arc, canon, budget, ctl, canon == root_part) + files_by_part[canon] = mf + ctl.progress("parsing", arc.model_read, arc.model_declared) + for item in mf.build: # a build item's p:path names the FILE its object lives in + if not arc.has(item.part): + raise SceneError("UNRESOLVED_REFERENCE", "A build item refers to a model part that is not in the archive.") + item.part = arc.index[item.part.casefold()][0] + if item.part not in files_by_part: + queue.append(item.part) + for obj in mf.objects.values(): + for comp in obj.components: + if not arc.has(comp.part): + raise SceneError("UNRESOLVED_REFERENCE", "A component refers to a model part that is not in the archive.") + comp.part = arc.index[comp.part.casefold()][0] + if comp.part not in files_by_part: + queue.append(comp.part) + settings = parse_settings(arc, ctl) + root = files_by_part[root_part] + bad_unit_parts = {mf.part for mf in files_by_part.values() if not mf.unit_recognized} + sources = [{"part": mf.part, "unit": mf.unit, "mm_per_unit": mf.mm_per_unit} + for mf in files_by_part.values()] + plates = [] + seen_ui = set() + for idx, plate in enumerate(settings.plates): + ui = plate["ui_number"] + if ui is not None and ui in seen_ui: + continue + seen_ui.add(ui) + plates.append({"id": f"p{ui}" if ui is not None else f"plate-{idx}", "ui_number": ui, "origin_mm": None}) + trav = Traversal(files_by_part, settings, root_part, ctl) + counts: dict[tuple[str, str], int] = {} + for item in root.build: + counts[(item.part, item.object_id)] = counts.get((item.part, item.object_id), 0) + 1 + part_by_bare: dict[str, set[str]] = {} + for item in root.build: + part_by_bare.setdefault(item.object_id, set()).add(item.part) + tops: list[TopContext] = [] + seen_n: dict[tuple[str, str], int] = {} + repeated_tops: list[str] = [] + ambiguous_ids: list[str] = [] + cross_file_ambiguous: list = [] + unknown_plate_ids: list[str] = [] + if root.build: + for bi, item in enumerate(root.build): + ikey = (item.part, item.object_id) + n = seen_n.get(ikey, 0) + seen_n[ikey] = n + 1 + plate, ambiguous = _plate_assignment(settings, item.object_id, n, counts[ikey]) + if len(part_by_bare[item.object_id]) > 1 and settings.plates: + # plate records name a BARE object id: when two build resources in different files share + # it, a record cannot prove membership for either of them + plate, ambiguous = {"state": "ambiguous", "plate_id": None, "source": "plate_config"}, False + cross_file_ambiguous.append(None) + ctx = TopContext(bi, bi, plate, True, counts[ikey] > 1) + trav.add_tree(ikey, item.transform, ctx) + tops.append(ctx) + if ctx.repeated: + repeated_tops.append(ctx.node.id) + if ambiguous: + ambiguous_ids.append(ctx.node.id) + elif plate["state"] == "unknown" and settings.plates: + unknown_plate_ids.append(ctx.node.id) + elif cross_file_ambiguous and plate["state"] == "ambiguous" and ctx.node.id not in unknown_plate_ids: + unknown_plate_ids.append(ctx.node.id) + else: + lims.add("NO_BUILD_ITEMS") + for ordinal, oid in enumerate(root.objects): + ctx = TopContext(ordinal, None, {"state": "unknown", "plate_id": None, "source": None}, False, False) + trav.add_tree((root_part, oid), None, ctx) + tops.append(ctx) + lims.add("UNKNOWN_PLACEMENT", [ctx.node.id]) + if bad_unit_parts: + # target_ids name the NODES that live in the affected part(s); for such a source `unit` and + # `mm_per_unit` in sources[] are the fallback millimetre values (the limitation says so) + # a build item's transform is written in the ROOT model's unit, so when the root is the bad part the + # top node of every item is affected too, even if its mesh lives in another file + lims.add("UNSUPPORTED_UNIT", [n.id for n in trav.nodes + if n.resource[0] in bad_unit_parts or (n.parent is None and root_part in bad_unit_parts)]) + if not settings.plates: + lims.add("PLATES_UNAVAILABLE") + if ambiguous_ids: + lims.add("PLATE_MEMBERSHIP_AMBIGUOUS", ambiguous_ids) + if unknown_plate_ids: + lims.add("PLATE_MEMBERSHIP_UNKNOWN", unknown_plate_ids) + return _finish(revision, sources, bed, rect, trav, tops, plates, len(plates), lims, ctl, + repeated_tops=repeated_tops) + finally: + arc.close() + + +def _build_stl(path: str, revision: str, ctl: Control) -> dict: + import os + import struct + + from .stl_io import StlTooLarge, decode_stl_stream + + lims = _Limitations() + bed, rect = _bed([]) + if bed["policy"] == "fallback": + lims.add("BED_TEMPLATE_UNAVAILABLE") + size = os.path.getsize(path) + ctl.progress("parsing", 0, size) + try: + with open(path, "rb") as fh: + # bounded and incremental: the binary header count is judged BEFORE any triangle is decoded, + # decoding runs in batches with the cancel/deadline hook between them, ASCII is line-streamed + verts, tris = decode_stl_stream(fh, size, max_triangles=L.MAX_PARSED_TRIANGLES, + max_vertices=L.MAX_VERTICES, check=ctl.check, + batch=L.GEOMETRY_BATCH_TRIANGLES) + except StlTooLarge as exc: + raise SceneError("LIMIT_EXCEEDED", f"The model has {exc.triangles:,} {exc.what}; Studio draws at most {exc.limit:,}.") from exc + except (SnapStudioError, ValueError, struct.error) as exc: + raise SceneError("INVALID_GEOMETRY", "This STL file could not be read.") from exc + mesh = MeshData() + for n, v in enumerate(verts): + if n % L.VERTEX_BATCH == 0: + ctl.check() + if not all(math.isfinite(c) and abs(c) <= L.MAX_COORDINATE_MM for c in v): + raise SceneError("INVALID_GEOMETRY", "A vertex coordinate is not usable.") + mesh.pos.extend(v) + for n, t in enumerate(tris): + if n % L.VERTEX_BATCH == 0: + ctl.check() + mesh.idx.extend(t) + mesh.nverts, mesh.ntris, mesh.max_index = len(verts), len(tris), len(verts) - 1 + part = "model.stl" + mf = ModelFile(part) + obj = ObjectDef("0") + obj.mesh = mesh + mf.objects["0"] = obj + trav = Traversal({part: mf}, SettingsInfo(), part, ctl) + ctx = TopContext(0, None, {"state": "unknown", "plate_id": None, "source": None}, False, False) + trav.add_tree((part, "0"), None, ctx) + lims.add("UNKNOWN_PLACEMENT", [ctx.node.id]) + lims.add("PLATES_UNAVAILABLE") + return _finish(revision, [{"part": part, "unit": "millimeter", "mm_per_unit": 1.0}], bed, rect, trav, [ctx], [], 0, + lims, ctl, repeated_tops=[]) + + +def build_scene_dict(path: str, revision: str | None = None, ctl: Control | None = None) -> dict: + """Build the scene/1 object for ``path`` (a private snapshot, when called by a job).""" + ctl = ctl or Control() + low = str(path).lower() + if not (low.endswith(".3mf") or low.endswith(".stl")): + raise SceneError("UNSUPPORTED_FORMAT", "Only .3mf and .stl files can be shown.") + revision = revision or _sha256_file(path) + if low.endswith(".stl"): + return _build_stl(path, revision, ctl) + return _build_3mf(path, revision, ctl) + + +def build_scene(path: str, revision: str | None = None, ctl: Control | None = None) -> bytes: + """The serialized scene (compact UTF-8 JSON), with the whole-body cap enforced.""" + return serialize(build_scene_dict(path, revision, ctl)) diff --git a/backend/snapstudio_core/scene_limits.py b/backend/snapstudio_core/scene_limits.py new file mode 100644 index 00000000..a493950c --- /dev/null +++ b/backend/snapstudio_core/scene_limits.py @@ -0,0 +1,64 @@ +"""Every numeric budget, cap, deadline and TTL behind the scene/1 contract, in one place. + +Nothing else in the scene code carries a literal limit. These are PROVISIONAL budgets (see +docs/testing/scene-contract/README.md for the corpus they were measured against). Each one is +enforced while the work happens, never after, and exceeding one is a clear LIMIT_EXCEEDED, never +a silently shortened scene. +""" +from __future__ import annotations + +MIB = 1024 * 1024 + +# --- what the engine will copy and read ------------------------------------- +SNAPSHOT_MAX_BYTES = 128 * MIB # hard ceiling on the private copy of the source file +SNAPSHOT_CHUNK_BYTES = 1 * MIB # streamed copy / archive read unit (bounded uninterruptible work) +DISK_HEADROOM_BYTES = 64 * MIB # free space required beyond the source size +ARCHIVE_EXPANSION_BYTES = 128 * MIB # total decompressed bytes the reader will pull from one archive +MODEL_XML_BYTES = 32 * MIB # decompressed bytes of all .model parts together +CONFIG_PART_BYTES = 8 * MIB # one metadata/config part (model_settings, Slic3r_PE_model, .rels) +ARCHIVE_MAX_ENTRIES = 20_000 # zip directory entries +MAX_MODEL_PARTS = 256 # distinct .model files parsed for one scene +MAX_RELS_PARTS = 64 # .rels parts examined + +# --- what the scene may contain --------------------------------------------- +MAX_RENDERED_TRIANGLES = 250_000 # counting every repetition of an instanced mesh +MAX_PARSED_TRIANGLES = 250_000 # distinct triangles accepted while parsing +MAX_VERTICES = 300_000 # distinct decoded vertices +MAX_NODES = 2_000 +MAX_OBJECT_DEFINITIONS = 20_000 # elements across all model parts (10x the node budget) +MAX_XML_ELEMENTS = 1_000_000 # every completed XML element, counted while parsing +MAX_PART_NAME_LENGTH = 256 # archive part names carried on nodes/meshes/sources (schema maxLength) +MAX_DEPTH = 64 +MAX_ID_LENGTH = 128 +MAX_FINDINGS = 500 +MAX_LIMITATIONS = 50 +MAX_VOLUMES_PER_MESH = 4_096 +MAX_RESPONSE_BYTES = 8 * MIB # the whole serialized body +MAX_COORDINATE_MM = 1.0e7 # keeps every coordinate representable as float32 + +# --- cooperative interruption (v4 item 2) ----------------------------------- +XML_EVENTS_PER_CHECK = 1_000 # lxml iterparse events between cancel/deadline checks +GEOMETRY_BATCH_TRIANGLES = 5_000 # decode/encode batch; 5_000 * 12 bytes is a multiple of 3 (base64-safe) +VERTEX_BATCH = 15_000 # transformed vertices between checks + +# --- jobs ------------------------------------------------------------------- +JOB_DEADLINE_SECONDS = 60.0 # wall clock from the moment the job becomes running +WEDGE_GRACE_SECONDS = 5.0 # worker still alive this long after cancel/TIMEOUT => WORKER_WEDGED +RESULT_TTL_SECONDS = 120.0 # a terminal job is kept this long +MAX_TERMINAL_JOBS = 8 # ... or until more than this many terminal jobs exist +MAX_REQUEST_ID_LENGTH = 64 +SNAPSHOT_DIR_MIN_AGE_SECONDS = 600 # a dead engine's scene-tmp folder is swept only once it is this old (a live engine's NEVER is) +MAX_PENDING_UNLINKS = 1024 # snapshot files that could not be deleted yet and are retried with back-off + +LIMITS_ECHO = { + "max_archive_bytes": ARCHIVE_EXPANSION_BYTES, + "max_model_xml_bytes": MODEL_XML_BYTES, + "max_rendered_triangles": MAX_RENDERED_TRIANGLES, + "max_vertices": MAX_VERTICES, + "max_nodes": MAX_NODES, + "max_depth": MAX_DEPTH, + "max_response_bytes": MAX_RESPONSE_BYTES, + "max_findings": MAX_FINDINGS, + "max_limitations": MAX_LIMITATIONS, + "max_id_length": MAX_ID_LENGTH, +} diff --git a/backend/snapstudio_core/stl_io.py b/backend/snapstudio_core/stl_io.py index 06928fc5..579e08a7 100644 --- a/backend/snapstudio_core/stl_io.py +++ b/backend/snapstudio_core/stl_io.py @@ -1,4 +1,5 @@ from __future__ import annotations +import re import struct from .errors import SnapStudioError @@ -46,3 +47,132 @@ def parse_stl(data: bytes): if not tris: raise SnapStudioError("STL contained no triangles") return verts, tris + + +# --- bounded, incremental decoder (additive; parse_stl above is unchanged) ----------------------------- + +class StlTooLarge(SnapStudioError): + """The file declares or contains more triangles (or vertices) than the caller allows.""" + + def __init__(self, triangles: int, limit: int, what: str = "triangles") -> None: + self.triangles, self.limit, self.what = triangles, limit, what + super().__init__(f"{triangles} {what}; limit {limit}") + + +_BATCH = 5000 + + +def _decode_binary_batch(raw: bytes, vmap: dict, verts: list, tris: list) -> None: + for rec in struct.iter_unpack("<12fH", raw): + tris.append((_add(vmap, verts, rec[3:6]), _add(vmap, verts, rec[6:9]), _add(vmap, verts, rec[9:12]))) + + +_CHUNK = 1024 * 1024 +_MAX_LINE = 4096 +_LINE_SPLIT = re.compile(rb"\r\n|\n|\r") # \n, \r\n and a lone \r, like parse_stl's splitlines + + +def _plain_text(head: bytes) -> bool: + """Plausibly TEXT: no NUL in the first KB and valid UTF-8 (a multi-byte character cut by the end of the + window is fine). A name such as ``solid Würfel`` is text; random binary is not.""" + first = head[:1024] + if b"\x00" in first: + return False + try: + first.decode("utf-8") + except UnicodeDecodeError as exc: + return exc.reason == "unexpected end of data" and exc.end >= len(first) + return True + + +def _looks_like_ascii_stl(head: bytes) -> bool: + """A "solid"/"facet" start on a file that is plausibly TEXT.""" + if not _plain_text(head): + return False + low = head.lstrip().lower() + return low.startswith(b"solid") or b"facet" in head.lower() + + +def decode_stl_stream(fh, size: int, *, max_triangles: int, max_vertices: int, check=lambda: None, + batch: int = _BATCH): + """Decode an STL from an open binary file without ever holding more than the caller's budget. + + The kind is decided from the 84-byte header AND the file size: a size equal to ``84 + 50 * N`` is + binary; so is a file whose first KB is not plausible text, and for those a declared ``N`` over + ``max_triangles`` is rejected IMMEDIATELY, before any triangle (or any large buffer) exists, whether or + not the size matches. Only plausible ASCII goes down the text path, which reads <= 1 MiB at a time, + refuses any line over 4 KiB, splits on LF, CRLF and a lone CR, keeps a running triangle counter and + calls ``check()`` (the cancel/deadline hook) per chunk. Binary decoding runs in batches of ``batch`` + triangles with ``check()`` between them. Vertices are de-duplicated exactly as ``parse_stl`` does. + Raises ``StlTooLarge`` or ``SnapStudioError``. + """ + head = fh.read(84) + verts: list[tuple] = [] + tris: list[tuple] = [] + vmap: dict = {} + if len(head) >= 84: + (n,) = struct.unpack_from(" max_triangles: + raise StlTooLarge(n, max_triangles) + if not consistent: + raise SnapStudioError("STL size does not match its header") + remaining = n + while remaining: + check() + take = min(batch, remaining) + raw = fh.read(take * 50) + if len(raw) != take * 50: + raise SnapStudioError("STL ended early") + _decode_binary_batch(raw, vmap, verts, tris) + if len(verts) > max_vertices: + raise StlTooLarge(len(verts), max_vertices, "vertices") + remaining -= take + if not tris: + raise SnapStudioError("STL contained no triangles") + return verts, tris + fh.seek(0) + head_for_text = first + else: + fh.seek(0) + head_for_text = head + if not _looks_like_ascii_stl(head_for_text): + raise SnapStudioError("unrecognized STL (not binary-size-consistent or ASCII)") + fh.seek(0) + cur: list[tuple] = [] + carry = b"" + + def feed(line: bytes) -> None: + nonlocal cur + if len(line) > _MAX_LINE: + raise SnapStudioError("STL has a line longer than 4096 bytes") + s = line.strip() + if s.startswith(b"vertex"): + _, x, y, z = s.split()[:4] + cur.append((float(x), float(y), float(z))) + if len(cur) == 3: + tris.append(tuple(_add(vmap, verts, p) for p in cur)) + cur = [] + if len(tris) > max_triangles: + raise StlTooLarge(len(tris), max_triangles) + if len(verts) > max_vertices: + raise StlTooLarge(len(verts), max_vertices, "vertices") + + while True: + check() + chunk = fh.read(_CHUNK) + if not chunk: + break + parts = _LINE_SPLIT.split(carry + chunk) + carry = parts.pop() + if len(carry) > _MAX_LINE: + raise SnapStudioError("STL has a line longer than 4096 bytes") + for line in parts: + feed(line) + if carry: + feed(carry) + if not tris: + raise SnapStudioError("STL contained no triangles") + return verts, tris diff --git a/backend/snapstudio_core/units.py b/backend/snapstudio_core/units.py new file mode 100644 index 00000000..a4413899 --- /dev/null +++ b/backend/snapstudio_core/units.py @@ -0,0 +1,102 @@ +"""One place that knows what a 3MF `unit` means in millimetres. + +A 3MF model part states its unit on the root `model` element; when the attribute is absent the format says +millimetre. Every number in that part (vertex coordinates AND the translation of any transform written in +it) is in that unit. Several Studio readers used to ignore the header, so a 12 inch part was reported as +12 mm. This table and the readers below are the single shared fix: nothing else may keep its own copy of +the factors, and the scene parser calls the SAME ``resolve_unit`` on the SAME root element. + +Three outcomes are kept apart on purpose: + +* **absent** attribute: the specification default, millimetre (``declared`` is False); +* **recognized** value: one of the six defined units; +* **unrecognized** value: NOT silently the same as absent. ``UnitInfo.recognized`` is False. Legacy + dimension readers still fall back to millimetres (they always have), the scene reports a limitation + (``UNSUPPORTED_UNIT``) and withholds fit findings. +""" +from __future__ import annotations + +import io +from typing import NamedTuple + +from lxml import etree + +#: The six units the 3MF core specification defines, in millimetres per unit. +MM_PER_UNIT: dict[str, float] = { + "micron": 0.001, + "millimeter": 1.0, + "centimeter": 10.0, + "inch": 25.4, + "foot": 304.8, + "meter": 1000.0, +} +DEFAULT_UNIT = "millimeter" + + +class UnitInfo(NamedTuple): + name: str # the declared value, or the default when absent + factor: float # millimetres per unit; 1.0 (the documented fallback) when not recognized + recognized: bool # False only for a declared value outside the six units + declared: bool # False when the attribute is absent (the default applies) + + +def resolve_unit(value: str | None) -> UnitInfo: + """Interpret the root element's ``unit`` attribute value (None = absent).""" + if value is None: + return UnitInfo(DEFAULT_UNIT, MM_PER_UNIT[DEFAULT_UNIT], True, False) + factor = MM_PER_UNIT.get(value) + if factor is None: + return UnitInfo(value, 1.0, False, True) + return UnitInfo(value, factor, True, True) + + +def read_unit(raw: bytes | str) -> UnitInfo: + """The unit declared by the REAL root element of a model part, read with the XML parser. + + Only the first start event is consumed, so comments, processing instructions, a DOCTYPE-free prolog + and a long preamble are all irrelevant, and a ```` inside a comment is not seen. A part + that cannot be parsed, or whose root is not ``model``, is NOT a legitimately absent attribute: it reads + as ``recognized=False`` (millimetre factor for the legacy readers, with the flag set); entities are never + resolved and the network is never touched. + """ + data = raw.encode("utf-8") if isinstance(raw, str) else raw + unreadable = UnitInfo(DEFAULT_UNIT, MM_PER_UNIT[DEFAULT_UNIT], False, False) + try: + for _event, elem in etree.iterparse(io.BytesIO(data), events=("start",), resolve_entities=False, + load_dtd=False, no_network=True, huge_tree=False): + if etree.QName(elem).localname != "model": + return unreadable + return resolve_unit(elem.get("unit")) + except (etree.LxmlError, ValueError): + pass + return unreadable + + +def unit_of(raw: bytes | str) -> tuple[str | None, float | None]: + """``(name, mm_per_unit)``; the factor is None for an unrecognized value.""" + info = read_unit(raw) + return info.name, (info.factor if info.recognized else None) + + +def mm_per_unit(raw: bytes | str) -> float: + """Millimetres per unit for a model part; an unrecognized unit reads as millimetres (see UnitInfo).""" + return read_unit(raw).factor + + +def scale_translation(transform, factor: float): + """A 3MF transform with its translation moved into millimetres (or back, with ``1/factor``). + + Accepts the 12-number list form (translation = entries 9..11) and the parsed row form + ``((a,b,c),(d,e,f),(g,h,i),(tx,ty,tz))``. Only the translation carries a length; the 3x3 part is + dimensionless. Factor 1.0 returns the input unchanged, so millimetre behaviour is exactly as before. + """ + if transform is None or factor == 1.0: + return transform + if isinstance(transform[0], (tuple, list)): + tx, ty, tz = transform[3] + return (transform[0], transform[1], transform[2], (tx * factor, ty * factor, tz * factor)) + out = list(transform) + out[9] *= factor + out[10] *= factor + out[11] *= factor + return out diff --git a/backend/tests/fixtures/scene/golden-scene-1.json b/backend/tests/fixtures/scene/golden-scene-1.json new file mode 100644 index 00000000..579dfc85 --- /dev/null +++ b/backend/tests/fixtures/scene/golden-scene-1.json @@ -0,0 +1,166 @@ +{ + "schema": "scene/1", + "revision": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "status": "complete", + "units": "mm", + "axes": "right-handed-z-up", + "sources": [ + { + "part": "3D/3dmodel.model", + "unit": "millimeter", + "mm_per_unit": 1.0 + } + ], + "bed": { + "polygon_mm": [ + [ + 0.5, + 1.0 + ], + [ + 270.5, + 1.0 + ], + [ + 270.5, + 271.0 + ], + [ + 0.5, + 271.0 + ] + ], + "height_mm": 270.05, + "edge_margin_mm": 0.5, + "policy": "u1_template" + }, + "meshes": [ + { + "key": { + "part": "3D/3dmodel.model", + "object_id": "1" + }, + "vertex_count": 8, + "triangle_count": 12, + "positions_f32le_base64": "AAAAAAAAAAAAAAAAAAAgQQAAAAAAAAAAAAAgQQAAIEEAAAAAAAAAAAAAIEEAAAAAAAAAAAAAAAAAACBBAAAgQQAAAAAAACBBAAAgQQAAIEEAACBBAAAAAAAAIEEAACBB", + "indices_u32le_base64": "AAAAAAEAAAACAAAAAAAAAAIAAAADAAAABAAAAAUAAAAGAAAABAAAAAYAAAAHAAAAAAAAAAEAAAAFAAAAAAAAAAUAAAAEAAAAAQAAAAIAAAAGAAAAAQAAAAYAAAAFAAAAAgAAAAMAAAAHAAAAAgAAAAcAAAAGAAAAAwAAAAAAAAAEAAAAAwAAAAQAAAAHAAAA", + "volumes": [ + { + "id": "v0", + "triangle_start": 0, + "triangle_count": 12, + "role": "part", + "source": "object" + } + ] + } + ], + "nodes": [ + { + "id": "b0", + "parent_id": null, + "resource": { + "part": "3D/3dmodel.model", + "object_id": "1" + }, + "mesh_key": { + "part": "3D/3dmodel.model", + "object_id": "1" + }, + "local_to_parent_mm": [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 50.0, + 60.0, + 0.0, + 1.0 + ], + "world_mm": [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 50.0, + 60.0, + 0.0, + 1.0 + ], + "mirrored": false, + "role_context": null, + "build_index": 0, + "instance_ref": { + "build_index": 0, + "component_path": [] + }, + "plate": { + "state": "unknown", + "plate_id": null, + "source": null + }, + "placement_state": "known", + "printable": true, + "has_non_part_volumes": false, + "bounds_mm": { + "min": [ + 50.0, + 60.0, + 0.0 + ], + "max": [ + 60.0, + 70.0, + 10.0 + ] + }, + "finding_ids": [] + } + ], + "plates": [], + "findings": [], + "limitations": [ + { + "code": "PLATES_UNAVAILABLE", + "target_ids": [] + } + ], + "counts": { + "nodes": 1, + "meshes": 1, + "vertices": 8, + "triangles": 12, + "rendered_triangles": 12, + "plates": 0, + "findings": 0, + "limitations": 1 + }, + "limits": { + "max_archive_bytes": 134217728, + "max_model_xml_bytes": 33554432, + "max_rendered_triangles": 250000, + "max_vertices": 300000, + "max_nodes": 2000, + "max_depth": 64, + "max_response_bytes": 8388608, + "max_findings": 500, + "max_limitations": 50, + "max_id_length": 128 + } +} diff --git a/backend/tests/fixtures/scene/make_golden.py b/backend/tests/fixtures/scene/make_golden.py new file mode 100644 index 00000000..ddbed05e --- /dev/null +++ b/backend/tests/fixtures/scene/make_golden.py @@ -0,0 +1,53 @@ +"""Writes golden-scene-1.json by hand, independently of snapstudio_core.scene. + +The golden scene is the frozen reference for scene/1: a 10 mm cube, authored in millimetres at +(50, 60, 0) in a plain 3MF with no slicer metadata. It is derived here from first principles (struct + +base64 + the U1 bed numbers), so a test can require the real builder to reproduce it exactly. + +Run: py -3.13 make_golden.py (rewrites golden-scene-1.json next to this file) +""" +import base64 +import json +import struct +from pathlib import Path + +REVISION = "0123456789abcdef" * 4 +VERTS = [(0, 0, 0), (10, 0, 0), (10, 10, 0), (0, 10, 0), (0, 0, 10), (10, 0, 10), (10, 10, 10), (0, 10, 10)] +TRIS = [(0, 1, 2), (0, 2, 3), (4, 5, 6), (4, 6, 7), (0, 1, 5), (0, 5, 4), + (1, 2, 6), (1, 6, 5), (2, 3, 7), (2, 7, 6), (3, 0, 4), (3, 4, 7)] +PART = "3D/3dmodel.model" + + +def b64(fmt, values): + return base64.b64encode(struct.pack("<" + fmt * len(values), *values)).decode("ascii") + + +matrix = [1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 50.0, 60.0, 0.0, 1.0] +scene = { + "schema": "scene/1", "revision": REVISION, "status": "complete", "units": "mm", "axes": "right-handed-z-up", + "sources": [{"part": PART, "unit": "millimeter", "mm_per_unit": 1.0}], + "bed": {"polygon_mm": [[0.5, 1.0], [270.5, 1.0], [270.5, 271.0], [0.5, 271.0]], + "height_mm": 270.05, "edge_margin_mm": 0.5, "policy": "u1_template"}, + "meshes": [{ + "key": {"part": PART, "object_id": "1"}, "vertex_count": 8, "triangle_count": 12, + "positions_f32le_base64": b64("f", [c for v in VERTS for c in v]), + "indices_u32le_base64": b64("I", [i for t in TRIS for i in t]), + "volumes": [{"id": "v0", "triangle_start": 0, "triangle_count": 12, "role": "part", "source": "object"}]}], + "nodes": [{ + "id": "b0", "parent_id": None, "resource": {"part": PART, "object_id": "1"}, + "mesh_key": {"part": PART, "object_id": "1"}, "local_to_parent_mm": matrix, "world_mm": matrix, + "mirrored": False, "role_context": None, "build_index": 0, "instance_ref": {"build_index": 0, "component_path": []}, + "plate": {"state": "unknown", "plate_id": None, "source": None}, "placement_state": "known", + "printable": True, "has_non_part_volumes": False, + "bounds_mm": {"min": [50.0, 60.0, 0.0], "max": [60.0, 70.0, 10.0]}, "finding_ids": []}], + "plates": [], "findings": [], + "limitations": [{"code": "PLATES_UNAVAILABLE", "target_ids": []}], + "counts": {"nodes": 1, "meshes": 1, "vertices": 8, "triangles": 12, "rendered_triangles": 12, + "plates": 0, "findings": 0, "limitations": 1}, + "limits": {"max_archive_bytes": 134217728, "max_model_xml_bytes": 33554432, "max_rendered_triangles": 250000, + "max_vertices": 300000, "max_nodes": 2000, "max_depth": 64, "max_response_bytes": 8388608, + "max_findings": 500, "max_limitations": 50, "max_id_length": 128}} +if __name__ == "__main__": + out = Path(__file__).with_name("golden-scene-1.json") + out.write_text(json.dumps(scene, indent=2) + "\n", encoding="utf-8") + print("wrote", out) diff --git a/backend/tests/scene_fixtures.py b/backend/tests/scene_fixtures.py new file mode 100644 index 00000000..a4c30b64 --- /dev/null +++ b/backend/tests/scene_fixtures.py @@ -0,0 +1,187 @@ +"""Authored 3MF / STL fixtures for the scene/1 tests and benchmark. + +Everything here is synthetic. Archives are written with ZIP_STORED and fixed timestamps so the bytes +are deterministic. Nothing here reads a real user file. +""" +from __future__ import annotations + +import struct +import zipfile +from pathlib import Path + +CORE = "http://schemas.microsoft.com/3dmanufacturing/core/2015/02" +PROD = "http://schemas.microsoft.com/3dmanufacturing/production/2015/06" +REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" +MODEL_REL = "http://schemas.microsoft.com/3dmanufacturing/2013/01/3dmodel" + +CUBE_TRIS = [(0, 1, 2), (0, 2, 3), (4, 5, 6), (4, 6, 7), (0, 1, 5), (0, 5, 4), + (1, 2, 6), (1, 6, 5), (2, 3, 7), (2, 7, 6), (3, 0, 4), (3, 4, 7)] + + +def cube_vertices(size: float = 10.0, origin=(0.0, 0.0, 0.0)): + ox, oy, oz = origin + s = size + return [(ox, oy, oz), (ox + s, oy, oz), (ox + s, oy + s, oz), (ox, oy + s, oz), + (ox, oy, oz + s), (ox + s, oy, oz + s), (ox + s, oy + s, oz + s), (ox, oy + s, oz + s)] + + +def mesh_xml(vertices, triangles) -> str: + v = "".join(f'' for x, y, z in vertices) + t = "".join(f'' for a, b, c in triangles) + return f"{v}{t}" + + +def cube_object(oid, size=10.0, origin=(0.0, 0.0, 0.0)) -> str: + return f'{mesh_xml(cube_vertices(size, origin), CUBE_TRIS)}' + + +def composite_object(oid, comps) -> str: + """comps: [(objectid, path|None, transform|None)]""" + inner = "" + for cid, path, tf in comps: + attrs = f'objectid="{cid}"' + if path is not None: + attrs += f' p:path="{path}"' + if tf: + attrs += f' transform="{tf}"' + inner += f"" + return f'{inner}' + + +def model_xml(objects, items, unit="millimeter", prolog='') -> str: + build = "" + for item in items: # (objectid, transform) or (objectid, transform, p:path) + oid, tf = item[0], item[1] + path = item[2] if len(item) > 2 else None + build += (f'") + return (f'{prolog}' + f"{''.join(objects)}{build}") + + +def sub_model_xml(objects, unit="millimeter") -> str: + return (f'' + f"{''.join(objects)}") + + +def rels_xml(target="/3D/3dmodel.model", extra="") -> str: + return (f'' + f'{extra}') + + +def write_zip(path, entries: dict, order=None) -> Path: + path = Path(path) + with zipfile.ZipFile(path, "w", zipfile.ZIP_STORED) as z: + for name in (order or list(entries)): + data = entries[name] + if isinstance(data, str): + data = data.encode("utf-8") + info = zipfile.ZipInfo(name, date_time=(2020, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_STORED + z.writestr(info, data) + return path + + +def three_mf(path, root_model: str, extra: dict | None = None, rels: str | None = None) -> Path: + entries = {"_rels/.rels": rels or rels_xml(), "3D/3dmodel.model": root_model} + entries.update(extra or {}) + return write_zip(path, entries) + + +IDENT = "1 0 0 0 1 0 0 0 1" + + +def tf(tx=0.0, ty=0.0, tz=0.0, lin=IDENT) -> str: + return f"{lin} {tx} {ty} {tz}" + + +def plain_cube_3mf(path, size=10.0, unit="millimeter", at=(50.0, 60.0, 0.0)) -> Path: + return three_mf(path, model_xml([cube_object("1", size)], [("1", tf(*at))], unit=unit)) + + +# --- Bambu/Orca style ------------------------------------------------------------------------ + +def model_settings_xml(objects: dict, plates: list) -> str: + """objects: {oid: [(part_id, subtype)]}; plates: [(plater_id, [(oid, instance_id|None)])]""" + out = [''] + for oid, parts in objects.items(): + out.append(f'') + for pid, sub in parts: + attr = f' subtype="{sub}"' if sub is not None else "" + out.append(f'') + out.append("") + for plater, members in plates: + out.append(f'') + for oid, inst in members: + inst_xml = f'' if inst is not None else "" + out.append(f'{inst_xml}') + out.append("") + out.append("") + return "".join(out) + + +def bambu_project(path, *, parts, items, plates, roles=None, unit="millimeter", sizes=None, origins=None, + omit_parts=()) -> Path: + """One composite root object `100` made of meshes 1..n in 3D/Objects/object_1.model. + + parts: number of mesh parts; roles: [subtype per part]; items: [(objectid, transform)]. + """ + roles = roles or ["normal_part"] * parts + sizes = sizes or [10.0] * parts + origins = origins or [(i * 20.0, 0.0, 0.0) for i in range(parts)] + sub = sub_model_xml([cube_object(str(i + 1), sizes[i], origins[i]) for i in range(parts)], unit=unit) + root = model_xml( + [composite_object("100", [(str(i + 1), "/3D/Objects/object_1.model", None) for i in range(parts)])], + items, unit=unit) + settings = model_settings_xml({"100": [(str(i + 1), roles[i]) for i in range(parts) if (i + 1) not in omit_parts]}, + plates) + rels3d = (f'' + f'') + return three_mf(path, root, {"3D/Objects/object_1.model": sub, "3D/_rels/3dmodel.model.rels": rels3d, + "Metadata/model_settings.config": settings}) + + +# --- Prusa style ----------------------------------------------------------------------------- + +def prusa_project(path, *, volumes, items=None, unit="millimeter", last_first=False) -> Path: + """One object `1` with 12 triangles per cube volume; volumes: [(first, last, volume_type)].""" + n = max(last for _first, last, _ in volumes) // 12 + 1 if volumes else 1 + verts, tris = [], [] + for k in range(n): + base = len(verts) + verts += cube_vertices(10.0, (k * 20.0, 0.0, 0.0)) + tris += [(a + base, b + base, c + base) for a, b, c in CUBE_TRIS] + obj = f'{mesh_xml(verts, tris)}' + root = model_xml([obj], items or [("1", tf(100, 100, 0))], unit=unit) + attrs = (lambda a, b: f'lastid="{b}" firstid="{a}"') if last_first else (lambda a, b: f'firstid="{a}" lastid="{b}"') + vol = "".join( + f'' + for a, b, vt in volumes) + cfg = f'{vol}' + return three_mf(path, root, {"Metadata/Slic3r_PE_model.config": cfg}) + + +# --- STL ------------------------------------------------------------------------------------- + +def binary_stl(path, size=10.0) -> Path: + verts = cube_vertices(size) + body = b"\x00" * 80 + struct.pack(" Path: + """A single mesh of about `triangles_target` triangles (a flat triangulated grid with shared vertices).""" + side = max(2, int((triangles_target / 2) ** 0.5)) + verts = [(x * 0.1, y * 0.1, 0.0) for y in range(side + 1) for x in range(side + 1)] + tris = [] + w = side + 1 + for y in range(side): + for x in range(side): + a = y * w + x + tris.append((a, a + 1, a + w)) + tris.append((a + 1, a + w + 1, a + w)) + return three_mf(path, model_xml([f'{mesh_xml(verts, tris)}'], + [("1", tf(*at))])) diff --git a/backend/tests/test_scene_api.py b/backend/tests/test_scene_api.py new file mode 100644 index 00000000..2be597b4 --- /dev/null +++ b/backend/tests/test_scene_api.py @@ -0,0 +1,168 @@ +"""The /scene/* HTTP surface: authentication, validation, status codes, pre-serialized body.""" +import http.client +import json +import threading +import time + +import pytest + +from snapstudio_api import scene_jobs as sj +from snapstudio_api.server import build_server +from snapstudio_core import scene_limits as L +from tests import scene_fixtures as fx +from tests.test_scene_contract import assert_valid, sub_validator + + +@pytest.fixture +def server(tmp_path, monkeypatch): + monkeypatch.setenv("SNAPSTUDIO_DATA_DIR", str(tmp_path / "data")) + sj.reset_default_for_tests() + httpd, token = build_server() + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + yield httpd.server_address[1], token + httpd.shutdown() + httpd.server_close() + sj.reset_default_for_tests() + + +def call(port, route, payload, token, *, raw=False): + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=20) + try: + headers = {"Content-Type": "application/json"} + if token: + headers["X-Auth-Token"] = token + conn.request("POST", route, body=json.dumps(payload).encode(), headers=headers) + resp = conn.getresponse() + body = resp.read() + assert resp.getheader("Content-Length") == str(len(body)) + assert resp.getheader("Content-Type") == "application/json" + return resp.status, (body if raw else json.loads(body)) + finally: + conn.close() + + +def poll(port, token, job_id, *states, timeout=15): + end = time.monotonic() + timeout + while time.monotonic() < end: + code, body = call(port, "/scene/status", {"job_id": job_id}, token) + assert code == 200 + if body["state"] in states: + return body + time.sleep(0.02) + raise AssertionError("timeout") + + +def test_every_scene_route_requires_the_token(server, tmp_path): + port, token = server + for route in sj.ROUTES: + assert call(port, route, {}, None)[0] == 401 + assert call(port, route, {}, "wrong-token")[0] == 401 + + +def test_full_flow_returns_the_exact_serialized_scene(server, tmp_path): + port, token = server + path = fx.bambu_project(tmp_path / "p.3mf", parts=2, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])]) + code, started = call(port, "/scene/start", {"path": str(path), "request_id": "req-flow-0001"}, token) + assert code == 200 and not list(sub_validator("job_start").iter_errors(started)) + status = poll(port, token, started["job_id"], "succeeded") + assert not list(sub_validator("job_status").iter_errors(status)) + code, raw = call(port, "/scene/result", {"job_id": started["job_id"], "expected_revision": status["revision"]}, token, raw=True) + assert code == 200 + sc = json.loads(raw) + assert_valid(sc) + assert raw == json.dumps(sc, separators=(",", ":")).encode() # compact, serialized once, no re-dump + assert len(raw) <= L.MAX_RESPONSE_BYTES + + +def test_error_statuses_and_bodies(server, tmp_path): + port, token = server + err = sub_validator("error_response") + cases = [ + ("/scene/start", {"request_id": "req-err-00001"}, 400, "INVALID_REQUEST"), + ("/scene/start", {"path": str(tmp_path / "nope.3mf"), "request_id": "req-err-00001"}, 400, "INVALID_REQUEST"), + ("/scene/start", {"path": "x.3mf", "request_id": "short"}, 400, "INVALID_REQUEST"), + ("/scene/start", {"path": "x.3mf", "request_id": "has spaces in it!"}, 400, "INVALID_REQUEST"), + ("/scene/start", {"path": str(tmp_path / "a.obj"), "request_id": "req-err-00002"}, 422, "UNSUPPORTED_FORMAT"), + ("/scene/status", {}, 400, "INVALID_REQUEST"), + ("/scene/status", {"job_id": "0" * 32}, 404, "EXPIRED"), + ("/scene/result", {"job_id": "0" * 32}, 404, "EXPIRED"), + ("/scene/result", {"job_id": "0" * 32, "expected_revision": "nothex"}, 400, "INVALID_REQUEST"), + ("/scene/cancel", {"job_id": "0" * 32}, 404, "EXPIRED"), + ] + (tmp_path / "a.obj").write_text("v") + for route, payload, status, code in cases: + got, body = call(port, route, payload, token) + assert (got, body["error"]) == (status, code), (route, payload, got, body) + assert not list(err.iter_errors(body)) + + +def test_failed_job_result_is_422_and_cancelled_is_409(server, tmp_path): + port, token = server + bad = tmp_path / "bad.3mf" + bad.write_bytes(b"junk" * 50) + _, started = call(port, "/scene/start", {"path": str(bad), "request_id": "req-bad-00001"}, token) + poll(port, token, started["job_id"], "failed") + code, body = call(port, "/scene/result", {"job_id": started["job_id"]}, token) + assert (code, body["error"]) == (422, "INVALID_ARCHIVE") + ok = fx.plain_cube_3mf(tmp_path / "ok.3mf") + _, second = call(port, "/scene/start", {"path": str(ok), "request_id": "req-ok-000001"}, token) + code, status = call(port, "/scene/cancel", {"job_id": second["job_id"]}, token) + assert code == 200 and status["state"] in ("cancelled", "succeeded") + if status["state"] == "cancelled": + code, body = call(port, "/scene/result", {"job_id": second["job_id"]}, token) + assert (code, body["error"]) == (409, "CANCELLED") + + +def test_request_id_conflict_and_idempotency_over_http(server, tmp_path): + port, token = server + a = fx.plain_cube_3mf(tmp_path / "a.3mf") + b = fx.plain_cube_3mf(tmp_path / "b.3mf", size=11) + _, one = call(port, "/scene/start", {"path": str(a), "request_id": "req-same-0001"}, token) + _, again = call(port, "/scene/start", {"path": str(a), "request_id": "req-same-0001"}, token) + assert one["job_id"] == again["job_id"] + code, body = call(port, "/scene/start", {"path": str(b), "request_id": "req-same-0001"}, token) + assert (code, body["error"], body["reason"]) == (409, "INVALID_REQUEST", "request_id reused for another path") + + +def test_not_ready_and_wedged_over_http(server, tmp_path, monkeypatch): + port, token = server + gate = threading.Event() + started = threading.Event() + + def stuck(path, revision, ctl): + started.set() + gate.wait(10) + return b"{}" + jobs = sj.SceneJobs(builder=stuck, wedge_grace=0.2) + monkeypatch.setattr(sj, "_default", jobs) + path = fx.plain_cube_3mf(tmp_path / "c.3mf") + _, a = call(port, "/scene/start", {"path": str(path), "request_id": "req-wedge-001"}, token) + started.wait(5) + code, body = call(port, "/scene/result", {"job_id": a["job_id"]}, token) + assert (code, body["error"], body["state"]) == (409, "NOT_READY", "running") + call(port, "/scene/cancel", {"job_id": a["job_id"]}, token) + time.sleep(0.4) + code, body = call(port, "/scene/start", {"path": str(path), "request_id": "req-wedge-002"}, token) + assert (code, body["error"]) == (503, "WORKER_WEDGED") + gate.set() + for _ in range(200): + if not jobs.wedged(): + break + time.sleep(0.02) + code, _ = call(port, "/scene/start", {"path": str(path), "request_id": "req-wedge-003"}, token) + assert code == 200 + jobs.close() + + +def test_other_routes_still_behave(server): + port, token = server + assert call(port, "/definitely_not_a_route", {}, token)[0] == 404 + + +@pytest.mark.parametrize("body", [[1, 2], "text", 7, None, [], True]) +def test_a14_a_non_object_json_body_is_a_400_not_a_500(server, body): + port, token = server + for route in sj.ROUTES: + code, resp = call(port, route, body, token) + assert code == 400 and resp["error"] == "INVALID_REQUEST", (route, body, code, resp) diff --git a/backend/tests/test_scene_build.py b/backend/tests/test_scene_build.py new file mode 100644 index 00000000..2a2882e1 --- /dev/null +++ b/backend/tests/test_scene_build.py @@ -0,0 +1,310 @@ +"""scene/1 builder: units, ids, instances, transforms, volumes, plates, STL, bed, comparisons. + +Every scene a test builds is also validated against the committed JSON Schema (real validator) and +the invariants JSON Schema cannot express. +""" +import json +import math +import os +from pathlib import Path + +import pytest + +from snapstudio_core import plate_placement, scene, units +from tests import scene_fixtures as fx +from tests.test_scene_contract import assert_valid + +REV = "ab" * 32 +EXAMPLES = Path(__file__).resolve().parents[2] / "examples" + + +def build(path) -> dict: + sc = scene.build_scene_dict(str(path), REV) + assert_valid(sc) + return sc + + +def node(sc, nid): + return next(n for n in sc["nodes"] if n["id"] == nid) + + +def codes(sc): + return [l["code"] for l in sc["limitations"]] + + +# ------------------------------------------------------------------------------ units + +@pytest.mark.parametrize("unit,factor", sorted(units.MM_PER_UNIT.items())) +def test_all_six_units_convert_to_millimetres(tmp_path, unit, factor): + # a 12-unit cube, built 2 units along x. Both the vertex coordinates and the transform's + # translation are in the model's unit. + path = fx.three_mf(tmp_path / "u.3mf", fx.model_xml([fx.cube_object("1", 12)], [("1", fx.tf(2, 0, 0))], unit=unit)) + sc = build(path) + assert sc["sources"] == [{"part": "3D/3dmodel.model", "unit": unit, "mm_per_unit": factor}] + b = node(sc, "b0")["bounds_mm"] + assert b["min"][0] == pytest.approx(2 * factor, abs=1e-5) + assert b["max"][0] == pytest.approx(14 * factor, abs=1e-5) + assert b["max"][2] == pytest.approx(12 * factor, abs=1e-5) + assert node(sc, "b0")["world_mm"][12] == pytest.approx(2 * factor, rel=1e-8) + if unit != "millimeter": + assert "NON_MM_SOURCE_UNIT" in codes(sc) and sc["status"] == "partial" + assert not [f for f in sc["findings"] if f["kind"] == "placement"] + + +def test_twelve_inch_part_is_304_8_mm(tmp_path): + sc = build(fx.three_mf(tmp_path / "i.3mf", fx.model_xml([fx.cube_object("1", 12)], [("1", None)], unit="inch"))) + b = node(sc, "b0")["bounds_mm"] + assert b["max"][0] - b["min"][0] == pytest.approx(304.8, abs=1e-4) + assert any(f["kind"] == "size" for f in sc["findings"]) # larger than the 270 mm plate + + +def test_unrecognised_unit_is_a_limitation_not_a_silent_millimetre(tmp_path): + # far off the plate: with an unknown unit no fit statement of any kind may be made + path = fx.three_mf(tmp_path / "x.3mf", fx.model_xml([fx.cube_object("1", 400)], [("1", fx.tf(900, 900, 0))], unit="yard")) + sc = build(path) + assert "UNSUPPORTED_UNIT" in codes(sc) and sc["status"] == "partial" + assert sc["findings"] == [] + assert sc["sources"] == [{"part": "3D/3dmodel.model", "unit": "millimeter", "mm_per_unit": 1.0}] # the documented fallback + absent = build(fx.three_mf(tmp_path / "y.3mf", fx.model_xml([fx.cube_object("1", 400)], [("1", fx.tf(900, 900, 0))]).replace(' unit="millimeter"', ""))) + assert "UNSUPPORTED_UNIT" not in codes(absent) and absent["findings"] # absent = millimetre, judged normally + + +def test_sub_model_in_a_different_unit_uses_its_own_unit(tmp_path): + sub = fx.sub_model_xml([fx.cube_object("1", 1)], unit="centimeter") + root = fx.model_xml([fx.composite_object("10", [("1", "/3D/Objects/a.model", None)])], [("10", fx.tf(5, 0, 0))]) + sc = build(fx.three_mf(tmp_path / "m.3mf", root, {"3D/Objects/a.model": sub})) + b = node(sc, "b0")["bounds_mm"] + assert b["min"][0] == pytest.approx(5.0) # root (mm) translation + assert b["max"][0] == pytest.approx(15.0) # 1 cm cube + assert {s["unit"] for s in sc["sources"]} == {"millimeter", "centimeter"} + + +# ------------------------------------------------------------------------------ ids and instances + +def test_cross_file_duplicate_object_ids_stay_distinct(tmp_path): + a = fx.sub_model_xml([fx.cube_object("1", 10)]) + b = fx.sub_model_xml([fx.cube_object("1", 30)]) + root = fx.model_xml([fx.composite_object("9", [("1", "/3D/Objects/a.model", None), + ("1", "/3D/Objects/b.model", fx.tf(100, 0, 0))])], [("9", None)]) + sc = build(fx.three_mf(tmp_path / "d.3mf", root, {"3D/Objects/a.model": a, "3D/Objects/b.model": b})) + keys = {(m["key"]["part"], m["key"]["object_id"]): m for m in sc["meshes"]} + assert set(keys) == {("3D/Objects/a.model", "1"), ("3D/Objects/b.model", "1")} + assert [n["id"] for n in sc["nodes"]] == ["b0", "b0.c0", "b0.c1"] + assert node(sc, "b0.c0")["bounds_mm"]["max"][0] == 10.0 + assert node(sc, "b0.c1")["bounds_mm"]["min"][0] == 100.0 and node(sc, "b0.c1")["bounds_mm"]["max"][0] == 130.0 + assert node(sc, "b0")["bounds_mm"] == {"min": [0.0, 0.0, 0.0], "max": [130.0, 30.0, 30.0]} + + +def test_repeated_instances_share_a_mesh_and_disable_engine_findings(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(100, 100, 0)), ("1", fx.tf(500, 100, 0))]) + sc = build(fx.three_mf(tmp_path / "r.3mf", root)) + assert len(sc["meshes"]) == 1 and [n["id"] for n in sc["nodes"]] == ["b0", "b1"] + assert sc["counts"]["rendered_triangles"] == 24 and sc["counts"]["triangles"] == 12 + assert node(sc, "b0")["bounds_mm"]["min"][0] == 100.0 + assert node(sc, "b1")["bounds_mm"]["min"][0] == 500.0 # geometry stays available + lim = next(l for l in sc["limitations"] if l["code"] == "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED") + assert lim["target_ids"] == ["b0", "b1"] and sc["status"] == "partial" + assert sc["findings"] == [] and all(n["finding_ids"] == [] for n in sc["nodes"]) + # the same cube on its own IS judged: the second instance really is off the plate + single = build(fx.three_mf(tmp_path / "s.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(500, 100, 0))]))) + assert [f["kind"] for f in single["findings"]] == ["placement"] + + +def test_nested_and_mirrored_transforms_compose(tmp_path): + inner = fx.composite_object("5", [("1", None, "-1 0 0 0 1 0 0 0 1 0 0 0")]) # mirror x + mesh = fx.cube_object("1", 10) + outer = fx.composite_object("6", [("5", None, fx.tf(30, 0, 0))]) + root = fx.model_xml([mesh, inner, outer], [("6", fx.tf(0, 50, 0))]) + sc = build(fx.three_mf(tmp_path / "n.3mf", root)) + assert [n["id"] for n in sc["nodes"]] == ["b0", "b0.c0", "b0.c0.c0"] + leaf = node(sc, "b0.c0.c0") + assert leaf["instance_ref"] == {"build_index": 0, "component_path": [0, 0]} + assert leaf["mirrored"] is True and node(sc, "b0")["mirrored"] is False + assert leaf["parent_id"] == "b0.c0" + # mirror -> x in [-10, 0]; +30 -> [20, 30]; y +50 + assert leaf["bounds_mm"] == {"min": [20.0, 50.0, 0.0], "max": [30.0, 60.0, 10.0]} + expected = scene.mat_mul(scene.mat_mul(scene.parse_transform(fx.tf(0, 50, 0), 1.0), + scene.parse_transform(fx.tf(30, 0, 0), 1.0)), + scene.parse_transform("-1 0 0 0 1 0 0 0 1 0 0 0", 1.0)) + assert leaf["world_mm"] == pytest.approx(expected) + + +def test_rotated_instance_bounds_are_tight_not_box_corners(tmp_path): + verts = [(0, 0, 0), (10, 0, 0), (0, 10, 0), (0, 0, 10)] + tris = [(0, 1, 2), (0, 1, 3), (0, 2, 3), (1, 2, 3)] + c = math.cos(math.pi / 4) + lin = f"{c} {c} 0 {-c} {c} 0 0 0 1" # 45 degrees about z + obj = f'{fx.mesh_xml(verts, tris)}' + sc = build(fx.three_mf(tmp_path / "rot.3mf", fx.model_xml([obj], [("1", fx.tf(0, 0, 0, lin))]))) + b = node(sc, "b0")["bounds_mm"] + assert b["max"][1] == pytest.approx(10 * c, abs=1e-4) # a box-corner estimate would give 14.14 + assert b["min"][0] == pytest.approx(-10 * c, abs=1e-4) and b["max"][0] == pytest.approx(10 * c, abs=1e-4) + + +# ------------------------------------------------------------------------------ roles and volumes + +def test_bambu_roles_and_non_part_volumes_do_not_count_for_bounds(tmp_path): + path = fx.bambu_project(tmp_path / "b.3mf", parts=3, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])], + roles=["normal_part", "modifier_part", "negative_part"], + origins=[(0, 0, 0), (500, 0, 0), (900, 0, 0)]) + sc = build(path) + roles = {n["resource"]["object_id"]: n["role_context"] for n in sc["nodes"] if n["mesh_key"]} + assert roles == {"1": "part", "2": "modifier", "3": "negative"} # roles follow the USE (node), not the mesh + assert node(sc, "b0.c0")["printable"] is True + top = node(sc, "b0") + assert top["has_non_part_volumes"] is True and top["printable"] is True + assert top["bounds_mm"] == {"min": [10.0, 10.0, 0.0], "max": [20.0, 20.0, 10.0]} # the modifier at x=500 is ignored + assert node(sc, "b0.c1")["printable"] is False and node(sc, "b0.c1")["bounds_mm"] is None + assert sc["findings"] == [] # fits: modifiers are excluded from fit + + +def test_prusa_ranges_partition_all_triangles(tmp_path): + path = fx.prusa_project(tmp_path / "p.3mf", volumes=[(0, 11, "ModelPart"), (12, 23, "ParameterModifier")]) + sc = build(path) + vols = sc["meshes"][0]["volumes"] + assert [(v["triangle_start"], v["triangle_count"], v["role"], v["source"]) for v in vols] == [ + (0, 12, "part", "prusa_range"), (12, 12, "modifier", "prusa_range")] + assert node(sc, "b0")["has_non_part_volumes"] is True + assert node(sc, "b0")["bounds_mm"]["max"][0] == pytest.approx(100 + 10) # only the first cube counts + + +def test_prusa_gap_and_overlap_never_guess(tmp_path): + gap = build(fx.prusa_project(tmp_path / "g.3mf", volumes=[(0, 5, "ModelPart"), (12, 23, "ModelPart")])) + assert [v["role"] for v in gap["meshes"][0]["volumes"]] == ["part", "unknown", "part"] + assert "UNKNOWN_VOLUME_ROLE" in codes(gap) and gap["status"] == "partial" + overlap = build(fx.prusa_project(tmp_path / "o.3mf", volumes=[(0, 12, "ModelPart"), (10, 23, "ModelPart")])) + assert [v["role"] for v in overlap["meshes"][0]["volumes"]] == ["unknown"] + assert node(overlap, "b0")["printable"] is None and overlap["findings"] == [] + + +def test_unrecognised_bambu_subtype_is_unknown_not_a_part(tmp_path): + sc = build(fx.bambu_project(tmp_path / "u.3mf", parts=1, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])], + roles=["totally_new_part"])) + assert node(sc, "b0.c0")["role_context"] == "unknown" + lim = next(l for l in sc["limitations"] if l["code"] == "UNKNOWN_VOLUME_ROLE") + assert lim["target_ids"] == ["b0.c0"] and node(sc, "b0")["printable"] is None + + +def test_plain_mesh_gets_one_synthetic_part_volume(tmp_path): + sc = build(fx.plain_cube_3mf(tmp_path / "c.3mf")) + assert sc["meshes"][0]["volumes"] == [ + {"id": "v0", "triangle_start": 0, "triangle_count": 12, "role": "part", "source": "object"}] + + +# ------------------------------------------------------------------------------ plates + +def test_two_instances_of_one_resource_on_different_plates(tmp_path): + path = fx.bambu_project(tmp_path / "pl.3mf", parts=1, items=[("100", fx.tf(10, 10, 0)), ("100", fx.tf(20, 10, 0))], + plates=[(1, [("100", 0)]), (2, [("100", 1)])]) + sc = build(path) + assert node(sc, "b0")["plate"] == {"state": "known", "plate_id": "p1", "source": "plate_config"} + assert node(sc, "b1")["plate"] == {"state": "known", "plate_id": "p2", "source": "plate_config"} + assert [p["id"] for p in sc["plates"]] == ["p1", "p2"] and all(p["origin_mm"] is None for p in sc["plates"]) + assert "MULTI_PLATE_PLACEMENT_UNCHECKED" in codes(sc) + assert sc["findings"] == [] + + +def test_membership_ambiguity_is_reported_not_guessed(tmp_path): + both = build(fx.bambu_project(tmp_path / "a.3mf", parts=1, items=[("100", fx.tf(10, 10, 0))], + plates=[(1, [("100", 0)]), (2, [("100", 0)])])) + assert node(both, "b0")["plate"]["state"] == "ambiguous" and node(both, "b0")["plate"]["plate_id"] is None + assert "PLATE_MEMBERSHIP_AMBIGUOUS" in codes(both) + unspecific = build(fx.bambu_project(tmp_path / "b.3mf", parts=1, items=[("100", fx.tf(10, 10, 0)), ("100", fx.tf(40, 10, 0))], + plates=[(1, [("100", None)])])) + assert [node(unspecific, i)["plate"]["state"] for i in ("b0", "b1")] == ["ambiguous", "ambiguous"] + absent = build(fx.bambu_project(tmp_path / "c.3mf", parts=1, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [])])) + assert node(absent, "b0")["plate"]["state"] == "unknown" + + +# ------------------------------------------------------------------------------ STL and no placement + +def test_stl_placement_is_unknown_and_never_judged(tmp_path): + path = fx.binary_stl(tmp_path / "huge.stl", size=900.0) + sc = build(path) + n = node(sc, "b0") + assert n["placement_state"] == "unknown" and n["plate"]["state"] == "unknown" and n["build_index"] is None + assert n["world_mm"] == scene.IDENTITY16 # an identity is a rendering default only + assert sc["findings"] == [] and "UNKNOWN_PLACEMENT" in codes(sc) and sc["status"] == "partial" + assert sc["sources"] == [{"part": "model.stl", "unit": "millimeter", "mm_per_unit": 1.0}] + assert sc["meshes"][0]["volumes"][0]["role"] == "part" + + +def test_model_without_build_items_has_unknown_placement(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], []) + sc = build(fx.three_mf(tmp_path / "nb.3mf", root)) + assert node(sc, "b0")["placement_state"] == "unknown" and node(sc, "b0")["build_index"] is None + assert {"NO_BUILD_ITEMS", "UNKNOWN_PLACEMENT"} <= set(codes(sc)) and sc["findings"] == [] + + +# ------------------------------------------------------------------------------ bed + +@pytest.mark.parametrize("breaker", ["empty", "raises"]) +def test_bed_template_problem_is_surfaced_and_suppresses_fit_findings(tmp_path, monkeypatch, breaker): + def broken(): + if breaker == "raises": + raise FileNotFoundError("template") + return [] + monkeypatch.setattr(plate_placement, "_u1_printable_area", broken) + sc = build(fx.three_mf(tmp_path / "f.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(900, 900, 0))]))) + assert sc["bed"]["policy"] == "fallback" and "BED_TEMPLATE_UNAVAILABLE" in codes(sc) + assert sc["findings"] == [] and sc["status"] == "partial" + + +def test_bed_matches_the_engines_u1_rectangle(): + lims = scene._Limitations() + bed, rect = scene._bed([]) + assert bed["policy"] == "u1_template" and rect == plate_placement.u1_bed_rect() + assert bed["polygon_mm"] == [[0.5, 1.0], [270.5, 1.0], [270.5, 271.0], [0.5, 271.0]] + assert bed["edge_margin_mm"] == plate_placement.EDGE_MARGIN_MM + + +# ------------------------------------------------------------------------------ scene vs the engine + +def _single_plate_cases(tmp_path): + cases = {} + for name, at in {"inside": (50, 60, 0), "past_right": (300, 60, 0), "edge_margin": (270.2, 60, 0), + "past_front": (50, -5, 0)}.items(): + cases[name] = fx.bambu_project(tmp_path / f"{name}.3mf", parts=1, items=[("100", fx.tf(*at))], plates=[(1, [("100", 0)])]) + cases["example_offplate"] = EXAMPLES / "demo_offplate_foreign.3mf" + cases["example_cube"] = EXAMPLES / "sample_cube_SnapmakerU1.3mf" + return cases + + +def test_scene_world_bounds_and_findings_agree_with_plate_placement_assess(tmp_path): + from snapstudio_core import geometry + for name, path in _single_plate_cases(tmp_path).items(): + sc = build(path) + assessed = plate_placement.assess(str(path)) + assert assessed["available"], name + dims = geometry.build_item_dims(str(path)) + assert len(dims) == 1 + top = node(sc, "b0") + assert top["bounds_mm"]["min"] == pytest.approx(list(dims[0]["bounds"]["min"]), abs=1e-4), name + assert top["bounds_mm"]["max"] == pytest.approx(list(dims[0]["bounds"]["max"]), abs=1e-4), name + scene_off = any(f["kind"] == "placement" for f in sc["findings"]) + assert scene_off == bool(assessed["off_plate"]), name + if scene_off: + f = next(f for f in sc["findings"] if f["kind"] == "placement") + assert f["value"]["overhang_mm"] == assessed["items"][0]["overhang_mm"], name + + +def test_repeated_instance_fixture_documents_why_no_finding_is_attached(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(500, 100, 0)), ("1", fx.tf(100, 100, 0))]) + path = fx.three_mf(tmp_path / "rep.3mf", root) + sc = build(path) + assert "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED" in codes(sc) + assert sc["findings"] == [] and not any(n["finding_ids"] for n in sc["nodes"]) + + +# ------------------------------------------------------------------------------ originals are never touched + +@pytest.mark.parametrize("make", ["3mf", "stl", "example"]) +def test_original_file_is_not_modified(tmp_path, make): + path = {"3mf": lambda: fx.bambu_project(tmp_path / "o.3mf", parts=2, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])]), + "stl": lambda: fx.binary_stl(tmp_path / "o.stl"), + "example": lambda: tmp_path.joinpath("e.3mf").write_bytes((EXAMPLES / "demo_offplate_foreign.3mf").read_bytes()) and tmp_path / "e.3mf"}[make]() + before = (Path(path).read_bytes(), os.stat(path).st_mtime_ns) + os.utime(path, ns=(before[1] - 10**9, before[1] - 10**9)) # make a rewrite visible even on coarse clocks + before = (Path(path).read_bytes(), os.stat(path).st_mtime_ns) + scene.build_scene(str(path)) + assert (Path(path).read_bytes(), os.stat(path).st_mtime_ns) == before diff --git a/backend/tests/test_scene_contract.py b/backend/tests/test_scene_contract.py new file mode 100644 index 00000000..79eacaa2 --- /dev/null +++ b/backend/tests/test_scene_contract.py @@ -0,0 +1,150 @@ +"""scene/1 contract: the committed JSON Schema, the golden scene, and the fixture matrix. + +The schema is validated with the real `jsonschema` validator (a test-only extra); only the +invariants JSON Schema cannot express (volumes partition all triangles, index ranges, matrix +finiteness, base64 byte lengths) are checked by the small helper below. +""" +import base64 +import json +import math +import struct +from pathlib import Path + +import jsonschema +import pytest + +from snapstudio_core import scene, scene_limits as L +from tests import scene_fixtures as fx + +FIX = Path(__file__).parent / "fixtures" / "scene" +GOLDEN_REV = "0123456789abcdef" * 4 + + +def validator(): + s = scene.load_schema() + jsonschema.Draft202012Validator.check_schema(s) + return jsonschema.Draft202012Validator(s) + + +def sub_validator(name): + s = scene.load_schema() + return jsonschema.Draft202012Validator({"$schema": s["$schema"], "$defs": s["$defs"], "$ref": f"#/$defs/{name}"}) + + +def check_invariants(sc: dict) -> None: + """What JSON Schema cannot say.""" + keys = set() + for mesh in sc["meshes"]: + key = (mesh["key"]["part"], mesh["key"]["object_id"]) + assert key not in keys + keys.add(key) + pos = base64.b64decode(mesh["positions_f32le_base64"], validate=True) + idx = base64.b64decode(mesh["indices_u32le_base64"], validate=True) + assert len(pos) == mesh["vertex_count"] * 12 + assert len(idx) == mesh["triangle_count"] * 12 + floats = struct.unpack(f"<{mesh['vertex_count'] * 3}f", pos) + assert all(math.isfinite(f) for f in floats) + ints = struct.unpack(f"<{mesh['triangle_count'] * 3}I", idx) + assert all(i < mesh["vertex_count"] for i in ints) + cursor = 0 + for vol in mesh["volumes"]: # partition: no gap, no overlap + assert vol["triangle_start"] == cursor + cursor += vol["triangle_count"] + assert cursor == mesh["triangle_count"] + ids = [n["id"] for n in sc["nodes"]] + assert len(set(ids)) == len(ids) + for n in sc["nodes"]: + assert all(math.isfinite(x) for x in n["world_mm"] + n["local_to_parent_mm"]) + assert n["parent_id"] is None or n["parent_id"] in ids + if n["mesh_key"] is not None: + assert (n["mesh_key"]["part"], n["mesh_key"]["object_id"]) in keys + assert len(n["id"]) <= L.MAX_ID_LENGTH + fids = {f["id"] for f in sc["findings"]} + assert len(fids) == len(sc["findings"]) + for n in sc["nodes"]: + assert set(n["finding_ids"]) <= fids + for f in sc["findings"]: + assert all(t in ids for t in f["target_ids"]) + assert sc["counts"]["nodes"] == len(sc["nodes"]) + assert sc["counts"]["rendered_triangles"] == sum( + next(m for m in sc["meshes"] if (m["key"]["part"], m["key"]["object_id"]) == (n["mesh_key"]["part"], n["mesh_key"]["object_id"]))["triangle_count"] + for n in sc["nodes"] if n["mesh_key"]) + + +def assert_valid(sc: dict) -> None: + errors = sorted(validator().iter_errors(sc), key=lambda e: list(e.path)) + assert not errors, errors[0].message + check_invariants(sc) + + +def test_schema_is_a_valid_closed_schema(): + s = scene.load_schema() + jsonschema.Draft202012Validator.check_schema(s) + assert s["additionalProperties"] is False + # every scene object definition is closed and requires every property it declares + for name in ("source", "bed", "mesh", "volume", "node", "plate", "finding", "limitation", "counts", "limits", + "plate_ref", "instance_ref", "resource_key"): + d = s["$defs"][name] + assert d["additionalProperties"] is False, name + assert set(d["required"]) == set(d["properties"]), name + + +def test_schema_enums_match_the_code(): + d = scene.load_schema()["$defs"] + assert tuple(d["error_code"]["enum"]) == scene.ERROR_CODES + assert "WORKER_WEDGED" in scene.ERROR_CODES and "BUSY" not in scene.ERROR_CODES + assert set(d["limitation_code"]["enum"]) == set(scene._Limitations.ORDER) + assert set(scene.PARTIAL_LIMITATIONS | scene.INFORMATIONAL_LIMITATIONS) == set(scene._Limitations.ORDER) + assert set(d["finding"]["properties"]["kind"]["enum"]) == {"placement", "size", "note"} + from snapstudio_core import units + assert set(d["unit_name"]["enum"]) == set(units.MM_PER_UNIT) + limits = d["limits"]["properties"] + assert set(limits) == set(L.LIMITS_ECHO) + + +def test_golden_scene_validates_and_is_reproduced_exactly(tmp_path): + golden = json.loads((FIX / "golden-scene-1.json").read_text("utf-8")) + assert_valid(golden) + path = fx.plain_cube_3mf(tmp_path / "cube.3mf", size=10.0, at=(50.0, 60.0, 0.0)) + built = scene.build_scene_dict(str(path), GOLDEN_REV) + assert built == golden + assert json.loads(scene.serialize(built)) == golden + + +def test_finding_kind_is_required_and_closed(): + v = sub_validator("finding") + good = {"id": "f0", "engine": "scene", "schema": "scene/1", "pointer": "/nodes/0", "scope": "instance", + "kind": "placement", "target_ids": ["b0"], "value": {"code": "PLACEMENT_OUTSIDE_BED", "overhang_mm": None}} + assert not list(v.iter_errors(good)) + bad = dict(good) + del bad["kind"] + assert list(v.iter_errors(bad)) + assert list(v.iter_errors({**good, "kind": "opinion"})) + assert list(v.iter_errors({**good, "extra": 1})) + + +@pytest.mark.parametrize("name", ["error_response"]) +def test_error_response_accepts_every_code(name): + v = sub_validator(name) + for code in scene.ERROR_CODES: + assert not list(v.iter_errors({"error": code, "message": "x"})) + assert list(v.iter_errors({"error": "BUSY", "message": "x"})) + + +def test_schema_rejects_mutations_of_the_golden(): + golden = json.loads((FIX / "golden-scene-1.json").read_text("utf-8")) + v = validator() + for mutate in ( + lambda s: s.pop("limits"), + lambda s: s.update(extra=1), + lambda s: s["nodes"][0].pop("placement_state"), + lambda s: s["nodes"][0].update(world_mm=[1.0] * 15), + lambda s: s["nodes"][0]["plate"].update(state="maybe"), + lambda s: s["meshes"][0]["volumes"][0].update(role="solid"), + lambda s: s["bed"].update(policy="guess"), + lambda s: s.update(status="done"), + lambda s: s["sources"][0].update(unit="yard"), + ): + broken = json.loads(json.dumps(golden)) + mutate(broken) + assert list(v.iter_errors(broken)) diff --git a/backend/tests/test_scene_hostile.py b/backend/tests/test_scene_hostile.py new file mode 100644 index 00000000..e2518e2a --- /dev/null +++ b/backend/tests/test_scene_hostile.py @@ -0,0 +1,355 @@ +"""scene/1 against hostile and broken input: every case ends in a clear, coded refusal -- never a +crash, a hang, a partial scene or a silently shortened one.""" +import time +import zipfile + +import pytest + +from snapstudio_core import scene, scene_limits as L +from tests import scene_fixtures as fx + +REV = "cd" * 32 + + +def refuse(path, code): + with pytest.raises(scene.SceneError) as e: + scene.build_scene(str(path), REV) + assert e.value.code == code, (e.value.code, e.value.message) + return e.value + + +def model(objects=None, items=None, **kw): + return fx.model_xml(objects or [fx.cube_object("1")], items or [("1", None)], **kw) + + +# ------------------------------------------------------------------------------ format + +def test_wrong_extension_and_not_an_archive(tmp_path): + p = tmp_path / "x.obj" + p.write_bytes(b"v 0 0 0") + refuse(p, "UNSUPPORTED_FORMAT") + q = tmp_path / "x.3mf" + q.write_bytes(b"this is not a zip file at all" * 10) + refuse(q, "INVALID_ARCHIVE") + + +def test_missing_root_model_and_unresolved_references(tmp_path): + refuse(fx.write_zip(tmp_path / "a.3mf", {"_rels/.rels": fx.rels_xml()}), "UNRESOLVED_REFERENCE") + missing_file = model([fx.composite_object("5", [("1", "/3D/Objects/nope.model", None)])], [("5", None)]) + refuse(fx.three_mf(tmp_path / "b.3mf", missing_file), "UNRESOLVED_REFERENCE") + missing_obj = model([fx.composite_object("5", [("77", None, None)])], [("5", None)]) + refuse(fx.three_mf(tmp_path / "c.3mf", missing_obj), "UNRESOLVED_REFERENCE") + missing_item = model([fx.cube_object("1")], [("42", None)]) + refuse(fx.three_mf(tmp_path / "d.3mf", missing_item), "UNRESOLVED_REFERENCE") + + +# ------------------------------------------------------------------------------ archive structure + +@pytest.mark.parametrize("name", ["../evil.model", "/abs.model", "a/../../b.model", "..\\win.model", "C:/x.model", + "a/\x01b"]) +def test_traversal_looking_entry_names_are_rejected(tmp_path, name): + entries = {"_rels/.rels": fx.rels_xml(), "3D/3dmodel.model": model(), name: "x"} + refuse(fx.write_zip(tmp_path / "t.3mf", entries), "INVALID_ARCHIVE") + + +@pytest.mark.parametrize("second", ["3d/3DMODEL.model", "3D\\3dmodel.model", "3D//3dmodel.model", "./3D/3dmodel.model"]) +def test_colliding_normalized_names_are_rejected(tmp_path, second): + import warnings + entries = {"_rels/.rels": fx.rels_xml(), "3D/3dmodel.model": model(), second: model()} + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + path = fx.write_zip(tmp_path / "c.3mf", entries) + refuse(path, "INVALID_ARCHIVE") + + +def test_exact_duplicate_entry_names_are_rejected(tmp_path): + import warnings + path = tmp_path / "dup.3mf" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with zipfile.ZipFile(path, "w", zipfile.ZIP_STORED) as z: + z.writestr("_rels/.rels", fx.rels_xml()) + z.writestr("3D/3dmodel.model", model()) + z.writestr("3D/3dmodel.model", model()) + refuse(path, "INVALID_ARCHIVE") + + +def test_external_relationships_are_rejected_anywhere(tmp_path): + ext = '' + refuse(fx.three_mf(tmp_path / "a.3mf", model(), rels=fx.rels_xml(extra=ext)), "INVALID_ARCHIVE") + rels3d = f'{ext}' + refuse(fx.three_mf(tmp_path / "b.3mf", model(), {"3D/_rels/3dmodel.model.rels": rels3d}), "INVALID_ARCHIVE") + + +def test_references_that_escape_the_package_are_rejected(tmp_path): + refuse(fx.three_mf(tmp_path / "a.3mf", model(), rels=fx.rels_xml(target="../../etc/passwd")), "INVALID_ARCHIVE") + escaping = model([fx.composite_object("5", [("1", "/../../x.model", None)])], [("5", None)]) + refuse(fx.three_mf(tmp_path / "b.3mf", escaping), "INVALID_ARCHIVE") + + +def test_relative_and_rooted_relationship_targets_resolve_but_urls_do_not(tmp_path): + ok = scene.build_scene_dict(str(fx.three_mf(tmp_path / "rel.3mf", model(), rels=fx.rels_xml(target="3D/3dmodel.model"))), REV) + assert ok["counts"]["nodes"] == 1 + for target in ("http://example.invalid/a.model", "file:///C:/x.model", "C:/x.model", "ftp://h/x"): + refuse(fx.three_mf(tmp_path / "url.3mf", model(), rels=fx.rels_xml(target=target)), "INVALID_ARCHIVE") + + +def test_only_the_needed_parts_are_ever_read(tmp_path): + big = "x" * (3 * 1024 * 1024) + path = fx.three_mf(tmp_path / "ok.3mf", model(), {"Metadata/thumbnail.png": big, "Metadata/plate_1.gcode": big, + "Metadata/project_settings.config": big}) + arcs = [] + real = scene.Archive.__init__ + + def spy(self, *a, **k): + real(self, *a, **k) + arcs.append(self) + scene.Archive.__init__ = spy + try: + scene.build_scene(str(path), REV) + finally: + scene.Archive.__init__ = real + assert arcs[0].total_read < 4096 # thumbnails, G-code and project settings were never read + + +# ------------------------------------------------------------------------------ XML hardening + +BILLION = (']>' + "&b;") + + +def test_dtd_and_entity_declarations_are_rejected_before_parsing(tmp_path): + refuse(fx.three_mf(tmp_path / "a.3mf", BILLION), "INVALID_ARCHIVE") + mixed = model(prolog=']>') + refuse(fx.three_mf(tmp_path / "b.3mf", mixed), "INVALID_ARCHIVE") + external = model(prolog='') + refuse(fx.three_mf(tmp_path / "c.3mf", external), "INVALID_ARCHIVE") + # the same refusal for the .rels and the settings parts + refuse(fx.three_mf(tmp_path / "d.3mf", model(), rels=BILLION), "INVALID_ARCHIVE") + settings = ']>' + refuse(fx.three_mf(tmp_path / "e.3mf", model(), {"Metadata/model_settings.config": settings}), "INVALID_ARCHIVE") + + +@pytest.mark.parametrize("encoding", ["utf-16", "utf-16-le", "utf-16-be", "utf-32"]) +def test_utf16_doctype_bomb_is_rejected_by_encoding_before_parsing(tmp_path, encoding): + bomb = BILLION.replace("", f'') + for with_bom in (True, False): + raw = bomb.encode(encoding) + if not with_bom and raw[:2] in (b"\xff\xfe", b"\xfe\xff") or (not with_bom and raw[:4] in (b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff")): + raw = raw[2:] if encoding != "utf-32" else raw[4:] + path = fx.write_zip(tmp_path / f"u-{encoding}-{with_bom}.3mf", + {"_rels/.rels": fx.rels_xml(), "3D/3dmodel.model": raw}) + err = refuse(path, "INVALID_ARCHIVE") + assert "UTF-8" in err.message + + +def test_declared_non_utf8_encoding_is_rejected(tmp_path): + xml = model(prolog='') + refuse(fx.three_mf(tmp_path / "a.3mf", xml), "INVALID_ARCHIVE") + ok = model(prolog='\ufeff') + scene.build_scene(str(fx.three_mf(tmp_path / "b.3mf", ok.encode("utf-8"))), REV) # UTF-8 with BOM is fine + + +def test_parser_level_doctype_check_is_a_second_line_of_defence(tmp_path, monkeypatch): + path = fx.three_mf(tmp_path / "x.3mf", BILLION.replace("", f'')) + original = scene.Archive.open + monkeypatch.setattr(scene.Archive, "open", lambda self, name, kind, scan=True: original(self, name, kind, scan=False)) + err = refuse(path, "INVALID_ARCHIVE") + assert "DTD" in err.message + + +def test_malformed_xml_is_a_clean_refusal(tmp_path): + refuse(fx.three_mf(tmp_path / "a.3mf", ""), "INVALID_ARCHIVE") + refuse(fx.three_mf(tmp_path / "b.3mf", ""), "INVALID_ARCHIVE") + refuse(fx.three_mf(tmp_path / "c.3mf", ""), "INVALID_ARCHIVE") + + +# ------------------------------------------------------------------------------ geometry validation + +def _mesh_model(vertices, triangles, transform=None): + obj = f'{fx.mesh_xml(vertices, triangles)}' + return fx.model_xml([obj], [("1", transform)]) + + +@pytest.mark.parametrize("bad", ["nan", "inf", "-inf", "abc", "", "1e999"]) +def test_non_finite_or_invalid_coordinates(tmp_path, bad): + v = [(0, 0, 0), (1, 0, 0), (0, bad, 0)] + refuse(fx.three_mf(tmp_path / "v.3mf", _mesh_model(v, [(0, 1, 2)])), "INVALID_GEOMETRY") + + +def test_coordinate_out_of_range(tmp_path): + refuse(fx.three_mf(tmp_path / "v.3mf", _mesh_model([(0, 0, 0), (1, 0, 0), (0, 1e30, 0)], [(0, 1, 2)])), "INVALID_GEOMETRY") + + +@pytest.mark.parametrize("tri", [(0, 1, 3), (0, 1, 99999999999), (0, -1, 2)]) +def test_invalid_triangle_indices(tmp_path, tri): + refuse(fx.three_mf(tmp_path / "t.3mf", _mesh_model([(0, 0, 0), (1, 0, 0), (0, 1, 0)], [tri])), "INVALID_GEOMETRY") + + +def test_missing_attributes_and_empty_meshes(tmp_path): + body = ('') + refuse(fx.three_mf(tmp_path / "a.3mf", fx.model_xml([body], [("1", None)])), "INVALID_GEOMETRY") + empty = '' + refuse(fx.three_mf(tmp_path / "b.3mf", fx.model_xml([empty], [("1", None)])), "INVALID_GEOMETRY") + nothing = '' + refuse(fx.three_mf(tmp_path / "c.3mf", fx.model_xml([nothing], [("1", None)])), "INVALID_GEOMETRY") + + +@pytest.mark.parametrize("transform", ["1 0 0 0 1 0 0 0 1 0 0 nan", "1 0 0 0 1 0 0 0 1 0 0 inf", "1 0 0 0 1 0 0 0 1 0 0", + "a b c d e f g h i j k l", "1 0 0 0 1 0 0 0 1 0 0 0 0"]) +def test_invalid_transforms(tmp_path, transform): + refuse(fx.three_mf(tmp_path / "x.3mf", model(items=[("1", transform)])), "INVALID_GEOMETRY") + + +def test_transform_overflow_in_composition_is_refused(tmp_path): + big = "1e200 0 0 0 1e200 0 0 0 1e200 0 0 0" + objs = [fx.cube_object("1"), fx.composite_object("2", [("1", None, big)]), fx.composite_object("3", [("2", None, big)])] + refuse(fx.three_mf(tmp_path / "o.3mf", fx.model_xml(objs, [("3", None)])), "INVALID_GEOMETRY") + + +def test_component_cycles_and_duplicate_ids(tmp_path): + cyc = [fx.composite_object("1", [("2", None, None)]), fx.composite_object("2", [("1", None, None)])] + refuse(fx.three_mf(tmp_path / "c.3mf", fx.model_xml(cyc, [("1", None)])), "INVALID_ARCHIVE") + selfref = [fx.composite_object("1", [("1", None, None)])] + refuse(fx.three_mf(tmp_path / "s.3mf", fx.model_xml(selfref, [("1", None)])), "INVALID_ARCHIVE") + dup = [fx.cube_object("1"), fx.cube_object("1")] + refuse(fx.three_mf(tmp_path / "d.3mf", fx.model_xml(dup, [("1", None)])), "INVALID_ARCHIVE") + both = '' + refuse(fx.three_mf(tmp_path / "b.3mf", fx.model_xml([both], [("1", None)])), "INVALID_ARCHIVE") + + +# ------------------------------------------------------------------------------ budgets (never truncate) + +def test_zip_bomb_is_refused_without_reading_it(tmp_path): + path = tmp_path / "bomb.3mf" + head = f'' + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as z: + z.writestr("_rels/.rels", fx.rels_xml()) + z.writestr("3D/3dmodel.model", head + " " * (40 * 1024 * 1024) + "") + assert path.stat().st_size < 200_000 + t = time.monotonic() + refuse(path, "LIMIT_EXCEEDED") + assert time.monotonic() - t < 2.0 + + +def test_total_expansion_budget_is_metered_while_reading(tmp_path, monkeypatch): + sub = fx.sub_model_xml([fx.cube_object("1")]) + root = model([fx.composite_object("9", [("1", "/3D/Objects/a.model", None)])], [("9", None)]) + path = fx.three_mf(tmp_path / "m.3mf", root, {"3D/Objects/a.model": sub}) + scene.build_scene(str(path), REV) + monkeypatch.setattr(L, "ARCHIVE_EXPANSION_BYTES", 1500) + refuse(path, "LIMIT_EXCEEDED") + monkeypatch.setattr(L, "ARCHIVE_EXPANSION_BYTES", 128 * 1024 * 1024) + monkeypatch.setattr(L, "MODEL_XML_BYTES", 1000) + refuse(path, "LIMIT_EXCEEDED") + + +def test_entry_count_and_part_count_budgets(tmp_path, monkeypatch): + path = fx.three_mf(tmp_path / "e.3mf", model(), {f"x/{i}.txt": "a" for i in range(20)}) + monkeypatch.setattr(L, "ARCHIVE_MAX_ENTRIES", 10) + refuse(path, "LIMIT_EXCEEDED") + + +@pytest.mark.parametrize("limit,value", [("MAX_PARSED_TRIANGLES", 11), ("MAX_VERTICES", 7)]) +def test_triangle_and_vertex_budgets_are_enforced_during_parsing(tmp_path, monkeypatch, limit, value): + path = fx.plain_cube_3mf(tmp_path / "c.3mf") + monkeypatch.setattr(L, limit, value) + refuse(path, "LIMIT_EXCEEDED") + + +def test_refusal_names_the_triangle_count_and_the_limit(tmp_path, monkeypatch): + path = fx.big_grid_3mf(tmp_path / "g.3mf", 400) + monkeypatch.setattr(L, "MAX_PARSED_TRIANGLES", 100) + err = refuse(path, "LIMIT_EXCEEDED") + assert "101" in err.message and "100" in err.message + monkeypatch.undo() + stl = fx.binary_stl(tmp_path / "c.stl") + monkeypatch.setattr(L, "MAX_PARSED_TRIANGLES", 11) + assert "12" in refuse(stl, "LIMIT_EXCEEDED").message + + +def test_rendered_triangles_count_every_repetition(tmp_path, monkeypatch): + path = fx.three_mf(tmp_path / "r.3mf", fx.model_xml([fx.cube_object("1")], [("1", fx.tf(i * 20, 0, 0)) for i in range(5)])) + monkeypatch.setattr(L, "MAX_RENDERED_TRIANGLES", 59) # 5 x 12 = 60 + refuse(path, "LIMIT_EXCEEDED") + monkeypatch.setattr(L, "MAX_RENDERED_TRIANGLES", 60) + assert scene.build_scene_dict(str(path), REV)["counts"]["rendered_triangles"] == 60 + + +def test_node_and_depth_budgets(tmp_path, monkeypatch): + path = fx.three_mf(tmp_path / "n.3mf", fx.model_xml([fx.cube_object("1")], [("1", fx.tf(i * 20, 0, 0)) for i in range(5)])) + monkeypatch.setattr(L, "MAX_NODES", 4) + refuse(path, "LIMIT_EXCEEDED") + chain = [fx.cube_object("0")] + [fx.composite_object(str(i), [(str(i - 1), None, None)]) for i in range(1, 70)] + deep = fx.three_mf(tmp_path / "d.3mf", fx.model_xml(chain, [("69", None)])) + refuse(deep, "LIMIT_EXCEEDED") + monkeypatch.setattr(L, "MAX_NODES", 2000) + shallow = fx.three_mf(tmp_path / "s.3mf", fx.model_xml(chain[:60], [("59", None)])) + assert scene.build_scene_dict(str(shallow), REV)["counts"]["nodes"] == 60 + + +def test_whole_body_cap_applies_to_the_exact_bytes(tmp_path, monkeypatch): + path = fx.plain_cube_3mf(tmp_path / "c.3mf") + size = len(scene.build_scene(str(path), REV)) + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", size) + assert len(scene.build_scene(str(path), REV)) == size + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", size - 1) + refuse(path, "LIMIT_EXCEEDED") + + +def test_geometry_preflight_rejects_before_any_encoding(tmp_path, monkeypatch): + path = fx.plain_cube_3mf(tmp_path / "c.3mf") + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", 100) + monkeypatch.setattr(scene, "_encode_f32", lambda *a: pytest.fail("encoded before the size check")) + refuse(path, "LIMIT_EXCEEDED") + + +def test_stl_budgets(tmp_path, monkeypatch): + stl = fx.binary_stl(tmp_path / "c.stl") + monkeypatch.setattr(L, "MAX_PARSED_TRIANGLES", 11) + refuse(stl, "LIMIT_EXCEEDED") + monkeypatch.undo() + text = tmp_path / "t.stl" + text.write_text("solid x\n" + "facet normal 0 0 0\nouter loop\nvertex 0 0 0\nvertex 1 0 0\nvertex 0 1 0\nendloop\nendfacet\n" * 5 + "endsolid\n") + assert scene.build_scene_dict(str(text), REV)["counts"]["triangles"] == 5 + monkeypatch.setattr(L, "MAX_PARSED_TRIANGLES", 4) + err = refuse(text, "LIMIT_EXCEEDED") # ASCII is line-streamed with a running counter + assert "5" in err.message and "4" in err.message + garbage = tmp_path / "g.stl" + garbage.write_bytes(b"not an stl at all" * 20) + monkeypatch.undo() + refuse(garbage, "INVALID_GEOMETRY") + + +# ------------------------------------------------------------------------------ cooperative interruption + +class CountingControl(scene.Control): + def __init__(self, cancel_at=None): + self.checks, self.cancel_at = 0, cancel_at + + def check(self): + self.checks += 1 + if self.cancel_at is not None and self.checks >= self.cancel_at: + raise scene.SceneCancelled() + + +def test_cancel_is_observed_inside_parsing_and_encoding(tmp_path): + path = fx.big_grid_3mf(tmp_path / "big.3mf", 30_000) + full = CountingControl() + scene.build_scene_dict(str(path), REV, full) + assert full.checks > 100 # checked far more often than once per phase + for at in (3, full.checks // 2, full.checks - 2): + c = CountingControl(cancel_at=at) + with pytest.raises(scene.SceneCancelled): + scene.build_scene_dict(str(path), REV, c) + assert c.checks == at # stops at the first check after the flag + + +def test_deadline_is_a_timeout_not_a_hang(tmp_path): + path = fx.big_grid_3mf(tmp_path / "big.3mf", 30_000) + + class Late(scene.Control): + def check(self): + raise scene.SceneTimeout() + with pytest.raises(scene.SceneTimeout): + scene.build_scene_dict(str(path), REV, Late()) diff --git a/backend/tests/test_scene_jobs.py b/backend/tests/test_scene_jobs.py new file mode 100644 index 00000000..c12bf68a --- /dev/null +++ b/backend/tests/test_scene_jobs.py @@ -0,0 +1,746 @@ +"""Scene jobs: state machine, worker model, snapshots, retention, cancellation, wedge backstop, leaks.""" +import hashlib +import json +import os +import subprocess +import sys +import threading +import time +from pathlib import Path + +import jsonschema +import pytest + +from snapstudio_api import scene_jobs as sj +from snapstudio_core import scene, scene_limits as L +from tests import scene_fixtures as fx +from tests.test_scene_contract import assert_valid, sub_validator + +RID = iter(range(10**6)) + + +def rid() -> str: + return f"req-{next(RID):08d}" + + +def validate(name, obj): + errors = list(sub_validator(name).iter_errors(obj)) + assert not errors, errors[0].message + + +def wait_for(fn, timeout=10.0, step=0.01): + end = time.monotonic() + timeout + while time.monotonic() < end: + value = fn() + if value: + return value + time.sleep(step) + raise AssertionError("timed out") + + +def wait_state(jobs, job_id, *states, timeout=10.0): + return wait_for(lambda: (lambda s: s if s["state"] in states else None)(jobs.status(job_id)), timeout) + + +def fake_builder(gate=None, honor_cancel=True, started=None, body=b'{"ok":true}'): + def builder(path, revision, ctl): + if started is not None: + started.set() + if gate is not None: + while not gate.wait(0.005): + if honor_cancel: + ctl.check() + return body + return builder + + +@pytest.fixture +def cube(tmp_path): + return str(fx.bambu_project(tmp_path / "cube.3mf", parts=2, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])])) + + +@pytest.fixture +def make(tmp_path): + made = [] + + def factory(**kw): + jobs = sj.SceneJobs(snapshot_root=str(tmp_path / "snap"), **kw) + made.append(jobs) + return jobs + yield factory + for jobs in made: + jobs.close() + + +def snapshot_files(jobs): + d = jobs.snapshot_dir() + return [] if d is None or not os.path.isdir(d) else os.listdir(d) + + +# ------------------------------------------------------------------------------ the happy path + +def test_start_status_result_and_snapshot_cleanup(make, cube): + jobs = make() + started = jobs.start(cube, rid()) + validate("job_start", started) + assert started["state"] in ("queued", "running") and started["replaced_job_id"] is None + status = wait_state(jobs, started["job_id"], "succeeded") + validate("job_status", status) + expected = hashlib.sha256(Path(cube).read_bytes()).hexdigest() + assert status["revision"] == expected and status["stage"] is None + code, body = jobs.result(started["job_id"]) + assert code == 200 and isinstance(body, bytes) + sc = json.loads(body) + assert_valid(sc) + assert sc["revision"] == expected and sc["counts"]["nodes"] == 3 + wait_for(lambda: not jobs.worker_alive()) + assert snapshot_files(jobs) == [] # the worker removed its own snapshot + assert body == scene.serialize(sc) # compact bytes, serialized exactly once + + +def test_original_file_bytes_and_mtime_unchanged_after_a_job(make, cube): + jobs = make() + old = os.stat(cube).st_mtime_ns - 5 * 10**9 + os.utime(cube, ns=(old, old)) + before = (Path(cube).read_bytes(), os.stat(cube).st_mtime_ns) + job = jobs.start(cube, rid()) + wait_state(jobs, job["job_id"], "succeeded") + jobs.result(job["job_id"]) + assert (Path(cube).read_bytes(), os.stat(cube).st_mtime_ns) == before + + +def test_result_while_not_ready_and_unknown_job(make, cube): + gate = threading.Event() + jobs = make(builder=fake_builder(gate)) + job = jobs.start(cube, rid()) + wait_state(jobs, job["job_id"], "running") + code, body = jobs.result(job["job_id"]) + assert code == 409 and body["error"] == "NOT_READY" and body["state"] == "running" + validate("error_response", body) + gate.set() + wait_state(jobs, job["job_id"], "succeeded") + with pytest.raises(sj.JobError) as e: + jobs.status("0" * 32) + assert (e.value.http, e.value.code) == (404, "EXPIRED") + + +def test_failed_job_reports_a_coded_error_with_422(make, tmp_path): + bad = tmp_path / "bad.3mf" + bad.write_bytes(b"not a zip" * 20) + jobs = make() + job = jobs.start(str(bad), rid()) + status = wait_state(jobs, job["job_id"], "failed") + assert status["error"]["code"] == "INVALID_ARCHIVE" + validate("job_status", status) + code, body = jobs.result(job["job_id"]) + assert code == 422 and body["error"] == "INVALID_ARCHIVE" + validate("error_response", body) + wait_for(lambda: not jobs.worker_alive()) + assert snapshot_files(jobs) == [] + + +def test_start_validation(make, tmp_path): + jobs = make() + txt = tmp_path / "a.txt" + txt.write_text("x") + with pytest.raises(sj.JobError) as e: + jobs.start(str(txt), rid()) + assert (e.value.http, e.value.code) == (422, "UNSUPPORTED_FORMAT") + with pytest.raises(sj.JobError) as e: + jobs.start(str(tmp_path / "missing.3mf"), rid()) + assert (e.value.http, e.value.code) == (400, "INVALID_REQUEST") + + +# ------------------------------------------------------------------------------ idempotency and request ids + +def test_same_request_id_and_path_returns_the_existing_job(make, cube): + jobs = make() + r = rid() + a = jobs.start(cube, r) + b = jobs.start(cube, r) + assert a["job_id"] == b["job_id"] and b["replaced_job_id"] is None + wait_state(jobs, a["job_id"], "succeeded") + c = jobs.start(cube, r) # still retained + assert c["job_id"] == a["job_id"] and c["state"] == "succeeded" + assert jobs.retained() == 1 + + +def test_request_id_reused_for_another_path_is_a_conflict(make, cube, tmp_path): + other = fx.plain_cube_3mf(tmp_path / "other.3mf") + jobs = make() + r = rid() + jobs.start(cube, r) + with pytest.raises(sj.JobError) as e: + jobs.start(str(other), r) + assert (e.value.http, e.value.code, e.value.extra["reason"]) == (409, "INVALID_REQUEST", "request_id reused for another path") + + +def test_reused_request_id_with_changed_file_is_source_changed(make, cube): + jobs = make() + r = rid() + job = jobs.start(cube, r) + wait_state(jobs, job["job_id"], "succeeded") + Path(cube).write_bytes(Path(cube).read_bytes() + b"\x00") + with pytest.raises(sj.JobError) as e: + jobs.start(cube, r) + assert (e.value.http, e.value.code) == (409, "SOURCE_CHANGED") + fresh = jobs.start(cube, rid()) # a fresh request_id starts a new scene + assert fresh["job_id"] != job["job_id"] + + +def test_expected_revision_mismatch_is_source_changed(make, cube): + jobs = make() + job = jobs.start(cube, rid()) + status = wait_state(jobs, job["job_id"], "succeeded") + assert jobs.result(job["job_id"], status["revision"])[0] == 200 + with pytest.raises(sj.JobError) as e: + jobs.result(job["job_id"], "f" * 64) + assert (e.value.http, e.value.code) == (409, "SOURCE_CHANGED") + + +def test_source_replaced_mid_job_cannot_tear_the_scene(make, cube): + """The snapshot is the single source: replacing the original after the copy changes nothing.""" + original = Path(cube).read_bytes() + seen = {} + gate = threading.Event() + started = threading.Event() + + def builder(path, revision, ctl): + started.set() + while not gate.wait(0.005): + ctl.check() + seen["snapshot"] = Path(path).read_bytes() + seen["revision"] = revision + return b"{}" + jobs = make(builder=builder) + job = jobs.start(cube, rid()) + started.wait(5) + Path(cube).write_bytes(b"replaced with something else entirely") + gate.set() + status = wait_state(jobs, job["job_id"], "succeeded") + assert seen["snapshot"] == original + assert seen["revision"] == status["revision"] == hashlib.sha256(original).hexdigest() + + +def test_file_changing_during_the_copy_is_source_changed(make, cube, monkeypatch): + monkeypatch.setattr(L, "SNAPSHOT_CHUNK_BYTES", 256) + calls = {"n": 0} + real = sj._Control.progress + + def tamper(self, stage, completed, total): + calls["n"] += 1 + if calls["n"] == 2 and stage == "reading": + with open(cube, "ab") as fh: + fh.write(b"appended while copying") + real(self, stage, completed, total) + monkeypatch.setattr(sj._Control, "progress", tamper) + jobs = make() + job = jobs.start(cube, rid()) + status = wait_state(jobs, job["job_id"], "failed") + assert status["error"]["code"] == "SOURCE_CHANGED" + wait_for(lambda: not jobs.worker_alive()) + assert snapshot_files(jobs) == [] + + +# ------------------------------------------------------------------------------ cancel, replace, races + +def test_replace_cancels_the_running_job_and_queues_behind_it(make, cube): + gate_a = threading.Event() + started = threading.Event() + order = [] + + def builder(path, revision, ctl): + order.append(os.path.basename(path)) + started.set() + if len(order) == 1: + while not gate_a.wait(0.005): + ctl.check() + return b"{}" + jobs = make(builder=builder) + a = jobs.start(cube, rid()) + started.wait(5) + b = jobs.start(cube, rid()) + assert b["replaced_job_id"] == a["job_id"] and b["state"] == "queued" + assert jobs.status(a["job_id"])["state"] == "cancelled" + assert jobs.result(a["job_id"])[1]["error"] == "CANCELLED" + wait_state(jobs, b["job_id"], "succeeded") # b ran only after a's thread observed the cancel + assert jobs.status(a["job_id"])["state"] == "cancelled" + assert len(order) == 2 and not gate_a.is_set() + + +def test_at_most_one_queued_job_newest_wins(make, cube): + gate = threading.Event() + started = threading.Event() + jobs = make(builder=fake_builder(gate, honor_cancel=False, started=started)) # holds the worker even when cancelled + a = jobs.start(cube, rid()) + started.wait(5) + b = jobs.start(cube, rid()) + c = jobs.start(cube, rid()) + assert c["replaced_job_id"] == b["job_id"] + assert jobs.status(b["job_id"])["state"] == "cancelled" and jobs.status(c["job_id"])["state"] == "queued" + assert jobs.status(a["job_id"])["state"] == "cancelled" + gate.set() + wait_state(jobs, c["job_id"], "succeeded") + + +def test_cancel_queued_running_and_terminal_is_idempotent(make, cube): + gate = threading.Event() + started = threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + a = jobs.start(cube, rid()) + started.wait(5) + s = jobs.cancel(a["job_id"]) + assert s["state"] == "cancelled" and s["error"]["code"] == "CANCELLED" + assert jobs.cancel(a["job_id"])["state"] == "cancelled" + wait_for(lambda: not jobs.worker_alive()) + b = jobs.start(cube, rid()) + gate.set() + wait_state(jobs, b["job_id"], "succeeded") + assert jobs.cancel(b["job_id"])["state"] == "succeeded" # a late cancel on a terminal job is a no-op + assert jobs.result(b["job_id"])[0] == 200 + + +def test_cancel_wins_the_race_against_a_worker_that_finishes_anyway(make, cube): + gate = threading.Event() + started = threading.Event() + jobs = make(builder=fake_builder(gate, honor_cancel=False, started=started, body=b'{"late":true}')) + a = jobs.start(cube, rid()) + started.wait(5) + jobs.cancel(a["job_id"]) + gate.set() # the worker now returns a result + wait_for(lambda: not jobs.worker_alive()) + assert jobs.status(a["job_id"])["state"] == "cancelled" # first terminal wins; the late result is discarded + assert jobs.result(a["job_id"]) == (409, {"error": "CANCELLED", "message": "That scene was cancelled."}) + assert jobs._jobs[a["job_id"]].result is None + + +def test_cancel_after_completion_does_not_change_the_result(make, cube): + jobs = make(builder=fake_builder()) + a = jobs.start(cube, rid()) + wait_state(jobs, a["job_id"], "succeeded") + jobs.cancel(a["job_id"]) + assert jobs.result(a["job_id"]) == (200, b'{"ok":true}') + + +def test_cancel_is_observed_within_two_seconds_on_a_real_scene(make, tmp_path): + big = fx.big_grid_3mf(tmp_path / "big.3mf", 90_000) + jobs = make() + job = jobs.start(str(big), rid()) + wait_for(lambda: jobs.status(job["job_id"])["state"] == "running" and jobs.status(job["job_id"])["stage"] in ("parsing", "encoding")) + t = time.monotonic() + jobs.cancel(job["job_id"]) + wait_for(lambda: not jobs.worker_alive(), timeout=5) + assert time.monotonic() - t <= 2.0 + assert jobs.status(job["job_id"])["state"] == "cancelled" + assert snapshot_files(jobs) == [] + + +# ------------------------------------------------------------------------------ deadline and wedge + +def test_deadline_is_a_timeout_and_the_clock_starts_when_running(make, cube): + gate = threading.Event() + started = threading.Event() + jobs = make(deadline=0.3, builder=fake_builder(gate, started=started)) + a = jobs.start(cube, rid()) + started.wait(5) + b = jobs.start(cube, rid()) # queued: its clock has not started + assert jobs.status(a["job_id"])["state"] == "cancelled" + wait_state(jobs, b["job_id"], "running") + time.sleep(0.05) + assert jobs.status(b["job_id"])["state"] == "running" + status = wait_state(jobs, b["job_id"], "failed", timeout=3) + assert status["error"]["code"] == "TIMEOUT" + assert jobs.result(b["job_id"])[0] == 422 + wait_for(lambda: not jobs.worker_alive()) + + +def test_worker_that_ignores_cancel_is_wedged_fails_closed_and_recovers(make, cube): + gate = threading.Event() + started = threading.Event() + jobs = make(wedge_grace=0.25, builder=fake_builder(gate, honor_cancel=False, started=started)) + a = jobs.start(cube, rid()) + started.wait(5) + jobs.cancel(a["job_id"]) + assert not jobs.wedged() + b = jobs.start(cube, rid()) # within the grace period: queued behind the cancelled job + assert b["state"] == "queued" + time.sleep(0.4) + assert jobs.wedged() + with pytest.raises(sj.JobError) as e: + jobs.start(cube, rid()) + assert (e.value.http, e.value.code) == (503, "WORKER_WEDGED") + validate("error_response", e.value.body()) + gate.set() # the stuck call finally returns + wait_state(jobs, b["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + assert not jobs.wedged() + assert jobs.start(cube, rid())["state"] in ("queued", "running") # recovered + + +def test_deadline_wedge_reports_timeout_and_rejects_new_starts(make, cube): + gate = threading.Event() + started = threading.Event() + jobs = make(deadline=0.15, wedge_grace=0.15, builder=fake_builder(gate, honor_cancel=False, started=started)) + a = jobs.start(cube, rid()) + started.wait(5) + wait_for(lambda: jobs.status(a["job_id"])["state"] == "failed", timeout=3) + assert jobs.status(a["job_id"])["error"]["code"] == "TIMEOUT" + assert jobs.wedged() + with pytest.raises(sj.JobError) as e: + jobs.start(cube, rid()) + assert e.value.code == "WORKER_WEDGED" + gate.set() + wait_for(lambda: not jobs.worker_alive()) + assert not jobs.wedged() and jobs.status(a["job_id"])["state"] == "failed" + + +# ------------------------------------------------------------------------------ retention + +def test_ttl_and_terminal_cap_with_an_injected_clock(make, cube): + now = [1000.0] + jobs = make(clock=lambda: now[0], builder=fake_builder(), ttl=120.0, max_terminal=8) + ids = [] + for _ in range(12): + j = jobs.start(cube, rid()) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + now[0] += 1.0 + ids.append(j["job_id"]) + assert jobs.retained() == 8 + with pytest.raises(sj.JobError) as e: + jobs.status(ids[0]) # oldest evicted + assert (e.value.http, e.value.code) == (404, "EXPIRED") + assert jobs.status(ids[-1])["state"] == "succeeded" + now[0] += 200.0 + with pytest.raises(sj.JobError): + jobs.status(ids[-1]) # TTL: 120 s after it became terminal + assert jobs.retained() == 0 + + +def test_failed_and_cancelled_jobs_are_retained_like_succeeded_ones(make, cube, tmp_path): + now = [0.0] + gate = threading.Event() + jobs = make(clock=lambda: now[0], builder=fake_builder(gate)) + a = jobs.start(cube, rid()) + wait_state(jobs, a["job_id"], "running") + jobs.cancel(a["job_id"]) + wait_for(lambda: not jobs.worker_alive()) + now[0] += 119.0 + assert jobs.status(a["job_id"])["state"] == "cancelled" + now[0] += 2.0 + with pytest.raises(sj.JobError): + jobs.status(a["job_id"]) + + +def test_eviction_removes_a_snapshot_the_worker_could_not_delete(make, cube, monkeypatch): + now = [0.0] + jobs = make(clock=lambda: now[0], builder=fake_builder(), ttl=10.0) + real_unlink = os.unlink + state = {"fail": True} + + def flaky(path, *a, **k): + if state["fail"] and str(path).endswith(".3mf"): + state["fail"] = False + raise PermissionError("sharing violation") + return real_unlink(path, *a, **k) + monkeypatch.setattr(sj.os, "unlink", flaky) + j = jobs.start(cube, rid()) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + assert len(snapshot_files(jobs)) == 1 # the worker's unlink failed + now[0] += 11.0 + with pytest.raises(sj.JobError): + jobs.status(j["job_id"]) + assert snapshot_files(jobs) == [] # eviction retried it (missing-file tolerant) + + +# ------------------------------------------------------------------------------ snapshot limits + +def test_snapshot_ceiling_and_disk_preflight(make, cube, monkeypatch): + jobs = make() + monkeypatch.setattr(L, "SNAPSHOT_MAX_BYTES", 1000) + a = jobs.start(cube, rid()) + assert wait_state(jobs, a["job_id"], "failed")["error"]["code"] == "LIMIT_EXCEEDED" + wait_for(lambda: not jobs.worker_alive()) + assert snapshot_files(jobs) == [] + monkeypatch.setattr(L, "SNAPSHOT_MAX_BYTES", 128 * 1024 * 1024) + monkeypatch.setattr(sj.shutil, "disk_usage", lambda p: type("U", (), {"free": 10})()) + b = jobs.start(cube, rid()) + status = wait_state(jobs, b["job_id"], "failed") + assert status["error"]["code"] == "INTERNAL" and "disk" in status["error"]["message"] + wait_for(lambda: not jobs.worker_alive()) + assert snapshot_files(jobs) == [] + + +def test_response_cap_surfaces_as_limit_exceeded(make, cube, monkeypatch): + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", 500) + jobs = make() + j = jobs.start(cube, rid()) + assert wait_state(jobs, j["job_id"], "failed")["error"]["code"] == "LIMIT_EXCEEDED" + assert jobs.result(j["job_id"])[0] == 422 + + +# ------------------------------------------------------------------------------ startup sweep + +def test_sweep_removes_only_dead_and_old_directories_and_never_a_live_ones(tmp_path): + root = tmp_path / "scene-tmp" + root.mkdir() + dead = subprocess.Popen([sys.executable, "-c", "pass"]) + dead.wait() + live = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + old = L.SNAPSHOT_DIR_MIN_AGE_SECONDS + 100 + try: + def mk(name, age_s=0): + d = root / name + d.mkdir() + (d / "x.3mf").write_bytes(b"x") + if age_s: + t = time.time() - age_s + os.utime(d, (t, t)) + return d + d_dead_old = mk(f"{dead.pid}-aaaa1111", old) + d_dead_young = mk(f"{dead.pid}-aaaa2222") + d_live = mk(f"{live.pid}-bbbb2222") + d_live_ancient = mk(f"{live.pid}-cccc3333", age_s=10 * 24 * 3600) # a live process's folder, however old + d_own = mk(f"{os.getpid()}-dddd4444", age_s=10 * 24 * 3600) + d_junk = mk("not-a-snapshot-dir", old) + removed = sj.sweep_stale(str(root)) + assert removed == [d_dead_old.name] + for survivor in (d_dead_young, d_live, d_live_ancient, d_own, d_junk): + assert survivor.exists(), survivor.name + finally: + live.kill() + live.wait() + assert sj.sweep_stale(str(tmp_path / "does-not-exist")) == [] + + +def test_sweep_tolerates_a_sharing_violation(tmp_path, monkeypatch): + root = tmp_path / "scene-tmp" + root.mkdir() + dead = subprocess.Popen([sys.executable, "-c", "pass"]) + dead.wait() + d = root / f"{dead.pid}-aaaa1111" + d.mkdir() + t = time.time() - L.SNAPSHOT_DIR_MIN_AGE_SECONDS - 10 + os.utime(d, (t, t)) + real = sj.shutil.rmtree + monkeypatch.setattr(sj.shutil, "rmtree", lambda *a, **k: (_ for _ in ()).throw(PermissionError("in use"))) + assert sj.sweep_stale(str(root)) == [] and d.exists() # no crash; left for the next start + monkeypatch.setattr(sj.shutil, "rmtree", real) + assert sj.sweep_stale(str(root)) == [d.name] + + +def test_posix_liveness_branch_by_monkeypatch(monkeypatch): + monkeypatch.setattr(sj.sys, "platform", "linux") + calls = [] + + def fake_kill(pid, sig): + calls.append((pid, sig)) + if pid == 111: + raise ProcessLookupError() + if pid == 222: + raise PermissionError() + monkeypatch.setattr(sj.os, "kill", fake_kill) + assert sj.pid_alive(111) is False # no such process + assert sj.pid_alive(222) is True # exists, not ours: alive + assert sj.pid_alive(333) is True + assert sj.pid_alive(os.getpid()) is True and sj.pid_alive(0) is False + assert (111, 0) in calls + + +def test_each_registry_uses_its_own_pid_nonce_directory(make, cube): + a, b = make(builder=fake_builder()), make(builder=fake_builder()) + for jobs in (a, b): + wait_state(jobs, jobs.start(cube, rid())["job_id"], "succeeded") + assert a.snapshot_dir() != b.snapshot_dir() + assert os.path.basename(a.snapshot_dir()).startswith(f"{os.getpid()}-") + + +# ------------------------------------------------------------------------------ leak gate + +def test_fifty_start_cancel_cycles_leave_nothing_behind(make, tmp_path): + big = str(fx.big_grid_3mf(tmp_path / "g.3mf", 20_000)) + small = str(fx.plain_cube_3mf(tmp_path / "s.3mf")) + baseline = threading.active_count() + jobs = make() + wedged_ever = False + for i in range(50): + job = jobs.start(big if i % 2 else small, rid()) + if i % 3 == 0: + wait_for(lambda: jobs.status(job["job_id"])["state"] != "queued") + jobs.cancel(job["job_id"]) + wedged_ever = wedged_ever or jobs.wedged() + assert jobs.retained() <= L.MAX_TERMINAL_JOBS + 2 # + the running/queued pair + jobs.close() + wait_for(lambda: not jobs.worker_alive(), timeout=10) + assert jobs.retained() <= L.MAX_TERMINAL_JOBS # retained jobs never exceed the cap once idle + assert all(j.state in sj.TERMINAL for j in jobs._jobs.values()) + assert snapshot_files(jobs) == [] # temp snapshots = 0 + assert all(j.snapshot is None for j in jobs._jobs.values()) + assert threading.active_count() == baseline # thread count back to baseline + assert not wedged_ever and not jobs.wedged() # WORKER_WEDGED never occurred + + +# ------------------------------------------------------------------------------ repair round 1 + +import contextlib +import errno + + +def poke(jobs): + """Any API call runs housekeeping; an unknown id is the cheapest one.""" + with contextlib.suppress(sj.JobError): + jobs.status("0" * 32) + + +def test_a10_a_retained_job_evicted_while_hashing_is_never_handed_back(make, cube, monkeypatch): + now = [0.0] + jobs = make(clock=lambda: now[0], builder=fake_builder(), ttl=120.0) + r = rid() + a = jobs.start(cube, r) + status = wait_state(jobs, a["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + + def hash_while_time_passes(path): + now[0] += 500.0 # the TTL expires during the hash + return status["revision"] + monkeypatch.setattr(jobs, "_current_hash", hash_while_time_passes) + b = jobs.start(cube, r) + assert b["job_id"] != a["job_id"] + assert jobs.status(b["job_id"])["request_id"] == r # the returned job is registered + with pytest.raises(sj.JobError): + jobs.status(a["job_id"]) + + +def test_a11_a_snapshot_that_resists_deletion_is_retried_with_backoff(make, cube, monkeypatch): + now = [0.0] + jobs = make(clock=lambda: now[0], builder=fake_builder(), ttl=10.0) + real = os.unlink + fails = {"n": 0} + + def flaky(path, *a, **k): + if str(path).endswith(".3mf") and fails["n"] < 3: + fails["n"] += 1 + raise PermissionError("sharing violation") + return real(path, *a, **k) + monkeypatch.setattr(sj.os, "unlink", flaky) + j = jobs.start(cube, rid()) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + assert len(snapshot_files(jobs)) == 1 # failure 1: the worker's own unlink + now[0] += 11.0 + poke(jobs) # TTL eviction: failure 2, path goes on the retry list + with pytest.raises(sj.JobError): + jobs.status(j["job_id"]) + assert len(snapshot_files(jobs)) == 1 and len(jobs._pending) == 1 + now[0] += 2.5 + poke(jobs) # first retry: failure 3, longer back-off + assert len(snapshot_files(jobs)) == 1 and jobs._pending[0][1] >= 2 + now[0] += 1.0 + poke(jobs) # not due yet: no attempt + assert fails["n"] == 3 + now[0] += 10.0 + poke(jobs) # now it succeeds and the path is forgotten + assert snapshot_files(jobs) == [] and jobs._pending == [] + + +def test_d2_the_snapshot_directory_is_recreated_for_every_job(make, cube): + jobs = make(builder=fake_builder()) + first = jobs.start(cube, rid()) + wait_state(jobs, first["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + import shutil + shutil.rmtree(jobs.snapshot_dir()) # something removed the per-process folder + second = jobs.start(cube, rid()) + assert wait_state(jobs, second["job_id"], "succeeded")["error"] is None + + +def test_d2_only_a_real_enospc_is_reported_as_a_disk_space_problem(make, cube, monkeypatch): + jobs = make(builder=fake_builder()) + real_open = open + + def failing(code): + def fake(path, mode="r", *a, **k): + if "w" in mode and str(path).endswith(".3mf"): + raise OSError(code, "boom") + return real_open(path, mode, *a, **k) + return fake + monkeypatch.setattr(sj, "open", failing(errno.EACCES), raising=False) + a = jobs.start(cube, rid()) + msg = wait_state(jobs, a["job_id"], "failed")["error"]["message"] + assert "disk space" not in msg and "temporary copy" in msg + wait_for(lambda: not jobs.worker_alive()) + monkeypatch.setattr(sj, "open", failing(errno.ENOSPC), raising=False) + b = jobs.start(cube, rid()) + assert "disk space" in wait_state(jobs, b["job_id"], "failed")["error"]["message"] + + +def test_d6a_a_cancelled_job_whose_worker_is_still_running_is_not_evicted(make, cube): + now = [0.0] + gate, started = threading.Event(), threading.Event() + jobs = make(clock=lambda: now[0], ttl=10.0, max_terminal=0, builder=fake_builder(gate, honor_cancel=False, started=started)) + a = jobs.start(cube, rid()) + started.wait(5) + jobs.cancel(a["job_id"]) + now[0] += 1000.0 + poke(jobs) + assert jobs.status(a["job_id"])["state"] == "cancelled" # worker not done: not evictable, whatever the TTL and cap + gate.set() + wait_for(lambda: not jobs.worker_alive()) + with pytest.raises(sj.JobError) as e: + jobs.status(a["job_id"]) + assert e.value.code == "EXPIRED" + + +def test_d6b_the_deadline_starts_when_the_job_runs_not_when_it_was_created(make, cube): + gate, started = threading.Event(), threading.Event() + calls = [] + + def builder(path, revision, ctl): + calls.append(1) + if len(calls) == 1: + started.set() + gate.wait(5) # the first job holds the worker, ignoring cancel + return b"{}" + ctl.check() + return b'{"second":true}' + jobs = make(deadline=0.4, wedge_grace=30.0, builder=builder) + a = jobs.start(cube, rid()) + started.wait(5) + b = jobs.start(cube, rid()) # queued behind a + time.sleep(0.7) # longer than the deadline, all of it spent queued + gate.set() + assert wait_state(jobs, b["job_id"], "succeeded", "failed")["state"] == "succeeded" + assert jobs.status(a["job_id"])["state"] == "cancelled" + + +def test_d6e_a_size_change_without_an_mtime_change_is_source_changed(make, cube, monkeypatch): + monkeypatch.setattr(L, "SNAPSHOT_CHUNK_BYTES", 256) + st = os.stat(cube) + calls = {"n": 0} + real = sj._Control.progress + + def tamper(self, stage, completed, total): + calls["n"] += 1 + if calls["n"] == 2 and stage == "reading": + with open(cube, "ab") as fh: + fh.write(b"grown") + os.utime(cube, ns=(st.st_atime_ns, st.st_mtime_ns)) # put the mtime back: only the size betrays it + real(self, stage, completed, total) + monkeypatch.setattr(sj._Control, "progress", tamper) + jobs = make() + job = jobs.start(cube, rid()) + assert wait_state(jobs, job["job_id"], "failed")["error"]["code"] == "SOURCE_CHANGED" + wait_for(lambda: not jobs.worker_alive()) + assert snapshot_files(jobs) == [] + + +def test_close_removes_the_per_process_directory_best_effort(make, cube): + jobs = make(builder=fake_builder()) + j = jobs.start(cube, rid()) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + d = jobs.snapshot_dir() + assert os.path.isdir(d) + jobs.close() + assert not os.path.exists(d) diff --git a/backend/tests/test_scene_repair.py b/backend/tests/test_scene_repair.py new file mode 100644 index 00000000..ef26252f --- /dev/null +++ b/backend/tests/test_scene_repair.py @@ -0,0 +1,703 @@ +"""Repair round 1: each test here fails without the corresponding fix (finding id in the docstring).""" +import hashlib +import json +import re +import struct +import time +import tracemalloc + +import pytest + +from snapstudio_core import scene, scene_limits as L, units +from tests import scene_fixtures as fx +from tests.test_scene_build import build, codes, node +from tests.test_scene_contract import assert_valid, sub_validator + +REV = "ef" * 32 + + +def refuse(path, code): + with pytest.raises(scene.SceneError) as e: + scene.build_scene(str(path), REV) + assert e.value.code == code, (e.value.code, e.value.message) + return e.value + + +# ------------------------------------------------------------------ A1: build items with p:path + +def test_a1_build_item_p_path_uses_the_referenced_file_not_the_root(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(5, 5, 0), "/3D/Objects/ext.model")]) + ext = fx.sub_model_xml([fx.cube_object("1", 30)]) + sc = build(fx.three_mf(tmp_path / "p.3mf", root, {"3D/Objects/ext.model": ext})) + n = node(sc, "b0") + assert n["resource"] == {"part": "3D/Objects/ext.model", "object_id": "1"} + assert n["bounds_mm"] == {"min": [5.0, 5.0, 0.0], "max": [35.0, 35.0, 30.0]} # the 30 mm cube + assert [m["key"]["part"] for m in sc["meshes"]] == ["3D/Objects/ext.model"] # the root cube is not substituted + + +def test_a1_unresolvable_p_path_is_an_error_not_a_fallback_to_the_root(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", None, "/3D/Objects/missing.model")]) + refuse(fx.three_mf(tmp_path / "a.3mf", root), "UNRESOLVED_REFERENCE") + ext = fx.sub_model_xml([fx.cube_object("7", 30)]) # file exists, object 1 does not + root2 = fx.model_xml([fx.cube_object("1", 10)], [("1", None, "/3D/Objects/ext.model")]) + refuse(fx.three_mf(tmp_path / "b.3mf", root2, {"3D/Objects/ext.model": ext}), "UNRESOLVED_REFERENCE") + + +def test_a1_same_object_id_in_two_files_is_not_a_repeated_instance(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(5, 5, 0), "/3D/Objects/ext.model"), ("1", fx.tf(100, 5, 0))]) + ext = fx.sub_model_xml([fx.cube_object("1", 30)]) + sc = build(fx.three_mf(tmp_path / "r.3mf", root, {"3D/Objects/ext.model": ext})) + assert "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED" not in codes(sc) + assert node(sc, "b0")["resource"]["part"] == "3D/Objects/ext.model" and node(sc, "b1")["resource"]["part"] == "3D/3dmodel.model" + twice = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(5, 5, 0), "/3D/Objects/ext.model"), ("1", fx.tf(100, 5, 0), "/3D/Objects/ext.model")]) + assert "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED" in codes(build(fx.three_mf(tmp_path / "t.3mf", twice, {"3D/Objects/ext.model": ext}))) + + +# ------------------------------------------------------------------ A2: the unit header, read from the real root + +CORE = fx.CORE + + +def _prefixed(xml: str) -> str: + xml = re.sub(r"<(/?)(model|resources|object|mesh|vertices|vertex|triangles|triangle|components|component|build|item)\b", + r"<\1m:\2", xml) + return xml.replace(f'xmlns="{CORE}"', f'xmlns:m="{CORE}"') + + +UNIT_DOCS = { + "prefixed": lambda: _prefixed(fx.model_xml([fx.cube_object("1", 12)], [("1", None)], unit="inch")), + "long_comment": lambda: fx.model_xml([fx.cube_object("1", 12)], [("1", None)], unit="inch", + prolog=''), + "pi_and_comments": lambda: fx.model_xml([fx.cube_object("1", 12)], [("1", None)], unit="inch", + prolog=''), + "decoy_comment": lambda: fx.model_xml([fx.cube_object("1", 12)], [("1", None)], unit="millimeter", + prolog=''), + "absent": lambda: fx.model_xml([fx.cube_object("1", 12)], [("1", None)]).replace(' unit="millimeter"', ""), +} +EXPECT = {"prefixed": "inch", "long_comment": "inch", "pi_and_comments": "inch", "decoy_comment": "millimeter", "absent": "millimeter"} + + +@pytest.mark.parametrize("name", sorted(UNIT_DOCS)) +def test_a2_the_unit_comes_from_the_real_root_element(tmp_path, name): + xml = UNIT_DOCS[name]() + want = EXPECT[name] + assert units.read_unit(xml.encode()).name == want + assert units.mm_per_unit(xml.encode()) == units.MM_PER_UNIT[want] + sc = build(fx.three_mf(tmp_path / f"{name}.3mf", xml)) + assert sc["sources"][0]["unit"] == want # the scene parser and the shared helper agree + b = node(sc, "b0")["bounds_mm"] + assert b["max"][0] == pytest.approx(12 * units.MM_PER_UNIT[want], abs=1e-4) + + +def test_a2_legacy_readers_see_the_same_unit_through_a_long_preamble(tmp_path): + from snapstudio_core.intelligence import project_info + xml = UNIT_DOCS["long_comment"]() + path = str(fx.three_mf(tmp_path / "l.3mf", xml)) + assert project_info(path)["dimensions_mm"] == {"x": 304.8, "y": 304.8, "z": 304.8} + decoy = str(fx.three_mf(tmp_path / "d.3mf", UNIT_DOCS["decoy_comment"]())) + assert project_info(decoy)["dimensions_mm"] == {"x": 12.0, "y": 12.0, "z": 12.0} + + +def test_a2_an_unrecognised_unit_is_not_the_same_as_an_absent_one(): + absent = units.read_unit(b'') + yard = units.read_unit(b'') + assert absent.recognized and not absent.declared and absent.factor == 1.0 + assert not yard.recognized and yard.declared and yard.name == "yard" and yard.factor == 1.0 # legacy fallback, flagged + assert units.unit_of(b'') == ("yard", None) + + +# ------------------------------------------------------------------ A4 / D4: metadata present but unmatched is never a printable part + +def test_a4_prusa_attribute_order_does_not_matter(tmp_path): + vols = [(0, 11, "ModelPart"), (12, 23, "ParameterModifier")] + normal = build(fx.prusa_project(tmp_path / "a.3mf", volumes=vols)) + swapped = build(fx.prusa_project(tmp_path / "b.3mf", volumes=vols, last_first=True)) + assert swapped["meshes"][0]["volumes"] == normal["meshes"][0]["volumes"] + assert [v["role"] for v in swapped["meshes"][0]["volumes"]] == ["part", "modifier"] + assert node(swapped, "b0")["has_non_part_volumes"] is True + + +def test_a4_bambu_part_missing_from_the_settings_is_unknown_not_printable(tmp_path): + path = fx.bambu_project(tmp_path / "m.3mf", parts=2, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])], + origins=[(0, 0, 0), (500, 0, 0)], omit_parts=(2,)) + sc = build(path) + missing = node(sc, "b0.c1") + assert missing["role_context"] == "unknown" and missing["printable"] is None and missing["bounds_mm"] is None + lim = next(l for l in sc["limitations"] if l["code"] == "UNKNOWN_VOLUME_ROLE") + assert lim["target_ids"] == ["b0.c1"] and sc["status"] == "partial" + assert node(sc, "b0")["bounds_mm"] == {"min": [10.0, 10.0, 0.0], "max": [20.0, 20.0, 10.0]} # the unknown part does not count + assert node(sc, "b0.c1")["finding_ids"] == [] and sc["findings"] == [] + + +def test_a4_an_object_with_no_metadata_at_all_keeps_the_synthetic_part(tmp_path): + sc = build(fx.plain_cube_3mf(tmp_path / "p.3mf")) + assert sc["meshes"][0]["volumes"][0]["role"] == "part" and node(sc, "b0")["role_context"] is None + + +# ------------------------------------------------------------------ A5: roles follow the traversal context + +def _shared_project(tmp_path, roles): + sub = fx.sub_model_xml([fx.cube_object("1", 10)]) + comp = [("1", "/3D/Objects/o.model", None)] + root = fx.model_xml([fx.composite_object("100", comp), fx.composite_object("101", comp)], + [("100", fx.tf(10, 10, 0)), ("101", fx.tf(60, 10, 0))]) + settings = fx.model_settings_xml({"100": [("1", roles[0])], "101": [("1", roles[1])]}, [(1, [("100", 0), ("101", 0)])]) + return fx.three_mf(tmp_path / "s.3mf", root, {"3D/Objects/o.model": sub, "Metadata/model_settings.config": settings}) + + +def test_a5_a_shared_mesh_keeps_each_use_s_role(tmp_path): + sc = build(_shared_project(tmp_path, ("negative_part", "normal_part"))) + assert len(sc["meshes"]) == 1 # one mesh, two uses + neg, part = node(sc, "b0.c0"), node(sc, "b1.c0") + assert (neg["role_context"], neg["printable"], neg["bounds_mm"]) == ("negative", False, None) + assert (part["role_context"], part["printable"]) == ("part", True) + assert part["bounds_mm"]["min"] == [60.0, 10.0, 0.0] + flipped = build(_shared_project(tmp_path, ("normal_part", "modifier_part"))) + assert (node(flipped, "b0.c0")["role_context"], node(flipped, "b1.c0")["role_context"]) == ("part", "modifier") + + +def test_a5_a_negative_part_over_an_assembly_applies_to_everything_beneath(tmp_path): + sub = fx.sub_model_xml([fx.cube_object("1", 10), fx.composite_object("2", [("1", None, None), ("1", None, fx.tf(20, 0, 0))])]) + root = fx.model_xml([fx.composite_object("100", [("2", "/3D/Objects/o.model", None)])], [("100", fx.tf(10, 10, 0))]) + settings = fx.model_settings_xml({"100": [("2", "negative_part")]}, [(1, [("100", 0)])]) + sc = build(fx.three_mf(tmp_path / "a.3mf", root, {"3D/Objects/o.model": sub, "Metadata/model_settings.config": settings})) + assert [n["id"] for n in sc["nodes"]] == ["b0", "b0.c0", "b0.c0.c0", "b0.c0.c1"] + for nid in ("b0.c0", "b0.c0.c0", "b0.c0.c1"): + n = node(sc, nid) + assert n["role_context"] == "negative" and n["printable"] is False and n["bounds_mm"] is None and n["has_non_part_volumes"] + assert node(sc, "b0")["printable"] is False and node(sc, "b0")["bounds_mm"] is None + assert sc["findings"] == [] + + +# ------------------------------------------------------------------ A7: the response cap never allocates the whole body first + +def test_a7_part_names_longer_than_the_schema_are_refused_before_they_are_placed(tmp_path): + name = "3D/Objects/" + "n" * 300 + ".model" + sub = fx.sub_model_xml([fx.cube_object("1", 10)]) + root = fx.model_xml([fx.composite_object("9", [("1", "/" + name, fx.tf(i % 50, i // 50, 0)) for i in range(2000)])], [("9", None)]) + path = fx.three_mf(tmp_path / "long.3mf", root, {name: sub}) + tracemalloc.start() + err = refuse(path, "LIMIT_EXCEEDED") + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + assert "part name" in err.message and peak < 50 * 1024 * 1024 + + +def test_a7_part_names_at_the_limit_still_yield_a_schema_valid_scene(tmp_path): + name = "3D/Objects/" + "n" * (L.MAX_PART_NAME_LENGTH - len("3D/Objects/.model")) + ".model" + assert len(name) == L.MAX_PART_NAME_LENGTH + sub = fx.sub_model_xml([fx.cube_object("1", 10)]) + root = fx.model_xml([fx.composite_object("9", [("1", "/" + name, fx.tf(i % 50, i // 50, 0)) for i in range(600)])], [("9", None)]) + sc = build(fx.three_mf(tmp_path / "ok.3mf", root, {name: sub})) + assert sc["counts"]["nodes"] == 601 and len(scene.serialize(sc)) <= L.MAX_RESPONSE_BYTES + + +def test_a7_serializer_checks_the_cap_while_building_and_never_dumps_the_whole_body(tmp_path, monkeypatch): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(i % 20 * 12, i // 20 * 12, 0)) for i in range(300)]) + sc = scene.build_scene_dict(str(fx.three_mf(tmp_path / "many.3mf", root)), REV) + full = len(json.dumps(sc, separators=(",", ":"))) + assert full > 100_000 + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", 30_000) + seen = [] + real = json.dumps + + def spy(value, *a, **k): + text = real(value, *a, **k) + seen.append(len(text)) + return text + monkeypatch.setattr(scene.json, "dumps", spy) + with pytest.raises(scene.SceneError) as e: + scene.serialize(sc) + assert e.value.code == "LIMIT_EXCEEDED" + assert max(seen) < 30_000 and sum(seen) < 30_000 + 5_000 # stopped one element past the cap + monkeypatch.undo() + size = len(scene.serialize(sc)) + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", size) + assert len(scene.serialize(sc)) == size # the accounting is exact + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", size - 1) + with pytest.raises(scene.SceneError): + scene.serialize(sc) + + +# ------------------------------------------------------------------ A8 / D3: parse-time memory bounds cover every element + +def test_a8_hundreds_of_thousands_of_unused_objects_are_rejected_early(tmp_path): + objects = "".join(f'' for i in range(2, 300_002)) + root = fx.model_xml([fx.cube_object("1", 10)], [("1", None)]).replace("", objects + "") + t = time.monotonic() + err = refuse(fx.three_mf(tmp_path / "objs.3mf", root), "LIMIT_EXCEEDED") + assert "objects" in err.message and time.monotonic() - t < 10 + + +def test_d3_junk_elements_are_capped_and_memory_stays_bounded(tmp_path): + junk = "" * 2_000_000 # 8 MB of unknown elements + root = fx.model_xml([fx.cube_object("1", 10)], [("1", None)]).replace("", junk + "") + path = fx.three_mf(tmp_path / "junk.3mf", root) + tracemalloc.start() + err = refuse(path, "LIMIT_EXCEEDED") + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + assert "elements" in err.message and peak <= 300 * 1024 * 1024 + + +# ------------------------------------------------------------------ A9: STL is decoded incrementally + +def _stl_with_header(path, n): + path.write_bytes(b"\0" * 80 + struct.pack("', + f'', +]) +def test_a12_external_relationship_is_rejected_whatever_the_namespace(tmp_path, rels): + path = fx.three_mf(tmp_path / "e.3mf", fx.model_xml([fx.cube_object("1")], [("1", None)]), + {"3D/_rels/3dmodel.model.rels": '' + rels}) + err = refuse(path, "INVALID_ARCHIVE") + assert "external" in err.message + + +def test_d6_the_byte_prescan_rejects_a_doctype_in_a_config_read_without_an_xml_parser(tmp_path): + cfg = ']>' + path = fx.three_mf(tmp_path / "c.3mf", fx.model_xml([fx.cube_object("1")], [("1", None)]), {"Metadata/Slic3r_PE_model.config": cfg}) + arc = scene.Archive(str(path), scene.Control()) + try: + with pytest.raises(scene.SceneError) as e: + arc.read_small("Metadata/Slic3r_PE_model.config") + assert e.value.code == "INVALID_ARCHIVE" + finally: + arc.close() + + +# ------------------------------------------------------------------ A13: ids fit by construction + +def test_a13_ids_fit_128_characters_at_the_deepest_allowed_nesting(tmp_path): + chain = [fx.cube_object("0")] + [fx.composite_object(str(i), [(str(i - 1), None, None)]) for i in range(1, L.MAX_DEPTH)] + path = fx.three_mf(tmp_path / "deep.3mf", fx.model_xml(chain, [(str(L.MAX_DEPTH - 1), None)])) + sc = build(path) + ids = [n["id"] for n in sc["nodes"]] + assert len(ids) == L.MAX_DEPTH == len(set(ids)) and max(map(len, ids)) <= 128 + assert any("~" in i for i in ids) # long ancestries are digested, not truncated + assert [n["id"] for n in build(path)["nodes"]] == ids # deterministic + parents = {n["id"]: n["parent_id"] for n in sc["nodes"]} + assert all(p is None or p in parents for p in parents.values()) + assert node(sc, ids[-1])["instance_ref"]["component_path"] == [0] * (L.MAX_DEPTH - 1) + assert hashlib.sha256(ids[-1].encode()).hexdigest() # (ids are plain strings) + + +# ------------------------------------------------------------------ D5: schema closure and plate state + +def test_d5_error_response_is_closed_and_a_known_plate_needs_an_id(): + err = sub_validator("error_response") + assert not list(err.iter_errors({"error": "EXPIRED", "message": "x"})) + assert list(err.iter_errors({"error": "EXPIRED", "message": "x", "extra": 1})) + plate = sub_validator("plate_ref") + assert not list(plate.iter_errors({"state": "known", "plate_id": "p1", "source": "plate_config"})) + assert list(plate.iter_errors({"state": "known", "plate_id": None, "source": "plate_config"})) + assert not list(plate.iter_errors({"state": "unknown", "plate_id": None, "source": None})) + + +def test_d5_a_node_on_no_plate_of_a_project_with_plates_gets_no_fit_finding_and_is_partial(tmp_path): + path = fx.bambu_project(tmp_path / "np.3mf", parts=1, items=[("100", fx.tf(900, 900, 0))], plates=[(1, [])]) + sc = build(path) + assert node(sc, "b0")["plate"]["state"] == "unknown" + assert sc["findings"] == [] and sc["status"] == "partial" and "PLATE_MEMBERSHIP_UNKNOWN" in codes(sc) + on_plate = build(fx.bambu_project(tmp_path / "op.3mf", parts=1, items=[("100", fx.tf(900, 900, 0))], plates=[(1, [("100", 0)])])) + assert [f["kind"] for f in on_plate["findings"]] == ["placement"] # with membership proven, the finding returns + + +# ================================================================== repair round 2 + +import os + + +def _config(tmp_path, name, settings_xml, model=None, extra=None): + root = model or fx.model_xml([fx.cube_object("1", 10)], [("1", None)]) + return fx.three_mf(tmp_path / name, root, dict({"Metadata/model_settings.config": settings_xml}, **(extra or {}))) + + +# ------------------------------------------------------------------ B1: STL kind and bounds + +def test_b1_a_large_solid_payload_with_no_newlines_is_rejected_without_allocating_it(tmp_path): + path = tmp_path / "bomb.stl" + with open(path, "wb") as fh: + fh.write(b"solid ") + for _ in range(64): + fh.write(b"a" * (1024 * 1024)) # 64 MiB, not one newline + tracemalloc.start() + t = time.monotonic() + err = refuse(path, "INVALID_GEOMETRY") + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + assert time.monotonic() - t < 5 and peak < 20 * 1024 * 1024, (peak, err.message) + + +def test_b1_a_header_over_the_budget_with_trailing_bytes_is_rejected_before_any_decoding(tmp_path, monkeypatch): + from snapstudio_core import stl_io + n = L.MAX_PARSED_TRIANGLES + 1 + path = tmp_path / "trail.stl" + path.write_bytes(b"solid mislabelled binary".ljust(80, b" ") + struct.pack("" + JUNK + "") + elif which == "prusa": + path = fx.three_mf(tmp_path / "p.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", None)]), + {"Metadata/Slic3r_PE_model.config": "" + JUNK + ""}) + else: + path = fx.three_mf(tmp_path / "r.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", None)]), + rels=f'' + JUNK + "") + tracemalloc.start() + err = refuse(path, "LIMIT_EXCEEDED") + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + assert "elements" in err.message and peak <= 300 * 1024 * 1024 + + +def test_b2_unrecognised_elements_are_released_as_they_complete(tmp_path, monkeypatch): + calls = [] + real = scene._free + monkeypatch.setattr(scene, "_free", lambda e: (calls.append(e.tag), real(e))[1]) + junk = "" * 3000 + scene.build_scene_dict(str(_config(tmp_path, "j.3mf", "" + junk + "")), REV) + assert sum(1 for t in calls if t == "x") >= 3000 + calls.clear() + root = fx.model_xml([fx.cube_object("1", 10)], [("1", None)]).replace("", junk + "") + scene.build_scene_dict(str(fx.three_mf(tmp_path / "m.3mf", root)), REV) + assert sum(1 for t in calls if t.endswith("}x") or t == "x") >= 3000 + + +# ------------------------------------------------------------------ B3: p:path rules + +def test_b3_a_relative_build_item_path_is_refused(tmp_path): + ext = fx.sub_model_xml([fx.cube_object("1", 30)]) + root = fx.model_xml([], [("1", None, "3D/Objects/ext.model")]) + refuse(fx.three_mf(tmp_path / "a.3mf", root, {"3D/Objects/ext.model": ext}), "UNRESOLVED_REFERENCE") + ok = fx.model_xml([], [("1", None, "/3D/Objects/ext.model")]) + assert scene.build_scene_dict(str(fx.three_mf(tmp_path / "b.3mf", ok, {"3D/Objects/ext.model": ext})), REV)["counts"]["nodes"] == 1 + + +def test_b3_a_relative_component_path_is_refused(tmp_path): + ext = fx.sub_model_xml([fx.cube_object("1", 30)]) + root = fx.model_xml([fx.composite_object("9", [("1", "3D/Objects/ext.model", None)])], [("9", None)]) + refuse(fx.three_mf(tmp_path / "a.3mf", root, {"3D/Objects/ext.model": ext}), "UNRESOLVED_REFERENCE") + + +def test_b3_component_paths_are_honoured_only_from_the_root_model(tmp_path): + leaf = fx.sub_model_xml([fx.cube_object("1", 10)]) + mid = fx.sub_model_xml([fx.composite_object("2", [("1", "/3D/Objects/leaf.model", None)])]) + root = fx.model_xml([fx.composite_object("9", [("2", "/3D/Objects/mid.model", None)])], [("9", None)]) + err = refuse(fx.three_mf(tmp_path / "n.3mf", root, {"3D/Objects/mid.model": mid, "3D/Objects/leaf.model": leaf}), + "UNRESOLVED_REFERENCE") + assert "root model" in err.message + + +# ------------------------------------------------------------------ B4: duplicate records never silently overwrite + +def _two_part_settings(records): + return ('' + records + + '' + '') + + +def _bambu_one_part(tmp_path, name, settings): + sub = fx.sub_model_xml([fx.cube_object("1", 10)]) + root = fx.model_xml([fx.composite_object("100", [("1", "/3D/Objects/o.model", None)])], [("100", fx.tf(10, 10, 0))]) + return fx.three_mf(tmp_path / name, root, {"3D/Objects/o.model": sub, "Metadata/model_settings.config": settings}) + + +def test_b4_duplicate_records_that_disagree_make_the_role_unknown(tmp_path): + neg = '' + part = '' + both = '' + for name, records in (("objects", neg + part), ("parts", both)): + sc = build(_bambu_one_part(tmp_path, name + ".3mf", _two_part_settings(records))) + n = node(sc, "b0.c0") + assert n["role_context"] == "unknown" and n["printable"] is None + assert sc["status"] == "partial" and "UNKNOWN_VOLUME_ROLE" in codes(sc) and sc["findings"] == [] + same = build(_bambu_one_part(tmp_path, "same.3mf", _two_part_settings(neg + neg))) + assert node(same, "b0.c0")["role_context"] == "negative" # identical duplicates are fine + + +# ------------------------------------------------------------------ B5: build objects in another file fail closed + +def test_b5_a_negative_part_assembly_in_another_file_is_unknown_and_never_judged(tmp_path): + a = fx.sub_model_xml([fx.cube_object("1", 10), fx.composite_object("2", [("1", None, None)])]) + root = fx.model_xml([], [("2", fx.tf(900, 100, 0), "/3D/Objects/a.model")]) + settings = fx.model_settings_xml({"2": [("1", "negative_part")]}, [(1, [("2", 0)])]) + sc = build(fx.three_mf(tmp_path / "x.3mf", root, {"3D/Objects/a.model": a, "Metadata/model_settings.config": settings})) + assert [n["role_context"] for n in sc["nodes"]] == ["unknown", "unknown"] + assert all(n["printable"] is None for n in sc["nodes"]) + assert sc["status"] == "partial" and "UNKNOWN_VOLUME_ROLE" in codes(sc) + assert sc["findings"] == [] + + +def test_b5_without_any_settings_a_file_qualified_build_object_is_still_ordinary(tmp_path): + root = fx.model_xml([], [("1", fx.tf(5, 5, 0), "/3D/Objects/a.model")]) + a = fx.sub_model_xml([fx.cube_object("1", 10)]) + sc = build(fx.three_mf(tmp_path / "plain.3mf", root, {"3D/Objects/a.model": a})) + assert node(sc, "b0")["printable"] is True and node(sc, "b0")["role_context"] is None + + +# ------------------------------------------------------------------ B6: plate matching is file-qualified + +def test_b6_a_bare_object_id_shared_across_files_cannot_prove_plate_membership(tmp_path): + ext = fx.sub_model_xml([fx.cube_object("1", 10)]) + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(900, 10, 0)), ("1", fx.tf(5, 5, 0), "/3D/Objects/ext.model")]) + settings = fx.model_settings_xml({}, [(1, [("1", 0)])]) + sc = build(fx.three_mf(tmp_path / "s.3mf", root, {"3D/Objects/ext.model": ext, "Metadata/model_settings.config": settings})) + assert [node(sc, "b0")["plate"]["state"], node(sc, "b1")["plate"]["state"]] == ["ambiguous", "ambiguous"] + assert all(node(sc, i)["plate"]["plate_id"] is None for i in ("b0", "b1")) + lim = next(l for l in sc["limitations"] if l["code"] == "PLATE_MEMBERSHIP_UNKNOWN") + assert set(lim["target_ids"]) == {"b0", "b1"} + assert sc["findings"] == [] and sc["status"] == "partial" # the root cube at X=900 is NOT reported + + +# ------------------------------------------------------------------ B7: unsupported unit names its nodes + +def test_b7_unsupported_unit_lists_the_affected_nodes(tmp_path): + sub = fx.sub_model_xml([fx.cube_object("1", 10)], unit="yard") + root = fx.model_xml([fx.composite_object("9", [("1", "/3D/Objects/y.model", None)]), fx.cube_object("2", 10)], + [("9", None), ("2", fx.tf(50, 0, 0))]) + sc = build(fx.three_mf(tmp_path / "u.3mf", root, {"3D/Objects/y.model": sub})) + lim = next(l for l in sc["limitations"] if l["code"] == "UNSUPPORTED_UNIT") + assert lim["target_ids"] == ["b0.c0"] # only the nodes living in the yard part + assert {s["part"]: s["unit"] for s in sc["sources"]}["3D/Objects/y.model"] == "millimeter" # the documented fallback + description = scene.load_schema()["$defs"]["source"]["description"] + assert "UNSUPPORTED_UNIT" in description and "fallback" in description.lower() + + +# ------------------------------------------------------------------ B8: the failure wording is pinned + +def test_b8_a_failed_move_says_it_kept_no_copy(tmp_path, monkeypatch): + from snapstudio_core import plate_placement + path = str(fx.three_mf(tmp_path / "m.3mf", fx.model_xml([fx.cube_object("1", 25.4)], [("1", fx.tf(290.0, 3.0, 0.0))]))) + monkeypatch.setattr(plate_placement, "_rewrite_items", lambda raw, offset_for: (raw, 1)) + monkeypatch.setattr(plate_placement, "verify_only_placement_moved", lambda a, b: {"passed": True, "checks": []}) + res = plate_placement.prepare_placed_copy(path, out_dir=str(tmp_path / "out")) + assert res["ok"] is False + assert res["reason"] == ("Studio could not move the objects onto the plate, so it did not keep a copy. " + "Open the original in Snapmaker Orca and use Arrange.") + + +# ------------------------------------------------------------------ B9: id collisions are detected + +def test_b9_a_node_id_collision_is_an_internal_error(tmp_path, monkeypatch): + monkeypatch.setattr(scene.Traversal, "_node_id", staticmethod(lambda ctx, parent, path: "same")) + root = fx.model_xml([fx.cube_object("1", 10)], [("1", None), ("1", fx.tf(20, 0, 0))]) + refuse(fx.three_mf(tmp_path / "c.3mf", root), "INTERNAL") + + +# ------------------------------------------------------------------ B10: surviving mutants + +def test_b10_an_assembly_role_is_inherited_under_a_root_file_parent_and_the_outer_role_wins(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10), fx.composite_object("50", [("1", None, None)]), + fx.composite_object("100", [("50", None, None)])], [("100", fx.tf(10, 10, 0))]) + settings = fx.model_settings_xml({"100": [("50", "negative_part")], "50": [("1", "normal_part")]}, [(1, [("100", 0)])]) + sc = build(fx.three_mf(tmp_path / "r.3mf", root, {"Metadata/model_settings.config": settings})) + assert [node(sc, i)["role_context"] for i in ("b0.c0", "b0.c0.c0")] == ["negative", "negative"] + assert node(sc, "b0.c0.c0")["printable"] is False and sc["findings"] == [] + + +def test_b10_an_unreadable_prusa_range_makes_the_whole_mesh_unknown_and_partial(tmp_path): + cfg = ('' + '' + '') + sc = build(fx.three_mf(tmp_path / "p.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(900, 900, 0))]), + {"Metadata/Slic3r_PE_model.config": cfg})) + assert [v["role"] for v in sc["meshes"][0]["volumes"]] == ["unknown"] + assert node(sc, "b0")["printable"] is None and sc["status"] == "partial" and sc["findings"] == [] + + +def test_b10_binary_stl_is_decoded_in_bounded_batches_with_the_hook_between_them(tmp_path, monkeypatch): + from snapstudio_core import stl_io + path = _stl_with_header(tmp_path / "b.stl", 12_000) + sizes, checks = [], [] + real = stl_io._decode_binary_batch + monkeypatch.setattr(stl_io, "_decode_binary_batch", lambda raw, *a: (sizes.append(len(raw) // 50), real(raw, *a))[1]) + with open(path, "rb") as fh: + stl_io.decode_stl_stream(fh, os.path.getsize(path), max_triangles=250_000, max_vertices=300_000, + check=lambda: checks.append(1)) + assert sizes == [5000, 5000, 2000] and len(checks) >= 3 + + +# ================================================================== repair round 3 + +def _bambu_part_at_900(tmp_path, name, object_xml, plate_extra=""): + settings = ('' + object_xml + + '' + '' + plate_extra + '') + sub = fx.sub_model_xml([fx.cube_object("1", 10)]) + root = fx.model_xml([fx.composite_object("100", [("1", "/3D/Objects/o.model", None)])], [("100", fx.tf(900, 10, 0))]) + return fx.three_mf(tmp_path / name, root, {"3D/Objects/o.model": sub, "Metadata/model_settings.config": settings}) + + +@pytest.mark.parametrize("name,obj", [ + ("after", ''), + ("between", ''), + ("leading", ''), +]) +def test_r3_1_an_unknown_element_next_to_a_part_record_never_changes_its_role(tmp_path, name, obj): + sc = build(_bambu_part_at_900(tmp_path, f"{name}.3mf", obj)) + n = node(sc, "b0.c0") + assert n["role_context"] == "negative" and n["printable"] is False + assert sc["findings"] == [] # a negative part is never judged against the bed + + +def test_r3_1_an_unknown_element_after_a_model_instance_keeps_plate_membership(tmp_path): + obj = '' + sc = build(_bambu_part_at_900(tmp_path, "pl.3mf", obj, plate_extra="")) + assert node(sc, "b0")["plate"] == {"state": "known", "plate_id": "p1", "source": "plate_config"} + assert [f["kind"] for f in sc["findings"]] == ["placement"] # and the part at X=900 is judged, as before + + +@pytest.mark.parametrize("junk", ["", "", ""]) +def test_r3_1_an_unknown_element_after_a_prusa_volume_never_drops_it(tmp_path, junk): + verts, tris = [], [] + for k in range(2): + base = len(verts) + verts += fx.cube_vertices(10.0, (k * 20.0, 0.0, 0.0)) + tris += [(a + base, b + base, c + base) for a, b, c in fx.CUBE_TRIS] + obj = f'{fx.mesh_xml(verts, tris)}' + vol = lambda a, b, vt: f'' + cfg = ('' + vol(0, 11, "ModelPart") + junk + vol(12, 23, "NegativeVolume") + junk + "" + junk + "") + sc = build(fx.three_mf(tmp_path / "p.3mf", fx.model_xml([obj], [("1", fx.tf(900, 100, 0))]), + {"Metadata/Slic3r_PE_model.config": cfg})) + assert [v["role"] for v in sc["meshes"][0]["volumes"]] == ["part", "negative"] + assert node(sc, "b0")["bounds_mm"]["max"][0] == pytest.approx(910.0) # the negative cube (to 930) does not count + + +# ------------------------------------------------------------------ R3-2: every completed element is released + +def test_r3_2_relationships_and_namespaced_settings_elements_are_released(tmp_path, monkeypatch): + seen = [] + real = scene._free + monkeypatch.setattr(scene, "_free", lambda e: (seen.append(e.tag), real(e))[1]) + rels = f'' + '' * 50 + "" + cfg = '' + "" * 50 + "" + path = fx.three_mf(tmp_path / "r.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", None)]), + {"3D/_rels/x.rels": rels, "Metadata/model_settings.config": cfg}) + scene.build_scene_dict(str(path), REV) + assert seen.count(f"{{{fx.REL_NS}}}Relationship") >= 50 + assert seen.count("{urn:x}object") >= 50 + + +def test_r3_2_hundreds_of_thousands_of_small_records_stay_within_a_memory_gate(tmp_path): + rels = f'' + '' * 300_000 + "" + assert len(rels) < L.CONFIG_PART_BYTES + cfg = '' + "" * 880_000 + "" + assert len(cfg) < L.CONFIG_PART_BYTES + path = fx.three_mf(tmp_path / "big.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", None)]), + {"3D/_rels/x.rels": rels, "Metadata/model_settings.config": cfg}) + tracemalloc.start() + sc = scene.build_scene_dict(str(path), REV) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + assert sc["counts"]["nodes"] == 1 and peak <= 300 * 1024 * 1024 + + +# ------------------------------------------------------------------ R3-3: a present-but-empty p:path is an error + +def test_r3_3_an_empty_build_item_path_is_not_treated_as_absent(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", None, "")]) # object 1 EXISTS in this model + refuse(fx.three_mf(tmp_path / "i.3mf", root), "UNRESOLVED_REFERENCE") + + +def test_r3_3_an_empty_component_path_is_not_treated_as_absent(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10), fx.composite_object("9", [("1", "", None)])], [("9", None)]) + refuse(fx.three_mf(tmp_path / "c.3mf", root), "UNRESOLVED_REFERENCE") + + +# ------------------------------------------------------------------ R3-4: a part record without an id still means "has metadata" + +def test_r3_4_a_part_record_without_an_id_makes_the_object_have_metadata(tmp_path): + sc = build(_bambu_part_at_900(tmp_path, "noid.3mf", '')) + n = node(sc, "b0.c0") + assert n["role_context"] == "unknown" and n["printable"] is None + assert sc["status"] == "partial" and "UNKNOWN_VOLUME_ROLE" in codes(sc) and sc["findings"] == [] + plain = build(_bambu_part_at_900(tmp_path, "nometa.3mf", '')) + assert node(plain, "b0.c0")["role_context"] is None # an object with no part records is unchanged + + +# ------------------------------------------------------------------ R3-5: the root unit reaches external build items + +def test_r3_5_a_bad_root_unit_names_the_external_build_item_nodes(tmp_path): + ext = fx.sub_model_xml([fx.cube_object("1", 10)]) + root = fx.model_xml([], [("1", fx.tf(5, 5, 0), "/3D/Objects/e.model")], unit="yard") + sc = build(fx.three_mf(tmp_path / "y.3mf", root, {"3D/Objects/e.model": ext})) + lim = next(l for l in sc["limitations"] if l["code"] == "UNSUPPORTED_UNIT") + assert lim["target_ids"] == ["b0"] # its translation is in the unrecognized unit + assert sc["findings"] == [] + + +# ------------------------------------------------------------------ R3-6: a non-ASCII solid name is text + +def test_r3_6_an_ascii_stl_with_a_non_ascii_name_loads(tmp_path): + facet = "facet normal 0 0 0\nouter loop\nvertex 0 0 0\nvertex 1 0 0\nvertex 0 1 0\nendloop\nendfacet\n" + path = tmp_path / "wuerfel.stl" + path.write_bytes(("solid Würfel_größe\n" + facet * 3 + "endsolid Würfel_größe\n").encode("utf-8")) + assert scene.build_scene_dict(str(path), REV)["counts"]["triangles"] == 3 + binary = tmp_path / "bin.stl" # high bytes that are NOT UTF-8 stay binary + binary.write_bytes(bytes(range(128, 256)) * 3 + struct.pack("') == ("inch", 25.4) + assert units.unit_of(b"") == ("foot", 304.8) + assert units.unit_of(b"") == ("millimeter", 1.0) # absent attribute = default + assert units.unit_of(b'') == ("yard", None) # unknown is surfaced, not guessed + assert units.mm_per_unit(b'') == 1.0 # legacy readers keep treating it as mm + info = units.read_unit(b'') # not a element + assert (info.recognized, info.declared, info.factor) == (False, False, 1.0) # unreadable is not "absent" + assert units.read_unit(b"this is not xml").recognized is False + assert units.read_unit(b"").recognized is False and units.read_unit(b"").recognized is True + assert units.scale_translation([1, 0, 0, 0, 1, 0, 0, 0, 1, 2, 3, 4], 25.4)[9:] == [2 * 25.4, 3 * 25.4, 4 * 25.4] + same = [1, 0, 0, 0, 1, 0, 0, 0, 1, 2, 3, 4] + assert units.scale_translation(same, 1.0) is same + + +# ------------------------------------------------------------------ every dimension source + +@pytest.mark.parametrize("unit,factor", UNITS) +def test_project_info_and_its_service_wrappers(tmp_path, unit, factor): + from snapstudio_api import service + from snapstudio_core.intelligence import project_info + path = project(tmp_path, unit) + want = {"x": 30.0, "y": 30.0, "z": 30.0} + assert project_info(path)["dimensions_mm"] == want + assert service.insights(path)["dimensions_mm"] == want # DesignHealth data path + assert service.strategy_recommend(path)["signals"]["dimensions_mm"] == want # strategy signals (~490/522/541) + + +@pytest.mark.parametrize("unit,factor", UNITS) +def test_canonical_and_layout_and_validation_report(tmp_path, unit, factor): + from snapstudio_api import service + from snapstudio_core import layout + path = project(tmp_path, unit) + assert service.canonical(path)["dimensions_mm"] == {"x": 30.0, "y": 30.0, "z": 30.0} # canonical.py:92 + rep = service.report(path) # validation_report.py:35 + fits = next(c for c in rep["checks"] if c["name"] == "Fits the print bed") + assert fits["status"] == "pass" and "30.0 × 30.0 × 30.0 mm" in fits["detail"] + lay = layout.assess_layout(path) # layout.py:55 + assert lay["status"] == "pass", lay + + +def test_twelve_inch_part_reads_304_8_mm_everywhere(tmp_path): + from snapstudio_api import service + from snapstudio_core import geometry, layout, plate_placement, placement, scale_doctor + from snapstudio_core.intelligence import project_info + path = twelve_inch(tmp_path) + full = {"x": 304.8, "y": 304.8, "z": 304.8} + assert project_info(path)["dimensions_mm"] == full + assert service.insights(path)["dimensions_mm"] == full + assert service.canonical(path)["dimensions_mm"] == full + assert service.strategy_recommend(path)["signals"]["dimensions_mm"] == full + # bed_fit (via its service caller): the model does NOT fit a 270 mm plate + bf = service.bed_fit(path) + assert bf["dims_mm"] == full and bf["overall_level"] == "risk" + assert any("305" in f["text"] for f in bf["findings"]) + # validation report + fits = next(c for c in service.report(path)["checks"] if c["name"] == "Fits the print bed") + assert fits["status"] == "warn" and "304.8" in fits["detail"] + # layout + assert layout.assess_layout(path)["status"] == "fail" + # geometry + item = geometry.build_item_dims(path)[0] + assert item["dimensions"] == {"x": 304.8, "y": 304.8, "z": 304.8} + mesh = geometry.load_mesh(path) + assert max(v[0] for v in mesh.verts) == pytest.approx(304.8) + # placement + obj = placement.read_objects(composite(tmp_path, "inch", 12, (0, 0, 0)))["objects"][0] + assert obj["footprint"]["width"] == pytest.approx(304.8) + assessed = plate_placement.assess(path) + assert assessed["items"][0]["dimensions"] == full and assessed["off_plate"] + # scale doctor + opts = scale_doctor.scale_options(path) + assert opts["current_parts"][0]["dimensions"] == full + prev = scale_doctor.preview(path, 100) + assert prev["original_dimensions"] == full and prev["fits_build_volume"] is False + + +@pytest.mark.parametrize("unit,factor", UNITS) +def test_build_item_translation_is_in_the_models_unit(tmp_path, unit, factor): + """A transform's translation is a length in the model's unit too (an inch offset is not 1 mm).""" + from snapstudio_core import geometry, placement + path = project(tmp_path, unit, size_mm=20.0, at_units=(10.0 / factor, 20.0 / factor, 0.0)) + item = geometry.build_item_dims(path)[0] + assert item["bounds"]["min"][0] == pytest.approx(10.0, abs=1e-6) + assert item["bounds"]["max"][1] == pytest.approx(40.0, abs=1e-6) + comp = composite(tmp_path, unit, 20.0 / factor, (10.0 / factor, 20.0 / factor, 0.0), name="comp.3mf") + fp = placement.read_objects(comp)["objects"][0]["footprint"] + assert (fp["min_x"], fp["max_y"]) == (pytest.approx(10.0, abs=1e-6), pytest.approx(40.0, abs=1e-6)) + + +def test_plate_placement_judges_an_inch_project_in_millimetres(tmp_path): + """3 inches in is 76.2 mm in: on the plate. 12 inches in is 304.8 mm in: off it.""" + from snapstudio_core import plate_placement + inside = project(tmp_path, "inch", size_mm=25.4, at_units=(3.0, 3.0, 0.0), name="a.3mf") + r = plate_placement.assess(inside) + assert not r["off_plate"] and r["items"][0]["position"] == {"x": 88.9, "y": 88.9} + outside = project(tmp_path, "inch", size_mm=25.4, at_units=(12.0, 3.0, 0.0), name="b.3mf") + assert plate_placement.assess(outside)["off_plate"] + + +def test_sub_model_units_are_per_part(tmp_path): + from snapstudio_core import geometry, placement + from snapstudio_core.intelligence import project_info + sub = fx.sub_model_xml([fx.cube_object("1", 2)], unit="centimeter") # a 20 mm cube + root = fx.model_xml([fx.composite_object("10", [("1", "/3D/Objects/a.model", None)])], [("10", fx.tf(5, 0, 0))]) + path = str(fx.three_mf(tmp_path / "m.3mf", root, {"3D/Objects/a.model": sub})) + assert project_info(path)["dimensions_mm"] == {"x": 20.0, "y": 20.0, "z": 20.0} + item = geometry.build_item_dims(path)[0] + assert item["bounds"]["min"][0] == pytest.approx(5.0) and item["bounds"]["max"][0] == pytest.approx(25.0) + assert placement.read_objects(path)["objects"][0]["footprint"]["width"] == pytest.approx(20.0) + + +# ------------------------------------------------------------------ millimetre behaviour is unchanged + +def test_millimetre_projects_read_exactly_as_before(tmp_path): + from snapstudio_core import geometry, placement + from snapstudio_core.intelligence import project_info + path = str(fx.three_mf(tmp_path / "mm.3mf", fx.model_xml([fx.cube_object("1", 12.3456)], [("1", fx.tf(1.25, 2.5, 0))]))) + assert project_info(path)["dimensions_mm"] == {"x": 12.3, "y": 12.3, "z": 12.3} + item = geometry.build_item_dims(path)[0] + assert item["bounds"] == {"min": (1.25, 2.5, 0.0), "max": (1.25 + 12.3456, 2.5 + 12.3456, 12.3456)} + fp = placement.read_objects(composite(tmp_path, "millimeter", 12.3456, (1.25, 2.5, 0)))["objects"][0]["footprint"] + assert (fp["min_x"], fp["min_y"]) == (1.25, 2.5) + # no unit attribute at all is millimetre + bare = fx.model_xml([fx.cube_object("1", 7)], [("1", None)]).replace(' unit="millimeter"', "") + assert project_info(str(fx.three_mf(tmp_path / "bare.3mf", bare)))["dimensions_mm"] == {"x": 7.0, "y": 7.0, "z": 7.0} + + +# ------------------------------------------------------------------ repair round 1: A3 / D1 / A15 + +def _root_translation(copy_path): + import re + import zipfile + with zipfile.ZipFile(copy_path) as z: + text = z.read("3D/3dmodel.model").decode() + return [float(x) for x in re.search(r"]*transform=\"([^\"]*)\"", text).group(1).split()] + + +def test_d1_the_placement_fix_moves_an_inch_model_by_the_right_amount(tmp_path): + from snapstudio_core import plate_placement + path = project(tmp_path, "inch", size_mm=25.4, at_units=(11.0, 3.0, 0.0)) + assert plate_placement.assess(path)["off_plate"] # 11 in + 1 in = 304.8 mm: off the 270 mm plate + res = plate_placement.prepare_placed_copy(path, out_dir=str(tmp_path / "out")) + assert res["ok"] is True, res.get("reason") + tx, ty, tz = _root_translation(res["output_path"])[9:12] + dx, dy = res["offset_mm"]["x"], res["offset_mm"]["y"] + assert tx == pytest.approx(11 + dx / 25.4, abs=1e-3) and ty == pytest.approx(3 + dy / 25.4, abs=1e-3) and tz == 0 + assert res["after"]["available"] and not res["after"]["off_plate"] # and it really lands on the plate + assert plate_placement.assess(res["output_path"])["off_plate"] == [] + + +def test_d1_a_move_that_did_not_work_leaves_no_copy_on_disk(tmp_path, monkeypatch): + from snapstudio_core import plate_placement + path = project(tmp_path, "inch", size_mm=25.4, at_units=(11.0, 3.0, 0.0)) + monkeypatch.setattr(plate_placement, "_rewrite_items", lambda raw, offset_for: (raw, 1)) # a move that moves nothing + monkeypatch.setattr(plate_placement, "verify_only_placement_moved", lambda a, b: {"passed": True, "checks": []}) + out = tmp_path / "out" + res = plate_placement.prepare_placed_copy(path, out_dir=str(out)) + assert res["ok"] is False and "output_path" not in res + assert not list(out.glob("*.3mf")) + + +def test_d1_millimetre_models_get_exactly_the_offset_they_always_got(tmp_path): + from snapstudio_core import plate_placement + path = str(fx.three_mf(tmp_path / "mm.3mf", fx.model_xml([fx.cube_object("1", 25.4)], [("1", fx.tf(290.0, 3.0, 0.0))]))) + res = plate_placement.prepare_placed_copy(path, out_dir=str(tmp_path / "out")) + assert res["ok"] is True + tx, ty, _ = _root_translation(res["output_path"])[9:12] + assert tx == pytest.approx(290.0 + res["offset_mm"]["x"], abs=1e-4) + + +def test_a15_placement_uses_the_shared_translation_scaler(): + from snapstudio_core import placement + assert not hasattr(placement, "_scale_translation") + rows = ((1, 0, 0), (0, 1, 0), (0, 0, 1), (1.0, 2.0, 3.0)) + assert units.scale_translation(rows, 25.4)[3] == pytest.approx((25.4, 50.8, 76.2)) + assert units.scale_translation(rows, 1.0) is rows + assert units.scale_translation(None, 25.4) is None diff --git a/backend/tools/scene_benchmark.py b/backend/tools/scene_benchmark.py new file mode 100644 index 00000000..fa66c50a --- /dev/null +++ b/backend/tools/scene_benchmark.py @@ -0,0 +1,303 @@ +"""Benchmark and budget-coverage report for scene/1. + + py -3.13 backend/tools/scene_benchmark.py --out docs/testing/scene-contract/results.json \ + [--real-world DIR] [--local-root DIR ...] [--local-sample 5] + +Measures, per corpus item: whether it fits the budgets, parse+encode time (cold and warm best-of-N), +peak extra memory (tracemalloc peak AND a sampled working-set delta), response size and counts. Then the +hard gates (100k-triangle fixture, cancel latency, 50-cycle leak check) and the coverage fraction. + +Privacy: items from outside the repository (``--local-root``) are recorded ONLY as anonymized ids +``P-01..P-05`` with byte size and triangle count. Names, paths and contents are never written. Nothing +here modifies any input; every file is only read. +""" +from __future__ import annotations + +import argparse +import json +import os +import platform +import statistics +import sys +import tempfile +import threading +import time +import tracemalloc +from pathlib import Path + +HERE = Path(__file__).resolve().parent +BACKEND = HERE.parent +REPO = BACKEND.parent +sys.path.insert(0, str(BACKEND)) + +from snapstudio_core import scene, scene_limits as L # noqa: E402 +from tests import scene_fixtures as fx # noqa: E402 + +MIB = 1024 * 1024 + + +def rss_bytes() -> int: + if sys.platform == "win32": + import ctypes + from ctypes import wintypes + + class PMC(ctypes.Structure): + _fields_ = [("cb", wintypes.DWORD), ("PageFaultCount", wintypes.DWORD), + ("PeakWorkingSetSize", ctypes.c_size_t), ("WorkingSetSize", ctypes.c_size_t), + ("QuotaPeakPagedPoolUsage", ctypes.c_size_t), ("QuotaPagedPoolUsage", ctypes.c_size_t), + ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t), ("QuotaNonPagedPoolUsage", ctypes.c_size_t), + ("PagefileUsage", ctypes.c_size_t), ("PeakPagefileUsage", ctypes.c_size_t)] + k32 = ctypes.WinDLL("kernel32") + psapi = ctypes.WinDLL("psapi") + k32.GetCurrentProcess.restype = wintypes.HANDLE + psapi.GetProcessMemoryInfo.argtypes = [wintypes.HANDLE, ctypes.POINTER(PMC), wintypes.DWORD] + counters = PMC() + counters.cb = ctypes.sizeof(PMC) + psapi.GetProcessMemoryInfo(k32.GetCurrentProcess(), ctypes.byref(counters), counters.cb) + return int(counters.WorkingSetSize) + with open("/proc/self/statm") as fh: + return int(fh.read().split()[1]) * os.sysconf("SC_PAGE_SIZE") + + +def machine() -> dict: + total = None + try: + if sys.platform == "win32": + import ctypes + + class MS(ctypes.Structure): + _fields_ = [("l", ctypes.c_ulong), ("m", ctypes.c_ulong), ("tp", ctypes.c_ulonglong), ("ap", ctypes.c_ulonglong), + ("tpf", ctypes.c_ulonglong), ("apf", ctypes.c_ulonglong), ("tv", ctypes.c_ulonglong), + ("av", ctypes.c_ulonglong), ("ae", ctypes.c_ulonglong)] + ms = MS() + ms.l = ctypes.sizeof(MS) + ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(ms)) + total = ms.tp + else: + total = os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") + except Exception: # noqa: BLE001 + pass + return {"platform": platform.platform(), "python": platform.python_version(), "cpu_count": os.cpu_count(), + "processor": platform.processor() or None, "ram_gib": round(total / 1024 ** 3, 1) if total else None} + + +class Sampler(threading.Thread): + """Samples the working set while a measurement runs.""" + + def __init__(self) -> None: + super().__init__(daemon=True) + self.stop = threading.Event() + self.peak = 0 + + def run(self) -> None: + while not self.stop.is_set(): + self.peak = max(self.peak, rss_bytes()) + time.sleep(0.005) + + +LIFTED = {"MAX_PARSED_TRIANGLES": 2_000_000, "MAX_VERTICES": 2_000_000, "MAX_RENDERED_TRIANGLES": 2_000_000, + "MAX_RESPONSE_BYTES": 256 * MIB, "MODEL_XML_BYTES": 512 * MIB, "ARCHIVE_EXPANSION_BYTES": 1024 * MIB, + "MAX_NODES": 50_000} + + +def lifted(path: str) -> dict: + """Evidence for a budget decision: what the refused file would cost if the budgets were raised + (to 2M triangles / 256 MiB). Reports what it measures; never ships those numbers as limits.""" + saved = {k: getattr(L, k) for k in LIFTED} + try: + for k, v in LIFTED.items(): + setattr(L, k, v) + t = time.perf_counter() + try: + sc = scene.build_scene_dict(path, "0" * 64) + body = scene.serialize(sc) + except scene.SceneError as exc: + return {"still_refused": exc.code, "message": exc.message} + return {"triangles": sc["counts"]["triangles"], "rendered_triangles": sc["counts"]["rendered_triangles"], + "vertices": sc["counts"]["vertices"], "nodes": sc["counts"]["nodes"], "response_bytes": len(body), + "time_s": round(time.perf_counter() - t, 2)} + finally: + for k, v in saved.items(): + setattr(L, k, v) + + +def measure(path: str, warm_runs: int) -> dict: + size = os.path.getsize(path) + rec: dict = {"bytes": size} + ceiling = size > L.SNAPSHOT_MAX_BYTES + rec["job_gate"] = "SNAPSHOT_CEILING" if ceiling else "ok" + try: + t = time.perf_counter() + body = scene.build_scene(path, "0" * 64) + cold = time.perf_counter() - t + except scene.SceneError as exc: + rec.update(result="refused", code=exc.code, message=exc.message, fits_budgets=False) + rec["lifted"] = lifted(path) + return rec + sc = json.loads(body) + rec.update(result="ok", code=None, status=sc["status"], fits_budgets=not ceiling, + triangles=sc["counts"]["triangles"], rendered_triangles=sc["counts"]["rendered_triangles"], + vertices=sc["counts"]["vertices"], nodes=sc["counts"]["nodes"], meshes=sc["counts"]["meshes"], + response_bytes=len(body), limitations=[l["code"] for l in sc["limitations"]], + cold_s=round(cold, 3)) + times = [] + for _ in range(warm_runs): + t = time.perf_counter() + scene.build_scene(path, "0" * 64) + times.append(time.perf_counter() - t) + rec["warm_best_s"] = round(min(times), 3) + rec["warm_median_s"] = round(statistics.median(times), 3) + base = rss_bytes() + sampler = Sampler() + sampler.start() + tracemalloc.start() + scene.build_scene(path, "0" * 64) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + sampler.stop.set() + sampler.join() + rec["tracemalloc_peak_mib"] = round(peak / MIB, 1) + rec["rss_delta_mib"] = round(max(0, sampler.peak - base) / MIB, 1) + return rec + + +def corpus(tmp: Path, args) -> list[tuple[str, str, str]]: + """(id, class, path). Synthetic fixtures are authored here; nothing private is named.""" + items: list[tuple[str, str, str]] = [] + items.append(("S-cube", "synthetic", str(fx.plain_cube_3mf(tmp / "s-cube.3mf")))) + items.append(("S-bambu-3parts", "synthetic", str(fx.bambu_project( + tmp / "s-bambu.3mf", parts=3, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])], + roles=["normal_part", "modifier_part", "negative_part"])))) + items.append(("S-prusa-volumes", "synthetic", str(fx.prusa_project( + tmp / "s-prusa.3mf", volumes=[(0, 11, "ModelPart"), (12, 23, "ParameterModifier")])))) + items.append(("S-stl", "synthetic", str(fx.binary_stl(tmp / "s.stl")))) + for n in (10_000, 50_000, 100_000): + items.append((f"S-grid-{n // 1000}k", "synthetic", str(fx.big_grid_3mf(tmp / f"s-grid-{n}.3mf", n)))) + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(i * 12.0 % 250, (i // 20) * 12.0, 0)) for i in range(300)]) + items.append(("S-instances-300", "synthetic", str(fx.three_mf(tmp / "s-inst.3mf", root)))) + items.append(("S-over-budget", "synthetic-over-budget", str(fx.big_grid_3mf(tmp / "s-over.3mf", 280_000)))) + for p in sorted((REPO / "examples").iterdir()): + if p.suffix.lower() in (".3mf", ".stl") and ".orig." not in p.name: + items.append((f"E-{p.stem}", "repo-example", str(p))) + rw = Path(args.real_world) if args.real_world else BACKEND / "tests" / "fixtures" / "real-world" + if rw.is_dir(): + for p in sorted(rw.glob("*.3mf")): + if ".orig." not in p.name: + items.append((f"R-{p.stem}", "real-world", str(p))) + pool = sorted((p for d in args.local_root or [] for p in Path(d).rglob("*") + if p.suffix.lower() in (".3mf", ".stl") and p.is_file()), + key=lambda p: (p.stat().st_size, str(p))) + if pool: # evenly spaced by size across ALL local roots; only the anonymized id is ever recorded + picks = [pool[int(i * (len(pool) - 1) / max(1, args.local_sample - 1))] for i in range(args.local_sample)] + for n, p in enumerate(dict.fromkeys(picks), 1): + items.append((f"P-{n:02d}", "local-anonymized", str(p))) + return items + + +def survey(roots: list[str]) -> dict: + """Size-only survey of local projects (no names): how many are over the snapshot ceiling.""" + sizes = [] + for d in roots: + for p in Path(d).rglob("*"): + if p.suffix.lower() == ".3mf" and p.is_file(): + sizes.append(p.stat().st_size) + if not sizes: + return {} + over = sum(1 for s in sizes if s > L.SNAPSHOT_MAX_BYTES) + sizes.sort() + return {"files": len(sizes), "over_snapshot_ceiling": over, "fraction_within_ceiling": round(1 - over / len(sizes), 3), + "median_mib": round(statistics.median(sizes) / MIB, 1), "p90_mib": round(sizes[int(0.9 * (len(sizes) - 1))] / MIB, 1)} + + +def gates(tmp: Path) -> dict: + sys.path.insert(0, str(BACKEND)) + from snapstudio_api import scene_jobs as sj + + out: dict = {} + big = fx.big_grid_3mf(tmp / "gate-100k.3mf", 100_000) + m = measure(str(big), 3) + out["fixture_100k_triangles"] = {"parse_encode_warm_s": m.get("warm_best_s"), "cold_s": m.get("cold_s"), + "extra_peak_mib_tracemalloc": m.get("tracemalloc_peak_mib"), + "extra_peak_mib_rss": m.get("rss_delta_mib"), "response_bytes": m.get("response_bytes"), + "gate_time_le_10s": (m.get("warm_best_s") or 1e9) <= 10, + "gate_mem_le_500mib": max(m.get("tracemalloc_peak_mib") or 0, m.get("rss_delta_mib") or 0) <= 500, + "target_time_le_3s": (m.get("warm_best_s") or 1e9) <= 3, + "target_mem_le_300mib": max(m.get("tracemalloc_peak_mib") or 0, m.get("rss_delta_mib") or 0) <= 300, + "gate_response_le_8mib": (m.get("response_bytes") or 1e9) <= L.MAX_RESPONSE_BYTES} + jobs = sj.SceneJobs(snapshot_root=str(tmp / "snap")) + latencies = [] + for _ in range(5): + job = jobs.start(str(big), f"bench-cancel-{time.monotonic_ns()}") + end = time.monotonic() + 10 + while time.monotonic() < end and jobs.status(job["job_id"])["stage"] not in ("parsing", "encoding"): + time.sleep(0.005) + time.sleep(0.15) + t = time.perf_counter() + jobs.cancel(job["job_id"]) + while jobs.worker_alive(): + time.sleep(0.002) + latencies.append(time.perf_counter() - t) + out["cancel_latency_s"] = {"max": round(max(latencies), 3), "median": round(statistics.median(latencies), 3), + "gate_le_2s": max(latencies) <= 2.0} + small = str(fx.plain_cube_3mf(tmp / "leak-small.3mf")) + mid = str(fx.big_grid_3mf(tmp / "leak-mid.3mf", 20_000)) + base_threads = threading.active_count() + wedged = False + leak = sj.SceneJobs(snapshot_root=str(tmp / "snap2")) + for i in range(50): + job = leak.start(mid if i % 2 else small, f"bench-leak-{i:04d}-{time.monotonic_ns()}") + if i % 3 == 0: + time.sleep(0.01) + leak.cancel(job["job_id"]) + wedged = wedged or leak.wedged() + leak.close() + end = time.monotonic() + 10 + while leak.worker_alive() and time.monotonic() < end: + time.sleep(0.01) + snaps = os.listdir(leak.snapshot_dir()) if leak.snapshot_dir() and os.path.isdir(leak.snapshot_dir()) else [] + out["leak_50_cycles"] = {"retained_jobs": leak.retained(), "cap": L.MAX_TERMINAL_JOBS, "temp_snapshots": len(snaps), + "threads_before": base_threads, "threads_after": threading.active_count(), + "worker_wedged_seen": wedged, + "gate_pass": leak.retained() <= L.MAX_TERMINAL_JOBS and not snaps + and threading.active_count() == base_threads and not wedged} + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--out", required=True) + ap.add_argument("--real-world", help="folder holding the fetched real-world 3MF fixtures") + ap.add_argument("--local-root", action="append", help="folder of local projects (anonymized ids only)") + ap.add_argument("--local-sample", type=int, default=5) + ap.add_argument("--warm-runs", type=int, default=3) + args = ap.parse_args() + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + rows = [] + for ident, klass, path in corpus(tmp, args): + rec = measure(path, args.warm_runs) + rec.update(id=ident, **{"class": klass}) + rows.append(rec) + print(f"{ident:24s} {klass:22s} {rec.get('result'):8s} {rec.get('code') or '':16s} " + f"{rec.get('warm_best_s', '-')!s:>7} s {rec.get('response_bytes', '-')!s:>9} B", flush=True) + results = {"schema": "scene-benchmark/1", "machine": machine(), "budgets": dict(L.LIMITS_ECHO), + "snapshot_ceiling_bytes": L.SNAPSHOT_MAX_BYTES, "items": rows} + within = [r for r in rows if r["class"] in ("repo-example", "real-world", "local-anonymized", "synthetic")] + results["coverage"] = { + "by_class": {k: {"items": sum(1 for r in rows if r["class"] == k), + "fit": sum(1 for r in rows if r["class"] == k and r.get("fits_budgets"))} + for k in sorted({r["class"] for r in rows})}, + "fraction_fitting_excluding_deliberate_over_budget": round( + sum(1 for r in within if r.get("fits_budgets")) / max(1, len(within)), 3)} + if args.local_root: + results["local_survey_3mf_sizes_only"] = survey(args.local_root) + results["gates"] = gates(tmp) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + print("wrote", out) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/testing/scene-contract/README.md b/docs/testing/scene-contract/README.md new file mode 100644 index 00000000..be80b37c --- /dev/null +++ b/docs/testing/scene-contract/README.md @@ -0,0 +1,291 @@ +# scene/1: the backend scene contract (PR 3a) + +The engine can now describe a project's geometry, placement and engine findings as one bounded, +machine-checked document (`scene/1`), built by a cancellable background job. This is the backend half +only. There is no viewer and no desktop change in this PR. + +* Contract: `backend/snapstudio_core/data/scene-1.schema.json` (JSON Schema 2020-12, frozen: every object + closed, every field required, unknown is `null`). +* Golden scene: `backend/tests/fixtures/scene/golden-scene-1.json`, written by hand and independently of the + builder by `make_golden.py` next to it. A test requires the real builder to reproduce it exactly. +* Builder: `backend/snapstudio_core/scene.py`. Every numeric budget lives in + `backend/snapstudio_core/scene_limits.py` and nowhere else. +* Jobs and routes: `backend/snapstudio_api/scene_jobs.py`; `POST /scene/start | status | result | cancel` + (existing `X-Auth-Token`; the result body is serialized once and sent by the new `_send_bytes`). +* Units fix: `backend/snapstudio_core/units.py`, applied to every dimension source (below). +* Measurements: `results.json` in this folder, produced by `backend/tools/scene_benchmark.py`. + +Not covered: any viewer, the Tauri window, WebKitGTK, a screen reader, a real printer, and Linux (see +"What was not run"). + +## What a scene is, and is not + +A scene is a **consistent snapshot of the file as it was copied for the job**. The job copies the source once +into a private file, hashes it (the `revision` is the SHA-256 of those bytes) and parses only the copy, so +replacing the original while a job runs cannot tear the scene. `SOURCE_CHANGED` applies only to (a) a file that +changed while it was being copied, (b) a reused `request_id`+path whose file now hashes differently from that +job's revision, and (c) an `expected_revision` that differs from the job's. It does **not** apply to edits made +after the snapshot was taken. The original file is only ever read (a test compares its bytes and mtime before and +after; so does the job test). + +Scenes say what the file proves and no more: + +* **Placement.** STL files, models with no build item and unresolved placement are `placement_state: "unknown"`. + Their `world_mm` is an identity used only so something can be drawn. No bed-fit or size statement of any kind is + attached to an unknown-placement node. +* **Plates.** Membership is per instance, from `model_settings.config` (`plate: known | ambiguous | unknown`). + Plate origins are never invented (`origin_mm` is always `null`). A multi-plate project gets + `MULTI_PLATE_PLACEMENT_UNCHECKED` and no placement findings, because the grid spacing is not in the file. +* **Volumes and roles.** Every mesh partitions *all* its triangles into volumes (`part`, `modifier`, `negative`, + `support_enforcer`, `support_blocker`, `unknown`). The synthetic whole-mesh `part` volume + (`source: "object"`) is used ONLY for an object that has no volume metadata at all. Prusa volume ranges are read + by attribute name (the order of `firstid`/`lastid` does not matter); an unmapped gap, an overlap or an unreadable + range is `unknown` (`UNKNOWN_VOLUME_ROLE`), never a guess. Bambu/Orca part records are **per use**, so the role + lives on the node (`role_context`), not on the shared mesh: a mesh can be a negative part under one object and an + ordinary part under another, and a role given to a component assembly applies to everything beneath it. Decision + (differs from `placement.read_objects`, which defaults to `normal_part`): an object that HAS part records but none + for one of its components gives that component `role_context: "unknown"`, `UNKNOWN_VOLUME_ROLE` and status + `partial`, and no fit finding. A node's bounds, `printable` and every fit statement use only role=part geometry; + non-part geometry is carried (`has_non_part_volumes`) and excluded from fit. +* **Production-extension build items.** A build item's `p:path` names the model FILE its object lives in. Node + identity, repeated-instance counting and lookup use the pair (part path, object id); an unresolvable `p:path` is + `UNRESOLVED_REFERENCE`, never a fall back to the root file. Per the Production Extension a `p:path` must be an + absolute package path (leading `/`), and component `p:path` references are honoured only from the root model + part; relative paths, and component paths in any other part, are `UNRESOLVED_REFERENCE`. +* **Fail closed across files and duplicates.** Where the file cannot prove an association the answer is + unknown, not a guess: (1) when the project has slicer settings, every part beneath a build object that lives in + another file gets `role_context: "unknown"` (the part records are matched against root-file objects), with + `UNKNOWN_VOLUME_ROLE`, status `partial` and no fit finding; (2) plate records name a bare object id, so when two + build resources in different files share an id, neither can be proven on a plate (`plate.state: "ambiguous"`, + `PLATE_MEMBERSHIP_UNKNOWN`, no finding); (3) two settings records for the same (object id, part id) that disagree + make the role unknown, identical duplicates are fine; (4) `UNSUPPORTED_UNIT` lists, in `target_ids`, the nodes + living in the affected part(s), and `sources[].unit`/`mm_per_unit` are then the fallback millimetre values. +* **Bounded parsing everywhere.** Model parts, the Bambu settings, the Prusa config and every `.rels` are parsed + with a counter on completed elements (1,000,000 each) and every completed element is released as soon as its data + is consumed (the element itself only, never its siblings: records a parent reads later, such as a negative + ``, are never lost to an unknown neighbour), so a part under its byte cap that is made of millions of tiny + elements is refused early. A part record without an `id` still counts as "this object has part metadata" (the + components it cannot cover are `unknown`). A present-but-empty `p:path` is `UNRESOLVED_REFERENCE`, not absent. + When the ROOT model's unit is unrecognized, the top node of every build item is listed in the `UNSUPPORTED_UNIT` + `target_ids`, since each item's translation is written in that unit. STL files: the kind is + decided from the header AND the size; a binary-looking header over the triangle budget is refused before any + decoding or large allocation whether or not the size matches; ASCII is read in 1 MiB chunks with a 4 KiB line cap, + split on LF, CRLF and a lone CR. +* **Bed.** The U1 rectangle comes from Studio's own template (`x 0.5..270.5, y 1..271`, 0.5 mm inward margin, + the same numbers `plate_placement` uses). If the template yields no usable rectangle, `bed.policy` is + `"fallback"`, `BED_TEMPLATE_UNAVAILABLE` is reported and **all fit findings are suppressed**. +* **Findings** carry `kind` (`placement | size | note`), `scope` (`project | plate | instance`), the engine + that produced them (`scene`), a JSON pointer into the scene and target node ids. Scene-produced + out-of-bounds and size statements are `placement` and `size`; the viewer copy rule (PR 3b) is that the word + "fits" is never derived from a scene. +* **Repeated instances.** Several build items of one object id get their own nodes and their own geometric + world bounds, but `REPEATED_INSTANCE_PLACEMENT_UNVERIFIED` is reported and no finding is attached, because the + engine's existing placement answer for repeated items is wrong on `main` (see "Known limitations"). +* **Non-millimetre sources.** Coordinates are converted to millimetres, and `sources[]` records each part's + unit. Because the contract was written before the conversion was shared, placement findings are still + disabled for any non-mm source (`NON_MM_SOURCE_UNIT`); size findings are not. A declared unit outside the six + defined ones is NOT treated like an absent attribute: the scene reports `UNSUPPORTED_UNIT`, falls back to + millimetres in `sources[]` (documented) and makes no fit statement. +* **Plates.** If a project has plates but an instance is on none, it is `PLATE_MEMBERSHIP_UNKNOWN`, status + `partial`, with no fit finding. + +## Units fix (a correction of wrong claims, not a new feature) + +Several readers ignored the 3MF `` header, so a 12 inch part was reported as 12 mm and +every size-based bed-fit statement for a non-millimetre project was wrong. One shared helper +(`snapstudio_core/units.py`: the six-unit `mm_per_unit` table, a header reader and a translation scaler) is now +applied to **every** dimension source. The header is read from the REAL root `model` element with the XML parser +(first start event; namespace prefixes, comments, processing instructions and a long preamble are irrelevant, and +a `` inside a comment is not seen), by the same `resolve_unit` function the scene parser calls. +Absent means millimetre; an unrecognized value is a distinct, flagged outcome (`UnitInfo.recognized`), with the +legacy readers keeping their old millimetre fallback: + +| Source | Change | +|---|---| +| `intelligence.project_info` (feeds DesignHealth, `service.insights`, strategy signals, `bed_fit`, `validation_report`, `canonical`, `layout`, `mm_doctor`, `toolhead_fit`) | vertex coordinates scaled per model part by that part's own unit | +| `geometry.build_item_dims` (feeds `layout`, `scale_doctor`, `plate_placement.assess`) | vertices scaled per part; component and build-item **translations** scaled too (a translation is a length in the model's unit) | +| `geometry.load_mesh` (feeds `scale_doctor.preview`, `mesh_diagnostics`) | vertices scaled per part | +| `placement.read_objects` | mesh points and transform translations scaled (through the shared helper); its repeated-instance behaviour is untouched | +| `plate_placement.prepare_placed_copy` (the "move onto the plate" fix) | the offset it writes is computed in millimetres and is now converted back to the ROOT model's unit before it is added to a build-item translation. A move that does not achieve its goal no longer leaves its copy on disk | + +Behaviour for millimetre files is unchanged: the scale factor is exactly 1.0 and the code skips the multiplication. +Evidence: outputs of `project_info`, `build_item_dims`, `load_mesh`, `read_objects`, `plate_placement.assess` and +`scale_doctor.scale_options` were hashed over all 50 fixtures and examples in the tree on `main` (b01f7d0) and on +this branch and compared: **identical** (sha256 `73a018e5...69ef` both sides). `tests/test_units_consumers.py` pins every consumer +named in the plan (`canonical`, `layout`, `validation_report`, `bed_fit` via `service.bed_fit`, strategy signals, +DesignHealth data path via `service.insights`, Scale Doctor, `plate_placement.assess`, `read_objects`) for all six +units, and asserts that a 12 inch part reads 304.8 mm everywhere. + +What this does **not** do (decision, plan v4 item 1): the size-based consumers still answer "does the model's +size fit the bed". They are not placement-aware. A small object translated off the plate, or separated instances +whose union is large, is still judged by size only. The scene's placement findings are the placement-aware +source. No consumer copy was edited here; qualifying those claims "by size" is PR 3c. + +## Jobs + +One worker thread, at most one queued job (newest wins; a running job is marked cancelled at once and the +replacement waits, queued, until the cancelled thread observes the flag and exits). States: +`queued | running | succeeded | failed | cancelled`. Terminal states are immutable and the first terminal wins +(a worker finishing after a cancel discards its result). 60 s deadline from the moment the job starts +running. A terminal job is kept 120 s or until more than 8 terminal jobs exist. An evicted or unknown id answers +404 `EXPIRED`; there is no stored "expired" state. + +| Route | Answers | +|---|---| +| `/scene/start {path, request_id}` | 200 job; 400 `INVALID_REQUEST`; 422 `UNSUPPORTED_FORMAT`; 409 `INVALID_REQUEST` (request_id reused for another path) or `SOURCE_CHANGED`; 503 `WORKER_WEDGED` | +| `/scene/status {job_id}` | 200 `{state, stage, completed, total, error, revision}`; 404 `EXPIRED` | +| `/scene/result {job_id, expected_revision?}` | 200 scene; 422 `{error: }` for a failed job; 409 `CANCELLED` / `NOT_READY` / `SOURCE_CHANGED`; 404 `EXPIRED` | +| `/scene/cancel {job_id}` | 200 status (idempotent; a late cancel on a terminal job is a no-op) | + +Error codes: `INVALID_REQUEST UNSUPPORTED_FORMAT INVALID_ARCHIVE INVALID_GEOMETRY UNRESOLVED_REFERENCE +LIMIT_EXCEEDED SOURCE_CHANGED CANCELLED TIMEOUT EXPIRED NOT_READY WORKER_WEDGED INTERNAL` (`BUSY` is not an +error code: a new start replaces the client's own earlier work). + +Interruptibility: archive reads are at most 1 MiB, lxml `iterparse` checks the cancel/deadline hook every 1,000 +events and on every read, geometry transform/encode runs in batches of at most 5,000 triangles. Backstop: if the +worker is still alive 5 s after a cancel or the deadline, the job is reported failed `TIMEOUT` (deadline case) and +every new start answers 503 `WORKER_WEDGED` until the thread exits. +**Accepted limitation:** a single stuck native or filesystem call cannot be interrupted in-process (there is no +multiprocessing in the frozen sidecar); the wedged state is the fail-closed backstop. + +Snapshots live in `/scene-tmp/-/`. The worker deletes its own snapshot in its `finally` before it +sets `worker_done`; eviction only touches terminal jobs whose worker is done (missing-file tolerant); cancel never +deletes a file. A snapshot file that cannot be deleted (a Windows sharing violation) stays on a bounded retry list +with back-off until it is gone, and the per-process folder is removed best-effort at shutdown. The per-process +folder is re-created for every job. A startup sweep removes a `-` folder ONLY when its owner pid is +not running AND the folder is at least 10 minutes old; a folder owned by a live process is never removed, +however old. Sharing violations are tolerated. + +## Budgets (provisional) and the coverage decision + +| Budget | Value | +|---|---| +| Snapshot copy ceiling / archive expansion | 128 MiB / 128 MiB | +| Model XML (all `.model` parts) | 32 MiB | +| Rendered triangles (every repetition counted) / distinct parsed triangles | 250,000 / 250,000 | +| Distinct vertices | 300,000 | +| Nodes / depth | 2,000 / 64 | +| Whole serialized response | 8 MiB (exact bytes; geometry pre-check refuses before encoding) | +| Ids | 128 characters, by construction (a long ancestry is replaced by a deterministic digest) | +| Part names carried on nodes / `` definitions / XML elements | 256 characters / 20,000 / 1,000,000, all counted while parsing | +| Findings / limitations | 500 / 50 | + +Exceeding any of these is `LIMIT_EXCEEDED`, never a shortened scene. + +### Corpus and results + +Qualifications: the four real-world fixtures are upstream files fetched by `fetch_real_world.py` and are absent from +this worktree (the benchmark read them from the main checkout); all five local samples are STL files (89% of the 11,604 local .stl/.3mf files are STL), so they exercise the STL path and say little about 3MF project structure; they +were chosen by size quantile, not at random; the fixture +called "100k" is 99,458 triangles; timings are one machine, one run. + +Machine: Windows 11, Intel64 Family 6 Model 183 (32 logical CPUs), 63.8 GiB RAM, Python 3.13.14. Numbers are one +run on a quiet machine; warm = best of 3. Corpus: authored synthetic fixtures, the repo examples, the four +real-world fixtures (fetched by `fetch_real_world.py`, not committed) and five local projects chosen by size +quantile and recorded **only** as `P-01..P-05` (names and contents are never written). All 19 repo examples +fit and are summarized in `results.json`; four representative ones are shown here. + +| id | class | size | result | triangles (rendered) | nodes | response | parse+encode warm | tracemalloc peak | RSS delta | +|---|---|---|---|---|---|---|---|---|---| +| S-cube | synthetic | 1.5 KiB | ok (complete) | 12 (12) | 1 | 2.0 KiB | 0.001 s | 0.1 MiB | 0.1 MiB | +| S-bambu-3parts | synthetic | 4.7 KiB | ok (complete) | 36 (36) | 4 | 5.0 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| S-prusa-volumes | synthetic | 2.6 KiB | ok (complete) | 24 (24) | 1 | 2.4 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| S-stl | synthetic | 0.7 KiB | ok (partial) | 12 (12) | 1 | 2.0 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| S-grid-10k | synthetic | 603.3 KiB | ok (complete) | 9,800 (9,800) | 1 | 233.6 KiB | 0.036 s | 1.3 MiB | 0.1 MiB | +| S-grid-50k | synthetic | 3.11 MiB | ok (complete) | 49,928 (49,928) | 1 | 1.15 MiB | 0.176 s | 5.6 MiB | 3.8 MiB | +| S-grid-100k | synthetic | 6.22 MiB | ok (complete) | 99,458 (99,458) | 1 | 2.28 MiB | 0.405 s | 10.9 MiB | 3.5 MiB | +| S-instances-300 | synthetic | 19.9 KiB | ok (partial) | 12 (3,600) | 300 | 187.3 KiB | 0.008 s | 1.8 MiB | 0.0 MiB | +| S-over-budget | synthetic (deliberately over) | 17.83 MiB | **refused `LIMIT_EXCEEDED`** | 279,752 | - | 6.42 MiB if budgets raised | 1.15 s if raised | - | - | +| E-demo_offplate_foreign | repo example | 9.7 KiB | ok (complete) | 12 (12) | 2 | 2.9 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| E-demo_u1_showcase | repo example | 1.7 KiB | ok (partial) | 12 (12) | 1 | 2.0 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| E-sample_cube | repo example | 0.7 KiB | ok (partial) | 12 (12) | 1 | 2.0 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| E-sample_cube_SnapmakerU1 | repo example | 9.7 KiB | ok (complete) | 12 (12) | 2 | 2.7 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| R-bambu-pa-pattern | real-world | 22.5 KiB | ok (complete) | 12 (12) | 2 | 2.8 KiB | 0.003 s | 0.1 MiB | 0.0 MiB | +| R-orca-badge | real-world | 797.9 KiB | ok (complete) | 72,586 (72,586) | 42 | 1.70 MiB | 0.274 s | 8.2 MiB | 3.0 MiB | +| R-orca-pa-line-dual | real-world | 29.0 KiB | ok (complete) | 24 (192) | 24 | 18.5 KiB | 0.002 s | 0.2 MiB | 0.0 MiB | +| R-prusa-seam-test | real-world | 2.40 MiB | ok (complete) | 225,154 (225,154) | 1 | 5.15 MiB | 0.87 s | 25.0 MiB | 30.2 MiB | +| P-01 | local | 0.5 KiB | ok (partial) | 8 (8) | 1 | 1.9 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| P-02 | local | 963.6 KiB | ok (partial) | 19,733 (19,733) | 1 | 464.3 KiB | 0.02 s | 4.5 MiB | 3.2 MiB | +| P-03 | local | 3.79 MiB | ok (partial) | 79,518 (79,518) | 1 | 1.82 MiB | 0.097 s | 18.4 MiB | 24.9 MiB | +| P-04 | local | 15.60 MiB | **refused `LIMIT_EXCEEDED`** | 327,141 | - | 7.49 MiB if budgets raised | 0.43 s if raised | - | - | +| P-05 | local | 1454.44 MiB | **refused `LIMIT_EXCEEDED`** (over the 128 MiB snapshot ceiling) | - | - | - | - | - | - | + +Coverage (fraction of the corpus that fits, excluding the one deliberately over-budget fixture): **0.944** +(repo examples 19/19, synthetic 8/8, real-world **4/4**, local **3/5**). A size-only survey of the 1,230 local `.3mf` +files under the maintainer's model folders found 126 (**10.2%**) larger than the 128 MiB snapshot ceiling (median 31.2 +MiB, p90 130.1 MiB); no names or paths are recorded. Some projects also exceed the triangle budget: the local `P-04` +has 327,141 triangles and `P-05` is over the ceiling (30.5 million triangles). + +### Budget decision (accepted) + +The first run, at the originally specified 100,000 triangles, fit only 3 of the 4 real-world fixtures: +`R-prusa-seam-test` has 225,154 triangles. Evidence: raised runs took about 1.2 s, and the response is 5.15 MiB. +The 8 MiB whole-body wire cap is the real wall, not time or memory. Decision: **`MAX_RENDERED_TRIANGLES` and +`MAX_PARSED_TRIANGLES` are now 250,000** (`scene_limits.py`, the only place they live); the 300,000 decoded-vertex +budget already covered the fixture (112,569 vertices) and is unchanged, as are the 8 MiB cap and every other gate. +Models over the budget stay refused with a message that names the count and the limit, for example +"The model has 327,141 triangles; Studio draws at most 250,000." (for a 3MF the count is "at least N", reported the +moment the budget is passed). `P-04` is such a model; its scene would be about 7.5 MiB. Chunked or binary mesh +transfer, or decimation, for larger models is a documented follow-up and is not part of this PR. The 128 MiB source +ceiling is unchanged; 10.2% of the surveyed local `.3mf` files exceed it and are refused. + +### Gates + +| Gate | Result | +|---|---| +| Whole-body response cap enforced on the exact bytes | pass (tests; largest measured scene 5.15 MiB) | +| Node / triangle / vertex / depth budgets enforced during traversal | pass (tests follow the constants) | +| Cancel observed within 2 s | pass: max 0.005 s over 5 cancels on the 100k fixture (and a test on a 90k scene) | +| 50 start/cancel cycles: retained <= cap, temp snapshots 0, threads at baseline | pass: retained 8 of 8, snapshots 0, threads 1 to 1 | +| `WORKER_WEDGED` never occurs in the 50-cycle run | pass (its fail-closed path is tested separately) | +| Repo examples fit | pass (19/19) | +| Four real-world fixtures fit | pass (4/4) after the budget decision above | +| 100k-triangle fixture: parse+encode <= 10 s, extra peak memory <= 500 MB | pass: 0.358 s, 11.0 MiB (tracemalloc) / 6.1 MiB (RSS) | +| Targets (not gates): <= 3 s and <= 300 MB on the 100k fixture | met | + +## Reproduce + +```text +cd backend +py -3.13 -m pip install -e ".[test]" +py -3.13 -m pytest -q tests/test_scene_contract.py tests/test_scene_build.py tests/test_scene_hostile.py \ + tests/test_scene_jobs.py tests/test_scene_api.py tests/test_units_consumers.py +py -3.13 tools/scene_benchmark.py --out ../docs/testing/scene-contract/results.json \ + --real-world tests/fixtures/real-world [--local-root ...] +``` + +## Known limitations (carry into the PR description) + +1. **Size-only consumers are not placement-aware.** DesignHealth, `bed_fit`, `validation_report` and the service + bed-fit path answer "does the SIZE fit". A small object translated off the plate is not detected by them, and + separated instances inflate the union. The scene's placement findings are the placement-aware source. Tracked + as issue #91; the copy change qualifying those claims "by size" is the planned PR 3c. +2. **Repeated build items of one object id are mis-placed by `plate_placement.assess` on `main`** + (`placement.py` keeps the last transform: a cube at X=500 and X=100 reports "inside"). Not fixed here (tracked as issue #93). The scene reports `REPEATED_INSTANCE_PLACEMENT_UNVERIFIED` and attaches no finding to those nodes. +3. **Plates.** Non-Bambu projects (Prusa, plain 3MF, STL) have no plate records (`PLATES_UNAVAILABLE`). Grid origins + of multi-plate projects are unknown, so multi-plate projects get no placement findings. +4. **Stuck native calls** cannot be interrupted in-process (fail-closed `WORKER_WEDGED` backstop, no multiprocessing). +5. **STL** is decoded incrementally: a binary header count over the budget is refused before any triangle is + decoded, decoding runs in batches of 5,000 triangles with the cancel hook between them, and ASCII is line-streamed + with a running counter. +6. **Budgets are provisional.** Models over 250,000 triangles (or over the 128 MiB source ceiling) are refused with `LIMIT_EXCEEDED`; chunked/binary transfer or decimation is a follow-up. +7. `jsonschema` is a test-only extra (`pip install -e ".[test]"`); the engine never imports it. + +## Deviations from the written plan + +* **Node ids.** `b.c.c...` does not fit 128 characters at 64 levels, so beyond 128 the ancestry is + replaced by a deterministic digest (`b3.~<32 hex>`); the 128-character contract is kept, not widened. The digest + (128 bits of SHA-256) gives practical collision resistance, not mathematical uniqueness; the builder asserts + uniqueness while building and fails with `INTERNAL` if two node ids ever collide. +* **Freeze order.** The JSON Schema was frozen first, but the golden scene and the fixture matrix were authored + after the first draft of the traversal, not before it. The golden is written independently of the builder and + the builder is held to it. +* **Size findings** are only emitted for nodes whose placement is known (plan v2: no bed-relative statement for + unknown placement), single-plate, template bed, non-repeated instances. +* **No `request_validation.py` change was needed**; the route validation reuses its existing helpers from + `scene_jobs.py`. + +## What was not run + +Linux (the POSIX branch of the pid-liveness check is exercised only by monkeypatch; the Linux CI job), the frozen sidecar, the packaged app, any +viewer, a real printer and any network. diff --git a/docs/testing/scene-contract/results.json b/docs/testing/scene-contract/results.json new file mode 100644 index 00000000..9629d874 --- /dev/null +++ b/docs/testing/scene-contract/results.json @@ -0,0 +1,915 @@ +{ + "schema": "scene-benchmark/1", + "machine": { + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.13.14", + "cpu_count": 32, + "processor": "Intel64 Family 6 Model 183 Stepping 1, GenuineIntel", + "ram_gib": 63.8 + }, + "budgets": { + "max_archive_bytes": 134217728, + "max_model_xml_bytes": 33554432, + "max_rendered_triangles": 250000, + "max_vertices": 300000, + "max_nodes": 2000, + "max_depth": 64, + "max_response_bytes": 8388608, + "max_findings": 500, + "max_limitations": 50, + "max_id_length": 128 + }, + "snapshot_ceiling_bytes": 134217728, + "items": [ + { + "bytes": 1553, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 1, + "meshes": 1, + "response_bytes": 2027, + "limitations": [ + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.005, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.1, + "id": "S-cube", + "class": "synthetic" + }, + { + "bytes": 4778, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 36, + "rendered_triangles": 36, + "vertices": 24, + "nodes": 4, + "meshes": 3, + "response_bytes": 5084, + "limitations": [], + "cold_s": 0.008, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "S-bambu-3parts", + "class": "synthetic" + }, + { + "bytes": 2690, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 24, + "rendered_triangles": 24, + "vertices": 16, + "nodes": 1, + "meshes": 1, + "response_bytes": 2454, + "limitations": [ + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "S-prusa-volumes", + "class": "synthetic" + }, + { + "bytes": 684, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 1, + "meshes": 1, + "response_bytes": 2049, + "limitations": [ + "UNKNOWN_PLACEMENT", + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.008, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "S-stl", + "class": "synthetic" + }, + { + "bytes": 617746, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 9800, + "rendered_triangles": 9800, + "vertices": 5041, + "nodes": 1, + "meshes": 1, + "response_bytes": 239176, + "limitations": [ + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.046, + "warm_best_s": 0.036, + "warm_median_s": 0.037, + "tracemalloc_peak_mib": 1.3, + "rss_delta_mib": 0.1, + "id": "S-grid-10k", + "class": "synthetic" + }, + { + "bytes": 3256096, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 49928, + "rendered_triangles": 49928, + "vertices": 25281, + "nodes": 1, + "meshes": 1, + "response_bytes": 1205070, + "limitations": [ + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.215, + "warm_best_s": 0.176, + "warm_median_s": 0.178, + "tracemalloc_peak_mib": 5.6, + "rss_delta_mib": 3.8, + "id": "S-grid-50k", + "class": "synthetic" + }, + { + "bytes": 6523679, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 99458, + "rendered_triangles": 99458, + "vertices": 50176, + "nodes": 1, + "meshes": 1, + "response_bytes": 2395870, + "limitations": [ + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.59, + "warm_best_s": 0.405, + "warm_median_s": 0.411, + "tracemalloc_peak_mib": 10.9, + "rss_delta_mib": 3.5, + "id": "S-grid-100k", + "class": "synthetic" + }, + { + "bytes": 20350, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 3600, + "vertices": 8, + "nodes": 300, + "meshes": 1, + "response_bytes": 191745, + "limitations": [ + "PLATES_UNAVAILABLE", + "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED" + ], + "cold_s": 0.019, + "warm_best_s": 0.008, + "warm_median_s": 0.011, + "tracemalloc_peak_mib": 1.8, + "rss_delta_mib": 0.0, + "id": "S-instances-300", + "class": "synthetic" + }, + { + "bytes": 18696925, + "job_gate": "ok", + "result": "refused", + "code": "LIMIT_EXCEEDED", + "message": "The project has at least 250,001 triangles; Studio draws at most 250,000.", + "fits_budgets": false, + "lifted": { + "triangles": 279752, + "rendered_triangles": 279752, + "vertices": 140625, + "nodes": 1, + "response_bytes": 6727764, + "time_s": 1.15 + }, + "id": "S-over-budget", + "class": "synthetic-over-budget" + }, + { + "bytes": 9899, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2974, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.002, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-demo_offplate_foreign", + "class": "repo-example" + }, + { + "bytes": 1790, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 1, + "meshes": 1, + "response_bytes": 2086, + "limitations": [ + "PLATE_MEMBERSHIP_UNKNOWN" + ], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.002, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-demo_u1_showcase", + "class": "repo-example" + }, + { + "bytes": 684, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 1, + "meshes": 1, + "response_bytes": 2049, + "limitations": [ + "UNKNOWN_PLACEMENT", + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.001, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube", + "class": "repo-example" + }, + { + "bytes": 9667, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.002, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_foreign", + "class": "repo-example" + }, + { + "bytes": 10157, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.001, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_foreign_SnapmakerU1", + "class": "repo-example" + }, + { + "bytes": 9951, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_scaled_100_U1", + "class": "repo-example" + }, + { + "bytes": 9950, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_SnapmakerU1", + "class": "repo-example" + }, + { + "bytes": 10155, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_SnapmakerU1_SnapmakerU1", + "class": "repo-example" + }, + { + "bytes": 9951, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.001, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.001, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_10", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.002, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_2", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.004, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_3", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_4", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_5", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.1, + "id": "E-sample_cube_U1_SnapmakerU1_6", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_7", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_8", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.004, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.1, + "id": "E-sample_cube_U1_SnapmakerU1_9", + "class": "repo-example" + }, + { + "bytes": 23059, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2875, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "R-bambu-pa-pattern", + "class": "real-world" + }, + { + "bytes": 817091, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 72586, + "rendered_triangles": 72586, + "vertices": 36353, + "nodes": 42, + "meshes": 39, + "response_bytes": 1786167, + "limitations": [], + "cold_s": 0.318, + "warm_best_s": 0.274, + "warm_median_s": 0.277, + "tracemalloc_peak_mib": 8.2, + "rss_delta_mib": 3.0, + "id": "R-orca-badge", + "class": "real-world" + }, + { + "bytes": 29649, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 24, + "rendered_triangles": 192, + "vertices": 16, + "nodes": 24, + "meshes": 2, + "response_bytes": 18971, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.002, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.2, + "rss_delta_mib": 0.0, + "id": "R-orca-pa-line-dual", + "class": "real-world" + }, + { + "bytes": 2514963, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 225154, + "rendered_triangles": 225154, + "vertices": 112569, + "nodes": 1, + "meshes": 1, + "response_bytes": 5405324, + "limitations": [ + "PLATES_UNAVAILABLE" + ], + "cold_s": 1.037, + "warm_best_s": 0.87, + "warm_median_s": 0.875, + "tracemalloc_peak_mib": 25.0, + "rss_delta_mib": 30.2, + "id": "R-prusa-seam-test", + "class": "real-world" + }, + { + "bytes": 484, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 8, + "rendered_triangles": 8, + "vertices": 6, + "nodes": 1, + "meshes": 1, + "response_bytes": 1988, + "limitations": [ + "UNKNOWN_PLACEMENT", + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.001, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "P-01", + "class": "local-anonymized" + }, + { + "bytes": 986734, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 19733, + "rendered_triangles": 19733, + "vertices": 9869, + "nodes": 1, + "meshes": 1, + "response_bytes": 475424, + "limitations": [ + "UNKNOWN_PLACEMENT", + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.021, + "warm_best_s": 0.02, + "warm_median_s": 0.021, + "tracemalloc_peak_mib": 4.5, + "rss_delta_mib": 3.2, + "id": "P-02", + "class": "local-anonymized" + }, + { + "bytes": 3975984, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 79518, + "rendered_triangles": 79518, + "vertices": 39546, + "nodes": 1, + "meshes": 1, + "response_bytes": 1906815, + "limitations": [ + "UNKNOWN_PLACEMENT", + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.097, + "warm_best_s": 0.097, + "warm_median_s": 0.097, + "tracemalloc_peak_mib": 18.4, + "rss_delta_mib": 24.9, + "id": "P-03", + "class": "local-anonymized" + }, + { + "bytes": 16357134, + "job_gate": "ok", + "result": "refused", + "code": "LIMIT_EXCEEDED", + "message": "The model has 327,141 triangles; Studio draws at most 250,000.", + "fits_budgets": false, + "lifted": { + "triangles": 327141, + "rendered_triangles": 327141, + "vertices": 163521, + "nodes": 1, + "response_bytes": 7852385, + "time_s": 0.43 + }, + "id": "P-04", + "class": "local-anonymized" + }, + { + "bytes": 1525086434, + "job_gate": "SNAPSHOT_CEILING", + "result": "refused", + "code": "LIMIT_EXCEEDED", + "message": "The model has 30,501,727 triangles; Studio draws at most 250,000.", + "fits_budgets": false, + "lifted": { + "still_refused": "LIMIT_EXCEEDED", + "message": "The model has 30,501,727 triangles; Studio draws at most 2,000,000." + }, + "id": "P-05", + "class": "local-anonymized" + } + ], + "coverage": { + "by_class": { + "local-anonymized": { + "items": 5, + "fit": 3 + }, + "real-world": { + "items": 4, + "fit": 4 + }, + "repo-example": { + "items": 19, + "fit": 19 + }, + "synthetic": { + "items": 8, + "fit": 8 + }, + "synthetic-over-budget": { + "items": 1, + "fit": 0 + } + }, + "fraction_fitting_excluding_deliberate_over_budget": 0.944 + }, + "local_survey_3mf_sizes_only": { + "files": 1230, + "over_snapshot_ceiling": 126, + "fraction_within_ceiling": 0.898, + "median_mib": 31.2, + "p90_mib": 130.1 + }, + "gates": { + "fixture_100k_triangles": { + "parse_encode_warm_s": 0.358, + "cold_s": 0.417, + "extra_peak_mib_tracemalloc": 11.0, + "extra_peak_mib_rss": 6.1, + "response_bytes": 2395870, + "gate_time_le_10s": true, + "gate_mem_le_500mib": true, + "target_time_le_3s": true, + "target_mem_le_300mib": true, + "gate_response_le_8mib": true + }, + "cancel_latency_s": { + "max": 0.005, + "median": 0.002, + "gate_le_2s": true + }, + "leak_50_cycles": { + "retained_jobs": 8, + "cap": 8, + "temp_snapshots": 0, + "threads_before": 1, + "threads_after": 1, + "worker_wedged_seen": false, + "gate_pass": true + } + } +} From df11a9dec478bd36f41c370a0be9e331997ff21d Mon Sep 17 00:00:00 2001 From: DeadlyVirusIn Date: Fri, 9 Oct 2026 12:01:37 -0400 Subject: [PATCH 2/6] fix(scene): order overlapping scene starts at the engine boundary An abandoned start that reached the engine after the start the client wanted could replace or cancel the wanted job. The engine now orders starts itself. - POST /scene/start accepts optional client_id and seq (both or neither). The engine keeps, under the registry lock, the highest accepted seq per client (16 clients, least recently used forgotten) and a bounded set of cancelled (client_id, request_id) pairs (64, oldest first). A start below the highest seq is refused with STALE_START; a start whose pair was cancelled is refused with CANCELLED_BEFORE_START. Nothing is registered, replaced or cancelled by a refused start. Equal seq with the same request id is idempotent; with a different request id it is rejected. - POST /scene/cancel accepts either {job_id} (unchanged) or {client_id, request_id}; the latter records the cancellation even when no job is registered yet, and only cancels a job that matches both ids exactly. - Starts without client_id and seq behave exactly as before. Error codes and schema updated; the contract README documents the shapes and limits. Deterministic tests (35) hold an abandoned start at the pre-registration seam with events, register the wanted start, then release the abandoned one; no sleep decides any outcome. Twelve mutants of the new rules are each caught. Verified: scene/unit files 268 passed; full backend 3126 passed, 49 skipped; millimeter-unchanged differential identical to base. Co-Authored-By: Claude Sonnet 5.5 --- backend/snapstudio_api/scene_jobs.py | 110 +++++- .../snapstudio_core/data/scene-1.schema.json | 5 +- backend/snapstudio_core/scene.py | 2 +- backend/tests/test_scene_ordering.py | 322 ++++++++++++++++++ docs/testing/scene-contract/README.md | 28 +- 5 files changed, 453 insertions(+), 14 deletions(-) create mode 100644 backend/tests/test_scene_ordering.py diff --git a/backend/snapstudio_api/scene_jobs.py b/backend/snapstudio_api/scene_jobs.py index 72368b1b..71ad19da 100644 --- a/backend/snapstudio_api/scene_jobs.py +++ b/backend/snapstudio_api/scene_jobs.py @@ -34,6 +34,7 @@ """ from __future__ import annotations +import collections import errno import hashlib import os @@ -54,6 +55,10 @@ TERMINAL = ("succeeded", "failed", "cancelled") _REQUEST_ID = re.compile(r"^[A-Za-z0-9_-]{8,%d}$" % L.MAX_REQUEST_ID_LENGTH) _REVISION = re.compile(r"^[0-9a-f]{64}$") +_CLIENT_ID = re.compile(r"^[A-Za-z0-9_-]{8,64}$") +MAX_SEQ = 2 ** 53 - 1 +MAX_CLIENTS = 16 # client_id -> highest accepted seq (LRU) +MAX_TOMBSTONES = 64 # cancelled (client_id, request_id) pairs (FIFO) class JobError(Exception): @@ -71,6 +76,7 @@ class Job: def __init__(self, job_id: str, request_id: str, path: str, now: float) -> None: self.job_id, self.request_id, self.path = job_id, request_id, path self.source = path # the path as the client gave it (for reading) + self.client_id: str | None = None # set when the start carried client_id/seq self.state = "queued" self.stage: str | None = None self.completed: int | None = None @@ -191,21 +197,58 @@ def __init__(self, *, clock=time.monotonic, builder=None, snapshot_root: str | N self._worker: threading.Thread | None = None self._dir: str | None = None self._pending: list[list] = [] # [path, attempts, next_try] snapshot files that resisted deletion + # Ordering at the engine boundary (all guarded by self._lock, the same lock that registers jobs): + # an abandoned start must never replace or cancel a newer one, however late it arrives. + self._clients: "collections.OrderedDict[str, tuple[int, str]]" = collections.OrderedDict() # client_id -> (highest seq, its request_id) + self._tombstones: "collections.OrderedDict[tuple[str, str], None]" = collections.OrderedDict() # ------------------------------------------------------------------ public API - def start(self, path: str, request_id: str) -> dict: + def _check_source(self, path: str) -> None: + """The pre-registration file check (a seam: it runs BEFORE the registry lock is taken).""" + if not os.path.isfile(path): + raise JobError(400, "INVALID_REQUEST", "That file could not be found.") + + def _admit_locked(self, client_id: str | None, seq: int | None, request_id: str) -> None: + """Ordering rules, evaluated under the registry lock, atomically with registration. + + (a) seq below the client's highest accepted seq: STALE_START, nothing changes. + (c) this (client_id, request_id) was cancelled before it arrived: CANCELLED_BEFORE_START. + (b) seq equal to the highest with a DIFFERENT request_id: INVALID_REQUEST (equal + same request_id is + the idempotent repeat and passes). Legacy starts (no client_id/seq) pass untouched. + """ + if client_id is None: + return + known = self._clients.get(client_id) + if known is not None: + if seq < known[0]: + raise JobError(409, "STALE_START", "A newer scene start from this client has already been accepted.") + if (client_id, request_id) in self._tombstones: + raise JobError(409, "CANCELLED_BEFORE_START", "That scene start was cancelled before it arrived.") + if known is not None and seq == known[0] and request_id != known[1]: + raise JobError(400, "INVALID_REQUEST", "That seq was already used for another request.", + reason="seq reused for another request_id") + + def _record_locked(self, client_id: str | None, seq: int | None, request_id: str) -> None: + if client_id is None: + return + self._clients[client_id] = (seq, request_id) + self._clients.move_to_end(client_id) + while len(self._clients) > MAX_CLIENTS: + self._clients.popitem(last=False) + + def start(self, path: str, request_id: str, client_id: str | None = None, seq: int | None = None) -> dict: key = os.path.normcase(os.path.abspath(path)) ext = os.path.splitext(key)[1].lower() if ext not in (".3mf", ".stl"): raise JobError(422, "UNSUPPORTED_FORMAT", "Only .3mf and .stl files can be shown.") - if not os.path.isfile(path): - raise JobError(400, "INVALID_REQUEST", "That file could not be found.") + self._check_source(path) for _attempt in range(2): with self._lock: self._housekeeping() + self._admit_locked(client_id, seq, request_id) existing = self._by_request.get(request_id) if existing is None: - return self._create_locked(path, key, request_id) + return self._create_locked(path, key, request_id, client_id, seq) self._same_path_or_raise(existing, key) revision = existing.revision # Hash OUTSIDE the lock (it can be slow), then re-check under it: the retained job may have @@ -214,18 +257,21 @@ def start(self, path: str, request_id: str) -> dict: raise JobError(409, "SOURCE_CHANGED", "The file changed since that scene was built. Start a new one.") with self._lock: self._housekeeping() + self._admit_locked(client_id, seq, request_id) # a newer start may have arrived meanwhile if self._jobs.get(existing.job_id) is existing and self._by_request.get(request_id) is existing: return self._start_body(existing, None) # evicted while hashing: go round once more and start a fresh job for the same request with self._lock: self._housekeeping() + self._admit_locked(client_id, seq, request_id) existing = self._by_request.get(request_id) if existing is not None: self._same_path_or_raise(existing, key) return self._start_body(existing, None) - return self._create_locked(path, key, request_id) + return self._create_locked(path, key, request_id, client_id, seq) - def _create_locked(self, path: str, key: str, request_id: str) -> dict: + def _create_locked(self, path: str, key: str, request_id: str, client_id: str | None = None, + seq: int | None = None) -> dict: if self._wedged(): raise JobError(503, "WORKER_WEDGED", "The scene worker is stuck. Try again after it recovers.") replaced = None @@ -238,6 +284,8 @@ def _create_locked(self, path: str, key: str, request_id: str) -> dict: self._cancel_locked(self._running) job = Job(uuid.uuid4().hex, request_id, key, self._clock()) job.source = path + job.client_id = client_id + self._record_locked(client_id, seq, request_id) self._jobs[job.job_id] = job self._by_request[request_id] = job self._queued = job @@ -265,6 +313,28 @@ def result(self, job_id: str, expected_revision: str | None = None) -> tuple[int return 409, {"error": "CANCELLED", "message": "That scene was cancelled."} return 409, {"error": "NOT_READY", "message": "That scene is not ready yet.", "state": job.state} + def cancel_request(self, client_id: str, request_id: str) -> dict: + """Cancel by (client_id, request_id) and remember it, even when no job is registered yet, so a start + that was abandoned and arrives LATE is refused (CANCELLED_BEFORE_START). Matches exactly: another + client's, or another request's, job is never touched.""" + with self._lock: + self._housekeeping() + self._tombstones[(client_id, request_id)] = None + while len(self._tombstones) > MAX_TOMBSTONES: + self._tombstones.popitem(last=False) + job = self._by_request.get(request_id) + if job is not None and job.client_id == client_id: + if job.state in ("queued", "running"): + if job is self._queued: + self._queued = None + self._cancel_locked(job) + body = self._status_body(job) + self._housekeeping() + return body + return {"job_id": None, "request_id": request_id, "state": "cancelled", "stage": None, + "completed": None, "total": None, "revision": None, + "error": {"code": "CANCELLED", "message": "That scene was cancelled."}} + def cancel(self, job_id: str) -> dict: with self._lock: self._housekeeping() @@ -602,6 +672,26 @@ def _request_id(data: dict) -> str: return value +def _client_id(data: dict) -> str: + value = rv.require_str(data, "client_id") + if not _CLIENT_ID.match(value): + raise rv.ValidationError("Invalid 'client_id'") + return value + + +def _ordering(data: dict) -> tuple[str | None, int | None]: + """client_id and seq travel together or not at all (an absent pair is the legacy, unordered start).""" + has_client, has_seq = data.get("client_id") is not None, data.get("seq") is not None + if not has_client and not has_seq: + return None, None + if has_client != has_seq: + raise rv.ValidationError("'client_id' and 'seq' must be given together") + seq = data["seq"] + if isinstance(seq, bool) or not isinstance(seq, int) or seq < 1 or seq > MAX_SEQ: + raise rv.ValidationError("Invalid 'seq'") + return _client_id(data), seq + + def _job_id(data: dict) -> str: value = rv.require_str(data, "job_id") if len(value) > 128: @@ -617,7 +707,8 @@ def handle(route: str, data: dict, jobs: SceneJobs | None = None) -> tuple[int, try: if route == "/scene/start": path = rv.require_bounded_str(data, "path", 4096) - return 200, jobs.start(path, _request_id(data)) + client_id, seq = _ordering(data) + return 200, jobs.start(path, _request_id(data), client_id, seq) if route == "/scene/status": return 200, jobs.status(_job_id(data)) if route == "/scene/result": @@ -626,6 +717,11 @@ def handle(route: str, data: dict, jobs: SceneJobs | None = None) -> tuple[int, raise rv.ValidationError("Invalid 'expected_revision'") return jobs.result(_job_id(data), expected) if route == "/scene/cancel": + if "job_id" in data and ("client_id" in data or "request_id" in data): + raise rv.ValidationError("Give either 'job_id' or 'client_id' and 'request_id', not both") + if "job_id" not in data and "client_id" in data: + client_id = _client_id(data) + return 200, jobs.cancel_request(client_id, _request_id(data)) return 200, jobs.cancel(_job_id(data)) except rv.ValidationError as exc: return 400, {"error": "INVALID_REQUEST", "message": str(exc)} diff --git a/backend/snapstudio_core/data/scene-1.schema.json b/backend/snapstudio_core/data/scene-1.schema.json index d96dcb13..c41efcad 100644 --- a/backend/snapstudio_core/data/scene-1.schema.json +++ b/backend/snapstudio_core/data/scene-1.schema.json @@ -183,7 +183,8 @@ }, "error_code": {"enum": [ "INVALID_REQUEST", "UNSUPPORTED_FORMAT", "INVALID_ARCHIVE", "INVALID_GEOMETRY", "UNRESOLVED_REFERENCE", - "LIMIT_EXCEEDED", "SOURCE_CHANGED", "CANCELLED", "TIMEOUT", "EXPIRED", "NOT_READY", "WORKER_WEDGED", "INTERNAL"]}, + "LIMIT_EXCEEDED", "SOURCE_CHANGED", "CANCELLED", "TIMEOUT", "EXPIRED", "NOT_READY", "WORKER_WEDGED", "INTERNAL", + "STALE_START", "CANCELLED_BEFORE_START"]}, "job_state": {"enum": ["queued", "running", "succeeded", "failed", "cancelled"]}, "error_response": { "type": "object", "additionalProperties": false, "required": ["error"], @@ -198,7 +199,7 @@ "type": "object", "additionalProperties": false, "required": ["job_id", "request_id", "state", "stage", "completed", "total", "error", "revision"], "properties": { - "job_id": {"$ref": "#/$defs/id"}, + "job_id": {"oneOf": [{"$ref": "#/$defs/id"}, {"type": "null"}]}, "request_id": {"$ref": "#/$defs/id"}, "state": {"$ref": "#/$defs/job_state"}, "stage": {"enum": ["reading", "parsing", "encoding", null]}, diff --git a/backend/snapstudio_core/scene.py b/backend/snapstudio_core/scene.py index d8240c11..96c700ed 100644 --- a/backend/snapstudio_core/scene.py +++ b/backend/snapstudio_core/scene.py @@ -76,7 +76,7 @@ ERROR_CODES = ( "INVALID_REQUEST", "UNSUPPORTED_FORMAT", "INVALID_ARCHIVE", "INVALID_GEOMETRY", "UNRESOLVED_REFERENCE", "LIMIT_EXCEEDED", "SOURCE_CHANGED", "CANCELLED", "TIMEOUT", - "EXPIRED", "NOT_READY", "WORKER_WEDGED", "INTERNAL") + "EXPIRED", "NOT_READY", "WORKER_WEDGED", "INTERNAL", "STALE_START", "CANCELLED_BEFORE_START") class SceneError(SnapStudioError): diff --git a/backend/tests/test_scene_ordering.py b/backend/tests/test_scene_ordering.py new file mode 100644 index 00000000..53c37783 --- /dev/null +++ b/backend/tests/test_scene_ordering.py @@ -0,0 +1,322 @@ +"""Start ordering at the engine boundary: an abandoned start can never replace or cancel a newer one. + +No sleeps decide anything here: the abandoned start A is held at the pre-registration seam +(`SceneJobs._check_source`, which runs before the registry lock) with a threading.Event until the test has +registered the wanted start B, then released, so it reaches the registry AFTER B every time. +""" +import threading + +import pytest + +from snapstudio_api import scene_jobs as sj +from tests import scene_fixtures as fx +from tests.test_scene_api import call, server # noqa: F401 (the loopback server fixture) +from tests.test_scene_contract import sub_validator +from tests.test_scene_jobs import fake_builder, wait_for, wait_state + +CLIENT = "client-aaaaaaaa" +OTHER = "client-bbbbbbbb" +_n = iter(range(1, 10**6)) + + +def rid() -> str: + return f"req-ord-{next(_n):08d}" + + +@pytest.fixture +def cube(tmp_path): + return str(fx.plain_cube_3mf(tmp_path / "cube.3mf")) + + +@pytest.fixture +def make(tmp_path): + made = [] + + def factory(**kw): + jobs = sj.SceneJobs(snapshot_root=str(tmp_path / "snap"), builder=kw.pop("builder", fake_builder()), **kw) + made.append(jobs) + return jobs + yield factory + for jobs in made: + jobs.close() + + +class HeldStart: + """Run jobs.start(...) on a thread that blocks at the pre-registration seam until released.""" + + def __init__(self, jobs, *args, **kwargs): + self.entered, self.release = threading.Event(), threading.Event() + self.result = self.error = None + real = jobs._check_source + + def seam(path): + real(path) + self.entered.set() + assert self.release.wait(10) + jobs._check_source = seam + self.thread = threading.Thread(target=self._run, args=(jobs, args, kwargs), daemon=True) + self.thread.start() + assert self.entered.wait(10) + jobs._check_source = real # later starts are not held + + def _run(self, jobs, args, kwargs): + try: + self.result = jobs.start(*args, **kwargs) + except sj.JobError as exc: + self.error = exc + + def finish(self): + self.release.set() + self.thread.join(10) + assert not self.thread.is_alive() + return self + + +# ------------------------------------------------------------------ the bug: A arrives after B + +def test_an_abandoned_start_arriving_after_the_wanted_one_is_refused_and_changes_nothing(make, cube): + jobs = make() + a = HeldStart(jobs, cube, "req-A-000001", CLIENT, 1) # sent first, abandoned, delayed in flight + b = jobs.start(cube, "req-B-000002", CLIENT, 2) # the wanted start arrives first + wait_state(jobs, b["job_id"], "succeeded") + before = jobs.retained() + a.finish() + assert a.error is not None and (a.error.http, a.error.code) == (409, "STALE_START") + assert not list(sub_validator("error_response").iter_errors(a.error.body())) + assert jobs.status(b["job_id"])["state"] == "succeeded" # B was never cancelled or replaced + assert jobs.retained() == before # and A registered nothing + assert jobs.start(cube, "req-B-000002", CLIENT, 2)["job_id"] == b["job_id"] + + +def test_the_stale_start_cannot_replace_a_RUNNING_wanted_start(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + a = HeldStart(jobs, cube, "req-A-000001", CLIENT, 1) + b = jobs.start(cube, "req-B-000002", CLIENT, 2) + assert started.wait(10) and jobs.status(b["job_id"])["state"] == "running" + a.finish() + assert a.error.code == "STALE_START" + assert jobs.status(b["job_id"])["state"] == "running" # not cancelled by the late arrival + gate.set() + assert wait_state(jobs, b["job_id"], "succeeded")["state"] == "succeeded" + + +# ------------------------------------------------------------------ cancel by (client_id, request_id) + +def test_a_cancel_that_arrives_before_the_start_tombstones_it(make, cube): + jobs = make() + out = jobs.cancel_request(CLIENT, "req-A-000001") # nothing registered yet + assert out["job_id"] is None and out["state"] == "cancelled" and out["error"]["code"] == "CANCELLED" + assert not list(sub_validator("job_status").iter_errors(out)) + with pytest.raises(sj.JobError) as e: + jobs.start(cube, "req-A-000001", CLIENT, 1) + assert (e.value.http, e.value.code) == (409, "CANCELLED_BEFORE_START") + assert jobs.retained() == 0 # nothing was registered + fresh = jobs.start(cube, "req-A-000002", CLIENT, 2) # a fresh request_id with a higher seq works + assert wait_state(jobs, fresh["job_id"], "succeeded")["state"] == "succeeded" + + +def test_cancelling_by_request_cancels_only_the_exact_job(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + a = jobs.start(cube, "req-A-000001", CLIENT, 1) + started.wait(10) + b = jobs.start(cube, "req-B-000002", CLIENT, 2) # replaces A as today + assert jobs.status(a["job_id"])["state"] == "cancelled" + jobs.cancel_request(CLIENT, "req-A-000001") # cancelling A (again)... + assert jobs.status(b["job_id"])["state"] in ("queued", "running") # ...never touches B + jobs.cancel_request(OTHER, "req-B-000002") # same request_id, DIFFERENT client: no match + assert jobs.status(b["job_id"])["state"] in ("queued", "running") + jobs.cancel_request(CLIENT, "req-C-nonexistent") # another request: no match + assert jobs.status(b["job_id"])["state"] in ("queued", "running") + out = jobs.cancel_request(CLIENT, "req-B-000002") # the exact pair + assert out["job_id"] == b["job_id"] and out["state"] == "cancelled" + gate.set() + + +def test_a_legacy_job_is_not_matched_by_a_client_cancel(make, cube): + jobs = make() + legacy = jobs.start(cube, "req-L-000001") + wait_state(jobs, legacy["job_id"], "succeeded") + out = jobs.cancel_request(CLIENT, "req-L-000001") + assert out["job_id"] is None + assert jobs.status(legacy["job_id"])["state"] == "succeeded" + + +def test_cancel_by_job_id_is_unchanged(make, cube): + jobs = make() + j = jobs.start(cube, "req-J-000001", CLIENT, 1) + wait_state(jobs, j["job_id"], "succeeded") + assert jobs.cancel(j["job_id"])["state"] == "succeeded" + with pytest.raises(sj.JobError) as e: + jobs.cancel("0" * 32) + assert e.value.code == "EXPIRED" + + +# ------------------------------------------------------------------ idempotency, remount, restart, retry, legacy + +def test_repeat_of_the_highest_seq_is_idempotent_and_a_different_request_is_refused(make, cube): + jobs = make() + b = jobs.start(cube, "req-B-000002", CLIENT, 2) + again = jobs.start(cube, "req-B-000002", CLIENT, 2) + assert again["job_id"] == b["job_id"] and again["replaced_job_id"] is None + with pytest.raises(sj.JobError) as e: + jobs.start(cube, "req-X-000003", CLIENT, 2) + assert (e.value.http, e.value.code) == (400, "INVALID_REQUEST") and e.value.extra["reason"] == "seq reused for another request_id" + assert jobs.status(b["job_id"])["state"] in ("queued", "running", "succeeded") + + +def test_remount_continues_the_seq_and_a_restarted_client_is_independent(make, cube): + jobs = make() + for seq in (1, 2, 3): + j = jobs.start(cube, rid(), CLIENT, seq) + wait_state(jobs, j["job_id"], "succeeded") + with pytest.raises(sj.JobError) as e: + jobs.start(cube, rid(), CLIENT, 2) # an old attempt from the same process + assert e.value.code == "STALE_START" + fresh_process = jobs.start(cube, rid(), OTHER, 1) # app restart: new random client_id, seq 1 + assert wait_state(jobs, fresh_process["job_id"], "succeeded")["state"] == "succeeded" + assert jobs.start(cube, rid(), CLIENT, 4)["state"] in ("queued", "running", "succeeded") + + +def test_retry_with_a_fresh_request_id_and_a_higher_seq_after_a_stale_refusal(make, cube): + jobs = make() + jobs.start(cube, "req-B-000002", CLIENT, 5) + with pytest.raises(sj.JobError): + jobs.start(cube, "req-A-000001", CLIENT, 4) + retry = jobs.start(cube, "req-A-000009", CLIENT, 6) + assert wait_state(jobs, retry["job_id"], "succeeded")["state"] == "succeeded" + + +def test_legacy_starts_without_the_fields_still_work_and_mix_freely(make, cube): + jobs = make() + a = jobs.start(cube, "req-L-000001") + wait_state(jobs, a["job_id"], "succeeded") + b = jobs.start(cube, "req-L-000002") + wait_state(jobs, b["job_id"], "succeeded") + c = jobs.start(cube, "req-O-000003", CLIENT, 7) + wait_state(jobs, c["job_id"], "succeeded") + d = jobs.start(cube, "req-L-000004") # legacy after ordered: not judged + assert wait_state(jobs, d["job_id"], "succeeded")["state"] == "succeeded" + + +def test_two_clients_are_independent_and_do_not_judge_each_other(make, cube): + jobs = make() + jobs.start(cube, "req-A-000001", CLIENT, 50) + b = jobs.start(cube, "req-B-000001", OTHER, 1) # a low seq for another client is fine + assert b["state"] in ("queued", "running", "succeeded") + held = HeldStart(jobs, cube, "req-B-000002", OTHER, 2) # concurrent, in flight + stale = jobs.start(cube, "req-A-000002", CLIENT, 51) + held.finish() + assert held.error is None and held.result is not None + assert stale["job_id"] + + +# ------------------------------------------------------------------ bounded maps + +def test_the_client_map_is_an_lru_and_the_live_client_keeps_its_ordering(make, cube): + jobs = make() + live = "client-live0001" + jobs.start(cube, rid(), live, 10) + for i in range(sj.MAX_CLIENTS + 5): # many other clients come and go + jobs.start(cube, rid(), f"client-{i:08d}", 1) + jobs.start(cube, rid(), live, 11 + i) # the live client keeps being used (touched) + assert len(jobs._clients) <= sj.MAX_CLIENTS + with pytest.raises(sj.JobError) as e: + jobs.start(cube, rid(), live, 10) + assert e.value.code == "STALE_START" # ordering for the live client survived eviction + assert "client-00000000" not in jobs._clients # an idle one was evicted (oldest first) + + +def test_tombstones_are_a_bounded_fifo(make, cube): + jobs = make() + for i in range(sj.MAX_TOMBSTONES + 1): + jobs.cancel_request(CLIENT, f"req-T-{i:08d}") + assert len(jobs._tombstones) == sj.MAX_TOMBSTONES + assert (CLIENT, "req-T-00000000") not in jobs._tombstones # first in, first out + assert (CLIENT, f"req-T-{sj.MAX_TOMBSTONES:08d}") in jobs._tombstones + with pytest.raises(sj.JobError) as e: + jobs.start(cube, f"req-T-{sj.MAX_TOMBSTONES:08d}", CLIENT, 1) + assert e.value.code == "CANCELLED_BEFORE_START" + + +# ------------------------------------------------------------------ the real HTTP route and body validation + +def test_http_stale_start_and_the_cancel_by_request_shape(server, tmp_path): # noqa: F811 + port, token = server + path = str(fx.plain_cube_3mf(tmp_path / "h.3mf")) + code, b = call(port, "/scene/start", {"path": path, "request_id": "req-B-http0002", "client_id": CLIENT, "seq": 2}, token) + assert code == 200 and not list(sub_validator("job_start").iter_errors(b)) + code, body = call(port, "/scene/start", {"path": path, "request_id": "req-A-http0001", "client_id": CLIENT, "seq": 1}, token) + assert (code, body["error"]) == (409, "STALE_START") + assert not list(sub_validator("error_response").iter_errors(body)) + code, status = call(port, "/scene/status", {"job_id": b["job_id"]}, token) + assert code == 200 and status["state"] != "cancelled" # B untouched + code, c = call(port, "/scene/cancel", {"client_id": CLIENT, "request_id": "req-C-http0003"}, token) + assert code == 200 and c["job_id"] is None and c["state"] == "cancelled" + assert not list(sub_validator("job_status").iter_errors(c)) + code, body = call(port, "/scene/start", {"path": path, "request_id": "req-C-http0003", "client_id": CLIENT, "seq": 3}, token) + assert (code, body["error"]) == (409, "CANCELLED_BEFORE_START") + code, hit = call(port, "/scene/cancel", {"client_id": CLIENT, "request_id": "req-B-http0002"}, token) + assert code == 200 and hit["job_id"] == b["job_id"] + code, legacy = call(port, "/scene/start", {"path": path, "request_id": "req-L-http0004"}, token) + assert code == 200 + + +@pytest.mark.parametrize("extra", [ + {"client_id": CLIENT, "seq": 0}, {"client_id": CLIENT, "seq": -3}, {"client_id": CLIENT, "seq": "1"}, + {"client_id": CLIENT, "seq": True}, {"client_id": CLIENT, "seq": 1.5}, {"client_id": CLIENT, "seq": 2 ** 60}, + {"client_id": "short", "seq": 1}, {"client_id": "has space in it", "seq": 1}, {"client_id": "x" * 65, "seq": 1}, + {"client_id": 12345678, "seq": 1}, {"client_id": CLIENT}, {"seq": 1}, +]) +def test_http_start_rejects_malformed_ordering_fields(server, tmp_path, extra): # noqa: F811 + port, token = server + path = str(fx.plain_cube_3mf(tmp_path / "v.3mf")) + code, body = call(port, "/scene/start", dict({"path": path, "request_id": "req-V-http0001"}, **extra), token) + assert (code, body["error"]) == (400, "INVALID_REQUEST"), (extra, code, body) + + +@pytest.mark.parametrize("body", [ + {"client_id": "short", "request_id": "req-C-http0001"}, {"client_id": CLIENT}, + {"client_id": CLIENT, "request_id": "bad id"}, {"job_id": "0" * 32, "client_id": CLIENT, "request_id": "req-C-http0001"}, +]) +def test_http_cancel_rejects_malformed_bodies(server, body): # noqa: F811 + port, token = server + code, resp = call(port, "/scene/cancel", body, token) + assert (code, resp["error"]) == (400, "INVALID_REQUEST"), (body, code, resp) + + +@pytest.mark.parametrize("body", [[1], "x", None, 7]) +def test_http_non_object_bodies_are_still_400(server, body): # noqa: F811 + port, token = server + for route in sj.ROUTES: + assert call(port, route, body, token)[0] == 400 + + +def test_a_repeat_of_an_old_start_that_is_still_hashing_is_refused_if_a_newer_one_arrived(make, cube, monkeypatch): + jobs = make() + a = jobs.start(cube, "req-A-000001", CLIENT, 1) + status = wait_state(jobs, a["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + inside, release = threading.Event(), threading.Event() + + def slow_hash(path): # the idempotent repeat hashes OUTSIDE the lock + inside.set() + assert release.wait(10) + return status["revision"] + monkeypatch.setattr(jobs, "_current_hash", slow_hash) + out = {} + + def repeat(): + try: + out["r"] = jobs.start(cube, "req-A-000001", CLIENT, 1) + except sj.JobError as exc: + out["e"] = exc + t = threading.Thread(target=repeat, daemon=True) + t.start() + assert inside.wait(10) + b = jobs.start(cube, "req-B-000002", CLIENT, 2) # the wanted start gets in while A's repeat hashes + release.set() + t.join(10) + assert "r" not in out and out["e"].code == "STALE_START" + assert jobs.status(b["job_id"])["state"] in ("queued", "running", "succeeded") diff --git a/docs/testing/scene-contract/README.md b/docs/testing/scene-contract/README.md index be80b37c..8c49b21a 100644 --- a/docs/testing/scene-contract/README.md +++ b/docs/testing/scene-contract/README.md @@ -132,14 +132,34 @@ running. A terminal job is kept 120 s or until more than 8 terminal jobs exist. | Route | Answers | |---|---| -| `/scene/start {path, request_id}` | 200 job; 400 `INVALID_REQUEST`; 422 `UNSUPPORTED_FORMAT`; 409 `INVALID_REQUEST` (request_id reused for another path) or `SOURCE_CHANGED`; 503 `WORKER_WEDGED` | +| `/scene/start {path, request_id, client_id?, seq?}` | 200 job; 400 `INVALID_REQUEST` (also: only one of `client_id`/`seq`, a bad `seq`/`client_id`, or the same `seq` reused for another request_id); 422 `UNSUPPORTED_FORMAT`; 409 `INVALID_REQUEST` (request_id reused for another path), `SOURCE_CHANGED`, `STALE_START` or `CANCELLED_BEFORE_START`; 503 `WORKER_WEDGED` | | `/scene/status {job_id}` | 200 `{state, stage, completed, total, error, revision}`; 404 `EXPIRED` | | `/scene/result {job_id, expected_revision?}` | 200 scene; 422 `{error: }` for a failed job; 409 `CANCELLED` / `NOT_READY` / `SOURCE_CHANGED`; 404 `EXPIRED` | -| `/scene/cancel {job_id}` | 200 status (idempotent; a late cancel on a terminal job is a no-op) | +| `/scene/cancel {job_id}` or `{client_id, request_id}` | 200 status (idempotent; a late cancel on a terminal job is a no-op). By request: cancels the job registered for exactly that client_id + request_id (if any) and ALWAYS records a tombstone; with no such job the body is a `job_status` with `job_id: null`, `state: "cancelled"` | Error codes: `INVALID_REQUEST UNSUPPORTED_FORMAT INVALID_ARCHIVE INVALID_GEOMETRY UNRESOLVED_REFERENCE -LIMIT_EXCEEDED SOURCE_CHANGED CANCELLED TIMEOUT EXPIRED NOT_READY WORKER_WEDGED INTERNAL` (`BUSY` is not an -error code: a new start replaces the client's own earlier work). +LIMIT_EXCEEDED SOURCE_CHANGED CANCELLED TIMEOUT EXPIRED NOT_READY WORKER_WEDGED INTERNAL STALE_START +CANCELLED_BEFORE_START` (`BUSY` is not an error code: a new start replaces the client's own earlier work). + +### Start ordering (an abandoned start can never replace a newer one) + +A client that aborts or times out a start cannot stop that request from still reaching the engine, possibly AFTER +the start it actually wants. The engine therefore orders starts itself, under the same lock that registers and +replaces jobs, so the decision is atomic with registration (nothing here uses timing): + +* `client_id` is 8-64 characters of `[A-Za-z0-9_-]`, random per app process; `seq` is an integer >= 1 that the + client increases for every start attempt. Both or neither; with neither the start behaves exactly as before. +* The engine remembers the highest accepted `seq` per `client_id` (LRU, 16 clients) and the cancelled + `(client_id, request_id)` pairs (FIFO, 64 tombstones). +* In order: `seq` below that client's highest -> 409 `STALE_START`, nothing changes (no replace, no cancel); + the pair is tombstoned -> 409 `CANCELLED_BEFORE_START`, nothing is registered; `seq` equal to the highest with + the same `request_id` -> the existing job (idempotent, as before); equal with a different `request_id` -> 400 + `INVALID_REQUEST`; otherwise the job is registered (replacing the older one as before) and `seq` is recorded. + An idempotent repeat that was still hashing the file is re-checked under the lock, so a newer start that arrived + meanwhile makes it `STALE_START` too. +* After `STALE_START` / `CANCELLED_BEFORE_START` the client retries with a FRESH `request_id` and a HIGHER `seq`. + A restarted app uses a new random `client_id` (seq starts again at 1); a remount in the same process just keeps + counting. If more than 16 clients are seen the least recently used one forgets its ordering (documented). Interruptibility: archive reads are at most 1 MiB, lxml `iterparse` checks the cancel/deadline hook every 1,000 events and on every read, geometry transform/encode runs in batches of at most 5,000 triangles. Backstop: if the From 3efb263fafb1f773787241fe4ed5c043b8522e7f Mon Sep 17 00:00:00 2001 From: DeadlyVirusIn Date: Fri, 9 Oct 2026 14:37:05 -0400 Subject: [PATCH 3/6] fix(scene): session-based start ordering with explicit invariants Replaces the bounded client map and tombstone set from the previous commit. Invariants, each enforced and tested: - Delayed starts cannot become valid through bookkeeping eviction. - Cancellation cannot be forgotten while the start remains admissible. - Idempotency is scoped to (session, request) and ownership is checked on every path. - Session expiry or capacity exhaustion fails explicitly; nothing old is silently revived. Design: POST /scene/session issues a session (32 at most, 15 minute idle timeout; a session that owns a registered job is never pruned; at capacity with nothing prunable the engine returns SESSION_LIMIT). Each session keeps only a start watermark, a cancelled-through sequence and its current request. A start with a lower sequence is STALE_START; one at or below the cancelled sequence is CANCELLED_BEFORE_START; an unknown or expired session is SESSION_EXPIRED and is never re-created by a late request. Cancelling by {client_id, request_id, seq} refuses that sequence and every lower one for the life of the session in constant memory. An exact retry of a processed start is refused; retries use a fresh request id and a higher sequence. Ordering is per session only; sessions end with the engine. Legacy starts without the fields behave as before in their own namespace. Tests (53) use events, barriers and a fake clock, with no sleeps deciding any outcome, and cover eviction pressure, concurrent sessions, cancellation before start and request-id collisions; 20 of 23 mutants are caught and the other three are redundant checks. Verified: eight scene/unit files 286 passed; full backend 3144 passed, 49 skipped; millimeter-unchanged differential identical to base. Co-Authored-By: Claude Sonnet 5.5 --- backend/snapstudio_api/scene_jobs.py | 201 ++++-- .../snapstudio_core/data/scene-1.schema.json | 9 +- backend/snapstudio_core/scene.py | 2 +- backend/snapstudio_core/scene_limits.py | 2 + backend/tests/test_scene_ordering.py | 627 +++++++++++++----- docs/testing/scene-contract/README.md | 84 ++- 6 files changed, 636 insertions(+), 289 deletions(-) diff --git a/backend/snapstudio_api/scene_jobs.py b/backend/snapstudio_api/scene_jobs.py index 71ad19da..b87f9403 100644 --- a/backend/snapstudio_api/scene_jobs.py +++ b/backend/snapstudio_api/scene_jobs.py @@ -34,7 +34,6 @@ """ from __future__ import annotations -import collections import errno import hashlib import os @@ -51,14 +50,13 @@ from snapstudio_core import scene_limits as L from . import request_validation as rv -ROUTES = ("/scene/start", "/scene/status", "/scene/result", "/scene/cancel") +ROUTES = ("/scene/session", "/scene/start", "/scene/status", "/scene/result", "/scene/cancel") TERMINAL = ("succeeded", "failed", "cancelled") _REQUEST_ID = re.compile(r"^[A-Za-z0-9_-]{8,%d}$" % L.MAX_REQUEST_ID_LENGTH) _REVISION = re.compile(r"^[0-9a-f]{64}$") _CLIENT_ID = re.compile(r"^[A-Za-z0-9_-]{8,64}$") MAX_SEQ = 2 ** 53 - 1 -MAX_CLIENTS = 16 # client_id -> highest accepted seq (LRU) -MAX_TOMBSTONES = 64 # cancelled (client_id, request_id) pairs (FIFO) +# (session capacity and idle TTL live in scene_limits: MAX_SESSIONS, SESSION_TTL_SECONDS) class JobError(Exception): @@ -72,6 +70,21 @@ def body(self) -> dict: return {"error": self.code, "message": self.message, **self.extra} +class Session: + """O(1) ordering state of one client session. Nothing else is remembered about its starts. + + ``watermark``: the highest admitted seq. ``dead_through``: the highest cancelled seq (every start with + seq <= dead_through is refused for as long as the session lives). ``current``: (seq, request_id, path_key, + job_id) of the entry at seq == watermark, for the idempotent repeat. + """ + + def __init__(self, client_id: str, now: float) -> None: + self.client_id, self.last_touch = client_id, now + self.watermark = 0 + self.dead_through = 0 + self.current: tuple[int, str, str, str | None] | None = None + + class Job: def __init__(self, job_id: str, request_id: str, path: str, now: float) -> None: self.job_id, self.request_id, self.path = job_id, request_id, path @@ -184,23 +197,23 @@ def sweep_stale(root: str | None = None, *, now: float | None = None) -> list[st class SceneJobs: def __init__(self, *, clock=time.monotonic, builder=None, snapshot_root: str | None = None, deadline: float = L.JOB_DEADLINE_SECONDS, wedge_grace: float = L.WEDGE_GRACE_SECONDS, - ttl: float = L.RESULT_TTL_SECONDS, max_terminal: int = L.MAX_TERMINAL_JOBS) -> None: + ttl: float = L.RESULT_TTL_SECONDS, max_terminal: int = L.MAX_TERMINAL_JOBS, + session_ttl: float = L.SESSION_TTL_SECONDS, max_sessions: int = L.MAX_SESSIONS) -> None: + self._session_ttl, self._max_sessions = session_ttl, max_sessions self._clock = clock self._builder = builder or scene.build_scene self._root = snapshot_root self._deadline, self._grace, self._ttl, self._max_terminal = deadline, wedge_grace, ttl, max_terminal self._lock = threading.RLock() self._jobs: dict[str, Job] = {} - self._by_request: dict[str, Job] = {} + self._by_request: dict[tuple[str | None, str], Job] = {} # (client_id | None, request_id) -> job self._queued: Job | None = None self._running: Job | None = None self._worker: threading.Thread | None = None self._dir: str | None = None self._pending: list[list] = [] # [path, attempts, next_try] snapshot files that resisted deletion - # Ordering at the engine boundary (all guarded by self._lock, the same lock that registers jobs): - # an abandoned start must never replace or cancel a newer one, however late it arrives. - self._clients: "collections.OrderedDict[str, tuple[int, str]]" = collections.OrderedDict() # client_id -> (highest seq, its request_id) - self._tombstones: "collections.OrderedDict[tuple[str, str], None]" = collections.OrderedDict() + # Ordering at the engine boundary (guarded by self._lock, the same lock that registers jobs). + self._sessions: dict[str, Session] = {} # ------------------------------------------------------------------ public API def _check_source(self, path: str) -> None: @@ -208,33 +221,66 @@ def _check_source(self, path: str) -> None: if not os.path.isfile(path): raise JobError(400, "INVALID_REQUEST", "That file could not be found.") - def _admit_locked(self, client_id: str | None, seq: int | None, request_id: str) -> None: - """Ordering rules, evaluated under the registry lock, atomically with registration. - - (a) seq below the client's highest accepted seq: STALE_START, nothing changes. - (c) this (client_id, request_id) was cancelled before it arrived: CANCELLED_BEFORE_START. - (b) seq equal to the highest with a DIFFERENT request_id: INVALID_REQUEST (equal + same request_id is - the idempotent repeat and passes). Legacy starts (no client_id/seq) pass untouched. + def open_session(self) -> dict: + """Issue a session (server-chosen id). I4: at capacity with nothing prunable it fails EXPLICITLY.""" + with self._lock: + self._housekeeping() + if len(self._sessions) >= self._max_sessions: + raise JobError(503, "SESSION_LIMIT", "Too many scene sessions are open. Try again later.") + client_id = secrets.token_urlsafe(18) # 24 characters of [A-Za-z0-9_-] + self._sessions[client_id] = Session(client_id, self._clock()) + return {"client_id": client_id, "ttl_s": int(self._session_ttl)} + + def _session_locked(self, client_id: str) -> Session: + """The live session or SESSION_EXPIRED (I1/I4: a late request from an expired or never-issued session is + refused, never re-created). Touches the session.""" + session = self._sessions.get(client_id) + if session is None: + raise JobError(409, "SESSION_EXPIRED", "That scene session is no longer open. Open a new session.") + session.last_touch = self._clock() + return session + + def _prune_sessions_locked(self) -> None: + """Remove only IDLE-EXPIRED sessions that own no registered job; a live session is never evicted.""" + now = self._clock() + owners = {key[0] for key in self._by_request} + for cid in [c for c, sess in self._sessions.items() + if now - sess.last_touch > self._session_ttl and c not in owners]: + del self._sessions[cid] + + def _admit_locked(self, client_id: str | None, seq: int | None, request_id: str, path_key: str) -> "Job | None": + """THE admission function for every start (new job, idempotent repeat), under the registry lock. + + Returns the registered job to hand back (idempotent), or None when a new job should be created; raises + otherwise. Legacy starts (no client_id) keep today's request_id-only behaviour in their own namespace. + For a session start, in order: unknown/expired session -> SESSION_EXPIRED; seq <= dead_through -> + CANCELLED_BEFORE_START; seq < watermark -> STALE_START; seq == watermark -> the same request_id and path + returns the job if it is still registered (otherwise STALE_START: an exact retry of a start that was + already processed is refused) and anything else is INVALID_REQUEST; seq > watermark -> new, except that + a request_id already registered for this session is INVALID_REQUEST (the old job is never handed back), + and that attempt still raises the watermark so an older attempt still in flight is refused. """ if client_id is None: - return - known = self._clients.get(client_id) - if known is not None: - if seq < known[0]: - raise JobError(409, "STALE_START", "A newer scene start from this client has already been accepted.") - if (client_id, request_id) in self._tombstones: + return self._by_request.get((None, request_id)) + session = self._session_locked(client_id) + if seq <= session.dead_through: raise JobError(409, "CANCELLED_BEFORE_START", "That scene start was cancelled before it arrived.") - if known is not None and seq == known[0] and request_id != known[1]: - raise JobError(400, "INVALID_REQUEST", "That seq was already used for another request.", - reason="seq reused for another request_id") - - def _record_locked(self, client_id: str | None, seq: int | None, request_id: str) -> None: - if client_id is None: - return - self._clients[client_id] = (seq, request_id) - self._clients.move_to_end(client_id) - while len(self._clients) > MAX_CLIENTS: - self._clients.popitem(last=False) + if seq < session.watermark: + raise JobError(409, "STALE_START", "A newer scene start from this session has already been accepted.") + existing = self._by_request.get((client_id, request_id)) + if seq == session.watermark: + cur = session.current + if cur is None or cur[1] != request_id or cur[2] != path_key: + raise JobError(400, "INVALID_REQUEST", "That seq was already used for another request.", + reason="seq reused for another request") + if existing is None or existing.job_id != cur[3]: + raise JobError(409, "STALE_START", "That scene start was already processed and its job is gone.") + return existing + if existing is not None: + session.watermark, session.current = seq, (seq, request_id, path_key, None) + raise JobError(400, "INVALID_REQUEST", "That request_id was already used by this session.", + reason="request id reused") + return None def start(self, path: str, request_id: str, client_id: str | None = None, seq: int | None = None) -> dict: key = os.path.normcase(os.path.abspath(path)) @@ -245,26 +291,26 @@ def start(self, path: str, request_id: str, client_id: str | None = None, seq: i for _attempt in range(2): with self._lock: self._housekeeping() - self._admit_locked(client_id, seq, request_id) - existing = self._by_request.get(request_id) + existing = self._admit_locked(client_id, seq, request_id, key) if existing is None: return self._create_locked(path, key, request_id, client_id, seq) self._same_path_or_raise(existing, key) revision = existing.revision - # Hash OUTSIDE the lock (it can be slow), then re-check under it: the retained job may have - # been evicted meanwhile, and an evicted job must never be handed back. - if revision is not None and self._current_hash(path) != revision: - raise JobError(409, "SOURCE_CHANGED", "The file changed since that scene was built. Start a new one.") + # Hash OUTSIDE the lock (it can be slow), then decide under it. The admission rules run FIRST: if a + # newer start, a cancel or the end of the session happened meanwhile the answer is STALE_START / + # CANCELLED_BEFORE_START / SESSION_EXPIRED, not SOURCE_CHANGED. + changed = revision is not None and self._current_hash(path) != revision with self._lock: self._housekeeping() - self._admit_locked(client_id, seq, request_id) # a newer start may have arrived meanwhile - if self._jobs.get(existing.job_id) is existing and self._by_request.get(request_id) is existing: + current = self._admit_locked(client_id, seq, request_id, key) + if current is existing and self._jobs.get(existing.job_id) is existing: + if changed: + raise JobError(409, "SOURCE_CHANGED", "The file changed since that scene was built. Start a new one.") return self._start_body(existing, None) - # evicted while hashing: go round once more and start a fresh job for the same request + # evicted while hashing (a legacy start only: a session start was refused above): start a fresh job with self._lock: self._housekeeping() - self._admit_locked(client_id, seq, request_id) - existing = self._by_request.get(request_id) + existing = self._admit_locked(client_id, seq, request_id, key) if existing is not None: self._same_path_or_raise(existing, key) return self._start_body(existing, None) @@ -285,19 +331,28 @@ def _create_locked(self, path: str, key: str, request_id: str, client_id: str | job = Job(uuid.uuid4().hex, request_id, key, self._clock()) job.source = path job.client_id = client_id - self._record_locked(client_id, seq, request_id) self._jobs[job.job_id] = job - self._by_request[request_id] = job + self._by_request[(client_id, request_id)] = job + if client_id is not None: # every registration records the session state atomically + session = self._sessions[client_id] + session.watermark, session.current = seq, (seq, request_id, key, job.job_id) self._queued = job if self._worker is None: self._worker = threading.Thread(target=self._worker_main, name="scene-worker", daemon=True) self._worker.start() return self._start_body(job, replaced) + def _touch_owner_locked(self, job: Job) -> None: + session = self._sessions.get(job.client_id) if job.client_id is not None else None + if session is not None: + session.last_touch = self._clock() + def status(self, job_id: str) -> dict: with self._lock: self._housekeeping() - return self._status_body(self._get(job_id)) + job = self._get(job_id) + self._touch_owner_locked(job) + return self._status_body(job) def result(self, job_id: str, expected_revision: str | None = None) -> tuple[int, bytes | dict]: with self._lock: @@ -313,17 +368,17 @@ def result(self, job_id: str, expected_revision: str | None = None) -> tuple[int return 409, {"error": "CANCELLED", "message": "That scene was cancelled."} return 409, {"error": "NOT_READY", "message": "That scene is not ready yet.", "state": job.state} - def cancel_request(self, client_id: str, request_id: str) -> dict: - """Cancel by (client_id, request_id) and remember it, even when no job is registered yet, so a start - that was abandoned and arrives LATE is refused (CANCELLED_BEFORE_START). Matches exactly: another - client's, or another request's, job is never touched.""" + def cancel_request(self, client_id: str, request_id: str, seq: int) -> dict: + """Cancel the start (client_id, request_id, seq). The session must exist (SESSION_EXPIRED otherwise). + ``dead_through`` rises to ``seq`` so that start, and every lower seq, stays refused for as long as the + session lives (O(1) memory: no per-request set), even when no job is registered yet. The registered job + is cancelled only if its client_id AND request_id both match exactly.""" with self._lock: self._housekeeping() - self._tombstones[(client_id, request_id)] = None - while len(self._tombstones) > MAX_TOMBSTONES: - self._tombstones.popitem(last=False) - job = self._by_request.get(request_id) - if job is not None and job.client_id == client_id: + session = self._session_locked(client_id) + session.dead_through = max(session.dead_through, seq) + job = self._by_request.get((client_id, request_id)) + if job is not None and job.client_id == client_id and job.request_id == request_id: if job.state in ("queued", "running"): if job is self._queued: self._queued = None @@ -417,6 +472,7 @@ def _wedged(self) -> bool: def _housekeeping(self) -> None: now = self._clock() + self._prune_sessions_locked() for entry in list(self._pending): if now >= entry[2]: self._try_unlink(entry) @@ -440,8 +496,8 @@ def _drop(self, job: Job) -> None: job.snapshot = None job.result = None self._jobs.pop(job.job_id, None) - if self._by_request.get(job.request_id) is job: - del self._by_request[job.request_id] + if self._by_request.get((job.client_id, job.request_id)) is job: + del self._by_request[(job.client_id, job.request_id)] def _defer_unlink(self, path: str) -> None: """Delete a snapshot file now; if Windows (or anything) still holds it, KEEP the path on a retry list @@ -667,24 +723,25 @@ def reset_default_for_tests() -> None: def _request_id(data: dict) -> str: value = rv.require_str(data, "request_id") - if not _REQUEST_ID.match(value): + if not _REQUEST_ID.fullmatch(value): # fullmatch: a trailing newline is not an id raise rv.ValidationError("Invalid 'request_id'") return value def _client_id(data: dict) -> str: value = rv.require_str(data, "client_id") - if not _CLIENT_ID.match(value): + if not _CLIENT_ID.fullmatch(value): raise rv.ValidationError("Invalid 'client_id'") return value def _ordering(data: dict) -> tuple[str | None, int | None]: - """client_id and seq travel together or not at all (an absent pair is the legacy, unordered start).""" - has_client, has_seq = data.get("client_id") is not None, data.get("seq") is not None + """client_id and seq travel together or not at all. Presence is tested by KEY: only both keys absent is the + legacy, unordered start; a null or half-present pair is a malformed request, never a legacy one.""" + has_client, has_seq = "client_id" in data, "seq" in data if not has_client and not has_seq: return None, None - if has_client != has_seq: + if not (has_client and has_seq): raise rv.ValidationError("'client_id' and 'seq' must be given together") seq = data["seq"] if isinstance(seq, bool) or not isinstance(seq, int) or seq < 1 or seq > MAX_SEQ: @@ -705,6 +762,8 @@ def handle(route: str, data: dict, jobs: SceneJobs | None = None) -> tuple[int, if not isinstance(data, dict): return 400, {"error": "INVALID_REQUEST", "message": "The request body must be a JSON object."} try: + if route == "/scene/session": + return 200, jobs.open_session() if route == "/scene/start": path = rv.require_bounded_str(data, "path", 4096) client_id, seq = _ordering(data) @@ -713,15 +772,17 @@ def handle(route: str, data: dict, jobs: SceneJobs | None = None) -> tuple[int, return 200, jobs.status(_job_id(data)) if route == "/scene/result": expected = data.get("expected_revision") - if expected is not None and (not isinstance(expected, str) or not _REVISION.match(expected)): + if expected is not None and (not isinstance(expected, str) or not _REVISION.fullmatch(expected)): raise rv.ValidationError("Invalid 'expected_revision'") return jobs.result(_job_id(data), expected) if route == "/scene/cancel": - if "job_id" in data and ("client_id" in data or "request_id" in data): - raise rv.ValidationError("Give either 'job_id' or 'client_id' and 'request_id', not both") - if "job_id" not in data and "client_id" in data: - client_id = _client_id(data) - return 200, jobs.cancel_request(client_id, _request_id(data)) + if "job_id" in data and ("client_id" in data or "request_id" in data or "seq" in data): + raise rv.ValidationError("Give either 'job_id' or 'client_id', 'request_id' and 'seq', not both") + if "job_id" not in data and ("client_id" in data or "seq" in data): + if not ("client_id" in data and "seq" in data and "request_id" in data): + raise rv.ValidationError("'client_id', 'request_id' and 'seq' must be given together") + client_id, seq = _ordering(data) + return 200, jobs.cancel_request(client_id, _request_id(data), seq) return 200, jobs.cancel(_job_id(data)) except rv.ValidationError as exc: return 400, {"error": "INVALID_REQUEST", "message": str(exc)} diff --git a/backend/snapstudio_core/data/scene-1.schema.json b/backend/snapstudio_core/data/scene-1.schema.json index c41efcad..dbebec7f 100644 --- a/backend/snapstudio_core/data/scene-1.schema.json +++ b/backend/snapstudio_core/data/scene-1.schema.json @@ -184,7 +184,7 @@ "error_code": {"enum": [ "INVALID_REQUEST", "UNSUPPORTED_FORMAT", "INVALID_ARCHIVE", "INVALID_GEOMETRY", "UNRESOLVED_REFERENCE", "LIMIT_EXCEEDED", "SOURCE_CHANGED", "CANCELLED", "TIMEOUT", "EXPIRED", "NOT_READY", "WORKER_WEDGED", "INTERNAL", - "STALE_START", "CANCELLED_BEFORE_START"]}, + "STALE_START", "CANCELLED_BEFORE_START", "SESSION_EXPIRED", "SESSION_LIMIT"]}, "job_state": {"enum": ["queued", "running", "succeeded", "failed", "cancelled"]}, "error_response": { "type": "object", "additionalProperties": false, "required": ["error"], @@ -210,6 +210,13 @@ "revision": {"oneOf": [{"$ref": "#/$defs/sha256"}, {"type": "null"}]} } }, + "session": { + "type": "object", "additionalProperties": false, "required": ["client_id", "ttl_s"], + "properties": { + "client_id": {"type": "string", "pattern": "^[A-Za-z0-9_-]{24}$"}, + "ttl_s": {"type": "integer", "minimum": 1} + } + }, "job_start": { "type": "object", "additionalProperties": false, "required": ["job_id", "request_id", "state", "revision", "replaced_job_id"], diff --git a/backend/snapstudio_core/scene.py b/backend/snapstudio_core/scene.py index 96c700ed..b4a61810 100644 --- a/backend/snapstudio_core/scene.py +++ b/backend/snapstudio_core/scene.py @@ -76,7 +76,7 @@ ERROR_CODES = ( "INVALID_REQUEST", "UNSUPPORTED_FORMAT", "INVALID_ARCHIVE", "INVALID_GEOMETRY", "UNRESOLVED_REFERENCE", "LIMIT_EXCEEDED", "SOURCE_CHANGED", "CANCELLED", "TIMEOUT", - "EXPIRED", "NOT_READY", "WORKER_WEDGED", "INTERNAL", "STALE_START", "CANCELLED_BEFORE_START") + "EXPIRED", "NOT_READY", "WORKER_WEDGED", "INTERNAL", "STALE_START", "CANCELLED_BEFORE_START", "SESSION_EXPIRED", "SESSION_LIMIT") class SceneError(SnapStudioError): diff --git a/backend/snapstudio_core/scene_limits.py b/backend/snapstudio_core/scene_limits.py index a493950c..c46b02a5 100644 --- a/backend/snapstudio_core/scene_limits.py +++ b/backend/snapstudio_core/scene_limits.py @@ -47,6 +47,8 @@ RESULT_TTL_SECONDS = 120.0 # a terminal job is kept this long MAX_TERMINAL_JOBS = 8 # ... or until more than this many terminal jobs exist MAX_REQUEST_ID_LENGTH = 64 +MAX_SESSIONS = 32 # open scene sessions (an idle-expired one that owns no job is pruned first) +SESSION_TTL_SECONDS = 15 * 60 # idle time after which a session without a registered job is pruned SNAPSHOT_DIR_MIN_AGE_SECONDS = 600 # a dead engine's scene-tmp folder is swept only once it is this old (a live engine's NEVER is) MAX_PENDING_UNLINKS = 1024 # snapshot files that could not be deleted yet and are retried with back-off diff --git a/backend/tests/test_scene_ordering.py b/backend/tests/test_scene_ordering.py index 53c37783..8bdb290a 100644 --- a/backend/tests/test_scene_ordering.py +++ b/backend/tests/test_scene_ordering.py @@ -1,27 +1,25 @@ -"""Start ordering at the engine boundary: an abandoned start can never replace or cancel a newer one. +"""Session-based start ordering at the engine boundary. -No sleeps decide anything here: the abandoned start A is held at the pre-registration seam -(`SceneJobs._check_source`, which runs before the registry lock) with a threading.Event until the test has -registered the wanted start B, then released, so it reaches the registry AFTER B every time. +Invariants (each test name carries its tag): + I1 Delayed starts cannot become valid through bookkeeping eviction. + I2 Cancellation cannot be forgotten while the corresponding start remains admissible. + I3 Idempotency is scoped to (session, request) and ownership is checked consistently on every path. + I4 Capacity exhaustion or session expiry fails EXPLICITLY; nothing old is ever silently revived. + +No sleep decides any outcome: the abandoned start A is held at the pre-registration seam +(`SceneJobs._check_source`, which runs before the registry lock) with a threading.Event, and time is a fake clock. """ import threading import pytest from snapstudio_api import scene_jobs as sj +from snapstudio_core import scene_limits as L from tests import scene_fixtures as fx from tests.test_scene_api import call, server # noqa: F401 (the loopback server fixture) from tests.test_scene_contract import sub_validator from tests.test_scene_jobs import fake_builder, wait_for, wait_state -CLIENT = "client-aaaaaaaa" -OTHER = "client-bbbbbbbb" -_n = iter(range(1, 10**6)) - - -def rid() -> str: - return f"req-ord-{next(_n):08d}" - @pytest.fixture def cube(tmp_path): @@ -41,6 +39,10 @@ def factory(**kw): jobs.close() +def sess(jobs) -> str: + return jobs.open_session()["client_id"] + + class HeldStart: """Run jobs.start(...) on a thread that blocks at the pre-registration seam until released.""" @@ -72,80 +74,257 @@ def finish(self): return self -# ------------------------------------------------------------------ the bug: A arrives after B +def refused(jobs, code, *args, http=None): + with pytest.raises(sj.JobError) as e: + jobs.start(*args) + assert e.value.code == code, (e.value.code, e.value.message) + if http: + assert e.value.http == http + assert not list(sub_validator("error_response").iter_errors(e.value.body())) + return e.value + + +# ------------------------------------------------------------------ the session + +def test_i4_open_session_shape_and_expiry_ttl(make): + jobs = make() + out = jobs.open_session() + assert set(out) == {"client_id", "ttl_s"} and len(out["client_id"]) == 24 and out["ttl_s"] == L.SESSION_TTL_SECONDS + assert not list(sub_validator("session").iter_errors(out)) + assert sess(jobs) != out["client_id"] + + +def test_the_per_session_state_is_constant_size(make, cube): + jobs = make() + s = sess(jobs) + for seq in range(1, 6): + jobs.start(cube, f"req-{seq:08d}", s, seq) + jobs.cancel_request(s, f"req-x-{seq:06d}", seq) + assert set(vars(jobs._sessions[s])) == {"client_id", "last_touch", "watermark", "dead_through", "current"} + assert not hasattr(jobs, "_tombstones") and not hasattr(jobs, "_clients") + + +# ------------------------------------------------------------------ I1: a delayed start never becomes valid -def test_an_abandoned_start_arriving_after_the_wanted_one_is_refused_and_changes_nothing(make, cube): +@pytest.mark.parametrize("wanted", ["running", "completed", "cancelled", "evicted"]) +def test_i1_a_delayed_abandoned_start_is_refused_whatever_became_of_the_wanted_one(make, cube, wanted): + now = [0.0] + gate, started = threading.Event(), threading.Event() + builder = fake_builder(gate, started=started) if wanted == "running" else fake_builder() + jobs = make(clock=lambda: now[0], builder=builder, ttl=10.0) + s = sess(jobs) + a = HeldStart(jobs, cube, "req-A-000001", s, 1) # abandoned, delayed in flight + b = jobs.start(cube, "req-B-000002", s, 2) # the wanted start gets in first + if wanted == "running": + assert started.wait(10) and jobs.status(b["job_id"])["state"] == "running" + else: + wait_state(jobs, b["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + if wanted == "cancelled": + jobs.cancel_request(s, "req-B-000002", 2) + if wanted == "evicted": + now[0] += 60.0 + with pytest.raises(sj.JobError): + jobs.status(b["job_id"]) # B's job is gone from the registry + a.finish() + assert a.error is not None and a.error.code in ("STALE_START", "CANCELLED_BEFORE_START") + if wanted == "cancelled": + assert a.error.code == "CANCELLED_BEFORE_START" + elif wanted == "evicted": + assert a.error.code == "STALE_START" # eviction did not make the late start valid + if wanted == "running": + assert jobs.status(b["job_id"])["state"] == "running" # B was neither cancelled nor replaced + gate.set() + wait_state(jobs, b["job_id"], "succeeded") + elif wanted == "completed": + assert jobs.status(b["job_id"])["state"] == "succeeded" + + +def test_i1_astra_repro_a_repeat_of_b_at_a_higher_seq_then_the_delayed_a(make, cube): jobs = make() - a = HeldStart(jobs, cube, "req-A-000001", CLIENT, 1) # sent first, abandoned, delayed in flight - b = jobs.start(cube, "req-B-000002", CLIENT, 2) # the wanted start arrives first + s = sess(jobs) + b = jobs.start(cube, "req-B-000001", s, 1) wait_state(jobs, b["job_id"], "succeeded") - before = jobs.retained() + a = HeldStart(jobs, cube, "req-A-000002", s, 2) + err = refused(jobs, "INVALID_REQUEST", cube, "req-B-000001", s, 3, http=400) # B repeated as seq 3 + assert err.extra["reason"] == "request id reused" a.finish() - assert a.error is not None and (a.error.http, a.error.code) == (409, "STALE_START") - assert not list(sub_validator("error_response").iter_errors(a.error.body())) - assert jobs.status(b["job_id"])["state"] == "succeeded" # B was never cancelled or replaced - assert jobs.retained() == before # and A registered nothing - assert jobs.start(cube, "req-B-000002", CLIENT, 2)["job_id"] == b["job_id"] + assert a.error.code == "STALE_START" # the watermark had already moved to 3 + assert jobs.status(b["job_id"])["state"] == "succeeded" + refused(jobs, "STALE_START", cube, "req-B-000001", s, 3) # the refused attempt is not an idempotent repeat + ok = jobs.start(cube, "req-B-000004", s, 4) # recovery: fresh request_id, higher seq + assert wait_state(jobs, ok["job_id"], "succeeded")["state"] == "succeeded" -def test_the_stale_start_cannot_replace_a_RUNNING_wanted_start(make, cube): +def test_i1_a_stale_start_cannot_replace_a_running_wanted_start(make, cube): gate, started = threading.Event(), threading.Event() jobs = make(builder=fake_builder(gate, started=started)) - a = HeldStart(jobs, cube, "req-A-000001", CLIENT, 1) - b = jobs.start(cube, "req-B-000002", CLIENT, 2) - assert started.wait(10) and jobs.status(b["job_id"])["state"] == "running" + s = sess(jobs) + a = HeldStart(jobs, cube, "req-A-000001", s, 1) + b = jobs.start(cube, "req-B-000002", s, 2) + started.wait(10) a.finish() - assert a.error.code == "STALE_START" - assert jobs.status(b["job_id"])["state"] == "running" # not cancelled by the late arrival + assert a.error.code == "STALE_START" and jobs.status(b["job_id"])["state"] == "running" + gate.set() + + +def test_i1_an_exact_retry_of_an_evicted_start_is_stale_and_replaces_nothing(make, cube): + now = [0.0] + gate, started = threading.Event(), threading.Event() + calls = [] + + def builder(path, revision, ctl): + calls.append(1) + if len(calls) == 1: + return b"{}" + started.set() + while not gate.wait(0.005): + ctl.check() + return b"{}" + jobs = make(clock=lambda: now[0], ttl=10.0, builder=builder) + s, t = sess(jobs), sess(jobs) + first = jobs.start(cube, "req-E-000001", s, 1) + wait_state(jobs, first["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + other = jobs.start(cube, "req-O-000001", t, 1) # someone else's job is running + assert started.wait(10) + now[0] += 60.0 + with pytest.raises(sj.JobError): + jobs.status(first["job_id"]) + err = refused(jobs, "STALE_START", cube, "req-E-000001", s, 1, http=409) + assert jobs.status(other["job_id"])["state"] == "running" # and it did not replace the running job gate.set() - assert wait_state(jobs, b["job_id"], "succeeded")["state"] == "succeeded" + wait_state(jobs, other["job_id"], "succeeded") + assert err + + +def test_i1_idempotent_repeat_while_registered_returns_the_same_job(make, cube): + jobs = make() + s = sess(jobs) + b = jobs.start(cube, "req-B-000002", s, 2) + again = jobs.start(cube, "req-B-000002", s, 2) + assert again["job_id"] == b["job_id"] and again["replaced_job_id"] is None + + +def test_i1_equal_seq_with_another_request_or_another_path_is_invalid(make, cube, tmp_path): + jobs = make() + s = sess(jobs) + jobs.start(cube, "req-B-000002", s, 2) + assert refused(jobs, "INVALID_REQUEST", cube, "req-X-000003", s, 2, http=400).extra["reason"] == "seq reused for another request" + other = str(fx.plain_cube_3mf(tmp_path / "other.3mf", size=11)) + refused(jobs, "INVALID_REQUEST", other, "req-B-000002", s, 2, http=400) + + +def test_i1_a_higher_seq_never_silently_returns_an_old_job(make, cube): + jobs = make() + s = sess(jobs) + j = jobs.start(cube, "req-R-000001", s, 1) + refused(jobs, "INVALID_REQUEST", cube, "req-R-000001", s, 2, http=400) + assert jobs.status(j["job_id"])["request_id"] == "req-R-000001" + + +def test_i1_stale_and_cancel_are_decided_before_source_changed(make, cube, monkeypatch): + for during, expected in (("newer", "STALE_START"), ("cancel", "CANCELLED_BEFORE_START"), ("expiry", "SESSION_EXPIRED")): + now = [0.0] + jobs = make(clock=lambda: now[0], session_ttl=100.0, ttl=10**9, max_terminal=100) + s = sess(jobs) + a = jobs.start(cube, "req-A-000001", s, 1) + wait_state(jobs, a["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + inside, release = threading.Event(), threading.Event() + + def hashing(path): # the file HAS changed, yet the answer is decided first + inside.set() + assert release.wait(10) + return "f" * 64 + monkeypatch.setattr(jobs, "_current_hash", hashing) + out = {} + + def repeat(): + try: + out["r"] = jobs.start(cube, "req-A-000001", s, 1) + except sj.JobError as exc: + out["e"] = exc + t = threading.Thread(target=repeat, daemon=True) + t.start() + assert inside.wait(10) + if during == "newer": + jobs.start(cube, "req-N-000002", s, 2) + elif during == "cancel": + jobs.cancel_request(s, "req-A-000001", 1) + else: + now[0] += 500.0 + jobs.cancel(a["job_id"]) # (the job is terminal; it still owns the session) + jobs._jobs.pop(a["job_id"]) + jobs._by_request.pop((s, "req-A-000001")) # nothing registered: the session may now expire + jobs._housekeeping() + release.set() + t.join(10) + assert out["e"].code == expected, (during, out) + + +def test_without_any_newer_start_a_changed_file_is_still_source_changed(make, cube): + jobs = make() + s = sess(jobs) + a = jobs.start(cube, "req-A-000001", s, 1) + wait_state(jobs, a["job_id"], "succeeded") + with open(cube, "ab") as fh: + fh.write(b"\0") + refused(jobs, "SOURCE_CHANGED", cube, "req-A-000001", s, 1) -# ------------------------------------------------------------------ cancel by (client_id, request_id) +# ------------------------------------------------------------------ I2: cancellation is not forgotten -def test_a_cancel_that_arrives_before_the_start_tombstones_it(make, cube): +def test_i2_a_cancel_that_arrives_before_the_start_refuses_it_and_every_lower_seq(make, cube): jobs = make() - out = jobs.cancel_request(CLIENT, "req-A-000001") # nothing registered yet + s = sess(jobs) + out = jobs.cancel_request(s, "req-A-000001", 3) # the client moved on from attempts 1..3 assert out["job_id"] is None and out["state"] == "cancelled" and out["error"]["code"] == "CANCELLED" assert not list(sub_validator("job_status").iter_errors(out)) - with pytest.raises(sj.JobError) as e: - jobs.start(cube, "req-A-000001", CLIENT, 1) - assert (e.value.http, e.value.code) == (409, "CANCELLED_BEFORE_START") + for seq, rid_ in ((3, "req-A-000001"), (2, "req-late-0002"), (1, "req-late-0001")): + refused(jobs, "CANCELLED_BEFORE_START", cube, rid_, s, seq, http=409) assert jobs.retained() == 0 # nothing was registered - fresh = jobs.start(cube, "req-A-000002", CLIENT, 2) # a fresh request_id with a higher seq works + fresh = jobs.start(cube, "req-A-000004", s, 4) # a fresh request_id with a higher seq works assert wait_state(jobs, fresh["job_id"], "succeeded")["state"] == "succeeded" -def test_cancelling_by_request_cancels_only_the_exact_job(make, cube): +def test_i2_a_cancelled_start_stays_refused_after_its_job_is_evicted_and_after_unrelated_activity(make, cube): + now = [0.0] + jobs = make(clock=lambda: now[0], ttl=1.0) + s = sess(jobs) + a = jobs.start(cube, "req-A-000001", s, 1) + jobs.cancel_request(s, "req-A-000001", 1) + wait_for(lambda: not jobs.worker_alive()) + now[0] += 100.0 + with pytest.raises(sj.JobError): + jobs.status(a["job_id"]) # the cancelled job is long gone + for i in range(200): # a great deal of unrelated cancel traffic + jobs.cancel_request(s, f"req-noise-{i:05d}", 1) + refused(jobs, "CANCELLED_BEFORE_START", cube, "req-A-000001", s, 1) + assert len(vars(jobs._sessions[s])) == 5 # and the memory is still O(1) + + +def test_i2_cancelling_a_never_touches_b(make, cube): gate, started = threading.Event(), threading.Event() jobs = make(builder=fake_builder(gate, started=started)) - a = jobs.start(cube, "req-A-000001", CLIENT, 1) + s = sess(jobs) + a = jobs.start(cube, "req-A-000001", s, 1) started.wait(10) - b = jobs.start(cube, "req-B-000002", CLIENT, 2) # replaces A as today + b = jobs.start(cube, "req-B-000002", s, 2) # replaces A as today assert jobs.status(a["job_id"])["state"] == "cancelled" - jobs.cancel_request(CLIENT, "req-A-000001") # cancelling A (again)... - assert jobs.status(b["job_id"])["state"] in ("queued", "running") # ...never touches B - jobs.cancel_request(OTHER, "req-B-000002") # same request_id, DIFFERENT client: no match + jobs.cancel_request(s, "req-A-000001", 1) # cancelling A... + assert jobs.status(b["job_id"])["state"] in ("queued", "running") # ...never touches B (exact request match) + jobs.cancel_request(s, "req-C-nonexistent", 1) assert jobs.status(b["job_id"])["state"] in ("queued", "running") - jobs.cancel_request(CLIENT, "req-C-nonexistent") # another request: no match - assert jobs.status(b["job_id"])["state"] in ("queued", "running") - out = jobs.cancel_request(CLIENT, "req-B-000002") # the exact pair + out = jobs.cancel_request(s, "req-B-000002", 2) assert out["job_id"] == b["job_id"] and out["state"] == "cancelled" gate.set() -def test_a_legacy_job_is_not_matched_by_a_client_cancel(make, cube): - jobs = make() - legacy = jobs.start(cube, "req-L-000001") - wait_state(jobs, legacy["job_id"], "succeeded") - out = jobs.cancel_request(CLIENT, "req-L-000001") - assert out["job_id"] is None - assert jobs.status(legacy["job_id"])["state"] == "succeeded" - - -def test_cancel_by_job_id_is_unchanged(make, cube): +def test_i2_cancel_by_job_id_is_unchanged(make, cube): jobs = make() - j = jobs.start(cube, "req-J-000001", CLIENT, 1) + s = sess(jobs) + j = jobs.start(cube, "req-J-000001", s, 1) wait_state(jobs, j["job_id"], "succeeded") assert jobs.cancel(j["job_id"])["state"] == "succeeded" with pytest.raises(sj.JobError) as e: @@ -153,170 +332,254 @@ def test_cancel_by_job_id_is_unchanged(make, cube): assert e.value.code == "EXPIRED" -# ------------------------------------------------------------------ idempotency, remount, restart, retry, legacy +# ------------------------------------------------------------------ I3: (session, request) scoping and ownership + +def test_i3_the_same_request_id_in_two_sessions_gives_each_its_own_job_and_cancel(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + s, t = sess(jobs), sess(jobs) + a = jobs.start(cube, "req-same-0001", s, 1) + started.wait(10) + b = jobs.start(cube, "req-same-0001", t, 1) # same request_id and path, another session + assert a["job_id"] != b["job_id"] and b["replaced_job_id"] == a["job_id"] + assert jobs.start(cube, "req-same-0001", t, 1)["job_id"] == b["job_id"] # idempotent inside its own scope + out = jobs.cancel_request(s, "req-same-0001", 1) + assert out["job_id"] == a["job_id"] + assert jobs.status(b["job_id"])["state"] in ("queued", "running") # t's job untouched: no leak across sessions + assert jobs.cancel_request(t, "req-same-0001", 1)["job_id"] == b["job_id"] + gate.set() + -def test_repeat_of_the_highest_seq_is_idempotent_and_a_different_request_is_refused(make, cube): +def test_i3_a_session_cannot_cancel_or_see_another_sessions_job_by_request(make, cube): jobs = make() - b = jobs.start(cube, "req-B-000002", CLIENT, 2) - again = jobs.start(cube, "req-B-000002", CLIENT, 2) - assert again["job_id"] == b["job_id"] and again["replaced_job_id"] is None - with pytest.raises(sj.JobError) as e: - jobs.start(cube, "req-X-000003", CLIENT, 2) - assert (e.value.http, e.value.code) == (400, "INVALID_REQUEST") and e.value.extra["reason"] == "seq reused for another request_id" - assert jobs.status(b["job_id"])["state"] in ("queued", "running", "succeeded") + s, t = sess(jobs), sess(jobs) + j = jobs.start(cube, "req-own-00001", s, 1) + wait_state(jobs, j["job_id"], "succeeded") + out = jobs.cancel_request(t, "req-own-00001", 1) + assert out["job_id"] is None + assert jobs.status(j["job_id"])["state"] == "succeeded" + assert jobs.start(cube, "req-own-00001", s, 1)["job_id"] == j["job_id"] + + +def test_i3_the_legacy_namespace_is_separate_and_unchanged(make, cube): + jobs = make() + s = sess(jobs) + legacy = jobs.start(cube, "req-ns-000001") + ordered = jobs.start(cube, "req-ns-000001", s, 1) + assert legacy["job_id"] != ordered["job_id"] + assert jobs.start(cube, "req-ns-000001")["job_id"] == legacy["job_id"] -def test_remount_continues_the_seq_and_a_restarted_client_is_independent(make, cube): +def test_i3_many_sessions_start_concurrently_and_independently(make, cube): jobs = make() - for seq in (1, 2, 3): - j = jobs.start(cube, rid(), CLIENT, seq) - wait_state(jobs, j["job_id"], "succeeded") + ids = [sess(jobs) for _ in range(8)] + barrier = threading.Barrier(len(ids)) + results, errors = {}, [] + + def go(i, cid): + barrier.wait(10) + try: + results[i] = jobs.start(cube, "req-same-0001", cid, 1) # every session reuses seq 1 AND the request_id + except sj.JobError as exc: + errors.append(exc) + threads = [threading.Thread(target=go, args=(i, c), daemon=True) for i, c in enumerate(ids)] + for t in threads: + t.start() + for t in threads: + t.join(10) + assert not errors and len(results) == len(ids) + assert len({r["job_id"] for r in results.values()}) == len(ids) + + +# ------------------------------------------------------------------ I4: expiry and capacity fail explicitly + +def test_i4_an_expired_session_refuses_start_and_cancel_and_a_late_delayed_start(make, cube): + now = [0.0] + jobs = make(clock=lambda: now[0], session_ttl=100.0) + s = sess(jobs) + late = HeldStart(jobs, cube, "req-A-000001", s, 1) # in flight when the session ends + now[0] += 500.0 + jobs.open_session() # any call prunes the idle-expired session + assert s not in jobs._sessions + late.finish() + assert late.error is not None and (late.error.http, late.error.code) == (409, "SESSION_EXPIRED") + assert s not in jobs._sessions and jobs.retained() == 0 # refused, never re-created, nothing registered + refused(jobs, "SESSION_EXPIRED", cube, "req-B-000002", s, 2) + with pytest.raises(sj.JobError) as e: + jobs.cancel_request(s, "req-B-000002", 2) + assert e.value.code == "SESSION_EXPIRED" + assert s not in jobs._sessions + + +@pytest.mark.parametrize("how", ["refused_start", "cancel", "status"]) +def test_i4_every_start_cancel_and_status_touches_the_session(make, cube, how): + """Each call keeps alive a session that would otherwise be idle-expired (the job it owned is already gone).""" + now = [0.0] + jobs = make(clock=lambda: now[0], session_ttl=100.0, ttl=100.0) + s = sess(jobs) + j = jobs.start(cube, "req-A-000002", s, 2) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + now[0] += 90.0 + if how == "status": + jobs.status(j["job_id"]) + elif how == "cancel": + jobs.cancel_request(s, "req-zzz-000001", 1) + else: + refused(jobs, "STALE_START", cube, "req-old-000001", s, 1) # a refused start still touches the session + now[0] += 90.0 # 180 s since creation, 90 s since the touch + jobs.open_session() # evicts the expired job (the session owned it until now) + jobs.open_session() # now the session is judged on its idle time alone + assert s in jobs._sessions + + +def test_i4_session_limit_is_explicit_when_nothing_is_prunable(make, cube): + jobs = make(max_sessions=3) + ids = [sess(jobs) for _ in range(3)] + with pytest.raises(sj.JobError) as e: + jobs.open_session() + assert (e.value.http, e.value.code) == (503, "SESSION_LIMIT") + assert not list(sub_validator("error_response").iter_errors(e.value.body())) + assert set(jobs._sessions) == set(ids) # a live session is never evicted to make room + + +def test_i4_an_idle_expired_session_without_a_job_is_pruned_to_make_room(make): + now = [0.0] + jobs = make(clock=lambda: now[0], max_sessions=2, session_ttl=100.0) + first, second = sess(jobs), sess(jobs) + now[0] += 50.0 + jobs._sessions[second].last_touch = now[0] # the second one stays active + now[0] += 60.0 # the first is idle 110 s (> ttl), the second 60 s + third = sess(jobs) + assert first not in jobs._sessions and second in jobs._sessions and third in jobs._sessions + + +def test_i4_a_session_that_owns_a_registered_job_is_never_pruned(make, cube): + now = [0.0] + jobs = make(clock=lambda: now[0], max_sessions=2, session_ttl=100.0, ttl=10**9, max_terminal=100) + s = sess(jobs) + j = jobs.start(cube, "req-A-000001", s, 1) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + other = sess(jobs) + now[0] += 10**6 # idle far beyond the ttl + with pytest.raises(sj.JobError) as e: + jobs.open_session() # `other` is prunable, `s` is not -> room for one + jobs.open_session() + assert e.value.code == "SESSION_LIMIT" + assert s in jobs._sessions and other not in jobs._sessions + + +def test_i4_an_engine_restart_drops_every_session_explicitly(make, cube): + old = make() + s = sess(old) + restarted = make() # a new SceneJobs is a new engine process + refused(restarted, "SESSION_EXPIRED", cube, "req-A-000001", s, 1, http=409) with pytest.raises(sj.JobError) as e: - jobs.start(cube, rid(), CLIENT, 2) # an old attempt from the same process - assert e.value.code == "STALE_START" - fresh_process = jobs.start(cube, rid(), OTHER, 1) # app restart: new random client_id, seq 1 - assert wait_state(jobs, fresh_process["job_id"], "succeeded")["state"] == "succeeded" - assert jobs.start(cube, rid(), CLIENT, 4)["state"] in ("queued", "running", "succeeded") + restarted.cancel_request(s, "req-A-000001", 1) + assert e.value.code == "SESSION_EXPIRED" + assert restarted.retained() == 0 and not restarted._sessions -def test_retry_with_a_fresh_request_id_and_a_higher_seq_after_a_stale_refusal(make, cube): +def test_i4_an_unknown_session_id_is_not_created_by_a_start(make, cube): jobs = make() - jobs.start(cube, "req-B-000002", CLIENT, 5) - with pytest.raises(sj.JobError): - jobs.start(cube, "req-A-000001", CLIENT, 4) - retry = jobs.start(cube, "req-A-000009", CLIENT, 6) - assert wait_state(jobs, retry["job_id"], "succeeded")["state"] == "succeeded" + refused(jobs, "SESSION_EXPIRED", cube, "req-A-000001", "never-issued-session-id", 1) + assert not jobs._sessions + +# ------------------------------------------------------------------ the legacy path -def test_legacy_starts_without_the_fields_still_work_and_mix_freely(make, cube): +def test_legacy_starts_without_the_fields_still_work(make, cube): jobs = make() a = jobs.start(cube, "req-L-000001") wait_state(jobs, a["job_id"], "succeeded") b = jobs.start(cube, "req-L-000002") wait_state(jobs, b["job_id"], "succeeded") - c = jobs.start(cube, "req-O-000003", CLIENT, 7) + s = sess(jobs) + c = jobs.start(cube, "req-O-000003", s, 7) wait_state(jobs, c["job_id"], "succeeded") - d = jobs.start(cube, "req-L-000004") # legacy after ordered: not judged + d = jobs.start(cube, "req-L-000004") assert wait_state(jobs, d["job_id"], "succeeded")["state"] == "succeeded" -def test_two_clients_are_independent_and_do_not_judge_each_other(make, cube): - jobs = make() - jobs.start(cube, "req-A-000001", CLIENT, 50) - b = jobs.start(cube, "req-B-000001", OTHER, 1) # a low seq for another client is fine - assert b["state"] in ("queued", "running", "succeeded") - held = HeldStart(jobs, cube, "req-B-000002", OTHER, 2) # concurrent, in flight - stale = jobs.start(cube, "req-A-000002", CLIENT, 51) - held.finish() - assert held.error is None and held.result is not None - assert stale["job_id"] - - -# ------------------------------------------------------------------ bounded maps +# ------------------------------------------------------------------ validation -def test_the_client_map_is_an_lru_and_the_live_client_keeps_its_ordering(make, cube): +@pytest.mark.parametrize("extra", [ + {"client_id": None, "seq": None}, {"client_id": "x" * 24, "seq": None}, {"client_id": None, "seq": 1}, + {"client_id": "x" * 24 + "\n", "seq": 1}, {"seq": None}, {"client_id": None}, {"seq": 1}, {"client_id": "x" * 24}, + {"client_id": "x" * 24, "seq": 0}, {"client_id": "x" * 24, "seq": -1}, {"client_id": "x" * 24, "seq": True}, + {"client_id": "x" * 24, "seq": 1.0}, {"client_id": "x" * 24, "seq": "1"}, {"client_id": "x" * 24, "seq": 2 ** 53}, + {"client_id": "short", "seq": 1}, {"client_id": 12345678, "seq": 1}, +]) +def test_validation_holes_are_closed_on_start(make, cube, extra): jobs = make() - live = "client-live0001" - jobs.start(cube, rid(), live, 10) - for i in range(sj.MAX_CLIENTS + 5): # many other clients come and go - jobs.start(cube, rid(), f"client-{i:08d}", 1) - jobs.start(cube, rid(), live, 11 + i) # the live client keeps being used (touched) - assert len(jobs._clients) <= sj.MAX_CLIENTS - with pytest.raises(sj.JobError) as e: - jobs.start(cube, rid(), live, 10) - assert e.value.code == "STALE_START" # ordering for the live client survived eviction - assert "client-00000000" not in jobs._clients # an idle one was evicted (oldest first) + status, body = sj.handle("/scene/start", dict({"path": cube, "request_id": "req-V-000001"}, **extra), jobs) + assert (status, body["error"]) == (400, "INVALID_REQUEST"), (extra, status, body) + assert jobs.retained() == 0 and not jobs._sessions -def test_tombstones_are_a_bounded_fifo(make, cube): +def test_validation_holes_are_closed_on_cancel_and_ids(make, cube): jobs = make() - for i in range(sj.MAX_TOMBSTONES + 1): - jobs.cancel_request(CLIENT, f"req-T-{i:08d}") - assert len(jobs._tombstones) == sj.MAX_TOMBSTONES - assert (CLIENT, "req-T-00000000") not in jobs._tombstones # first in, first out - assert (CLIENT, f"req-T-{sj.MAX_TOMBSTONES:08d}") in jobs._tombstones - with pytest.raises(sj.JobError) as e: - jobs.start(cube, f"req-T-{sj.MAX_TOMBSTONES:08d}", CLIENT, 1) - assert e.value.code == "CANCELLED_BEFORE_START" + s = sess(jobs) + bad = [ + {"client_id": s, "request_id": "req-V-000001"}, # seq missing + {"client_id": s, "request_id": "req-V-000001\n", "seq": 1}, {"client_id": s + "\n", "request_id": "req-V-000001", "seq": 1}, + {"client_id": s, "request_id": "req-V-000001", "seq": None}, {"client_id": None, "request_id": "req-V-000001", "seq": 1}, + {"client_id": s, "request_id": "req-V-000001", "seq": True}, {"client_id": s, "request_id": "req-V-000001", "seq": 1.5}, + {"job_id": "0" * 32, "client_id": s, "request_id": "req-V-000001", "seq": 1}, + {"request_id": "req-V-000001", "seq": 1}, + ] + for body in bad: + status, resp = sj.handle("/scene/cancel", body, jobs) + assert (status, resp["error"]) == (400, "INVALID_REQUEST"), (body, status, resp) + status, resp = sj.handle("/scene/result", {"job_id": "0" * 32, "expected_revision": "a" * 64 + "\n"}, jobs) + assert status == 400 + assert jobs._sessions[s].dead_through == 0 # no malformed cancel changed any state + + +def test_seq_upper_bound_is_two_to_the_53_minus_1(make, cube): + jobs = make() + s = sess(jobs) + status, _ = sj.handle("/scene/start", {"path": cube, "request_id": "req-S-000001", "client_id": s, "seq": 2 ** 53 - 1}, jobs) + assert status == 200 -# ------------------------------------------------------------------ the real HTTP route and body validation +# ------------------------------------------------------------------ the real HTTP route -def test_http_stale_start_and_the_cancel_by_request_shape(server, tmp_path): # noqa: F811 +def test_http_session_start_stale_cancel_and_expiry_shapes(server, tmp_path): # noqa: F811 port, token = server path = str(fx.plain_cube_3mf(tmp_path / "h.3mf")) - code, b = call(port, "/scene/start", {"path": path, "request_id": "req-B-http0002", "client_id": CLIENT, "seq": 2}, token) + code, opened = call(port, "/scene/session", {}, token) + assert code == 200 and set(opened) == {"client_id", "ttl_s"} and not list(sub_validator("session").iter_errors(opened)) + cid = opened["client_id"] + code, b = call(port, "/scene/start", {"path": path, "request_id": "req-B-http0002", "client_id": cid, "seq": 2}, token) assert code == 200 and not list(sub_validator("job_start").iter_errors(b)) - code, body = call(port, "/scene/start", {"path": path, "request_id": "req-A-http0001", "client_id": CLIENT, "seq": 1}, token) - assert (code, body["error"]) == (409, "STALE_START") - assert not list(sub_validator("error_response").iter_errors(body)) + code, body = call(port, "/scene/start", {"path": path, "request_id": "req-A-http0001", "client_id": cid, "seq": 1}, token) + assert (code, body["error"]) == (409, "STALE_START") and not list(sub_validator("error_response").iter_errors(body)) code, status = call(port, "/scene/status", {"job_id": b["job_id"]}, token) - assert code == 200 and status["state"] != "cancelled" # B untouched - code, c = call(port, "/scene/cancel", {"client_id": CLIENT, "request_id": "req-C-http0003"}, token) - assert code == 200 and c["job_id"] is None and c["state"] == "cancelled" - assert not list(sub_validator("job_status").iter_errors(c)) - code, body = call(port, "/scene/start", {"path": path, "request_id": "req-C-http0003", "client_id": CLIENT, "seq": 3}, token) + assert code == 200 and status["state"] != "cancelled" + code, c = call(port, "/scene/cancel", {"client_id": cid, "request_id": "req-C-http0003", "seq": 3}, token) + assert code == 200 and c["job_id"] is None and c["state"] == "cancelled" and not list(sub_validator("job_status").iter_errors(c)) + code, body = call(port, "/scene/start", {"path": path, "request_id": "req-C-http0003", "client_id": cid, "seq": 3}, token) assert (code, body["error"]) == (409, "CANCELLED_BEFORE_START") - code, hit = call(port, "/scene/cancel", {"client_id": CLIENT, "request_id": "req-B-http0002"}, token) - assert code == 200 and hit["job_id"] == b["job_id"] - code, legacy = call(port, "/scene/start", {"path": path, "request_id": "req-L-http0004"}, token) + code, body = call(port, "/scene/start", {"path": path, "request_id": "req-D-http0004", "client_id": "never-issued-session-id", "seq": 1}, token) + assert (code, body["error"]) == (409, "SESSION_EXPIRED") + code, body = call(port, "/scene/cancel", {"client_id": "never-issued-session-id", "request_id": "req-D-http0004", "seq": 1}, token) + assert (code, body["error"]) == (409, "SESSION_EXPIRED") + code, legacy = call(port, "/scene/start", {"path": path, "request_id": "req-L-http0005"}, token) assert code == 200 -@pytest.mark.parametrize("extra", [ - {"client_id": CLIENT, "seq": 0}, {"client_id": CLIENT, "seq": -3}, {"client_id": CLIENT, "seq": "1"}, - {"client_id": CLIENT, "seq": True}, {"client_id": CLIENT, "seq": 1.5}, {"client_id": CLIENT, "seq": 2 ** 60}, - {"client_id": "short", "seq": 1}, {"client_id": "has space in it", "seq": 1}, {"client_id": "x" * 65, "seq": 1}, - {"client_id": 12345678, "seq": 1}, {"client_id": CLIENT}, {"seq": 1}, -]) -def test_http_start_rejects_malformed_ordering_fields(server, tmp_path, extra): # noqa: F811 - port, token = server - path = str(fx.plain_cube_3mf(tmp_path / "v.3mf")) - code, body = call(port, "/scene/start", dict({"path": path, "request_id": "req-V-http0001"}, **extra), token) - assert (code, body["error"]) == (400, "INVALID_REQUEST"), (extra, code, body) - - -@pytest.mark.parametrize("body", [ - {"client_id": "short", "request_id": "req-C-http0001"}, {"client_id": CLIENT}, - {"client_id": CLIENT, "request_id": "bad id"}, {"job_id": "0" * 32, "client_id": CLIENT, "request_id": "req-C-http0001"}, -]) -def test_http_cancel_rejects_malformed_bodies(server, body): # noqa: F811 +def test_http_session_limit_is_503(server): # noqa: F811 port, token = server - code, resp = call(port, "/scene/cancel", body, token) - assert (code, resp["error"]) == (400, "INVALID_REQUEST"), (body, code, resp) + codes = [call(port, "/scene/session", {}, token)[0] for _ in range(L.MAX_SESSIONS + 1)] + assert codes[:L.MAX_SESSIONS] == [200] * L.MAX_SESSIONS and codes[-1] == 503 + code, body = call(port, "/scene/session", {}, token) + assert (code, body["error"]) == (503, "SESSION_LIMIT") -@pytest.mark.parametrize("body", [[1], "x", None, 7]) -def test_http_non_object_bodies_are_still_400(server, body): # noqa: F811 +def test_http_routes_reject_non_object_bodies_and_need_the_token(server): # noqa: F811 port, token = server for route in sj.ROUTES: - assert call(port, route, body, token)[0] == 400 - - -def test_a_repeat_of_an_old_start_that_is_still_hashing_is_refused_if_a_newer_one_arrived(make, cube, monkeypatch): - jobs = make() - a = jobs.start(cube, "req-A-000001", CLIENT, 1) - status = wait_state(jobs, a["job_id"], "succeeded") - wait_for(lambda: not jobs.worker_alive()) - inside, release = threading.Event(), threading.Event() - - def slow_hash(path): # the idempotent repeat hashes OUTSIDE the lock - inside.set() - assert release.wait(10) - return status["revision"] - monkeypatch.setattr(jobs, "_current_hash", slow_hash) - out = {} - - def repeat(): - try: - out["r"] = jobs.start(cube, "req-A-000001", CLIENT, 1) - except sj.JobError as exc: - out["e"] = exc - t = threading.Thread(target=repeat, daemon=True) - t.start() - assert inside.wait(10) - b = jobs.start(cube, "req-B-000002", CLIENT, 2) # the wanted start gets in while A's repeat hashes - release.set() - t.join(10) - assert "r" not in out and out["e"].code == "STALE_START" - assert jobs.status(b["job_id"])["state"] in ("queued", "running", "succeeded") + assert call(port, route, None, token)[0] == 400 + assert call(port, route, {}, None)[0] == 401 diff --git a/docs/testing/scene-contract/README.md b/docs/testing/scene-contract/README.md index 8c49b21a..9da43fb9 100644 --- a/docs/testing/scene-contract/README.md +++ b/docs/testing/scene-contract/README.md @@ -132,49 +132,63 @@ running. A terminal job is kept 120 s or until more than 8 terminal jobs exist. | Route | Answers | |---|---| -| `/scene/start {path, request_id, client_id?, seq?}` | 200 job; 400 `INVALID_REQUEST` (also: only one of `client_id`/`seq`, a bad `seq`/`client_id`, or the same `seq` reused for another request_id); 422 `UNSUPPORTED_FORMAT`; 409 `INVALID_REQUEST` (request_id reused for another path), `SOURCE_CHANGED`, `STALE_START` or `CANCELLED_BEFORE_START`; 503 `WORKER_WEDGED` | +| `/scene/session {}` | 200 `{client_id, ttl_s}` (server-issued 24-character id); 503 `SESSION_LIMIT` | +| `/scene/start {path, request_id, client_id?, seq?}` | 200 job; 400 `INVALID_REQUEST` (also: only one of `client_id`/`seq`, null or malformed values, `seq` reused for another request, `request_id` reused at a higher `seq`); 422 `UNSUPPORTED_FORMAT`; 409 `INVALID_REQUEST` (request_id reused for another path), `SOURCE_CHANGED`, `STALE_START`, `CANCELLED_BEFORE_START` or `SESSION_EXPIRED`; 503 `WORKER_WEDGED` | | `/scene/status {job_id}` | 200 `{state, stage, completed, total, error, revision}`; 404 `EXPIRED` | | `/scene/result {job_id, expected_revision?}` | 200 scene; 422 `{error: }` for a failed job; 409 `CANCELLED` / `NOT_READY` / `SOURCE_CHANGED`; 404 `EXPIRED` | -| `/scene/cancel {job_id}` or `{client_id, request_id}` | 200 status (idempotent; a late cancel on a terminal job is a no-op). By request: cancels the job registered for exactly that client_id + request_id (if any) and ALWAYS records a tombstone; with no such job the body is a `job_status` with `job_id: null`, `state: "cancelled"` | +| `/scene/cancel {job_id}` or `{client_id, request_id, seq}` | 200 status (idempotent; a late cancel on a terminal job is a no-op). By request: 409 `SESSION_EXPIRED` for an unknown session; otherwise `dead_through` rises to `seq` and the job registered for exactly that client_id + request_id (if any) is cancelled; with no such job the body is a `job_status` with `job_id: null`, `state: "cancelled"` | Error codes: `INVALID_REQUEST UNSUPPORTED_FORMAT INVALID_ARCHIVE INVALID_GEOMETRY UNRESOLVED_REFERENCE LIMIT_EXCEEDED SOURCE_CHANGED CANCELLED TIMEOUT EXPIRED NOT_READY WORKER_WEDGED INTERNAL STALE_START -CANCELLED_BEFORE_START` (`BUSY` is not an error code: a new start replaces the client's own earlier work). +CANCELLED_BEFORE_START SESSION_EXPIRED SESSION_LIMIT` (`BUSY` is not an error code: a new start replaces the client's own earlier work). -### Start ordering (an abandoned start can never replace a newer one) +### Start ordering by session A client that aborts or times out a start cannot stop that request from still reaching the engine, possibly AFTER -the start it actually wants. The engine therefore orders starts itself, under the same lock that registers and -replaces jobs, so the decision is atomic with registration (nothing here uses timing): - -* `client_id` is 8-64 characters of `[A-Za-z0-9_-]`, random per app process; `seq` is an integer >= 1 that the - client increases for every start attempt. Both or neither; with neither the start behaves exactly as before. -* The engine remembers the highest accepted `seq` per `client_id` (LRU, 16 clients) and the cancelled - `(client_id, request_id)` pairs (FIFO, 64 tombstones). -* In order: `seq` below that client's highest -> 409 `STALE_START`, nothing changes (no replace, no cancel); - the pair is tombstoned -> 409 `CANCELLED_BEFORE_START`, nothing is registered; `seq` equal to the highest with - the same `request_id` -> the existing job (idempotent, as before); equal with a different `request_id` -> 400 - `INVALID_REQUEST`; otherwise the job is registered (replacing the older one as before) and `seq` is recorded. - An idempotent repeat that was still hashing the file is re-checked under the lock, so a newer start that arrived - meanwhile makes it `STALE_START` too. -* After `STALE_START` / `CANCELLED_BEFORE_START` the client retries with a FRESH `request_id` and a HIGHER `seq`. - A restarted app uses a new random `client_id` (seq starts again at 1); a remount in the same process just keeps - counting. If more than 16 clients are seen the least recently used one forgets its ordering (documented). - -Interruptibility: archive reads are at most 1 MiB, lxml `iterparse` checks the cancel/deadline hook every 1,000 -events and on every read, geometry transform/encode runs in batches of at most 5,000 triangles. Backstop: if the -worker is still alive 5 s after a cancel or the deadline, the job is reported failed `TIMEOUT` (deadline case) and -every new start answers 503 `WORKER_WEDGED` until the thread exits. -**Accepted limitation:** a single stuck native or filesystem call cannot be interrupted in-process (there is no -multiprocessing in the frozen sidecar); the wedged state is the fail-closed backstop. - -Snapshots live in `/scene-tmp/-/`. The worker deletes its own snapshot in its `finally` before it -sets `worker_done`; eviction only touches terminal jobs whose worker is done (missing-file tolerant); cancel never -deletes a file. A snapshot file that cannot be deleted (a Windows sharing violation) stays on a bounded retry list -with back-off until it is gone, and the per-process folder is removed best-effort at shutdown. The per-process -folder is re-created for every job. A startup sweep removes a `-` folder ONLY when its owner pid is -not running AND the folder is at least 10 minutes old; a folder owned by a live process is never removed, -however old. Sharing violations are tolerated. +the start it actually wants. The engine therefore orders a client's starts itself. The design is a **session** with +O(1) state, not a set of remembered requests. Four invariants, each pinned by tests named `test_i1_...` to +`test_i4_...`: + +* **I1: delayed starts cannot become valid through bookkeeping eviction.** A start is judged against the + session's watermark, not against whether its job is still retained, so an evicted job does not make an old + start admissible again. +* **I2: cancellation cannot be forgotten while the start remains admissible.** A cancel raises the session's + `dead_through` mark, so that start and every lower `seq` stay refused for as long as the session lives. +* **I3: idempotency is scoped to (session, request) and ownership is checked consistently on every path.** Jobs + are keyed by `(client_id, request_id)`; two sessions may use the same request_id and path and each gets, and + can cancel, its own job; cancel by request matches both ids exactly. +* **I4: capacity exhaustion or session expiry fails explicitly; nothing old is silently revived.** + +**Protocol.** `POST /scene/session {}` returns `{client_id, ttl_s}`; the engine chooses the id (24 characters of +`[A-Za-z0-9_-]`). Starts and by-request cancels carry that `client_id` and a `seq`, an integer from 1 to 2**53-1 that +the client treats as a strictly increasing generation counter for the session (the client only ever abandons lower +seqs when it moves to a higher one). Both keys or neither: null or half-present pairs are `400 INVALID_REQUEST`, +never a legacy start; ids must match in full (a trailing newline is rejected). With neither key the start behaves +exactly as before, in its own request_id-only namespace. + +**State per session** (nothing else): `watermark` (highest admitted seq), `dead_through` (highest cancelled seq), +`current` = (seq, request_id, path, job_id) of the entry at the watermark. + +**Rules, in order, under the registry lock that also registers jobs** (one admission function for every +successful start): unknown or expired `client_id` -> 409 `SESSION_EXPIRED` (a late start from an expired session +is refused, never re-created); `seq <= dead_through` -> 409 `CANCELLED_BEFORE_START`; `seq < watermark` -> 409 +`STALE_START`; `seq == watermark` with the same request_id and path -> the registered job (idempotent), or 409 +`STALE_START` if that job is no longer registered (an exact retry of an already-processed start is refused); equal +with anything else -> 400 `INVALID_REQUEST`; `seq > watermark` -> register (replacing the older job as before) and +set the watermark, except that a request_id already registered for the session is 400 `INVALID_REQUEST` +("request id reused": the old job is never handed back), and that attempt still raises the watermark so an older +attempt still in flight is refused. An idempotent repeat that was still hashing the file is re-judged by the same +rules BEFORE `SOURCE_CHANGED` is reported. After any refusal the client retries with a FRESH `request_id` and a +HIGHER `seq`. + +**Sessions.** At most 32 (`MAX_SESSIONS`), idle TTL 15 minutes (`SESSION_TTL_SECONDS`), touched by every start, +cancel and status. Only idle-expired sessions that own no registered job are pruned; a live session is never +evicted. At capacity with nothing prunable, opening a session is 503 `SESSION_LIMIT`. + +**Limits, stated plainly.** Ordering is provided PER SESSION. It is not provided across sessions: a start from +another session replaces the previous job exactly as before. An engine restart drops every session, so the first +start from an old `client_id` answers `SESSION_EXPIRED` and the client must open a new session. Session expiry +likewise ends that session's ordering (explicitly: `SESSION_EXPIRED`, never a silent revival). ## Budgets (provisional) and the coverage decision From 0b69bac6d07e499ba1976149b8a0e46ebf9624f5 Mon Sep 17 00:00:00 2001 From: DeadlyVirusIn Date: Fri, 9 Oct 2026 14:59:05 -0400 Subject: [PATCH 4/6] fix(scene): session-own job ids; advance ordering before post-admission failures - status, result and cancel by job id now require the owning session's client_id. A missing, foreign or never-issued id answers the same 404 as an unknown job id, so existence is never confirmed. replaced_job_id in a start response is only returned when the replaced job has the same owner. Legacy jobs behave as before. Cancel, result and status touch the owner session. - The session watermark advances atomically with admission, before any rejection that happens after a start was admitted (worker wedged, unsupported format, missing file), so a delayed older start cannot be accepted after a newer attempt was processed. After such a rejection the client retries with a fresh request id and a higher sequence. - Cancel by request also requires the job's sequence to be at or below the cancelled sequence, so a delayed cancel for an old attempt cannot cancel a newer job that reused the request id after eviction. - Sessions are pruned after job eviction, so an expired session whose last job was just evicted does not cause a spurious SESSION_LIMIT. Tests use events, barriers and a fake clock; 11 of 12 new-rule mutants are caught and the survivor is a redundant guard. Verified: eight scene/unit files 296 passed; full backend 3154 passed, 49 skipped; millimeter-unchanged differential identical to base. Co-Authored-By: Claude Sonnet 5.5 --- backend/snapstudio_api/scene_jobs.py | 104 +++++++---- backend/tests/test_scene_ordering.py | 253 +++++++++++++++++++++++--- docs/testing/scene-contract/README.md | 17 +- 3 files changed, 315 insertions(+), 59 deletions(-) diff --git a/backend/snapstudio_api/scene_jobs.py b/backend/snapstudio_api/scene_jobs.py index b87f9403..4bae11ab 100644 --- a/backend/snapstudio_api/scene_jobs.py +++ b/backend/snapstudio_api/scene_jobs.py @@ -89,7 +89,8 @@ class Job: def __init__(self, job_id: str, request_id: str, path: str, now: float) -> None: self.job_id, self.request_id, self.path = job_id, request_id, path self.source = path # the path as the client gave it (for reading) - self.client_id: str | None = None # set when the start carried client_id/seq + self.client_id: str | None = None # the OWNING session (None for a legacy job) + self.seq: int | None = None # the start's seq inside that session self.state = "queued" self.stage: str | None = None self.completed: int | None = None @@ -282,12 +283,35 @@ def _admit_locked(self, client_id: str | None, seq: int | None, request_id: str, reason="request id reused") return None + def _advance_locked(self, client_id: str | None, seq: int | None, request_id: str, path_key: str, + job_id: str | None = None) -> None: + """Record an attempt that has been (or is being) processed, so no delayed OLDER attempt can overtake it. + Called for every admitted start BEFORE anything that can still reject it (WORKER_WEDGED, an unsupported + format, a missing file), and again with the job id once it is registered.""" + if client_id is None: + return + session = self._sessions.get(client_id) + if session is not None and seq > session.dead_through and seq >= session.watermark: + session.watermark, session.current = seq, (seq, request_id, path_key, job_id) + + def _precheck(self, path: str, key: str, request_id: str, client_id: str | None, seq: int | None) -> None: + """The checks that run before registration. A rejection here still counts as a processed attempt for its + session (its seq is recorded), so a delayed older start cannot be accepted behind it.""" + try: + if os.path.splitext(key)[1].lower() not in (".3mf", ".stl"): + raise JobError(422, "UNSUPPORTED_FORMAT", "Only .3mf and .stl files can be shown.") + self._check_source(path) + except JobError: + if client_id is not None: + with self._lock: + self._housekeeping() + self._admit_locked(client_id, seq, request_id, key) # stale / cancelled / expired are reported first + self._advance_locked(client_id, seq, request_id, key) + raise + def start(self, path: str, request_id: str, client_id: str | None = None, seq: int | None = None) -> dict: key = os.path.normcase(os.path.abspath(path)) - ext = os.path.splitext(key)[1].lower() - if ext not in (".3mf", ".stl"): - raise JobError(422, "UNSUPPORTED_FORMAT", "Only .3mf and .stl files can be shown.") - self._check_source(path) + self._precheck(path, key, request_id, client_id, seq) for _attempt in range(2): with self._lock: self._housekeeping() @@ -318,24 +342,22 @@ def start(self, path: str, request_id: str, client_id: str | None = None, seq: i def _create_locked(self, path: str, key: str, request_id: str, client_id: str | None = None, seq: int | None = None) -> dict: + self._advance_locked(client_id, seq, request_id, key) # H2: BEFORE anything below can reject if self._wedged(): raise JobError(503, "WORKER_WEDGED", "The scene worker is stuck. Try again after it recovers.") replaced = None - if self._queued is not None: - replaced = self._queued.job_id - self._cancel_locked(self._queued) - self._queued = None - if self._running is not None and self._running.state == "running": - replaced = self._running.job_id - self._cancel_locked(self._running) + for old in (self._queued, self._running if self._running is not None and self._running.state == "running" else None): + if old is not None: + if old.client_id == client_id: # H1: another session's job id is never revealed + replaced = old.job_id + self._cancel_locked(old) + self._queued = None job = Job(uuid.uuid4().hex, request_id, key, self._clock()) job.source = path - job.client_id = client_id + job.client_id, job.seq = client_id, seq self._jobs[job.job_id] = job self._by_request[(client_id, request_id)] = job - if client_id is not None: # every registration records the session state atomically - session = self._sessions[client_id] - session.watermark, session.current = seq, (seq, request_id, key, job.job_id) + self._advance_locked(client_id, seq, request_id, key, job.job_id) # every registration records session state atomically self._queued = job if self._worker is None: self._worker = threading.Thread(target=self._worker_main, name="scene-worker", daemon=True) @@ -347,17 +369,26 @@ def _touch_owner_locked(self, job: Job) -> None: if session is not None: session.last_touch = self._clock() - def status(self, job_id: str) -> dict: + def _get_owned(self, job_id: str, client_id: str | None) -> Job: + """Resolve a job id for a caller. A job owned by a session needs that session's client_id; a missing or + different one answers EXACTLY like an unknown id (404 EXPIRED), so existence is never confirmed. A legacy + job (no owner) keeps today's behaviour.""" + job = self._jobs.get(job_id) + if job is None or (job.client_id is not None and job.client_id != client_id): + raise JobError(404, "EXPIRED", "That scene is no longer available.") + self._touch_owner_locked(job) + return job + + def status(self, job_id: str, client_id: str | None = None) -> dict: with self._lock: self._housekeeping() - job = self._get(job_id) - self._touch_owner_locked(job) - return self._status_body(job) + return self._status_body(self._get_owned(job_id, client_id)) - def result(self, job_id: str, expected_revision: str | None = None) -> tuple[int, bytes | dict]: + def result(self, job_id: str, expected_revision: str | None = None, + client_id: str | None = None) -> tuple[int, bytes | dict]: with self._lock: self._housekeeping() - job = self._get(job_id) + job = self._get_owned(job_id, client_id) if expected_revision is not None and job.revision is not None and job.revision != expected_revision: raise JobError(409, "SOURCE_CHANGED", "That scene was built from a different version of the file.") if job.state == "succeeded": @@ -372,13 +403,14 @@ def cancel_request(self, client_id: str, request_id: str, seq: int) -> dict: """Cancel the start (client_id, request_id, seq). The session must exist (SESSION_EXPIRED otherwise). ``dead_through`` rises to ``seq`` so that start, and every lower seq, stays refused for as long as the session lives (O(1) memory: no per-request set), even when no job is registered yet. The registered job - is cancelled only if its client_id AND request_id both match exactly.""" + is cancelled only if its client_id AND request_id match exactly AND it was started at or before ``seq`` + (a delayed cancel for an OLD attempt can never cancel a newer job that reused the request id).""" with self._lock: self._housekeeping() session = self._session_locked(client_id) session.dead_through = max(session.dead_through, seq) job = self._by_request.get((client_id, request_id)) - if job is not None and job.client_id == client_id and job.request_id == request_id: + if job is not None and job.client_id == client_id and job.request_id == request_id and job.seq <= seq: if job.state in ("queued", "running"): if job is self._queued: self._queued = None @@ -390,10 +422,10 @@ def cancel_request(self, client_id: str, request_id: str, seq: int) -> dict: "completed": None, "total": None, "revision": None, "error": {"code": "CANCELLED", "message": "That scene was cancelled."}} - def cancel(self, job_id: str) -> dict: + def cancel(self, job_id: str, client_id: str | None = None) -> dict: with self._lock: self._housekeeping() - job = self._get(job_id) + job = self._get_owned(job_id, client_id) # touches the owner session too if job.state in ("queued", "running"): if job is self._queued: self._queued = None @@ -472,7 +504,6 @@ def _wedged(self) -> bool: def _housekeeping(self) -> None: now = self._clock() - self._prune_sessions_locked() for entry in list(self._pending): if now >= entry[2]: self._try_unlink(entry) @@ -489,6 +520,7 @@ def _housekeeping(self) -> None: if excess > 0: for j in sorted((j for j in terminal if j.worker_done.is_set()), key=lambda j: j.terminal_at)[:excess]: self._drop(j) + self._prune_sessions_locked() # AFTER the evictions: a session whose last job was just evicted can go now def _drop(self, job: Job) -> None: if job.snapshot: @@ -749,6 +781,12 @@ def _ordering(data: dict) -> tuple[str | None, int | None]: return _client_id(data), seq +def _credential(data: dict) -> str | None: + """The optional session credential for job-id routes: a job owned by a session needs it. Presence is tested + by key, so a null or malformed client_id is a 400, never silently 'no credential'.""" + return _client_id(data) if "client_id" in data else None + + def _job_id(data: dict) -> str: value = rv.require_str(data, "job_id") if len(value) > 128: @@ -769,21 +807,21 @@ def handle(route: str, data: dict, jobs: SceneJobs | None = None) -> tuple[int, client_id, seq = _ordering(data) return 200, jobs.start(path, _request_id(data), client_id, seq) if route == "/scene/status": - return 200, jobs.status(_job_id(data)) + return 200, jobs.status(_job_id(data), _credential(data)) if route == "/scene/result": expected = data.get("expected_revision") if expected is not None and (not isinstance(expected, str) or not _REVISION.fullmatch(expected)): raise rv.ValidationError("Invalid 'expected_revision'") - return jobs.result(_job_id(data), expected) + return jobs.result(_job_id(data), expected, _credential(data)) if route == "/scene/cancel": - if "job_id" in data and ("client_id" in data or "request_id" in data or "seq" in data): - raise rv.ValidationError("Give either 'job_id' or 'client_id', 'request_id' and 'seq', not both") - if "job_id" not in data and ("client_id" in data or "seq" in data): + if "job_id" in data and ("request_id" in data or "seq" in data): + raise rv.ValidationError("Give either 'job_id' (with 'client_id' for a session job) or 'client_id', 'request_id' and 'seq'") + if "job_id" not in data and ("client_id" in data or "seq" in data or "request_id" in data): if not ("client_id" in data and "seq" in data and "request_id" in data): raise rv.ValidationError("'client_id', 'request_id' and 'seq' must be given together") client_id, seq = _ordering(data) return 200, jobs.cancel_request(client_id, _request_id(data), seq) - return 200, jobs.cancel(_job_id(data)) + return 200, jobs.cancel(_job_id(data), _credential(data)) except rv.ValidationError as exc: return 400, {"error": "INVALID_REQUEST", "message": str(exc)} except JobError as exc: diff --git a/backend/tests/test_scene_ordering.py b/backend/tests/test_scene_ordering.py index 8bdb290a..6c9e48d2 100644 --- a/backend/tests/test_scene_ordering.py +++ b/backend/tests/test_scene_ordering.py @@ -18,7 +18,25 @@ from tests import scene_fixtures as fx from tests.test_scene_api import call, server # noqa: F401 (the loopback server fixture) from tests.test_scene_contract import sub_validator -from tests.test_scene_jobs import fake_builder, wait_for, wait_state +from tests.test_scene_jobs import fake_builder, wait_for + + +def _owner(jobs, job_id): + job = jobs._jobs.get(job_id) + return job.client_id if job is not None else None + + +def st(jobs, job_id): + """status() presenting the job's own session credential (looked up here, in the TEST, not by the engine).""" + return jobs.status(job_id, _owner(jobs, job_id)) + + +def cn(jobs, job_id): + return jobs.cancel(job_id, _owner(jobs, job_id)) + + +def wait_state(jobs, job_id, *states, timeout=10.0): + return wait_for(lambda: (lambda r: r if r["state"] in states else None)(st(jobs, job_id)), timeout) @pytest.fixture @@ -116,7 +134,7 @@ def test_i1_a_delayed_abandoned_start_is_refused_whatever_became_of_the_wanted_o a = HeldStart(jobs, cube, "req-A-000001", s, 1) # abandoned, delayed in flight b = jobs.start(cube, "req-B-000002", s, 2) # the wanted start gets in first if wanted == "running": - assert started.wait(10) and jobs.status(b["job_id"])["state"] == "running" + assert started.wait(10) and st(jobs, b["job_id"])["state"] == "running" else: wait_state(jobs, b["job_id"], "succeeded") wait_for(lambda: not jobs.worker_alive()) @@ -125,7 +143,7 @@ def test_i1_a_delayed_abandoned_start_is_refused_whatever_became_of_the_wanted_o if wanted == "evicted": now[0] += 60.0 with pytest.raises(sj.JobError): - jobs.status(b["job_id"]) # B's job is gone from the registry + st(jobs, b["job_id"]) # B's job is gone from the registry a.finish() assert a.error is not None and a.error.code in ("STALE_START", "CANCELLED_BEFORE_START") if wanted == "cancelled": @@ -133,11 +151,11 @@ def test_i1_a_delayed_abandoned_start_is_refused_whatever_became_of_the_wanted_o elif wanted == "evicted": assert a.error.code == "STALE_START" # eviction did not make the late start valid if wanted == "running": - assert jobs.status(b["job_id"])["state"] == "running" # B was neither cancelled nor replaced + assert st(jobs, b["job_id"])["state"] == "running" # B was neither cancelled nor replaced gate.set() wait_state(jobs, b["job_id"], "succeeded") elif wanted == "completed": - assert jobs.status(b["job_id"])["state"] == "succeeded" + assert st(jobs, b["job_id"])["state"] == "succeeded" def test_i1_astra_repro_a_repeat_of_b_at_a_higher_seq_then_the_delayed_a(make, cube): @@ -150,7 +168,7 @@ def test_i1_astra_repro_a_repeat_of_b_at_a_higher_seq_then_the_delayed_a(make, c assert err.extra["reason"] == "request id reused" a.finish() assert a.error.code == "STALE_START" # the watermark had already moved to 3 - assert jobs.status(b["job_id"])["state"] == "succeeded" + assert st(jobs, b["job_id"])["state"] == "succeeded" refused(jobs, "STALE_START", cube, "req-B-000001", s, 3) # the refused attempt is not an idempotent repeat ok = jobs.start(cube, "req-B-000004", s, 4) # recovery: fresh request_id, higher seq assert wait_state(jobs, ok["job_id"], "succeeded")["state"] == "succeeded" @@ -164,7 +182,7 @@ def test_i1_a_stale_start_cannot_replace_a_running_wanted_start(make, cube): b = jobs.start(cube, "req-B-000002", s, 2) started.wait(10) a.finish() - assert a.error.code == "STALE_START" and jobs.status(b["job_id"])["state"] == "running" + assert a.error.code == "STALE_START" and st(jobs, b["job_id"])["state"] == "running" gate.set() @@ -190,9 +208,9 @@ def builder(path, revision, ctl): assert started.wait(10) now[0] += 60.0 with pytest.raises(sj.JobError): - jobs.status(first["job_id"]) + st(jobs, first["job_id"]) err = refused(jobs, "STALE_START", cube, "req-E-000001", s, 1, http=409) - assert jobs.status(other["job_id"])["state"] == "running" # and it did not replace the running job + assert st(jobs, other["job_id"])["state"] == "running" # and it did not replace the running job gate.set() wait_state(jobs, other["job_id"], "succeeded") assert err @@ -220,7 +238,7 @@ def test_i1_a_higher_seq_never_silently_returns_an_old_job(make, cube): s = sess(jobs) j = jobs.start(cube, "req-R-000001", s, 1) refused(jobs, "INVALID_REQUEST", cube, "req-R-000001", s, 2, http=400) - assert jobs.status(j["job_id"])["request_id"] == "req-R-000001" + assert st(jobs, j["job_id"])["request_id"] == "req-R-000001" def test_i1_stale_and_cancel_are_decided_before_source_changed(make, cube, monkeypatch): @@ -254,7 +272,6 @@ def repeat(): jobs.cancel_request(s, "req-A-000001", 1) else: now[0] += 500.0 - jobs.cancel(a["job_id"]) # (the job is terminal; it still owns the session) jobs._jobs.pop(a["job_id"]) jobs._by_request.pop((s, "req-A-000001")) # nothing registered: the session may now expire jobs._housekeeping() @@ -297,7 +314,7 @@ def test_i2_a_cancelled_start_stays_refused_after_its_job_is_evicted_and_after_u wait_for(lambda: not jobs.worker_alive()) now[0] += 100.0 with pytest.raises(sj.JobError): - jobs.status(a["job_id"]) # the cancelled job is long gone + st(jobs, a["job_id"]) # the cancelled job is long gone for i in range(200): # a great deal of unrelated cancel traffic jobs.cancel_request(s, f"req-noise-{i:05d}", 1) refused(jobs, "CANCELLED_BEFORE_START", cube, "req-A-000001", s, 1) @@ -311,11 +328,11 @@ def test_i2_cancelling_a_never_touches_b(make, cube): a = jobs.start(cube, "req-A-000001", s, 1) started.wait(10) b = jobs.start(cube, "req-B-000002", s, 2) # replaces A as today - assert jobs.status(a["job_id"])["state"] == "cancelled" + assert st(jobs, a["job_id"])["state"] == "cancelled" jobs.cancel_request(s, "req-A-000001", 1) # cancelling A... - assert jobs.status(b["job_id"])["state"] in ("queued", "running") # ...never touches B (exact request match) + assert st(jobs, b["job_id"])["state"] in ("queued", "running") # ...never touches B (exact request match) jobs.cancel_request(s, "req-C-nonexistent", 1) - assert jobs.status(b["job_id"])["state"] in ("queued", "running") + assert st(jobs, b["job_id"])["state"] in ("queued", "running") out = jobs.cancel_request(s, "req-B-000002", 2) assert out["job_id"] == b["job_id"] and out["state"] == "cancelled" gate.set() @@ -326,9 +343,9 @@ def test_i2_cancel_by_job_id_is_unchanged(make, cube): s = sess(jobs) j = jobs.start(cube, "req-J-000001", s, 1) wait_state(jobs, j["job_id"], "succeeded") - assert jobs.cancel(j["job_id"])["state"] == "succeeded" + assert cn(jobs, j["job_id"])["state"] == "succeeded" with pytest.raises(sj.JobError) as e: - jobs.cancel("0" * 32) + cn(jobs, "0" * 32) assert e.value.code == "EXPIRED" @@ -341,11 +358,12 @@ def test_i3_the_same_request_id_in_two_sessions_gives_each_its_own_job_and_cance a = jobs.start(cube, "req-same-0001", s, 1) started.wait(10) b = jobs.start(cube, "req-same-0001", t, 1) # same request_id and path, another session - assert a["job_id"] != b["job_id"] and b["replaced_job_id"] == a["job_id"] + assert a["job_id"] != b["job_id"] and b["replaced_job_id"] is None # H1: s's job id is not revealed to t + assert st(jobs, a["job_id"])["state"] == "cancelled" # (the newest start still replaces it, as before) assert jobs.start(cube, "req-same-0001", t, 1)["job_id"] == b["job_id"] # idempotent inside its own scope out = jobs.cancel_request(s, "req-same-0001", 1) assert out["job_id"] == a["job_id"] - assert jobs.status(b["job_id"])["state"] in ("queued", "running") # t's job untouched: no leak across sessions + assert st(jobs, b["job_id"])["state"] in ("queued", "running") # t's job untouched: no leak across sessions assert jobs.cancel_request(t, "req-same-0001", 1)["job_id"] == b["job_id"] gate.set() @@ -357,7 +375,7 @@ def test_i3_a_session_cannot_cancel_or_see_another_sessions_job_by_request(make, wait_state(jobs, j["job_id"], "succeeded") out = jobs.cancel_request(t, "req-own-00001", 1) assert out["job_id"] is None - assert jobs.status(j["job_id"])["state"] == "succeeded" + assert st(jobs, j["job_id"])["state"] == "succeeded" assert jobs.start(cube, "req-own-00001", s, 1)["job_id"] == j["job_id"] @@ -422,7 +440,7 @@ def test_i4_every_start_cancel_and_status_touches_the_session(make, cube, how): wait_for(lambda: not jobs.worker_alive()) now[0] += 90.0 if how == "status": - jobs.status(j["job_id"]) + st(jobs, j["job_id"]) elif how == "cancel": jobs.cancel_request(s, "req-zzz-000001", 1) else: @@ -556,7 +574,7 @@ def test_http_session_start_stale_cancel_and_expiry_shapes(server, tmp_path): # assert code == 200 and not list(sub_validator("job_start").iter_errors(b)) code, body = call(port, "/scene/start", {"path": path, "request_id": "req-A-http0001", "client_id": cid, "seq": 1}, token) assert (code, body["error"]) == (409, "STALE_START") and not list(sub_validator("error_response").iter_errors(body)) - code, status = call(port, "/scene/status", {"job_id": b["job_id"]}, token) + code, status = call(port, "/scene/status", {"job_id": b["job_id"], "client_id": cid}, token) assert code == 200 and status["state"] != "cancelled" code, c = call(port, "/scene/cancel", {"client_id": cid, "request_id": "req-C-http0003", "seq": 3}, token) assert code == 200 and c["job_id"] is None and c["state"] == "cancelled" and not list(sub_validator("job_status").iter_errors(c)) @@ -583,3 +601,194 @@ def test_http_routes_reject_non_object_bodies_and_need_the_token(server): # noq for route in sj.ROUTES: assert call(port, route, None, token)[0] == 400 assert call(port, route, {}, None)[0] == 401 + + +# ================================================================== round: job-id ownership, post-admission failures + +def _unknown_body(jobs, call): + with pytest.raises(sj.JobError) as e: + call() + return (e.value.http, e.value.body()) + + +def test_h1_job_id_operations_are_session_owned_and_refused_exactly_like_unknown_ids(make, cube): + jobs = make() + a, b = sess(jobs), sess(jobs) + j = jobs.start(cube, "req-own-00001", a, 1) + wait_state(jobs, j["job_id"], "succeeded") + unknown = "0" * 32 + for call_for in (lambda jid, cid: jobs.status(jid, cid), lambda jid, cid: jobs.result(jid, None, cid), + lambda jid, cid: jobs.cancel(jid, cid)): + baseline = _unknown_body(jobs, lambda: call_for(unknown, b)) + assert baseline[0] == 404 and baseline[1]["error"] == "EXPIRED" + assert _unknown_body(jobs, lambda: call_for(j["job_id"], b)) == baseline # another session + assert _unknown_body(jobs, lambda: call_for(j["job_id"], None)) == baseline # no credential at all + assert _unknown_body(jobs, lambda: call_for(j["job_id"], "x" * 24)) == baseline # a never-issued id + assert jobs.status(j["job_id"], a)["state"] == "succeeded" # the owner still can + assert jobs.result(j["job_id"], None, a)[0] == 200 and jobs.cancel(j["job_id"], a)["state"] == "succeeded" + + +def test_h1_a_foreign_cancel_by_job_id_does_not_cancel_anything(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + a, b = sess(jobs), sess(jobs) + j = jobs.start(cube, "req-own-00001", a, 1) + started.wait(10) + with pytest.raises(sj.JobError): + jobs.cancel(j["job_id"], b) + assert jobs.status(j["job_id"], a)["state"] == "running" + gate.set() + + +def test_h1_replaced_job_id_never_leaks_across_sessions(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, honor_cancel=True, started=started)) + a, b = sess(jobs), sess(jobs) + ja = jobs.start(cube, "req-a-000001", a, 1) + started.wait(10) + jb = jobs.start(cube, "req-b-000001", b, 1) # another session replaces A's running job + assert jb["replaced_job_id"] is None # ...and learns nothing about it + assert jobs.status(ja["job_id"], a)["state"] == "cancelled" + jb2 = jobs.start(cube, "req-b-000002", b, 2) # the same session replacing its own job + assert jb2["replaced_job_id"] == jb["job_id"] + legacy = jobs.start(cube, "req-l-000001") # a legacy caller replacing a session job + assert legacy["replaced_job_id"] is None + legacy2 = jobs.start(cube, "req-l-000002") # legacy replacing legacy: as before + assert legacy2["replaced_job_id"] == legacy["job_id"] + sj_after_legacy = jobs.start(cube, "req-b-000003", b, 3) # a session replacing a legacy job + assert sj_after_legacy["replaced_job_id"] is None + gate.set() + + +def test_h1_legacy_jobs_keep_todays_behaviour_for_job_id_operations(make, cube): + jobs = make() + j = jobs.start(cube, "req-l-000001") + wait_state(jobs, j["job_id"], "succeeded") + assert jobs.status(j["job_id"])["state"] == "succeeded" + assert jobs.status(j["job_id"], sess(jobs))["state"] == "succeeded" # no owner: any caller, as today + assert jobs.result(j["job_id"])[0] == 200 and jobs.cancel(j["job_id"])["state"] == "succeeded" + + +def test_h1_http_routes_carry_the_credential(server, tmp_path): # noqa: F811 + port, token = server + path = str(fx.plain_cube_3mf(tmp_path / "h1.3mf")) + _, a = call(port, "/scene/session", {}, token) + _, b = call(port, "/scene/session", {}, token) + _, job = call(port, "/scene/start", {"path": path, "request_id": "req-h1-000001", "client_id": a["client_id"], "seq": 1}, token) + for route in ("/scene/status", "/scene/result", "/scene/cancel"): + missing = call(port, route, {"job_id": job["job_id"]}, token) + foreign = call(port, route, {"job_id": job["job_id"], "client_id": b["client_id"]}, token) + unknown = call(port, route, {"job_id": "0" * 32, "client_id": b["client_id"]}, token) + assert missing == foreign == unknown and missing[0] == 404 and missing[1]["error"] == "EXPIRED" + bad = call(port, route, {"job_id": job["job_id"], "client_id": None}, token) + assert bad[0] == 400 + code, ok = call(port, "/scene/status", {"job_id": job["job_id"], "client_id": a["client_id"]}, token) + assert code == 200 and ok["job_id"] == job["job_id"] + code, both = call(port, "/scene/cancel", {"job_id": job["job_id"], "client_id": a["client_id"], "request_id": "req-h1-000001"}, token) + assert code == 400 + + +def test_h1_job_id_cancel_and_result_touch_the_owner_session(make, cube): + for how in ("cancel_job", "result"): + now = [0.0] + jobs = make(clock=lambda: now[0], session_ttl=100.0, ttl=100.0) + s = sess(jobs) + j = jobs.start(cube, "req-A-000002", s, 2) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + now[0] += 90.0 + (jobs.cancel(j["job_id"], s) if how == "cancel_job" else jobs.result(j["job_id"], None, s)) + now[0] += 90.0 + jobs.open_session() + jobs.open_session() + assert s in jobs._sessions, how + + +def _wedge_then_recover(make, cube): + """A cancelled running job whose worker ignores the cancel, past the grace period: the engine is wedged.""" + now = [0.0] + gate, started = threading.Event(), threading.Event() + jobs = make(clock=lambda: now[0], wedge_grace=10.0, builder=fake_builder(gate, honor_cancel=False, started=started)) + s = sess(jobs) + late = HeldStart(jobs, cube, "req-late-0002", s, 2) # attempt 2 is delayed in flight + first = jobs.start(cube, "req-one-00001", s, 1) + assert started.wait(10) + jobs.cancel_request(s, "req-one-00001", 1) # seq 1 running AND cancelled... + now[0] += 100.0 + assert jobs.wedged() # ...and the worker is stuck past the grace + return jobs, s, gate, late, first + + +def test_h2_a_wedged_rejection_still_advances_the_watermark(make, cube): + jobs, s, gate, late, _ = _wedge_then_recover(make, cube) + refused(jobs, "WORKER_WEDGED", cube, "req-three-0003", s, 3, http=503) + gate.set() # the stuck call returns: the worker recovers + wait_for(lambda: not jobs.worker_alive()) + late.finish() # the delayed seq 2 finally arrives + assert late.error is not None and late.error.code == "STALE_START" + assert jobs.retained() == 1 # only the seq-1 job: seq 2 registered nothing + refused(jobs, "STALE_START", cube, "req-three-0003", s, 3) # an exact retry of the refused attempt is stale + ok = jobs.start(cube, "req-four-0004", s, 4) # the documented recovery: fresh request_id, higher seq + assert ok["state"] in ("queued", "running") + + +def test_h2_unsupported_format_and_missing_file_also_advance_the_watermark(make, cube, tmp_path): + jobs = make() + s = sess(jobs) + late = HeldStart(jobs, cube, "req-late-0002", s, 2) + text = tmp_path / "notes.txt" + text.write_text("x") + refused(jobs, "UNSUPPORTED_FORMAT", str(text), "req-three-0003", s, 3, http=422) + late.finish() + assert late.error.code == "STALE_START" + late2 = HeldStart(jobs, cube, "req-late-0004", s, 4) + refused(jobs, "INVALID_REQUEST", str(tmp_path / "missing.3mf"), "req-five-0005", s, 5, http=400) + late2.finish() + assert late2.error.code == "STALE_START" + refused(jobs, "STALE_START", str(text), "req-old-00001", s, 1) # staleness is reported first and moves nothing + assert jobs._sessions[s].watermark == 5 + assert jobs.retained() == 0 + + +# ------------------------------------------------------------------ Astra optionals + +def test_a_delayed_cancel_for_an_old_attempt_cannot_cancel_a_newer_job_that_reused_the_request_id(make, cube): + now = [0.0] + gate, started = threading.Event(), threading.Event() + calls = [] + + def builder(path, revision, ctl): + calls.append(1) + if len(calls) == 1: + return b"{}" + started.set() + while not gate.wait(0.005): + ctl.check() + return b"{}" + jobs = make(clock=lambda: now[0], ttl=10.0, builder=builder) + s = sess(jobs) + old = jobs.start(cube, "req-R-000001", s, 1) + wait_state(jobs, old["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + now[0] += 60.0 + with pytest.raises(sj.JobError): + st(jobs, old["job_id"]) # the old job was evicted + new = jobs.start(cube, "req-R-000001", s, 5) # the request id is reused at a higher seq + assert started.wait(10) and st(jobs, new["job_id"])["state"] == "running" + out = jobs.cancel_request(s, "req-R-000001", 1) # the delayed cancel for the OLD attempt + assert out["job_id"] is None + assert st(jobs, new["job_id"])["state"] == "running" # the newer job is untouched + assert jobs.cancel_request(s, "req-R-000001", 5)["job_id"] == new["job_id"] + gate.set() + + +def test_an_expired_session_whose_last_job_was_just_evicted_does_not_cause_session_limit(make, cube): + now = [0.0] + jobs = make(clock=lambda: now[0], max_sessions=1, session_ttl=50.0, ttl=50.0) + s = sess(jobs) + j = jobs.start(cube, "req-A-000001", s, 1) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + now[0] += 500.0 # the job AND the session expire together + fresh = jobs.open_session() # one call: evict the job, then prune the session + assert fresh["client_id"] != s and s not in jobs._sessions diff --git a/docs/testing/scene-contract/README.md b/docs/testing/scene-contract/README.md index 9da43fb9..08555c0d 100644 --- a/docs/testing/scene-contract/README.md +++ b/docs/testing/scene-contract/README.md @@ -134,9 +134,9 @@ running. A terminal job is kept 120 s or until more than 8 terminal jobs exist. |---|---| | `/scene/session {}` | 200 `{client_id, ttl_s}` (server-issued 24-character id); 503 `SESSION_LIMIT` | | `/scene/start {path, request_id, client_id?, seq?}` | 200 job; 400 `INVALID_REQUEST` (also: only one of `client_id`/`seq`, null or malformed values, `seq` reused for another request, `request_id` reused at a higher `seq`); 422 `UNSUPPORTED_FORMAT`; 409 `INVALID_REQUEST` (request_id reused for another path), `SOURCE_CHANGED`, `STALE_START`, `CANCELLED_BEFORE_START` or `SESSION_EXPIRED`; 503 `WORKER_WEDGED` | -| `/scene/status {job_id}` | 200 `{state, stage, completed, total, error, revision}`; 404 `EXPIRED` | -| `/scene/result {job_id, expected_revision?}` | 200 scene; 422 `{error: }` for a failed job; 409 `CANCELLED` / `NOT_READY` / `SOURCE_CHANGED`; 404 `EXPIRED` | -| `/scene/cancel {job_id}` or `{client_id, request_id, seq}` | 200 status (idempotent; a late cancel on a terminal job is a no-op). By request: 409 `SESSION_EXPIRED` for an unknown session; otherwise `dead_through` rises to `seq` and the job registered for exactly that client_id + request_id (if any) is cancelled; with no such job the body is a `job_status` with `job_id: null`, `state: "cancelled"` | +| `/scene/status {job_id, client_id?}` | 200 `{state, stage, completed, total, error, revision}`; 404 `EXPIRED`. A job started in a session needs that session's `client_id`; a missing or different one answers exactly like an unknown id (404 `EXPIRED`), so existence is never confirmed | +| `/scene/result {job_id, client_id?, expected_revision?}` | 200 scene; 422 `{error: }` for a failed job; 409 `CANCELLED` / `NOT_READY` / `SOURCE_CHANGED`; 404 `EXPIRED` | +| `/scene/cancel {job_id, client_id?}` or `{client_id, request_id, seq}` | 200 status (idempotent; a late cancel on a terminal job is a no-op; for a session job `client_id` is required, as for status and result). By request: 409 `SESSION_EXPIRED` for an unknown session; otherwise `dead_through` rises to `seq` and the job registered for exactly that client_id + request_id (if any) is cancelled, but only if it was started at or before that `seq` (a delayed cancel for an old attempt cannot cancel a newer job that reused the request id); with no such job the body is a `job_status` with `job_id: null`, `state: "cancelled"` | Error codes: `INVALID_REQUEST UNSUPPORTED_FORMAT INVALID_ARCHIVE INVALID_GEOMETRY UNRESOLVED_REFERENCE LIMIT_EXCEEDED SOURCE_CHANGED CANCELLED TIMEOUT EXPIRED NOT_READY WORKER_WEDGED INTERNAL STALE_START @@ -181,8 +181,17 @@ attempt still in flight is refused. An idempotent repeat that was still hashing rules BEFORE `SOURCE_CHANGED` is reported. After any refusal the client retries with a FRESH `request_id` and a HIGHER `seq`. +**Ownership (I3).** Every job records the session that started it. Status, result and cancel-by-job_id on a session +job need that session's `client_id` (404 `EXPIRED`, identical to an unknown id, otherwise), and `replaced_job_id` +in a start response names a replaced job only when it belonged to the same session (or both are legacy); in every +other case it is `null`. Legacy jobs keep today's behaviour. + +**After an admitted start that is then rejected** (`WORKER_WEDGED`, an unsupported format, a missing file) the +session's watermark has already advanced to that `seq`, so a delayed older attempt cannot be accepted behind it. +The client retries with a fresh `request_id` and a higher `seq`; an exact retry is `STALE_START`. + **Sessions.** At most 32 (`MAX_SESSIONS`), idle TTL 15 minutes (`SESSION_TTL_SECONDS`), touched by every start, -cancel and status. Only idle-expired sessions that own no registered job are pruned; a live session is never +cancel (by request or by job_id), status and result. Sessions are pruned after expired jobs are evicted in the same pass. Only idle-expired sessions that own no registered job are pruned; a live session is never evicted. At capacity with nothing prunable, opening a session is 503 `SESSION_LIMIT`. **Limits, stated plainly.** Ordering is provided PER SESSION. It is not provided across sessions: a start from From 46589959d5c9f9b3accd16b8b50eba322caf48f9 Mon Sep 17 00:00:00 2001 From: DeadlyVirusIn Date: Fri, 9 Oct 2026 15:29:11 -0400 Subject: [PATCH 5/6] fix(scene): keep an exact retry's job reference; cancel by request binds to its exact attempt - A failed precheck on an exact retry no longer clears the session's current entry. Session state is advanced only for a new attempt (strictly higher sequence), so later exact retries still return the original job while it exists. - Cancel by request cancels a job only when client_id, request_id and the attempt's sequence all match exactly. The cancelled sequence still rises, so that sequence and every lower one stay refused. Tests use real threads and events: a retry mid-hash while another retry fails its precheck, a cancel at a higher sequence leaving the older job running, and only the exact attempt being cancelled. Verified: eight scene/unit files 300 passed; full backend 3158 passed, 49 skipped; millimeter-unchanged differential identical to base. Co-Authored-By: Claude Sonnet 5.5 --- backend/snapstudio_api/scene_jobs.py | 19 ++++--- backend/tests/test_scene_ordering.py | 82 +++++++++++++++++++++++++++ docs/testing/scene-contract/README.md | 6 +- 3 files changed, 99 insertions(+), 8 deletions(-) diff --git a/backend/snapstudio_api/scene_jobs.py b/backend/snapstudio_api/scene_jobs.py index 4bae11ab..78259d0d 100644 --- a/backend/snapstudio_api/scene_jobs.py +++ b/backend/snapstudio_api/scene_jobs.py @@ -291,7 +291,9 @@ def _advance_locked(self, client_id: str | None, seq: int | None, request_id: st if client_id is None: return session = self._sessions.get(client_id) - if session is not None and seq > session.dead_through and seq >= session.watermark: + # STRICTLY greater: only a NEW attempt sets `current`. An exact retry (seq == watermark) never replaces the + # job reference of the entry it repeats, whatever else happens to that retry. + if session is not None and seq > session.dead_through and seq > session.watermark: session.watermark, session.current = seq, (seq, request_id, path_key, job_id) def _precheck(self, path: str, key: str, request_id: str, client_id: str | None, seq: int | None) -> None: @@ -305,8 +307,9 @@ def _precheck(self, path: str, key: str, request_id: str, client_id: str | None, if client_id is not None: with self._lock: self._housekeeping() - self._admit_locked(client_id, seq, request_id, key) # stale / cancelled / expired are reported first - self._advance_locked(client_id, seq, request_id, key) + existing = self._admit_locked(client_id, seq, request_id, key) # stale / cancelled / expired first + if existing is None: # a NEW attempt: record it as processed + self._advance_locked(client_id, seq, request_id, key) raise def start(self, path: str, request_id: str, client_id: str | None = None, seq: int | None = None) -> dict: @@ -357,7 +360,8 @@ def _create_locked(self, path: str, key: str, request_id: str, client_id: str | job.client_id, job.seq = client_id, seq self._jobs[job.job_id] = job self._by_request[(client_id, request_id)] = job - self._advance_locked(client_id, seq, request_id, key, job.job_id) # every registration records session state atomically + if client_id is not None: # every registration records the job reference atomically + self._sessions[client_id].current = (seq, request_id, key, job.job_id) self._queued = job if self._worker is None: self._worker = threading.Thread(target=self._worker_main, name="scene-worker", daemon=True) @@ -403,14 +407,15 @@ def cancel_request(self, client_id: str, request_id: str, seq: int) -> dict: """Cancel the start (client_id, request_id, seq). The session must exist (SESSION_EXPIRED otherwise). ``dead_through`` rises to ``seq`` so that start, and every lower seq, stays refused for as long as the session lives (O(1) memory: no per-request set), even when no job is registered yet. The registered job - is cancelled only if its client_id AND request_id match exactly AND it was started at or before ``seq`` - (a delayed cancel for an OLD attempt can never cancel a newer job that reused the request id).""" + is cancelled only if it is the job of that EXACT attempt: client_id, request_id AND seq all equal (a delayed + cancel for an old attempt can never cancel a newer job that reused the request id, and a cancel for a + higher seq does not cancel an older job; the start of that seq and every lower one stays refused).""" with self._lock: self._housekeeping() session = self._session_locked(client_id) session.dead_through = max(session.dead_through, seq) job = self._by_request.get((client_id, request_id)) - if job is not None and job.client_id == client_id and job.request_id == request_id and job.seq <= seq: + if job is not None and job.client_id == client_id and job.request_id == request_id and job.seq == seq: if job.state in ("queued", "running"): if job is self._queued: self._queued = None diff --git a/backend/tests/test_scene_ordering.py b/backend/tests/test_scene_ordering.py index 6c9e48d2..eb35d30d 100644 --- a/backend/tests/test_scene_ordering.py +++ b/backend/tests/test_scene_ordering.py @@ -9,6 +9,7 @@ No sleep decides any outcome: the abandoned start A is held at the pre-registration seam (`SceneJobs._check_source`, which runs before the registry lock) with a threading.Event, and time is a fake clock. """ +import os import threading import pytest @@ -792,3 +793,84 @@ def test_an_expired_session_whose_last_job_was_just_evicted_does_not_cause_sessi now[0] += 500.0 # the job AND the session expire together fresh = jobs.open_session() # one call: evict the job, then prune the session assert fresh["client_id"] != s and s not in jobs._sessions + + +# ================================================================== round: exact retries and exact-attempt cancels + +def test_b1_a_failed_precheck_on_an_exact_retry_keeps_the_original_job_reference(make, cube, tmp_path): + jobs = make() + s = sess(jobs) + b = jobs.start(cube, "req-B-000002", s, 2) + wait_state(jobs, b["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + hidden = cube + ".away" + os.replace(cube, hidden) # the file vanishes: the retry's precheck fails + refused(jobs, "INVALID_REQUEST", cube, "req-B-000002", s, 2, http=400) + assert jobs._sessions[s].current[3] == b["job_id"] # the retained job's reference was NOT cleared + os.replace(hidden, cube) + again = jobs.start(cube, "req-B-000002", s, 2) # a later exact retry still gets the job + assert again["job_id"] == b["job_id"] + + +def test_b1_concurrent_exact_retries_one_hashing_one_failing_its_precheck(make, cube): + jobs = make() + s = sess(jobs) + b = jobs.start(cube, "req-B-000002", s, 2) + status = wait_state(jobs, b["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + inside, release = threading.Event(), threading.Event() + real_hash = jobs._current_hash + + def slow(path): # retry R1 is mid-hash, outside the lock + inside.set() + assert release.wait(10) + return status["revision"] + jobs._current_hash = slow + out = {} + + def retry(): + try: + out["r"] = jobs.start(cube, "req-B-000002", s, 2) + except sj.JobError as exc: + out["e"] = exc + t = threading.Thread(target=retry, daemon=True) + t.start() + assert inside.wait(10) + hidden = cube + ".away" + os.replace(cube, hidden) + refused(jobs, "INVALID_REQUEST", cube, "req-B-000002", s, 2, http=400) # R2 fails its precheck meanwhile + os.replace(hidden, cube) + release.set() + t.join(10) + jobs._current_hash = real_hash + assert "e" not in out, out.get("e") and out["e"].code + assert out["r"]["job_id"] == b["job_id"] # R1 still gets the job that still exists + + +def test_b2_a_cancel_at_a_higher_seq_does_not_cancel_the_older_job_of_the_same_request(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + s = sess(jobs) + five = jobs.start(cube, "req-R-000001", s, 5) + started.wait(10) + out = jobs.cancel_request(s, "req-R-000001", 6) # a cancel for seq 6, not for this attempt + assert out["job_id"] is None + assert st(jobs, five["job_id"])["state"] == "running" # the seq-5 job is untouched + assert jobs._sessions[s].dead_through == 6 # but seq 6 and everything below stays refused + refused(jobs, "CANCELLED_BEFORE_START", cube, "req-R-000002", s, 6) + gate.set() + + +def test_b2_only_the_exact_attempt_is_cancelled(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + s = sess(jobs) + five = jobs.start(cube, "req-R-000001", s, 5) + started.wait(10) + assert jobs.cancel_request(s, "req-R-000001", 4)["job_id"] is None # an older attempt's cancel: no match + assert st(jobs, five["job_id"])["state"] == "running" + assert jobs._sessions[s].dead_through == 4 + out = jobs.cancel_request(s, "req-R-000001", 5) # the exact attempt + assert out["job_id"] == five["job_id"] and out["state"] == "cancelled" + assert jobs._sessions[s].dead_through == 5 + gate.set() diff --git a/docs/testing/scene-contract/README.md b/docs/testing/scene-contract/README.md index 08555c0d..76566254 100644 --- a/docs/testing/scene-contract/README.md +++ b/docs/testing/scene-contract/README.md @@ -136,7 +136,7 @@ running. A terminal job is kept 120 s or until more than 8 terminal jobs exist. | `/scene/start {path, request_id, client_id?, seq?}` | 200 job; 400 `INVALID_REQUEST` (also: only one of `client_id`/`seq`, null or malformed values, `seq` reused for another request, `request_id` reused at a higher `seq`); 422 `UNSUPPORTED_FORMAT`; 409 `INVALID_REQUEST` (request_id reused for another path), `SOURCE_CHANGED`, `STALE_START`, `CANCELLED_BEFORE_START` or `SESSION_EXPIRED`; 503 `WORKER_WEDGED` | | `/scene/status {job_id, client_id?}` | 200 `{state, stage, completed, total, error, revision}`; 404 `EXPIRED`. A job started in a session needs that session's `client_id`; a missing or different one answers exactly like an unknown id (404 `EXPIRED`), so existence is never confirmed | | `/scene/result {job_id, client_id?, expected_revision?}` | 200 scene; 422 `{error: }` for a failed job; 409 `CANCELLED` / `NOT_READY` / `SOURCE_CHANGED`; 404 `EXPIRED` | -| `/scene/cancel {job_id, client_id?}` or `{client_id, request_id, seq}` | 200 status (idempotent; a late cancel on a terminal job is a no-op; for a session job `client_id` is required, as for status and result). By request: 409 `SESSION_EXPIRED` for an unknown session; otherwise `dead_through` rises to `seq` and the job registered for exactly that client_id + request_id (if any) is cancelled, but only if it was started at or before that `seq` (a delayed cancel for an old attempt cannot cancel a newer job that reused the request id); with no such job the body is a `job_status` with `job_id: null`, `state: "cancelled"` | +| `/scene/cancel {job_id, client_id?}` or `{client_id, request_id, seq}` | 200 status (idempotent; a late cancel on a terminal job is a no-op; for a session job `client_id` is required, as for status and result). By request: 409 `SESSION_EXPIRED` for an unknown session; otherwise `dead_through` rises to `seq` and the job of that EXACT attempt (client_id, request_id and `seq` all equal) is cancelled, if any, so a delayed cancel for an old attempt cannot cancel a newer job that reused the request id, and a cancel for a higher `seq` does not cancel an older job; that `seq` and every lower one stay refused for the life of the session; with no such job the body is a `job_status` with `job_id: null`, `state: "cancelled"` | Error codes: `INVALID_REQUEST UNSUPPORTED_FORMAT INVALID_ARCHIVE INVALID_GEOMETRY UNRESOLVED_REFERENCE LIMIT_EXCEEDED SOURCE_CHANGED CANCELLED TIMEOUT EXPIRED NOT_READY WORKER_WEDGED INTERNAL STALE_START @@ -186,6 +186,10 @@ job need that session's `client_id` (404 `EXPIRED`, identical to an unknown id, in a start response names a replaced job only when it belonged to the same session (or both are legacy); in every other case it is `null`. Legacy jobs keep today's behaviour. +**An exact retry never loses its job.** A retry of the entry at the watermark (same `seq`, request_id and path) that is +rejected before registration (for example the file has since vanished) leaves the session's reference to that +entry's job untouched, so later and concurrent exact retries still get the job for as long as it is registered. + **After an admitted start that is then rejected** (`WORKER_WEDGED`, an unsupported format, a missing file) the session's watermark has already advanced to that `seq`, so a delayed older attempt cannot be accepted behind it. The client retries with a fresh `request_id` and a higher `seq`; an exact retry is `STALE_START`. From 2544de9b44dc91178a44bbb9d27e08e8ee06fd02 Mon Sep 17 00:00:00 2001 From: DeadlyVirusIn Date: Wed, 21 Oct 2026 18:03:06 -0400 Subject: [PATCH 6/6] fix(scene): keep main's geometry guard; scene normalises units only The unit scaling #97 added to geometry, intelligence, placement and plate_placement contradicts the RC2 geometry_basis guard (non-millimetre = not checked), so those consumers stay as on main. The scene contract and the shared units table are unchanged. Tests that pinned the removed consumer behaviour and the Arrange-remedy wording are removed or pointed at the units helper. Co-Authored-By: Claude Sonnet 5.5 --- backend/snapstudio_core/geometry.py | 23 +-- backend/snapstudio_core/placement.py | 20 +-- backend/snapstudio_core/plate_placement.py | 18 +- backend/tests/test_scene_repair.py | 27 +-- backend/tests/test_units_consumers.py | 198 +-------------------- 5 files changed, 27 insertions(+), 259 deletions(-) diff --git a/backend/snapstudio_core/geometry.py b/backend/snapstudio_core/geometry.py index 94861085..ea5eb845 100644 --- a/backend/snapstudio_core/geometry.py +++ b/backend/snapstudio_core/geometry.py @@ -14,7 +14,6 @@ from .container import ThreeMF from .config_io import load_model_settings -from . import units as _units # Mirror intelligence.py's byte guard; plus a triangle cap for analysis cost. _MAX_BYTES = 80 * 1024 * 1024 @@ -45,7 +44,6 @@ def _parse_3mf_model(raw: bytes, verts: list, faces: list) -> None: hardened XML parser (no XXE/DTD/network). Triangle indices are local to each , so we offset by the vertex count at the start of that mesh.""" root = load_model_settings(raw) - scale = _units.mm_per_unit(raw) # the part's declared unit -> millimetres (1.0 for mm) for mesh in root.iter(f"{_3MF_CORE_NS}mesh"): vstart = len(verts) vnode = mesh.find(f"{_3MF_CORE_NS}vertices") @@ -53,11 +51,7 @@ def _parse_3mf_model(raw: bytes, verts: list, faces: list) -> None: if vnode is None or tnode is None: continue for v in vnode.iterfind(f"{_3MF_CORE_NS}vertex"): - if scale == 1.0: - verts.append((float(v.get("x", 0)), float(v.get("y", 0)), float(v.get("z", 0)))) - else: - verts.append((float(v.get("x", 0)) * scale, float(v.get("y", 0)) * scale, - float(v.get("z", 0)) * scale)) + verts.append((float(v.get("x", 0)), float(v.get("y", 0)), float(v.get("z", 0)))) for t in tnode.iterfind(f"{_3MF_CORE_NS}triangle"): faces.append((vstart + int(t.get("v1")), vstart + int(t.get("v2")), vstart + int(t.get("v3")))) if len(faces) > _MAX_TRIANGLES: @@ -119,11 +113,8 @@ def build_item_dims(path: str) -> list[dict]: # object_id -> (verts, [(component_objectid, component_path, component_xform)]), per file parsed: dict[str, dict[str, tuple]] = {} - unit_scale: dict[str, float] = {} for fname, raw in model_files.items(): root = load_model_settings(raw) - scale = _units.mm_per_unit(raw) # this part's declared unit -> millimetres - unit_scale[fname] = scale objs: dict[str, tuple] = {} for ob in root.iter(f"{_3MF_CORE_NS}object"): oid = ob.get("id") @@ -133,18 +124,14 @@ def build_item_dims(path: str) -> list[dict]: vn = mesh.find(f"{_3MF_CORE_NS}vertices") if vn is not None: for v in vn.iterfind(f"{_3MF_CORE_NS}vertex"): - if scale == 1.0: - verts.append((float(v.get("x", 0)), float(v.get("y", 0)), float(v.get("z", 0)))) - else: - verts.append((float(v.get("x", 0)) * scale, float(v.get("y", 0)) * scale, - float(v.get("z", 0)) * scale)) + verts.append((float(v.get("x", 0)), float(v.get("y", 0)), float(v.get("z", 0)))) comps = [] cn = ob.find(f"{_3MF_CORE_NS}components") if cn is not None: for c in cn.iterfind(f"{_3MF_CORE_NS}component"): comps.append((c.get("objectid"), c.get(f"{_3MF_PROD_NS}path"), - _units.scale_translation(_xform(c.get("transform")), scale))) + _xform(c.get("transform")))) objs[oid] = (verts, comps) parsed[fname] = objs @@ -183,9 +170,7 @@ def collect(objid, prefer_file, xform, acc, seen): for it in load_model_settings(model_files[root_file]).iter(f"{_3MF_CORE_NS}item"): oid = it.get("objectid") acc: list = [] - collect(oid, root_file, - _units.scale_translation(_xform(it.get("transform")), unit_scale[root_file]), - acc, frozenset()) + collect(oid, root_file, _xform(it.get("transform")), acc, frozenset()) if not acc: continue xs = [p[0] for p in acc]; ys = [p[1] for p in acc]; zs = [p[2] for p in acc] diff --git a/backend/snapstudio_core/placement.py b/backend/snapstudio_core/placement.py index 9a5d3b99..8f2915c2 100644 --- a/backend/snapstudio_core/placement.py +++ b/backend/snapstudio_core/placement.py @@ -25,8 +25,6 @@ import re import zipfile -from . import units as _units - SCHEMA_VERSION = "placement/1" #: The states an object's footprint can be in against the target polygon. They are @@ -228,7 +226,6 @@ def read_objects(path: str) -> dict: with zipfile.ZipFile(path) as archive: names = archive.namelist() root = _read(archive, ROOT_MODEL) - root_scale = _units.mm_per_unit(root) # the unit the root model declares -> mm settings_text = _read(archive, MODEL_SETTINGS) try: project = json.loads(_read(archive, PROJECT_SETTINGS) or "{}") @@ -237,16 +234,15 @@ def read_objects(path: str) -> dict: meshes: dict[str, list[tuple]] = {} for name in sorted(n for n in names if n.startswith(OBJECTS_DIR)): body = _read(archive, name) - part_scale = _units.mm_per_unit(body) # each part's own declared unit for mesh_id in re.findall(r'', body, re.S) if block: - meshes[mesh_id] = _scaled(_points(block.group(0)), part_scale) + meshes[mesh_id] = _points(block.group(0)) if not meshes: # A project whose geometry is still in the root model. for mesh_id, block in zip(_OBJECT_WITH_ID.findall(root), _OBJECT_BLOCK.findall(root)): - meshes[mesh_id] = _scaled(_points(block), root_scale) + meshes[mesh_id] = _points(block) polygon = polygon_of(project) parts_by_object = multipart._parts_by_object(settings_text) if settings_text else {} @@ -255,7 +251,7 @@ def read_objects(path: str) -> dict: objects = [] for object_id, transform_text in placements.items(): - item = _units.scale_translation(parse_transform(transform_text), root_scale) + item = parse_transform(transform_text) part_ids = [part_id for part_id, _subtype in parts_by_object.get(object_id, [])] roles = dict(parts_by_object.get(object_id, [])) own_components = components.get(object_id) or [ @@ -266,8 +262,7 @@ def read_objects(path: str) -> dict: points = meshes.get(mesh_id) if points is None: continue - whole = compose(item, _units.scale_translation(parse_transform(component_transform), - root_scale)) + whole = compose(item, parse_transform(component_transform)) moved = [apply(whole, point) for point in points] every.extend(moved) if roles.get(mesh_id, "normal_part") == "normal_part": @@ -287,13 +282,6 @@ def read_objects(path: str) -> dict: "polygon_source": PROJECT_SETTINGS if polygon else None} -def _scaled(points: list[tuple], factor: float) -> list[tuple]: - """Mesh points in millimetres. A millimetre part is returned untouched.""" - if factor == 1.0: - return points - return [(x * factor, y * factor, z * factor) for x, y, z in points] - - def _components_by_object(root: str) -> dict: out: dict[str, list] = {} for match in re.finditer( diff --git a/backend/snapstudio_core/plate_placement.py b/backend/snapstudio_core/plate_placement.py index e007f9cd..574fe055 100644 --- a/backend/snapstudio_core/plate_placement.py +++ b/backend/snapstudio_core/plate_placement.py @@ -32,7 +32,6 @@ from importlib.resources import files from pathlib import Path -from . import units as _units from .container import ThreeMF SCHEMA_VERSION = "placement/1" @@ -607,11 +606,7 @@ def prepare_placed_copy(path: str, out_dir: str | None = None, "before": before} offset = before["suggested_offset"] - # The geometry above is measured in millimetres, but a build item's translation is written in the - # ROOT model's own unit. Convert the offset back before it is added, or an inch project would be - # shifted by 25.4 times too little (and the move would silently not land). - root_scale = _units.mm_per_unit(tm.read_part(ROOT_MODEL)) - offset_for = _uniform_offset(offset["x"] / root_scale, offset["y"] / root_scale) + offset_for = _uniform_offset(offset["x"], offset["y"]) try: rewritten, moved = _rewrite_items(tm.read_part(ROOT_MODEL), offset_for) @@ -648,13 +643,14 @@ def prepare_placed_copy(path: str, out_dir: str | None = None, } ok = bool(after.get("available")) and not after.get("off_plate") if not ok: - # A copy that did not achieve what it was supposed to is not kept on disk: the - # result says plainly that the move did not work and names no file. - out.unlink(missing_ok=True) + # The copy is left in place so a user can inspect it, but the result says + # plainly that the move did not achieve what it was supposed to. return { "schema_version": SCHEMA_VERSION, "ok": False, - "reason": ("Studio could not move the objects onto the plate, so it did not " - "keep a copy. Open the original in Snapmaker Orca and use Arrange."), + "reason": ("Studio moved the arrangement but the copy still has objects " + "off the plate, so it is not safe to rely on. Open the original " + "in Snapmaker Orca and use Arrange."), + "output_path": str(out), "output_name": out.name, "before": before, "after": after, } diff --git a/backend/tests/test_scene_repair.py b/backend/tests/test_scene_repair.py index ef26252f..7b32b79e 100644 --- a/backend/tests/test_scene_repair.py +++ b/backend/tests/test_scene_repair.py @@ -89,13 +89,13 @@ def test_a2_the_unit_comes_from_the_real_root_element(tmp_path, name): assert b["max"][0] == pytest.approx(12 * units.MM_PER_UNIT[want], abs=1e-4) -def test_a2_legacy_readers_see_the_same_unit_through_a_long_preamble(tmp_path): - from snapstudio_core.intelligence import project_info - xml = UNIT_DOCS["long_comment"]() - path = str(fx.three_mf(tmp_path / "l.3mf", xml)) - assert project_info(path)["dimensions_mm"] == {"x": 304.8, "y": 304.8, "z": 304.8} - decoy = str(fx.three_mf(tmp_path / "d.3mf", UNIT_DOCS["decoy_comment"]())) - assert project_info(decoy)["dimensions_mm"] == {"x": 12.0, "y": 12.0, "z": 12.0} +def test_a2_the_shared_unit_reader_sees_the_unit_through_a_long_preamble(): + # The size consumers on main stay behind geometry_basis (not checked for non-mm); only the scene + # contract and this helper normalise units, so the helper is asserted directly. + def raw(xml): + return xml if isinstance(xml, bytes) else xml.encode("utf-8") + assert units.unit_of(raw(UNIT_DOCS["long_comment"]())) == ("inch", 25.4) + assert units.unit_of(raw(UNIT_DOCS["decoy_comment"]())) == ("millimeter", 1.0) def test_a2_an_unrecognised_unit_is_not_the_same_as_an_absent_one(): @@ -528,19 +528,6 @@ def test_b7_unsupported_unit_lists_the_affected_nodes(tmp_path): assert "UNSUPPORTED_UNIT" in description and "fallback" in description.lower() -# ------------------------------------------------------------------ B8: the failure wording is pinned - -def test_b8_a_failed_move_says_it_kept_no_copy(tmp_path, monkeypatch): - from snapstudio_core import plate_placement - path = str(fx.three_mf(tmp_path / "m.3mf", fx.model_xml([fx.cube_object("1", 25.4)], [("1", fx.tf(290.0, 3.0, 0.0))]))) - monkeypatch.setattr(plate_placement, "_rewrite_items", lambda raw, offset_for: (raw, 1)) - monkeypatch.setattr(plate_placement, "verify_only_placement_moved", lambda a, b: {"passed": True, "checks": []}) - res = plate_placement.prepare_placed_copy(path, out_dir=str(tmp_path / "out")) - assert res["ok"] is False - assert res["reason"] == ("Studio could not move the objects onto the plate, so it did not keep a copy. " - "Open the original in Snapmaker Orca and use Arrange.") - - # ------------------------------------------------------------------ B9: id collisions are detected def test_b9_a_node_id_collision_is_an_internal_error(tmp_path, monkeypatch): diff --git a/backend/tests/test_units_consumers.py b/backend/tests/test_units_consumers.py index 393009c3..5b21dca9 100644 --- a/backend/tests/test_units_consumers.py +++ b/backend/tests/test_units_consumers.py @@ -1,35 +1,12 @@ -"""The 3MF `unit` header, applied once (snapstudio_core.units) and checked at every consumer. +"""The 3MF `unit` header table (snapstudio_core.units), as the scene contract reads it. -Before this fix several readers ignored the header, so a 12 inch part was reported as 12 mm and bed-fit -statements were wrong for any non-millimetre project. A part that is 12 inches must read 304.8 mm -EVERYWHERE; millimetre projects must read exactly as they always did. +The scene contract (scene.py) normalises a part's declared unit to millimetres through this table. The size-dependent +verdicts elsewhere (bed fit, readiness, success estimate, first-layer footprint, placement) do NOT consume it: on main +they stay behind `geometry_basis`, which reports "not checked" for a non-millimetre project rather than converting. A +shared physical-geometry normaliser for those consumers is a separate, later change. """ -import pytest - from snapstudio_core import units -from tests import scene_fixtures as fx - -UNITS = sorted(units.MM_PER_UNIT.items()) - - -def project(tmp_path, unit, size_mm=30.0, at_units=(0.0, 0.0, 0.0), name="u.3mf"): - """A cube whose true edge is `size_mm`, written in `unit`, placed `at_units` (in that unit).""" - f = units.MM_PER_UNIT[unit] - return str(fx.three_mf(tmp_path / name, fx.model_xml([fx.cube_object("1", size_mm / f)], [("1", fx.tf(*at_units))], unit=unit))) - - -def composite(tmp_path, unit, size_units, at_units, name="c.3mf"): - """The Bambu/Orca shape (one composite object over a mesh in 3D/Objects/): the only shape - placement.read_objects reads.""" - return str(fx.bambu_project(tmp_path / name, parts=1, items=[("100", fx.tf(*at_units))], plates=[(1, [("100", 0)])], - unit=unit, sizes=[size_units], origins=[(0.0, 0.0, 0.0)])) - - -def twelve_inch(tmp_path, at=(0.0, 0.0, 0.0)): - return str(fx.three_mf(tmp_path / "in.3mf", fx.model_xml([fx.cube_object("1", 12)], [("1", fx.tf(*at))], unit="inch"))) - -# ------------------------------------------------------------------ the shared table def test_table_has_the_six_units_and_reads_the_header(): assert units.MM_PER_UNIT == {"micron": 0.001, "millimeter": 1.0, "centimeter": 10.0, "inch": 25.4, @@ -46,168 +23,3 @@ def test_table_has_the_six_units_and_reads_the_header(): assert units.scale_translation([1, 0, 0, 0, 1, 0, 0, 0, 1, 2, 3, 4], 25.4)[9:] == [2 * 25.4, 3 * 25.4, 4 * 25.4] same = [1, 0, 0, 0, 1, 0, 0, 0, 1, 2, 3, 4] assert units.scale_translation(same, 1.0) is same - - -# ------------------------------------------------------------------ every dimension source - -@pytest.mark.parametrize("unit,factor", UNITS) -def test_project_info_and_its_service_wrappers(tmp_path, unit, factor): - from snapstudio_api import service - from snapstudio_core.intelligence import project_info - path = project(tmp_path, unit) - want = {"x": 30.0, "y": 30.0, "z": 30.0} - assert project_info(path)["dimensions_mm"] == want - assert service.insights(path)["dimensions_mm"] == want # DesignHealth data path - assert service.strategy_recommend(path)["signals"]["dimensions_mm"] == want # strategy signals (~490/522/541) - - -@pytest.mark.parametrize("unit,factor", UNITS) -def test_canonical_and_layout_and_validation_report(tmp_path, unit, factor): - from snapstudio_api import service - from snapstudio_core import layout - path = project(tmp_path, unit) - assert service.canonical(path)["dimensions_mm"] == {"x": 30.0, "y": 30.0, "z": 30.0} # canonical.py:92 - rep = service.report(path) # validation_report.py:35 - fits = next(c for c in rep["checks"] if c["name"] == "Fits the print bed") - assert fits["status"] == "pass" and "30.0 × 30.0 × 30.0 mm" in fits["detail"] - lay = layout.assess_layout(path) # layout.py:55 - assert lay["status"] == "pass", lay - - -def test_twelve_inch_part_reads_304_8_mm_everywhere(tmp_path): - from snapstudio_api import service - from snapstudio_core import geometry, layout, plate_placement, placement, scale_doctor - from snapstudio_core.intelligence import project_info - path = twelve_inch(tmp_path) - full = {"x": 304.8, "y": 304.8, "z": 304.8} - assert project_info(path)["dimensions_mm"] == full - assert service.insights(path)["dimensions_mm"] == full - assert service.canonical(path)["dimensions_mm"] == full - assert service.strategy_recommend(path)["signals"]["dimensions_mm"] == full - # bed_fit (via its service caller): the model does NOT fit a 270 mm plate - bf = service.bed_fit(path) - assert bf["dims_mm"] == full and bf["overall_level"] == "risk" - assert any("305" in f["text"] for f in bf["findings"]) - # validation report - fits = next(c for c in service.report(path)["checks"] if c["name"] == "Fits the print bed") - assert fits["status"] == "warn" and "304.8" in fits["detail"] - # layout - assert layout.assess_layout(path)["status"] == "fail" - # geometry - item = geometry.build_item_dims(path)[0] - assert item["dimensions"] == {"x": 304.8, "y": 304.8, "z": 304.8} - mesh = geometry.load_mesh(path) - assert max(v[0] for v in mesh.verts) == pytest.approx(304.8) - # placement - obj = placement.read_objects(composite(tmp_path, "inch", 12, (0, 0, 0)))["objects"][0] - assert obj["footprint"]["width"] == pytest.approx(304.8) - assessed = plate_placement.assess(path) - assert assessed["items"][0]["dimensions"] == full and assessed["off_plate"] - # scale doctor - opts = scale_doctor.scale_options(path) - assert opts["current_parts"][0]["dimensions"] == full - prev = scale_doctor.preview(path, 100) - assert prev["original_dimensions"] == full and prev["fits_build_volume"] is False - - -@pytest.mark.parametrize("unit,factor", UNITS) -def test_build_item_translation_is_in_the_models_unit(tmp_path, unit, factor): - """A transform's translation is a length in the model's unit too (an inch offset is not 1 mm).""" - from snapstudio_core import geometry, placement - path = project(tmp_path, unit, size_mm=20.0, at_units=(10.0 / factor, 20.0 / factor, 0.0)) - item = geometry.build_item_dims(path)[0] - assert item["bounds"]["min"][0] == pytest.approx(10.0, abs=1e-6) - assert item["bounds"]["max"][1] == pytest.approx(40.0, abs=1e-6) - comp = composite(tmp_path, unit, 20.0 / factor, (10.0 / factor, 20.0 / factor, 0.0), name="comp.3mf") - fp = placement.read_objects(comp)["objects"][0]["footprint"] - assert (fp["min_x"], fp["max_y"]) == (pytest.approx(10.0, abs=1e-6), pytest.approx(40.0, abs=1e-6)) - - -def test_plate_placement_judges_an_inch_project_in_millimetres(tmp_path): - """3 inches in is 76.2 mm in: on the plate. 12 inches in is 304.8 mm in: off it.""" - from snapstudio_core import plate_placement - inside = project(tmp_path, "inch", size_mm=25.4, at_units=(3.0, 3.0, 0.0), name="a.3mf") - r = plate_placement.assess(inside) - assert not r["off_plate"] and r["items"][0]["position"] == {"x": 88.9, "y": 88.9} - outside = project(tmp_path, "inch", size_mm=25.4, at_units=(12.0, 3.0, 0.0), name="b.3mf") - assert plate_placement.assess(outside)["off_plate"] - - -def test_sub_model_units_are_per_part(tmp_path): - from snapstudio_core import geometry, placement - from snapstudio_core.intelligence import project_info - sub = fx.sub_model_xml([fx.cube_object("1", 2)], unit="centimeter") # a 20 mm cube - root = fx.model_xml([fx.composite_object("10", [("1", "/3D/Objects/a.model", None)])], [("10", fx.tf(5, 0, 0))]) - path = str(fx.three_mf(tmp_path / "m.3mf", root, {"3D/Objects/a.model": sub})) - assert project_info(path)["dimensions_mm"] == {"x": 20.0, "y": 20.0, "z": 20.0} - item = geometry.build_item_dims(path)[0] - assert item["bounds"]["min"][0] == pytest.approx(5.0) and item["bounds"]["max"][0] == pytest.approx(25.0) - assert placement.read_objects(path)["objects"][0]["footprint"]["width"] == pytest.approx(20.0) - - -# ------------------------------------------------------------------ millimetre behaviour is unchanged - -def test_millimetre_projects_read_exactly_as_before(tmp_path): - from snapstudio_core import geometry, placement - from snapstudio_core.intelligence import project_info - path = str(fx.three_mf(tmp_path / "mm.3mf", fx.model_xml([fx.cube_object("1", 12.3456)], [("1", fx.tf(1.25, 2.5, 0))]))) - assert project_info(path)["dimensions_mm"] == {"x": 12.3, "y": 12.3, "z": 12.3} - item = geometry.build_item_dims(path)[0] - assert item["bounds"] == {"min": (1.25, 2.5, 0.0), "max": (1.25 + 12.3456, 2.5 + 12.3456, 12.3456)} - fp = placement.read_objects(composite(tmp_path, "millimeter", 12.3456, (1.25, 2.5, 0)))["objects"][0]["footprint"] - assert (fp["min_x"], fp["min_y"]) == (1.25, 2.5) - # no unit attribute at all is millimetre - bare = fx.model_xml([fx.cube_object("1", 7)], [("1", None)]).replace(' unit="millimeter"', "") - assert project_info(str(fx.three_mf(tmp_path / "bare.3mf", bare)))["dimensions_mm"] == {"x": 7.0, "y": 7.0, "z": 7.0} - - -# ------------------------------------------------------------------ repair round 1: A3 / D1 / A15 - -def _root_translation(copy_path): - import re - import zipfile - with zipfile.ZipFile(copy_path) as z: - text = z.read("3D/3dmodel.model").decode() - return [float(x) for x in re.search(r"]*transform=\"([^\"]*)\"", text).group(1).split()] - - -def test_d1_the_placement_fix_moves_an_inch_model_by_the_right_amount(tmp_path): - from snapstudio_core import plate_placement - path = project(tmp_path, "inch", size_mm=25.4, at_units=(11.0, 3.0, 0.0)) - assert plate_placement.assess(path)["off_plate"] # 11 in + 1 in = 304.8 mm: off the 270 mm plate - res = plate_placement.prepare_placed_copy(path, out_dir=str(tmp_path / "out")) - assert res["ok"] is True, res.get("reason") - tx, ty, tz = _root_translation(res["output_path"])[9:12] - dx, dy = res["offset_mm"]["x"], res["offset_mm"]["y"] - assert tx == pytest.approx(11 + dx / 25.4, abs=1e-3) and ty == pytest.approx(3 + dy / 25.4, abs=1e-3) and tz == 0 - assert res["after"]["available"] and not res["after"]["off_plate"] # and it really lands on the plate - assert plate_placement.assess(res["output_path"])["off_plate"] == [] - - -def test_d1_a_move_that_did_not_work_leaves_no_copy_on_disk(tmp_path, monkeypatch): - from snapstudio_core import plate_placement - path = project(tmp_path, "inch", size_mm=25.4, at_units=(11.0, 3.0, 0.0)) - monkeypatch.setattr(plate_placement, "_rewrite_items", lambda raw, offset_for: (raw, 1)) # a move that moves nothing - monkeypatch.setattr(plate_placement, "verify_only_placement_moved", lambda a, b: {"passed": True, "checks": []}) - out = tmp_path / "out" - res = plate_placement.prepare_placed_copy(path, out_dir=str(out)) - assert res["ok"] is False and "output_path" not in res - assert not list(out.glob("*.3mf")) - - -def test_d1_millimetre_models_get_exactly_the_offset_they_always_got(tmp_path): - from snapstudio_core import plate_placement - path = str(fx.three_mf(tmp_path / "mm.3mf", fx.model_xml([fx.cube_object("1", 25.4)], [("1", fx.tf(290.0, 3.0, 0.0))]))) - res = plate_placement.prepare_placed_copy(path, out_dir=str(tmp_path / "out")) - assert res["ok"] is True - tx, ty, _ = _root_translation(res["output_path"])[9:12] - assert tx == pytest.approx(290.0 + res["offset_mm"]["x"], abs=1e-4) - - -def test_a15_placement_uses_the_shared_translation_scaler(): - from snapstudio_core import placement - assert not hasattr(placement, "_scale_translation") - rows = ((1, 0, 0), (0, 1, 0), (0, 0, 1), (1.0, 2.0, 3.0)) - assert units.scale_translation(rows, 25.4)[3] == pytest.approx((25.4, 50.8, 76.2)) - assert units.scale_translation(rows, 1.0) is rows - assert units.scale_translation(None, 25.4) is None