diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77a9d89a..a0cd76e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,8 +39,7 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -e . - pip install pytest + pip install -e ".[test]" - name: Test run: python -m pytest -q diff --git a/backend/pyproject.toml b/backend/pyproject.toml index a7e3219f..64d76645 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -22,6 +22,8 @@ classifiers = [ [project.optional-dependencies] dev = ["pytest>=8"] +# Test-only: validates the committed scene/1 JSON Schema with a real validator (never imported by the engine). +test = ["pytest>=8", "jsonschema>=4.18,<5"] [project.scripts] u1convert = "u1convert.cli:cli" diff --git a/backend/snapstudio_api/scene_jobs.py b/backend/snapstudio_api/scene_jobs.py new file mode 100644 index 00000000..78259d0d --- /dev/null +++ b/backend/snapstudio_api/scene_jobs.py @@ -0,0 +1,834 @@ +"""Scene jobs: the bounded, cancellable background build behind /scene/start|status|result|cancel. + +Model (frozen by plan v3/v4/v5/v6): + +* **One worker thread, at most one queued job.** A new start cancels and replaces the client's own + earlier work (the newest queued job wins; a running job is marked cancelled at once). The worker + slot is released only when the thread has actually exited; a replacement therefore waits, queued, + until the cancelled thread observes the flag (bounded by the cooperative-interruption units in + ``snapstudio_core.scene_limits``). +* **States** ``queued | running | succeeded | failed | cancelled``. Terminal states are immutable and + the FIRST terminal wins: a worker finishing after a cancel discards its result. Cancel is legal from + queued|running and a late cancel on a terminal job is a no-op. There is no stored "expired" state: + an evicted or unknown id answers 404 EXPIRED. +* **Retention.** A terminal job is kept for ``RESULT_TTL_SECONDS`` or until more than + ``MAX_TERMINAL_JOBS`` terminal jobs exist (oldest first). Only jobs whose worker has exited are evicted. +* **Deadline.** 60 s wall clock from the moment the job becomes ``running``, observed cooperatively. + Accepted limitation: a single stuck native/filesystem call cannot be interrupted in-process (there is + no multiprocessing in the frozen sidecar). The backstop is fail-closed: if the worker is still alive + ``WEDGE_GRACE_SECONDS`` after a cancel or after the deadline, the job is reported failed with TIMEOUT + (when it was the deadline) and every new start answers 503 WORKER_WEDGED until the thread exits. +* **Snapshot.** The source is copied once into a private file under + ``/scene-tmp/-/`` (streamed 1 MiB chunks, hard ceiling), hashed (the revision is the + SHA-256 of those bytes), and ONLY the copy is parsed. A scene is a consistent snapshot of the file as + copied. ``SOURCE_CHANGED`` therefore applies only to (a) a file that changed while it was being + copied, (b) a reused request_id+path whose file now hashes differently from that job's revision, and + (c) an ``expected_revision`` that differs from the job's revision. It does NOT apply to edits made + after the snapshot was taken. +* **Snapshot lifetime.** The WORKER deletes its own snapshot in its ``finally`` BEFORE setting + ``worker_done``. Eviction removes only the snapshot of a terminal job whose worker is done, with a + missing-file-tolerant unlink. Cancel never deletes a file. A startup sweep removes only + ``scene-tmp/-`` directories whose pid is dead (or older than 24 h) and tolerates sharing + violations; it never touches a live process's directory. +* **Nothing here touches the original file except to read it.** +""" +from __future__ import annotations + +import errno +import hashlib +import os +import re +import secrets +import shutil +import sys +import threading +import time +import uuid + +from snapstudio_core import paths as _paths +from snapstudio_core import scene +from snapstudio_core import scene_limits as L +from . import request_validation as rv + +ROUTES = ("/scene/session", "/scene/start", "/scene/status", "/scene/result", "/scene/cancel") +TERMINAL = ("succeeded", "failed", "cancelled") +_REQUEST_ID = re.compile(r"^[A-Za-z0-9_-]{8,%d}$" % L.MAX_REQUEST_ID_LENGTH) +_REVISION = re.compile(r"^[0-9a-f]{64}$") +_CLIENT_ID = re.compile(r"^[A-Za-z0-9_-]{8,64}$") +MAX_SEQ = 2 ** 53 - 1 +# (session capacity and idle TTL live in scene_limits: MAX_SESSIONS, SESSION_TTL_SECONDS) + + +class JobError(Exception): + """An answer the API gives instead of a result: HTTP status, stable code, safe message.""" + + def __init__(self, http: int, code: str, message: str, **extra) -> None: + self.http, self.code, self.message, self.extra = http, code, message, extra + super().__init__(message) + + def body(self) -> dict: + return {"error": self.code, "message": self.message, **self.extra} + + +class Session: + """O(1) ordering state of one client session. Nothing else is remembered about its starts. + + ``watermark``: the highest admitted seq. ``dead_through``: the highest cancelled seq (every start with + seq <= dead_through is refused for as long as the session lives). ``current``: (seq, request_id, path_key, + job_id) of the entry at seq == watermark, for the idempotent repeat. + """ + + def __init__(self, client_id: str, now: float) -> None: + self.client_id, self.last_touch = client_id, now + self.watermark = 0 + self.dead_through = 0 + self.current: tuple[int, str, str, str | None] | None = None + + +class Job: + def __init__(self, job_id: str, request_id: str, path: str, now: float) -> None: + self.job_id, self.request_id, self.path = job_id, request_id, path + self.source = path # the path as the client gave it (for reading) + self.client_id: str | None = None # the OWNING session (None for a legacy job) + self.seq: int | None = None # the start's seq inside that session + self.state = "queued" + self.stage: str | None = None + self.completed: int | None = None + self.total: int | None = None + self.error: dict | None = None + self.revision: str | None = None + self.result: bytes | None = None + self.snapshot: str | None = None + self.created = now + self.deadline: float | None = None + self.terminal_at: float | None = None + self.cancelled_at: float | None = None + self.worker_done = threading.Event() + self.cancel_flag = threading.Event() + + +class _Control(scene.Control): + def __init__(self, registry: "SceneJobs", job: Job) -> None: + self.registry, self.job = registry, job + + def check(self) -> None: + job = self.job + if job.cancel_flag.is_set() or job.state != "running": + raise scene.SceneCancelled() + if self.registry._clock() > job.deadline: + raise scene.SceneTimeout() + + def progress(self, stage, completed, total) -> None: + self.job.stage, self.job.completed, self.job.total = stage, completed, total + + +def pid_alive(pid: int) -> bool: + """Whether a process with this pid exists. Errs towards 'alive' when it cannot tell.""" + if pid == os.getpid(): + return True + if pid <= 0: + return False + if sys.platform == "win32": + import ctypes + from ctypes import wintypes + k32 = ctypes.WinDLL("kernel32", use_last_error=True) # type: ignore[attr-defined] + k32.OpenProcess.restype = wintypes.HANDLE + k32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + k32.GetExitCodeProcess.argtypes = [wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD)] + k32.CloseHandle.argtypes = [wintypes.HANDLE] + handle = k32.OpenProcess(0x1000, False, pid) # PROCESS_QUERY_LIMITED_INFORMATION + if not handle: + return ctypes.get_last_error() == 5 # access denied: the process exists + try: + code = wintypes.DWORD() + if not k32.GetExitCodeProcess(handle, ctypes.byref(code)): + return True + return code.value == 259 # STILL_ACTIVE + finally: + k32.CloseHandle(handle) + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return True + return True + + +def sweep_stale(root: str | None = None, *, now: float | None = None) -> list[str]: + """Remove scene-tmp directories left by engines that are gone. Returns what it removed. + + A ``-`` folder is removed only when its owner pid is NOT running AND the folder is older + than ``SNAPSHOT_DIR_MIN_AGE_SECONDS``. A folder owned by a live process is never touched, however old + (pid reuse is an accepted cost: a stale folder next to a recycled pid waits until that pid exits). + Sharing violations are tolerated and retried at the next start.""" + base = root or os.path.join(_paths.data_dir(create=False), "scene-tmp") + removed: list[str] = [] + try: + names = os.listdir(base) + except OSError: + return removed + now = time.time() if now is None else now + for name in names: + m = re.match(r"^(\d+)-([0-9a-f]+)$", name) + full = os.path.join(base, name) + if not m or not os.path.isdir(full): + continue + pid = int(m.group(1)) + if pid == os.getpid(): + continue # our own directory, whatever else is true + try: + age = now - os.stat(full).st_mtime + except OSError: + continue + if pid_alive(pid): + continue # NEVER a live process's folder, whatever its age + if age < L.SNAPSHOT_DIR_MIN_AGE_SECONDS: + continue # a just-exited engine's folder is left alone a while + try: + shutil.rmtree(full) + removed.append(name) + except OSError: + pass # sharing violation / in use: leave it for next time + return removed + + +class SceneJobs: + def __init__(self, *, clock=time.monotonic, builder=None, snapshot_root: str | None = None, + deadline: float = L.JOB_DEADLINE_SECONDS, wedge_grace: float = L.WEDGE_GRACE_SECONDS, + ttl: float = L.RESULT_TTL_SECONDS, max_terminal: int = L.MAX_TERMINAL_JOBS, + session_ttl: float = L.SESSION_TTL_SECONDS, max_sessions: int = L.MAX_SESSIONS) -> None: + self._session_ttl, self._max_sessions = session_ttl, max_sessions + self._clock = clock + self._builder = builder or scene.build_scene + self._root = snapshot_root + self._deadline, self._grace, self._ttl, self._max_terminal = deadline, wedge_grace, ttl, max_terminal + self._lock = threading.RLock() + self._jobs: dict[str, Job] = {} + self._by_request: dict[tuple[str | None, str], Job] = {} # (client_id | None, request_id) -> job + self._queued: Job | None = None + self._running: Job | None = None + self._worker: threading.Thread | None = None + self._dir: str | None = None + self._pending: list[list] = [] # [path, attempts, next_try] snapshot files that resisted deletion + # Ordering at the engine boundary (guarded by self._lock, the same lock that registers jobs). + self._sessions: dict[str, Session] = {} + + # ------------------------------------------------------------------ public API + def _check_source(self, path: str) -> None: + """The pre-registration file check (a seam: it runs BEFORE the registry lock is taken).""" + if not os.path.isfile(path): + raise JobError(400, "INVALID_REQUEST", "That file could not be found.") + + def open_session(self) -> dict: + """Issue a session (server-chosen id). I4: at capacity with nothing prunable it fails EXPLICITLY.""" + with self._lock: + self._housekeeping() + if len(self._sessions) >= self._max_sessions: + raise JobError(503, "SESSION_LIMIT", "Too many scene sessions are open. Try again later.") + client_id = secrets.token_urlsafe(18) # 24 characters of [A-Za-z0-9_-] + self._sessions[client_id] = Session(client_id, self._clock()) + return {"client_id": client_id, "ttl_s": int(self._session_ttl)} + + def _session_locked(self, client_id: str) -> Session: + """The live session or SESSION_EXPIRED (I1/I4: a late request from an expired or never-issued session is + refused, never re-created). Touches the session.""" + session = self._sessions.get(client_id) + if session is None: + raise JobError(409, "SESSION_EXPIRED", "That scene session is no longer open. Open a new session.") + session.last_touch = self._clock() + return session + + def _prune_sessions_locked(self) -> None: + """Remove only IDLE-EXPIRED sessions that own no registered job; a live session is never evicted.""" + now = self._clock() + owners = {key[0] for key in self._by_request} + for cid in [c for c, sess in self._sessions.items() + if now - sess.last_touch > self._session_ttl and c not in owners]: + del self._sessions[cid] + + def _admit_locked(self, client_id: str | None, seq: int | None, request_id: str, path_key: str) -> "Job | None": + """THE admission function for every start (new job, idempotent repeat), under the registry lock. + + Returns the registered job to hand back (idempotent), or None when a new job should be created; raises + otherwise. Legacy starts (no client_id) keep today's request_id-only behaviour in their own namespace. + For a session start, in order: unknown/expired session -> SESSION_EXPIRED; seq <= dead_through -> + CANCELLED_BEFORE_START; seq < watermark -> STALE_START; seq == watermark -> the same request_id and path + returns the job if it is still registered (otherwise STALE_START: an exact retry of a start that was + already processed is refused) and anything else is INVALID_REQUEST; seq > watermark -> new, except that + a request_id already registered for this session is INVALID_REQUEST (the old job is never handed back), + and that attempt still raises the watermark so an older attempt still in flight is refused. + """ + if client_id is None: + return self._by_request.get((None, request_id)) + session = self._session_locked(client_id) + if seq <= session.dead_through: + raise JobError(409, "CANCELLED_BEFORE_START", "That scene start was cancelled before it arrived.") + if seq < session.watermark: + raise JobError(409, "STALE_START", "A newer scene start from this session has already been accepted.") + existing = self._by_request.get((client_id, request_id)) + if seq == session.watermark: + cur = session.current + if cur is None or cur[1] != request_id or cur[2] != path_key: + raise JobError(400, "INVALID_REQUEST", "That seq was already used for another request.", + reason="seq reused for another request") + if existing is None or existing.job_id != cur[3]: + raise JobError(409, "STALE_START", "That scene start was already processed and its job is gone.") + return existing + if existing is not None: + session.watermark, session.current = seq, (seq, request_id, path_key, None) + raise JobError(400, "INVALID_REQUEST", "That request_id was already used by this session.", + reason="request id reused") + return None + + def _advance_locked(self, client_id: str | None, seq: int | None, request_id: str, path_key: str, + job_id: str | None = None) -> None: + """Record an attempt that has been (or is being) processed, so no delayed OLDER attempt can overtake it. + Called for every admitted start BEFORE anything that can still reject it (WORKER_WEDGED, an unsupported + format, a missing file), and again with the job id once it is registered.""" + if client_id is None: + return + session = self._sessions.get(client_id) + # STRICTLY greater: only a NEW attempt sets `current`. An exact retry (seq == watermark) never replaces the + # job reference of the entry it repeats, whatever else happens to that retry. + if session is not None and seq > session.dead_through and seq > session.watermark: + session.watermark, session.current = seq, (seq, request_id, path_key, job_id) + + def _precheck(self, path: str, key: str, request_id: str, client_id: str | None, seq: int | None) -> None: + """The checks that run before registration. A rejection here still counts as a processed attempt for its + session (its seq is recorded), so a delayed older start cannot be accepted behind it.""" + try: + if os.path.splitext(key)[1].lower() not in (".3mf", ".stl"): + raise JobError(422, "UNSUPPORTED_FORMAT", "Only .3mf and .stl files can be shown.") + self._check_source(path) + except JobError: + if client_id is not None: + with self._lock: + self._housekeeping() + existing = self._admit_locked(client_id, seq, request_id, key) # stale / cancelled / expired first + if existing is None: # a NEW attempt: record it as processed + self._advance_locked(client_id, seq, request_id, key) + raise + + def start(self, path: str, request_id: str, client_id: str | None = None, seq: int | None = None) -> dict: + key = os.path.normcase(os.path.abspath(path)) + self._precheck(path, key, request_id, client_id, seq) + for _attempt in range(2): + with self._lock: + self._housekeeping() + existing = self._admit_locked(client_id, seq, request_id, key) + if existing is None: + return self._create_locked(path, key, request_id, client_id, seq) + self._same_path_or_raise(existing, key) + revision = existing.revision + # Hash OUTSIDE the lock (it can be slow), then decide under it. The admission rules run FIRST: if a + # newer start, a cancel or the end of the session happened meanwhile the answer is STALE_START / + # CANCELLED_BEFORE_START / SESSION_EXPIRED, not SOURCE_CHANGED. + changed = revision is not None and self._current_hash(path) != revision + with self._lock: + self._housekeeping() + current = self._admit_locked(client_id, seq, request_id, key) + if current is existing and self._jobs.get(existing.job_id) is existing: + if changed: + raise JobError(409, "SOURCE_CHANGED", "The file changed since that scene was built. Start a new one.") + return self._start_body(existing, None) + # evicted while hashing (a legacy start only: a session start was refused above): start a fresh job + with self._lock: + self._housekeeping() + existing = self._admit_locked(client_id, seq, request_id, key) + if existing is not None: + self._same_path_or_raise(existing, key) + return self._start_body(existing, None) + return self._create_locked(path, key, request_id, client_id, seq) + + def _create_locked(self, path: str, key: str, request_id: str, client_id: str | None = None, + seq: int | None = None) -> dict: + self._advance_locked(client_id, seq, request_id, key) # H2: BEFORE anything below can reject + if self._wedged(): + raise JobError(503, "WORKER_WEDGED", "The scene worker is stuck. Try again after it recovers.") + replaced = None + for old in (self._queued, self._running if self._running is not None and self._running.state == "running" else None): + if old is not None: + if old.client_id == client_id: # H1: another session's job id is never revealed + replaced = old.job_id + self._cancel_locked(old) + self._queued = None + job = Job(uuid.uuid4().hex, request_id, key, self._clock()) + job.source = path + job.client_id, job.seq = client_id, seq + self._jobs[job.job_id] = job + self._by_request[(client_id, request_id)] = job + if client_id is not None: # every registration records the job reference atomically + self._sessions[client_id].current = (seq, request_id, key, job.job_id) + self._queued = job + if self._worker is None: + self._worker = threading.Thread(target=self._worker_main, name="scene-worker", daemon=True) + self._worker.start() + return self._start_body(job, replaced) + + def _touch_owner_locked(self, job: Job) -> None: + session = self._sessions.get(job.client_id) if job.client_id is not None else None + if session is not None: + session.last_touch = self._clock() + + def _get_owned(self, job_id: str, client_id: str | None) -> Job: + """Resolve a job id for a caller. A job owned by a session needs that session's client_id; a missing or + different one answers EXACTLY like an unknown id (404 EXPIRED), so existence is never confirmed. A legacy + job (no owner) keeps today's behaviour.""" + job = self._jobs.get(job_id) + if job is None or (job.client_id is not None and job.client_id != client_id): + raise JobError(404, "EXPIRED", "That scene is no longer available.") + self._touch_owner_locked(job) + return job + + def status(self, job_id: str, client_id: str | None = None) -> dict: + with self._lock: + self._housekeeping() + return self._status_body(self._get_owned(job_id, client_id)) + + def result(self, job_id: str, expected_revision: str | None = None, + client_id: str | None = None) -> tuple[int, bytes | dict]: + with self._lock: + self._housekeeping() + job = self._get_owned(job_id, client_id) + if expected_revision is not None and job.revision is not None and job.revision != expected_revision: + raise JobError(409, "SOURCE_CHANGED", "That scene was built from a different version of the file.") + if job.state == "succeeded": + return 200, job.result + if job.state == "failed": + return 422, {"error": job.error["code"], "message": job.error["message"]} + if job.state == "cancelled": + return 409, {"error": "CANCELLED", "message": "That scene was cancelled."} + return 409, {"error": "NOT_READY", "message": "That scene is not ready yet.", "state": job.state} + + def cancel_request(self, client_id: str, request_id: str, seq: int) -> dict: + """Cancel the start (client_id, request_id, seq). The session must exist (SESSION_EXPIRED otherwise). + ``dead_through`` rises to ``seq`` so that start, and every lower seq, stays refused for as long as the + session lives (O(1) memory: no per-request set), even when no job is registered yet. The registered job + is cancelled only if it is the job of that EXACT attempt: client_id, request_id AND seq all equal (a delayed + cancel for an old attempt can never cancel a newer job that reused the request id, and a cancel for a + higher seq does not cancel an older job; the start of that seq and every lower one stays refused).""" + with self._lock: + self._housekeeping() + session = self._session_locked(client_id) + session.dead_through = max(session.dead_through, seq) + job = self._by_request.get((client_id, request_id)) + if job is not None and job.client_id == client_id and job.request_id == request_id and job.seq == seq: + if job.state in ("queued", "running"): + if job is self._queued: + self._queued = None + self._cancel_locked(job) + body = self._status_body(job) + self._housekeeping() + return body + return {"job_id": None, "request_id": request_id, "state": "cancelled", "stage": None, + "completed": None, "total": None, "revision": None, + "error": {"code": "CANCELLED", "message": "That scene was cancelled."}} + + def cancel(self, job_id: str, client_id: str | None = None) -> dict: + with self._lock: + self._housekeeping() + job = self._get_owned(job_id, client_id) # touches the owner session too + if job.state in ("queued", "running"): + if job is self._queued: + self._queued = None + self._cancel_locked(job) + body = self._status_body(job) + self._housekeeping() # a cancelled-while-queued job can push past the cap + return body + + # ------------------------------------------------------------------ introspection (tests, benchmark) + def snapshot_dir(self) -> str | None: + return self._dir + + def retained(self) -> int: + with self._lock: + return len(self._jobs) + + def wedged(self) -> bool: + with self._lock: + return self._wedged() + + def worker_alive(self) -> bool: + with self._lock: + return self._worker is not None and self._worker.is_alive() + + def close(self, timeout: float = 5.0) -> None: + """Cancel everything and wait for the worker to exit (tests / orderly shutdown).""" + with self._lock: + for job in list(self._jobs.values()): + if job.state in ("queued", "running"): + if job is self._queued: + self._queued = None + self._cancel_locked(job) + worker = self._worker + if worker is not None: + worker.join(timeout) + if worker.is_alive(): + return + with self._lock: + for entry in list(self._pending): + self._try_unlink(entry) + if self._dir and not self._pending: + shutil.rmtree(self._dir, ignore_errors=True) # best effort; a later start sweeps what is left + + # ------------------------------------------------------------------ internals (lock held) + def _same_path_or_raise(self, job: Job, key: str) -> None: + if job.path != key: + raise JobError(409, "INVALID_REQUEST", "That request_id is already used for another file.", + reason="request_id reused for another path") + + def _get(self, job_id: str) -> Job: + job = self._jobs.get(job_id) + if job is None: + raise JobError(404, "EXPIRED", "That scene is no longer available.") + return job + + def _cancel_locked(self, job: Job) -> None: + if job.state not in ("queued", "running"): + return + was_queued = job.state == "queued" + now = self._clock() + job.state = "cancelled" + job.error = {"code": "CANCELLED", "message": "That scene was cancelled."} + job.terminal_at = job.cancelled_at = now + job.cancel_flag.set() + if was_queued: + job.worker_done.set() # it never had a worker or a snapshot + + def _wedged(self) -> bool: + job = self._running + if job is None or job.worker_done.is_set(): + return False + now = self._clock() + if job.state == "cancelled" and job.cancelled_at is not None and now >= job.cancelled_at + self._grace: + return True + return job.deadline is not None and now >= job.deadline + self._grace + + def _housekeeping(self) -> None: + now = self._clock() + for entry in list(self._pending): + if now >= entry[2]: + self._try_unlink(entry) + job = self._running + if (job is not None and job.state == "running" and job.deadline is not None + and now >= job.deadline + self._grace and not job.worker_done.is_set()): + self._fail_locked(job, "TIMEOUT", "Building the scene took too long.") # backstop; the thread is wedged + evictable = [j for j in self._jobs.values() if j.state in TERMINAL and j.worker_done.is_set()] + for j in evictable: + if now - j.terminal_at > self._ttl: + self._drop(j) + terminal = [j for j in self._jobs.values() if j.state in TERMINAL] + excess = len(terminal) - self._max_terminal + if excess > 0: + for j in sorted((j for j in terminal if j.worker_done.is_set()), key=lambda j: j.terminal_at)[:excess]: + self._drop(j) + self._prune_sessions_locked() # AFTER the evictions: a session whose last job was just evicted can go now + + def _drop(self, job: Job) -> None: + if job.snapshot: + self._defer_unlink(job.snapshot) + job.snapshot = None + job.result = None + self._jobs.pop(job.job_id, None) + if self._by_request.get((job.client_id, job.request_id)) is job: + del self._by_request[(job.client_id, job.request_id)] + + def _defer_unlink(self, path: str) -> None: + """Delete a snapshot file now; if Windows (or anything) still holds it, KEEP the path on a retry list + with back-off. It is forgotten only when it is gone, or when the process exits (the startup sweep + of a later engine then removes the folder).""" + entry = [path, 0, self._clock()] + if not self._try_unlink(entry) and len(self._pending) < L.MAX_PENDING_UNLINKS: + self._pending.append(entry) + + def _try_unlink(self, entry: list) -> bool: + try: + os.unlink(entry[0]) + except FileNotFoundError: + pass + except OSError: + entry[1] += 1 + entry[2] = self._clock() + min(2.0 ** entry[1], 30.0) # 2 s, 4 s, ... capped at 30 s + return False + if entry in self._pending: + self._pending.remove(entry) + return True + + def _fail_locked(self, job: Job, code: str, message: str) -> None: + if job.state in TERMINAL: + return # the first terminal state wins + job.state = "failed" + job.error = {"code": code, "message": message} + job.terminal_at = self._clock() + job.cancel_flag.set() + + def _status_body(self, job: Job) -> dict: + running = job.state == "running" + return {"job_id": job.job_id, "request_id": job.request_id, "state": job.state, + "stage": job.stage if running else None, + "completed": job.completed if running else None, + "total": job.total if running else None, + "error": job.error, "revision": job.revision} + + def _start_body(self, job: Job, replaced: str | None) -> dict: + return {"job_id": job.job_id, "request_id": job.request_id, "state": job.state, + "revision": job.revision, "replaced_job_id": replaced} + + # ------------------------------------------------------------------ worker + def _worker_main(self) -> None: + try: + while True: + with self._lock: + job = self._queued + if job is None: + self._worker = None # the slot is released only here, as the thread exits + self._housekeeping() + return + self._queued = None + now = self._clock() + job.state = "running" + job.deadline = now + self._deadline # the clock starts when the job starts running + self._running = job + try: + self._execute(job) + finally: + with self._lock: + if self._running is job: + self._running = None + self._housekeeping() # retention does not depend on a client polling + except BaseException: # noqa: BLE001 - never leave the slot taken + with self._lock: + self._worker = None + raise + + def _execute(self, job: Job) -> None: + ctl = _Control(self, job) + try: + job.stage, job.completed, job.total = "reading", 0, None + snap = self._snapshot(job, ctl) + body = self._builder(snap, job.revision, ctl) + with self._lock: + if job.state == "running": + job.state = "succeeded" + job.result = body + job.terminal_at = self._clock() + # else: cancelled / failed first -- the late result is discarded + except scene.SceneCancelled: + with self._lock: + if job.state == "running": + self._cancel_locked(job) + except scene.SceneTimeout: + with self._lock: + self._fail_locked(job, "TIMEOUT", "Building the scene took too long.") + except scene.SceneError as exc: + with self._lock: + self._fail_locked(job, exc.code, exc.message) + except JobError as exc: + with self._lock: + self._fail_locked(job, exc.code, exc.message) + except BaseException as exc: # noqa: BLE001 + try: + from . import server + server._log_unexpected(exc) + except Exception: # noqa: BLE001 + pass + with self._lock: + self._fail_locked(job, "INTERNAL", "The scene could not be built.") + finally: + snap_path = job.snapshot + if snap_path: + try: + os.unlink(snap_path) # the worker deletes its own file ... + job.snapshot = None + except FileNotFoundError: + job.snapshot = None + except OSError: + pass # ... and eviction retries if Windows still holds it + job.worker_done.set() # ... BEFORE it signals that it is done + + # ------------------------------------------------------------------ snapshot + def _snapshot_directory(self) -> str: + with self._lock: + if self._dir is None: + base = self._root or os.path.join(_paths.data_dir(), "scene-tmp") + self._dir = os.path.join(base, f"{os.getpid()}-{secrets.token_hex(4)}") + os.makedirs(self._dir, exist_ok=True) # on EVERY job: something may have removed it + return self._dir + + def _snapshot(self, job: Job, ctl: _Control) -> str: + src = job.source + directory = self._snapshot_directory() + ext = os.path.splitext(src)[1].lower() + dest = os.path.join(directory, job.job_id + ext) + try: + fh = open(src, "rb") + except OSError as exc: + raise JobError(400, "INVALID_REQUEST", "That file could not be read.") from exc + try: + st0 = os.fstat(fh.fileno()) + size0 = st0.st_size + if size0 > L.SNAPSHOT_MAX_BYTES: + raise scene.SceneError("LIMIT_EXCEEDED", "The file is larger than Studio will open here.") + try: + free = shutil.disk_usage(directory).free + except OSError: + free = None + if free is not None and free < size0 + L.DISK_HEADROOM_BYTES: + raise scene.SceneError("INTERNAL", "There is not enough free disk space to read this file.") + digest = hashlib.sha256() + count = 0 + job.snapshot = dest + try: + with open(dest, "wb") as out: + while True: + ctl.check() + chunk = fh.read(L.SNAPSHOT_CHUNK_BYTES) + if not chunk: + break + count += len(chunk) + if count > L.SNAPSHOT_MAX_BYTES or count > size0: + break + digest.update(chunk) + out.write(chunk) + ctl.progress("reading", count, size0) + except OSError as exc: + if exc.errno == errno.ENOSPC: + raise scene.SceneError("INTERNAL", "There is not enough free disk space to read this file.") from exc + raise scene.SceneError("INTERNAL", "Studio could not write its temporary copy of this file.") from exc + st1 = os.fstat(fh.fileno()) + finally: + fh.close() + try: + st2 = os.stat(src) + except OSError: + st2 = None + changed = (count != size0 + or (st1.st_size, st1.st_mtime_ns) != (size0, st0.st_mtime_ns) + or st2 is None or (st2.st_size, st2.st_mtime_ns) != (size0, st0.st_mtime_ns) + or (st0.st_ino and st2 is not None and st2.st_ino and st0.st_ino != st2.st_ino)) + if changed: + raise scene.SceneError("SOURCE_CHANGED", "The file changed while Studio was reading it.") + job.revision = digest.hexdigest() + return dest + + @staticmethod + def _current_hash(path: str) -> str | None: + """SHA-256 of the file as it is now (None if it cannot be read or is over the ceiling).""" + digest = hashlib.sha256() + total = 0 + try: + with open(path, "rb") as fh: + while True: + chunk = fh.read(L.SNAPSHOT_CHUNK_BYTES) + if not chunk: + break + total += len(chunk) + if total > L.SNAPSHOT_MAX_BYTES: + return None + digest.update(chunk) + except OSError: + return None + return digest.hexdigest() + + +# ---------------------------------------------------------------------------------------------- +# HTTP glue (kept out of server.py so that file only gains a route branch) +# ---------------------------------------------------------------------------------------------- + +_default: SceneJobs | None = None +_default_lock = threading.Lock() + + +def registry() -> SceneJobs: + global _default + with _default_lock: + if _default is None: + _default = SceneJobs() + return _default + + +def reset_default_for_tests() -> None: + global _default + with _default_lock: + if _default is not None: + _default.close() + _default = None + + +def _request_id(data: dict) -> str: + value = rv.require_str(data, "request_id") + if not _REQUEST_ID.fullmatch(value): # fullmatch: a trailing newline is not an id + raise rv.ValidationError("Invalid 'request_id'") + return value + + +def _client_id(data: dict) -> str: + value = rv.require_str(data, "client_id") + if not _CLIENT_ID.fullmatch(value): + raise rv.ValidationError("Invalid 'client_id'") + return value + + +def _ordering(data: dict) -> tuple[str | None, int | None]: + """client_id and seq travel together or not at all. Presence is tested by KEY: only both keys absent is the + legacy, unordered start; a null or half-present pair is a malformed request, never a legacy one.""" + has_client, has_seq = "client_id" in data, "seq" in data + if not has_client and not has_seq: + return None, None + if not (has_client and has_seq): + raise rv.ValidationError("'client_id' and 'seq' must be given together") + seq = data["seq"] + if isinstance(seq, bool) or not isinstance(seq, int) or seq < 1 or seq > MAX_SEQ: + raise rv.ValidationError("Invalid 'seq'") + return _client_id(data), seq + + +def _credential(data: dict) -> str | None: + """The optional session credential for job-id routes: a job owned by a session needs it. Presence is tested + by key, so a null or malformed client_id is a 400, never silently 'no credential'.""" + return _client_id(data) if "client_id" in data else None + + +def _job_id(data: dict) -> str: + value = rv.require_str(data, "job_id") + if len(value) > 128: + raise rv.ValidationError("Invalid 'job_id'") + return value + + +def handle(route: str, data: dict, jobs: SceneJobs | None = None) -> tuple[int, bytes | dict]: + """Run one /scene/* request. Returns (HTTP status, dict body or pre-serialized bytes).""" + jobs = jobs or registry() + if not isinstance(data, dict): + return 400, {"error": "INVALID_REQUEST", "message": "The request body must be a JSON object."} + try: + if route == "/scene/session": + return 200, jobs.open_session() + if route == "/scene/start": + path = rv.require_bounded_str(data, "path", 4096) + client_id, seq = _ordering(data) + return 200, jobs.start(path, _request_id(data), client_id, seq) + if route == "/scene/status": + return 200, jobs.status(_job_id(data), _credential(data)) + if route == "/scene/result": + expected = data.get("expected_revision") + if expected is not None and (not isinstance(expected, str) or not _REVISION.fullmatch(expected)): + raise rv.ValidationError("Invalid 'expected_revision'") + return jobs.result(_job_id(data), expected, _credential(data)) + if route == "/scene/cancel": + if "job_id" in data and ("request_id" in data or "seq" in data): + raise rv.ValidationError("Give either 'job_id' (with 'client_id' for a session job) or 'client_id', 'request_id' and 'seq'") + if "job_id" not in data and ("client_id" in data or "seq" in data or "request_id" in data): + if not ("client_id" in data and "seq" in data and "request_id" in data): + raise rv.ValidationError("'client_id', 'request_id' and 'seq' must be given together") + client_id, seq = _ordering(data) + return 200, jobs.cancel_request(client_id, _request_id(data), seq) + return 200, jobs.cancel(_job_id(data), _credential(data)) + except rv.ValidationError as exc: + return 400, {"error": "INVALID_REQUEST", "message": str(exc)} + except JobError as exc: + return exc.http, exc.body() + return 404, {"error": "not found"} diff --git a/backend/snapstudio_api/server.py b/backend/snapstudio_api/server.py index 3c57eb90..0886b0e4 100644 --- a/backend/snapstudio_api/server.py +++ b/backend/snapstudio_api/server.py @@ -20,6 +20,7 @@ from snapstudio_core.material_mapping import MappingFileUnavailable from . import _lifeline from . import service +from . import scene_jobs from . import request_validation as rv from .request_validation import ValidationError @@ -185,7 +186,11 @@ def _cors(self): self.send_header("Access-Control-Allow-Headers", "Content-Type, X-Auth-Token") def _send(self, code: int, obj: dict): - body = json.dumps(obj).encode("utf-8") + self._send_bytes(code, json.dumps(obj).encode("utf-8")) + + def _send_bytes(self, code: int, body: bytes): + """Send an already-serialized JSON body. The scene routes use this so a body that was + size-checked once (<= 8 MiB) is not serialized a second time.""" self.send_response(code) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) @@ -1344,6 +1349,17 @@ def _or_none(v): self._send(404, {"error": "unknown job"}) else: self._send(200, status) + elif self.path in scene_jobs.ROUTES: + # scene/1: bounded, cancellable, authenticated (the token was checked above). + try: + status, body = scene_jobs.handle(self.path, data) + if isinstance(body, bytes): + self._send_bytes(status, body) + else: + self._send(status, body) + except Exception as exc: + _log_unexpected(exc) + self._send(500, {"error": "INTERNAL", "message": "The scene could not be built."}) elif self.path == "/shutdown": # Graceful half of the L4 zero-orphan sequence (see # sidecar::shutdown_sidecar): reply first, THEN stop the @@ -1408,6 +1424,10 @@ def serve(host: str = "127.0.0.1", port: int = 0) -> None: _watch_parent_then_exit() _lifeline.start_parent_lifeline() httpd, token = build_server(host, port) + try: + scene_jobs.sweep_stale() # leftovers of engines that are gone; never a live engine's folder + except Exception: # noqa: BLE001 - housekeeping must not stop the engine starting + pass actual_port = httpd.server_address[1] print(json.dumps({"port": actual_port, "token": token}), flush=True) # handshake line try: diff --git a/backend/snapstudio_core/data/scene-1.schema.json b/backend/snapstudio_core/data/scene-1.schema.json new file mode 100644 index 00000000..dbebec7f --- /dev/null +++ b/backend/snapstudio_core/data/scene-1.schema.json @@ -0,0 +1,232 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://snapmaker-studio.invalid/schema/scene-1.schema.json", + "title": "Snapmaker Studio scene/1", + "description": "Read-only description of a project's geometry, placement and engine findings. FROZEN: every scene object is closed (additionalProperties false), every field is required, and unknown is null. Millimetres, right-handed, Z up. A scene is a consistent snapshot of the file as it was copied for the job. Matrices (local_to_parent_mm, world_mm) are 16 numbers in COLUMN-MAJOR order acting on COLUMN vectors (x' = M * x; the translation is elements 12..14). Mesh coordinates are LOCAL to the mesh resource (before any node matrix), in millimetres. A node's bounds_mm and printable concern its role=part volumes only. An instance's effective volume role is node.role_context when that is not null (Bambu/Orca part records are per use), otherwise the roles of the mesh volumes. PLACEMENT_OUTSIDE_BED.overhang_mm measures how far the printable bounds intrude past the bed polygon shrunk inward by edge_margin_mm, in X and Y only (never height).", + "type": "object", + "additionalProperties": false, + "required": ["schema", "revision", "status", "units", "axes", "sources", "bed", "meshes", "nodes", "plates", "findings", "limitations", "counts", "limits"], + "properties": { + "schema": {"const": "scene/1"}, + "revision": {"$ref": "#/$defs/sha256"}, + "status": {"enum": ["complete", "partial"]}, + "units": {"const": "mm"}, + "axes": {"const": "right-handed-z-up"}, + "sources": {"type": "array", "maxItems": 256, "items": {"$ref": "#/$defs/source"}}, + "bed": {"$ref": "#/$defs/bed"}, + "meshes": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/mesh"}}, + "nodes": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/node"}}, + "plates": {"type": "array", "maxItems": 256, "items": {"$ref": "#/$defs/plate"}}, + "findings": {"type": "array", "maxItems": 500, "items": {"$ref": "#/$defs/finding"}}, + "limitations": {"type": "array", "maxItems": 50, "items": {"$ref": "#/$defs/limitation"}}, + "counts": {"$ref": "#/$defs/counts"}, + "limits": {"$ref": "#/$defs/limits"} + }, + "$defs": { + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "id": {"type": "string", "minLength": 1, "maxLength": 128}, + "uint": {"type": "integer", "minimum": 0, "maximum": 4294967295}, + "vec3": {"type": "array", "minItems": 3, "maxItems": 3, "items": {"type": "number"}}, + "matrix16": {"type": "array", "minItems": 16, "maxItems": 16, "items": {"type": "number"}}, + "resource_key": { + "type": "object", "additionalProperties": false, "required": ["part", "object_id"], + "properties": {"part": {"type": "string", "minLength": 1, "maxLength": 256}, "object_id": {"$ref": "#/$defs/id"}} + }, + "unit_name": {"enum": ["micron", "millimeter", "centimeter", "inch", "foot", "meter"]}, + "source": { + "description": "One model part that was read. When the part declares a unit outside the six defined ones, unit and mm_per_unit are the FALLBACK millimetre values, and the UNSUPPORTED_UNIT limitation (whose target_ids are the nodes living in that part) says so; no fit statement is made for such a scene.", + "type": "object", "additionalProperties": false, "required": ["part", "unit", "mm_per_unit"], + "properties": { + "part": {"type": "string", "minLength": 1, "maxLength": 256}, + "unit": {"$ref": "#/$defs/unit_name"}, + "mm_per_unit": {"type": "number", "exclusiveMinimum": 0} + } + }, + "bed": { + "type": "object", "additionalProperties": false, "required": ["polygon_mm", "height_mm", "edge_margin_mm", "policy"], + "properties": { + "polygon_mm": {"type": "array", "minItems": 3, "maxItems": 16, + "items": {"type": "array", "minItems": 2, "maxItems": 2, "items": {"type": "number"}}}, + "height_mm": {"type": ["number", "null"]}, + "edge_margin_mm": {"type": "number", "minimum": 0}, + "policy": {"enum": ["u1_template", "fallback"]} + } + }, + "volume_role": {"enum": ["part", "modifier", "negative", "support_enforcer", "support_blocker", "unknown"]}, + "volume": { + "type": "object", "additionalProperties": false, "required": ["id", "triangle_start", "triangle_count", "role", "source"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "triangle_start": {"$ref": "#/$defs/uint"}, + "triangle_count": {"$ref": "#/$defs/uint"}, + "role": {"$ref": "#/$defs/volume_role"}, + "source": {"enum": ["prusa_range", "bambu_part", "object", null]} + } + }, + "mesh": { + "type": "object", "additionalProperties": false, + "required": ["key", "vertex_count", "triangle_count", "positions_f32le_base64", "indices_u32le_base64", "volumes"], + "properties": { + "key": {"$ref": "#/$defs/resource_key"}, + "vertex_count": {"$ref": "#/$defs/uint"}, + "triangle_count": {"$ref": "#/$defs/uint"}, + "positions_f32le_base64": {"type": "string", "pattern": "^[A-Za-z0-9+/]*={0,2}$"}, + "indices_u32le_base64": {"type": "string", "pattern": "^[A-Za-z0-9+/]*={0,2}$"}, + "volumes": {"type": "array", "minItems": 1, "maxItems": 4096, "items": {"$ref": "#/$defs/volume"}} + } + }, + "plate_ref": { + "type": "object", "additionalProperties": false, "required": ["state", "plate_id", "source"], + "properties": { + "state": {"enum": ["known", "ambiguous", "unknown"]}, + "plate_id": {"type": ["string", "null"], "maxLength": 128}, + "source": {"enum": ["plate_config", null]} + }, + "if": {"properties": {"state": {"const": "known"}}}, + "then": {"properties": {"plate_id": {"type": "string"}, "source": {"const": "plate_config"}}} + }, + "instance_ref": { + "type": "object", "additionalProperties": false, "required": ["build_index", "component_path"], + "properties": { + "build_index": {"type": ["integer", "null"], "minimum": 0}, + "component_path": {"type": "array", "maxItems": 64, "items": {"type": "integer", "minimum": 0}} + } + }, + "bounds": { + "type": ["object", "null"], "additionalProperties": false, "required": ["min", "max"], + "properties": {"min": {"$ref": "#/$defs/vec3"}, "max": {"$ref": "#/$defs/vec3"}} + }, + "node": { + "type": "object", "additionalProperties": false, + "required": ["id", "parent_id", "resource", "mesh_key", "local_to_parent_mm", "world_mm", "mirrored", "role_context", "build_index", "instance_ref", + "plate", "placement_state", "printable", "has_non_part_volumes", "bounds_mm", "finding_ids"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "parent_id": {"type": ["string", "null"], "maxLength": 128}, + "resource": {"$ref": "#/$defs/resource_key"}, + "mesh_key": {"oneOf": [{"$ref": "#/$defs/resource_key"}, {"type": "null"}]}, + "local_to_parent_mm": {"$ref": "#/$defs/matrix16"}, + "world_mm": {"$ref": "#/$defs/matrix16"}, + "mirrored": {"type": "boolean"}, + "role_context": {"enum": ["part", "modifier", "negative", "support_enforcer", "support_blocker", "unknown", null]}, + "build_index": {"type": ["integer", "null"], "minimum": 0}, + "instance_ref": {"$ref": "#/$defs/instance_ref"}, + "plate": {"$ref": "#/$defs/plate_ref"}, + "placement_state": {"enum": ["known", "unknown"]}, + "printable": {"type": ["boolean", "null"]}, + "has_non_part_volumes": {"type": "boolean"}, + "bounds_mm": {"$ref": "#/$defs/bounds"}, + "finding_ids": {"type": "array", "maxItems": 500, "items": {"$ref": "#/$defs/id"}} + } + }, + "plate": { + "type": "object", "additionalProperties": false, "required": ["id", "ui_number", "origin_mm"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "ui_number": {"type": ["integer", "null"], "minimum": 0}, + "origin_mm": {"oneOf": [{"$ref": "#/$defs/vec3"}, {"type": "null"}]} + } + }, + "finding": { + "type": "object", "additionalProperties": false, "required": ["id", "engine", "schema", "pointer", "scope", "kind", "target_ids", "value"], + "properties": { + "id": {"$ref": "#/$defs/id"}, + "engine": {"type": "string", "minLength": 1, "maxLength": 64}, + "schema": {"type": "string", "minLength": 1, "maxLength": 64}, + "pointer": {"type": "string", "maxLength": 128}, + "scope": {"enum": ["project", "plate", "instance"]}, + "kind": {"enum": ["placement", "size", "note"]}, + "target_ids": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/id"}}, + "value": { + "type": "object", "additionalProperties": false, "required": ["code", "overhang_mm"], + "properties": { + "code": {"enum": ["PLACEMENT_OUTSIDE_BED", "SIZE_EXCEEDS_BED"]}, + "overhang_mm": { + "type": ["object", "null"], "additionalProperties": false, "required": ["left", "right", "front", "back"], + "properties": {"left": {"type": "number"}, "right": {"type": "number"}, "front": {"type": "number"}, "back": {"type": "number"}} + } + } + } + } + }, + "limitation_code": {"enum": [ + "UNKNOWN_VOLUME_ROLE", "UNKNOWN_PLACEMENT", "PLATE_MEMBERSHIP_AMBIGUOUS", "PLATE_MEMBERSHIP_UNKNOWN", + "PLATES_UNAVAILABLE", "BED_TEMPLATE_UNAVAILABLE", "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED", + "NON_MM_SOURCE_UNIT", "UNSUPPORTED_UNIT", "MULTI_PLATE_PLACEMENT_UNCHECKED", "NO_BUILD_ITEMS"]}, + "limitation": { + "type": "object", "additionalProperties": false, "required": ["code", "target_ids"], + "properties": { + "code": {"$ref": "#/$defs/limitation_code"}, + "target_ids": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/id"}} + } + }, + "counts": { + "type": "object", "additionalProperties": false, + "required": ["nodes", "meshes", "vertices", "triangles", "rendered_triangles", "plates", "findings", "limitations"], + "properties": { + "nodes": {"$ref": "#/$defs/uint"}, "meshes": {"$ref": "#/$defs/uint"}, "vertices": {"$ref": "#/$defs/uint"}, + "triangles": {"$ref": "#/$defs/uint"}, "rendered_triangles": {"$ref": "#/$defs/uint"}, + "plates": {"$ref": "#/$defs/uint"}, "findings": {"$ref": "#/$defs/uint"}, "limitations": {"$ref": "#/$defs/uint"} + } + }, + "limits": { + "type": "object", "additionalProperties": false, + "required": ["max_archive_bytes", "max_model_xml_bytes", "max_rendered_triangles", "max_vertices", "max_nodes", "max_depth", + "max_response_bytes", "max_findings", "max_limitations", "max_id_length"], + "properties": { + "max_archive_bytes": {"$ref": "#/$defs/uint"}, "max_model_xml_bytes": {"$ref": "#/$defs/uint"}, + "max_rendered_triangles": {"$ref": "#/$defs/uint"}, "max_vertices": {"$ref": "#/$defs/uint"}, + "max_nodes": {"$ref": "#/$defs/uint"}, "max_depth": {"$ref": "#/$defs/uint"}, + "max_response_bytes": {"$ref": "#/$defs/uint"}, "max_findings": {"$ref": "#/$defs/uint"}, + "max_limitations": {"$ref": "#/$defs/uint"}, "max_id_length": {"$ref": "#/$defs/uint"} + } + }, + "error_code": {"enum": [ + "INVALID_REQUEST", "UNSUPPORTED_FORMAT", "INVALID_ARCHIVE", "INVALID_GEOMETRY", "UNRESOLVED_REFERENCE", + "LIMIT_EXCEEDED", "SOURCE_CHANGED", "CANCELLED", "TIMEOUT", "EXPIRED", "NOT_READY", "WORKER_WEDGED", "INTERNAL", + "STALE_START", "CANCELLED_BEFORE_START", "SESSION_EXPIRED", "SESSION_LIMIT"]}, + "job_state": {"enum": ["queued", "running", "succeeded", "failed", "cancelled"]}, + "error_response": { + "type": "object", "additionalProperties": false, "required": ["error"], + "properties": { + "error": {"$ref": "#/$defs/error_code"}, + "message": {"type": "string", "maxLength": 512}, + "state": {"$ref": "#/$defs/job_state"}, + "reason": {"type": "string", "maxLength": 256} + } + }, + "job_status": { + "type": "object", "additionalProperties": false, + "required": ["job_id", "request_id", "state", "stage", "completed", "total", "error", "revision"], + "properties": { + "job_id": {"oneOf": [{"$ref": "#/$defs/id"}, {"type": "null"}]}, + "request_id": {"$ref": "#/$defs/id"}, + "state": {"$ref": "#/$defs/job_state"}, + "stage": {"enum": ["reading", "parsing", "encoding", null]}, + "completed": {"type": ["integer", "null"], "minimum": 0}, + "total": {"type": ["integer", "null"], "minimum": 0}, + "error": {"type": ["object", "null"], "additionalProperties": false, "required": ["code", "message"], + "properties": {"code": {"$ref": "#/$defs/error_code"}, "message": {"type": "string", "maxLength": 512}}}, + "revision": {"oneOf": [{"$ref": "#/$defs/sha256"}, {"type": "null"}]} + } + }, + "session": { + "type": "object", "additionalProperties": false, "required": ["client_id", "ttl_s"], + "properties": { + "client_id": {"type": "string", "pattern": "^[A-Za-z0-9_-]{24}$"}, + "ttl_s": {"type": "integer", "minimum": 1} + } + }, + "job_start": { + "type": "object", "additionalProperties": false, + "required": ["job_id", "request_id", "state", "revision", "replaced_job_id"], + "properties": { + "job_id": {"$ref": "#/$defs/id"}, + "request_id": {"$ref": "#/$defs/id"}, + "state": {"$ref": "#/$defs/job_state"}, + "revision": {"oneOf": [{"$ref": "#/$defs/sha256"}, {"type": "null"}]}, + "replaced_job_id": {"type": ["string", "null"], "maxLength": 128} + } + } + } +} diff --git a/backend/snapstudio_core/scene.py b/backend/snapstudio_core/scene.py new file mode 100644 index 00000000..b4a61810 --- /dev/null +++ b/backend/snapstudio_core/scene.py @@ -0,0 +1,1459 @@ +"""scene/1 -- a read-only, bounded description of a project's geometry, placement and findings. + +The contract is the committed JSON Schema ``data/scene-1.schema.json`` (frozen: closed objects, +every field required, unknown is null). This module is the only producer of it. + +What a scene IS: a consistent snapshot of the file as it was copied for the job. It is not a live +view. ``SOURCE_CHANGED`` therefore only ever applies to a reused request_id+path or an +``expected_revision`` mismatch (see ``snapstudio_api.scene_jobs``); it does not apply to edits made +after the snapshot was taken. + +Design rules, in order of importance: + +* **Untrusted input.** The archive is opened by a SELECTIVE reader that owns its own checks: + colliding or traversal-looking entry names, external relationships, DTD/ENTITY declarations and + any text encoding other than UTF-8 are rejected BEFORE parsing; only the ``.model``, ``.rels`` and + the two metadata configs the scene needs are ever read (never thumbnails or embedded G-code); + every read is metered against the budgets in ``scene_limits`` while it happens. + ``ThreeMF.open()`` and ``config_io`` are deliberately untouched. +* **Cooperative interruption.** Archive reads are <= 1 MiB, lxml ``iterparse`` checks the cancel / + deadline hook every <= 1,000 events (and on every read), geometry encode/transform runs in + batches of <= 5,000 triangles. The worst uninterruptible call is one such unit. Accepted + limitation: a single stuck native/filesystem call cannot be interrupted in-process; the backstop + is the job registry's fail-closed WORKER_WEDGED state (no multiprocessing in v1). +* **Honest unknowns.** Placement, plate membership and volume roles that the file does not prove + are reported as ``unknown``/``ambiguous`` with a limitation, never guessed. Bed fit is only + claimed when the bed is Studio's U1 template, placement is known and the instance is not one of + several build items of the same object id (the engine's own placement answer for those is + unverified, see ``REPEATED_INSTANCE_PLACEMENT_UNVERIFIED``). +* **Never silent truncation.** Over any budget the build stops with ``LIMIT_EXCEEDED``. +""" +from __future__ import annotations + +import base64 +import hashlib +import io +import json +import math +import posixpath +import re +import sys +import zipfile +import zlib +from array import array +from importlib.resources import files + +from lxml import etree + +from . import scene_limits as L +from . import units as _units +from .errors import SnapStudioError + +SCHEMA_ID = "scene/1" +SCHEMA_RESOURCE = "scene-1.schema.json" + +CORE_NS = "http://schemas.microsoft.com/3dmanufacturing/core/2015/02" +PROD_NS = "http://schemas.microsoft.com/3dmanufacturing/production/2015/06" +_T = {n: f"{{{CORE_NS}}}{n}" for n in ( + "model", "resources", "object", "mesh", "vertices", "vertex", "triangles", "triangle", + "components", "component", "build", "item")} +_PROD_PATH = f"{{{PROD_NS}}}path" + +ROOT_MODEL_DEFAULT = "3D/3dmodel.model" +MODEL_SETTINGS = "Metadata/model_settings.config" +PRUSA_CONFIG = "Metadata/Slic3r_PE_model.config" +_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" +_MODEL_REL_TYPE = "http://schemas.microsoft.com/3dmanufacturing/2013/01/3dmodel" + +# limitation codes that make a scene "partial" rather than "complete" +PARTIAL_LIMITATIONS = frozenset({ + "UNKNOWN_VOLUME_ROLE", "UNKNOWN_PLACEMENT", "PLATE_MEMBERSHIP_AMBIGUOUS", "PLATE_MEMBERSHIP_UNKNOWN", "UNSUPPORTED_UNIT", + "BED_TEMPLATE_UNAVAILABLE", "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED", "NON_MM_SOURCE_UNIT", + "NO_BUILD_ITEMS"}) +#: Information only: the scene is still as complete as the file allows. +INFORMATIONAL_LIMITATIONS = frozenset({"PLATES_UNAVAILABLE", "MULTI_PLATE_PLACEMENT_UNCHECKED"}) + +ERROR_CODES = ( + "INVALID_REQUEST", "UNSUPPORTED_FORMAT", "INVALID_ARCHIVE", "INVALID_GEOMETRY", + "UNRESOLVED_REFERENCE", "LIMIT_EXCEEDED", "SOURCE_CHANGED", "CANCELLED", "TIMEOUT", + "EXPIRED", "NOT_READY", "WORKER_WEDGED", "INTERNAL", "STALE_START", "CANCELLED_BEFORE_START", "SESSION_EXPIRED", "SESSION_LIMIT") + + +class SceneError(SnapStudioError): + """A refusal with a stable machine code (one of ERROR_CODES) and a message safe to show.""" + + def __init__(self, code: str, message: str) -> None: + assert code in ERROR_CODES, code + self.code = code + self.message = message + super().__init__(message) + + +class SceneCancelled(Exception): + """Raised by the job control hook when the job was cancelled.""" + + +class SceneTimeout(Exception): + """Raised by the job control hook when the job's deadline passed.""" + + +class Control: + """The cancel/deadline/progress hook. The default does nothing.""" + + def check(self) -> None: # pragma: no cover - trivial + return None + + def progress(self, stage: str, completed: int | None, total: int | None) -> None: # pragma: no cover + return None + + +def load_schema() -> dict: + return json.loads((files("snapstudio_core.data") / SCHEMA_RESOURCE).read_text("utf-8")) + + +# -------------------------------------------------------------------------------------------- +# Selective archive reader +# -------------------------------------------------------------------------------------------- + +_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]") +_DRIVE = re.compile(r"^[A-Za-z]:") +_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*:") # http:, file:, C: ... never a package-relative path +_XML_DECL_ENCODING = re.compile(rb"""^\s*<\?xml[^>]*?encoding\s*=\s*["']([^"']+)["']""", re.I) +_BOMS_REJECTED = (b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff", b"\xff\xfe", b"\xfe\xff") +_READ_ERRORS = (zipfile.BadZipFile, NotImplementedError, zlib.error, EOFError, RuntimeError, OSError) + + +def normalize_entry_name(name: str) -> tuple[str, bool]: + """A zip entry name as a normalized archive-relative path, and whether it is a directory. + + Rejected: control characters, rooted names, drive letters, any ``..`` segment. Backslashes are + read as separators, so ``a\\b`` and ``a/b`` collide (and are rejected as a pair by the caller). + """ + n = name.replace("\\", "/") + if _CONTROL_CHARS.search(n): + raise SceneError("INVALID_ARCHIVE", "The archive has an entry name with control characters.") + if n.startswith("/") or _DRIVE.match(n): + raise SceneError("INVALID_ARCHIVE", "The archive has an entry with an absolute path.") + segments = n.split("/") + if ".." in segments: + raise SceneError("INVALID_ARCHIVE", "The archive has an entry that escapes its own folder.") + clean = [s for s in segments if s not in ("", ".")] + if not clean: + raise SceneError("INVALID_ARCHIVE", "The archive has an entry with an empty name.") + return "/".join(clean), n.endswith("/") + + +def resolve_reference(target: str, base_dir: str = "") -> str: + """Resolve a relationship Target / component path to a normalized part name. + + A leading ``/`` means "from the package root", which is how every slicer writes them, so it is + accepted. A reference that climbs above the package root is rejected. + """ + t = target.replace("\\", "/") + if _CONTROL_CHARS.search(t) or _SCHEME.match(t): + raise SceneError("INVALID_ARCHIVE", "The project has a malformed internal reference.") + parts = [] if t.startswith("/") else [s for s in base_dir.split("/") if s] + for seg in t.split("/"): + if seg in ("", "."): + continue + if seg == "..": + if not parts: + raise SceneError("INVALID_ARCHIVE", "A reference in the project escapes the archive.") + parts.pop() + else: + parts.append(seg) + if not parts: + raise SceneError("INVALID_ARCHIVE", "The project has an empty internal reference.") + return "/".join(parts) + + +class GuardedStream: + """A file-like over one archive member that meters, scans and honours cancellation. + + Reads at most ``CHUNK`` bytes per call (bounded uninterruptible work). The DTD/ENTITY scan and + the encoding check run on the raw bytes BEFORE the parser sees them. + """ + + def __init__(self, archive: "Archive", info: zipfile.ZipInfo, kind: str, cap: int, scan: bool = True): + self.archive, self.kind, self.cap, self.scan = archive, kind, cap, scan + self.bytes_read = 0 + self._carry = b"" + self._first = True + try: + self._fh = archive.zf.open(info, "r") + except _READ_ERRORS as exc: + raise SceneError("INVALID_ARCHIVE", "A part of the archive could not be opened.") from exc + + def close(self) -> None: + try: + self._fh.close() + except Exception: # noqa: BLE001 + pass + + def read(self, size: int = -1) -> bytes: + self.archive.ctl.check() + n = L.SNAPSHOT_CHUNK_BYTES if size is None or size < 0 else min(size, L.SNAPSHOT_CHUNK_BYTES) + try: + chunk = self._fh.read(n) + if self._first and chunk and len(chunk) < 4: + chunk += self._fh.read(4 - len(chunk)) + except _READ_ERRORS as exc: + raise SceneError("INVALID_ARCHIVE", "A part of the archive is damaged.") from exc + if not chunk: + return chunk + self.bytes_read += len(chunk) + self.archive.account(len(chunk), self.kind) + if self.bytes_read > self.cap: + raise SceneError("LIMIT_EXCEEDED", "A part of the archive expands past what Studio will read.") + if self.scan: + if self._first: + self._check_encoding(chunk) + hay = (self._carry + chunk).lower() + if b" None: + if head.startswith(b"\xef\xbb\xbf"): + head = head[3:] + elif head.startswith(_BOMS_REJECTED) or b"\x00" in head[:4]: + raise SceneError("INVALID_ARCHIVE", "The project is not UTF-8 text, which is the only encoding Studio accepts.") + found = _XML_DECL_ENCODING.match(head) + if found and found.group(1).strip().lower() not in (b"utf-8", b"utf8"): + raise SceneError("INVALID_ARCHIVE", "The project is not UTF-8 text, which is the only encoding Studio accepts.") + + +class Archive: + """Read-only, selective, metered access to one 3MF.""" + + def __init__(self, path: str, ctl: Control): + self.ctl = ctl + self.total_read = 0 + self.model_read = 0 + self.model_declared = 0 + try: + self.zf = zipfile.ZipFile(path) + except (zipfile.BadZipFile, OSError, EOFError) as exc: + raise SceneError("INVALID_ARCHIVE", "This file is not a readable 3MF archive.") from exc + infos = self.zf.infolist() + if len(infos) > L.ARCHIVE_MAX_ENTRIES: + raise SceneError("LIMIT_EXCEEDED", "The archive has more entries than Studio will read.") + self.index: dict[str, tuple[str, zipfile.ZipInfo]] = {} + for info in infos: + norm, is_dir = normalize_entry_name(info.filename) + key = norm.casefold() + if key in self.index: + raise SceneError("INVALID_ARCHIVE", "The archive has duplicate or colliding entry names.") + if info.flag_bits & 0x1: + raise SceneError("INVALID_ARCHIVE", "The archive is encrypted.") + self.index[key] = (norm, info) + if is_dir: + self.index[key] = (norm + "/", info) + + def close(self) -> None: + self.zf.close() + + def account(self, n: int, kind: str) -> None: + self.total_read += n + if self.total_read > L.ARCHIVE_EXPANSION_BYTES: + raise SceneError("LIMIT_EXCEEDED", "The archive expands past what Studio will read.") + if kind == "model": + self.model_read += n + if self.model_read > L.MODEL_XML_BYTES: + raise SceneError("LIMIT_EXCEEDED", "The model data is larger than Studio will read.") + + def has(self, name: str) -> bool: + entry = self.index.get(name.casefold()) + return entry is not None and not entry[0].endswith("/") + + def names(self) -> list[str]: + return [norm for norm, _ in self.index.values() if not norm.endswith("/")] + + def open(self, name: str, kind: str, scan: bool = True) -> GuardedStream: + entry = self.index.get(name.casefold()) + if entry is None or entry[0].endswith("/"): + raise SceneError("UNRESOLVED_REFERENCE", "The project refers to a part that is not in the archive.") + info = entry[1] + if len(entry[0]) > L.MAX_PART_NAME_LENGTH: + # part names are carried on nodes, meshes and sources: bound them before they are placed + raise SceneError("LIMIT_EXCEEDED", f"A part name is longer than the {L.MAX_PART_NAME_LENGTH} characters Studio carries.") + cap = L.MODEL_XML_BYTES if kind == "model" else L.CONFIG_PART_BYTES + if info.file_size > cap: + raise SceneError("LIMIT_EXCEEDED", "A part of the archive is larger than Studio will read.") + if kind == "model": + self.model_declared += info.file_size + return GuardedStream(self, info, kind, cap, scan) + + def read_small(self, name: str) -> bytes: + stream = self.open(name, "config") + try: + out = [] + while True: + chunk = stream.read(L.SNAPSHOT_CHUNK_BYTES) + if not chunk: + break + out.append(chunk) + return b"".join(out) + finally: + stream.close() + + +def iter_xml(stream: GuardedStream, ctl: Control, known=None, consume=frozenset(), local=False): + """Yield lxml ``(event, element)`` pairs with the hardened options. + + The parser is also asked, on its first event, whether it saw a doctype / internal DTD or a + non-UTF-8 declaration, so rejection does not depend on the raw byte scan alone. + + Every completed element is COUNTED against ``MAX_XML_ELEMENTS`` (so a part under its byte cap that is + made of millions of tiny elements is refused early). When ``known`` (the local names the caller reads) + is given, any completed element outside it is released as soon as the caller has seen it, so unknown + junk never accumulates (an element in the namespace the caller does not read is unknown too), and so is + any element named in ``consume`` (self-contained records the caller has read at their own end event); + the caller releases the recognized parents it consumed. ``local=True`` matches by local name. + """ + try: + it = etree.iterparse(stream, events=("start", "end"), resolve_entities=False, + load_dtd=False, no_network=True, huge_tree=False) + first, count, elements = True, 0, 0 + for event, elem in it: + if first: + first = False + info = elem.getroottree().docinfo + if info.doctype or info.internalDTD is not None or info.system_url or info.public_id: + raise SceneError("INVALID_ARCHIVE", "The project contains a DTD, which Studio does not accept.") + if (info.encoding or "UTF-8").upper() not in ("UTF-8", "UTF8"): + raise SceneError("INVALID_ARCHIVE", "The project is not UTF-8 text, which is the only encoding Studio accepts.") + count += 1 + if count % L.XML_EVENTS_PER_CHECK == 0: + ctl.check() + if event == "end": + elements += 1 + if elements > L.MAX_XML_ELEMENTS: + raise SceneError("LIMIT_EXCEEDED", f"A part of the project has more than {L.MAX_XML_ELEMENTS:,} elements.") + yield event, elem + if known is not None and event == "end": + tag = etree.QName(elem).localname if local else elem.tag + if tag not in known or tag in consume: + _free(elem) + except etree.LxmlError as exc: + raise SceneError("INVALID_ARCHIVE", "A part of the project is not well-formed XML.") from exc + finally: + stream.close() + + +def _free(elem) -> None: + """Release ONE completed element: clear it and detach it from its parent. + + Never its siblings. Records that a parent consumes later (the ````/````/```` + children of a settings ````/````) are siblings of whatever unknown element happens to follow + them, and deleting them would silently turn a negative part into a printable one. The caller must have + read everything it needs from ``elem`` (as plain data) BEFORE calling this. + """ + elem.clear() + parent = elem.getparent() + if parent is not None: + parent.remove(elem) + + +# -------------------------------------------------------------------------------------------- +# Matrices (column-major 4x4 as 16 floats; a 3MF transform is 12 numbers: 3 columns + translation) +# -------------------------------------------------------------------------------------------- + +IDENTITY16 = [1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0] + + +def parse_transform(text: str | None, mm_per_unit: float) -> list[float] | None: + """A 3MF ``transform`` attribute as a column-major 16-float matrix in millimetres.""" + if text is None: + return None + parts = text.split() + if len(parts) != 12: + raise SceneError("INVALID_GEOMETRY", "A transform does not have twelve numbers.") + try: + v = [float(p) for p in parts] + except ValueError as exc: + raise SceneError("INVALID_GEOMETRY", "A transform has a value that is not a number.") from exc + if not all(math.isfinite(x) for x in v): + raise SceneError("INVALID_GEOMETRY", "A transform has a value that is not finite.") + return [v[0], v[1], v[2], 0.0, v[3], v[4], v[5], 0.0, v[6], v[7], v[8], 0.0, + v[9] * mm_per_unit, v[10] * mm_per_unit, v[11] * mm_per_unit, 1.0] + + +def mat_mul(a: list[float], b: list[float]) -> list[float]: + out = [0.0] * 16 + for c in range(4): + for r in range(4): + out[c * 4 + r] = (a[r] * b[c * 4] + a[4 + r] * b[c * 4 + 1] + + a[8 + r] * b[c * 4 + 2] + a[12 + r] * b[c * 4 + 3]) + return out + + +def mat_det3(m: list[float]) -> float: + return (m[0] * (m[5] * m[10] - m[9] * m[6]) - m[4] * (m[1] * m[10] - m[9] * m[2]) + + m[8] * (m[1] * m[6] - m[5] * m[2])) + + +def _round_matrix(m: list[float]) -> list[float]: + return [float(f"{x:.9g}") + 0.0 for x in m] # + 0.0 turns -0.0 into 0.0 + + +# -------------------------------------------------------------------------------------------- +# Model parts +# -------------------------------------------------------------------------------------------- + +class Budget: + """Counters shared by every model part of one scene, enforced WHILE parsing (never after).""" + + def __init__(self) -> None: + self.vertices = 0 + self.triangles = 0 + self.elements = 0 + self.objects = 0 + + +class MeshData: + __slots__ = ("pos", "idx", "nverts", "ntris", "max_index") + + def __init__(self) -> None: + self.pos = array("d") + self.idx = array("I") + self.nverts = 0 + self.ntris = 0 + self.max_index = -1 + + +class Component: + __slots__ = ("object_id", "part", "transform") + + def __init__(self, object_id: str, part: str, transform): + self.object_id, self.part, self.transform = object_id, part, transform + + +class ObjectDef: + __slots__ = ("id", "mesh", "components") + + def __init__(self, oid: str): + self.id = oid + self.mesh: MeshData | None = None + self.components: list[Component] = [] + + +class BuildItem: + """A build item. ``part`` is the model FILE its object lives in: the root unless the item carries a + production-extension ``p:path`` (so identity everywhere is the pair part + object id).""" + __slots__ = ("object_id", "part", "transform") + + def __init__(self, object_id: str, part: str, transform): + self.object_id, self.part, self.transform = object_id, part, transform + + +class ModelFile: + def __init__(self, part: str): + self.part = part + self.unit = _units.DEFAULT_UNIT + self.mm_per_unit = 1.0 + self.unit_recognized = True + self.objects: dict[str, ObjectDef] = {} + self.build: list[BuildItem] = [] + + +def _check_id(value: str | None, what: str) -> str: + if value is None or not value.strip(): + raise SceneError("INVALID_ARCHIVE", f"A {what} has no id.") + if len(value) > L.MAX_ID_LENGTH: + raise SceneError("LIMIT_EXCEEDED", f"A {what} id is longer than Studio will carry.") + return value + + +def _package_path(ref: str) -> str: + """A production-extension ``p:path``: an ABSOLUTE package path (leading ``/``); relative ones are refused.""" + if not ref.startswith("/"): + raise SceneError("UNRESOLVED_REFERENCE", "A p:path reference must be an absolute package path.") + return resolve_reference(ref) + + +def parse_model(arc: Archive, part: str, budget: Budget, ctl: Control, is_root: bool) -> ModelFile: + """Stream one ``.model`` part into objects, meshes, components and (root only) build items.""" + mf = ModelFile(part) + stream = arc.open(part, "model") + cur: ObjectDef | None = None + mesh: MeshData | None = None + saw_root = False + t_vertex, t_triangle, t_component, t_item = _T["vertex"], _T["triangle"], _T["component"], _T["item"] + t_object, t_mesh, t_model = _T["object"], _T["mesh"], _T["model"] + limit_v, limit_t, max_coord = L.MAX_VERTICES, L.MAX_PARSED_TRIANGLES, L.MAX_COORDINATE_MM + for event, elem in iter_xml(stream, ctl): + tag = elem.tag + if event == "start": + if not saw_root: + saw_root = True + if tag != t_model: + raise SceneError("INVALID_ARCHIVE", "A model part does not start with a model element.") + # The SAME function the legacy readers use, on the real root element. + info = _units.resolve_unit(elem.get("unit")) + mf.unit_recognized = info.recognized + mf.unit = info.name if info.recognized else _units.DEFAULT_UNIT # documented fallback + mf.mm_per_unit = info.factor + elif tag == t_object: + budget.objects += 1 + if budget.objects > L.MAX_OBJECT_DEFINITIONS: + raise SceneError("LIMIT_EXCEEDED", f"The project defines more than {L.MAX_OBJECT_DEFINITIONS:,} objects.") + oid = _check_id(elem.get("id"), "object") + if oid in mf.objects: + raise SceneError("INVALID_ARCHIVE", "A model part defines the same object id twice.") + cur = ObjectDef(oid) + mf.objects[oid] = cur + elif tag == t_mesh and cur is not None: + mesh = MeshData() + continue + # --- end events + budget.elements += 1 + if budget.elements > L.MAX_XML_ELEMENTS: + raise SceneError("LIMIT_EXCEEDED", f"The model data has more than {L.MAX_XML_ELEMENTS:,} elements.") + if tag == t_vertex: + if mesh is None: + continue + try: + x, y, z = float(elem.get("x")), float(elem.get("y")), float(elem.get("z")) + except (TypeError, ValueError) as exc: + raise SceneError("INVALID_GEOMETRY", "A vertex has a coordinate that is missing or not a number.") from exc + scale = mf.mm_per_unit + x, y, z = x * scale, y * scale, z * scale + if not (math.isfinite(x) and math.isfinite(y) and math.isfinite(z)): + raise SceneError("INVALID_GEOMETRY", "A vertex has a coordinate that is not finite.") + if abs(x) > max_coord or abs(y) > max_coord or abs(z) > max_coord: + raise SceneError("INVALID_GEOMETRY", "A vertex coordinate is outside the range Studio can represent.") + budget.vertices += 1 + if budget.vertices > limit_v: + raise SceneError("LIMIT_EXCEEDED", "The project has more vertices than Studio will draw.") + mesh.pos.append(x) + mesh.pos.append(y) + mesh.pos.append(z) + mesh.nverts += 1 + elif tag == t_triangle: + if mesh is None: + continue + try: + a, b, c = int(elem.get("v1")), int(elem.get("v2")), int(elem.get("v3")) + except (TypeError, ValueError) as exc: + raise SceneError("INVALID_GEOMETRY", "A triangle has an index that is missing or not a whole number.") from exc + if a < 0 or b < 0 or c < 0 or a > 0xFFFFFFFF or b > 0xFFFFFFFF or c > 0xFFFFFFFF: + raise SceneError("INVALID_GEOMETRY", "A triangle has a vertex index outside the supported range.") + budget.triangles += 1 + if budget.triangles > limit_t: + raise SceneError("LIMIT_EXCEEDED", f"The project has at least {budget.triangles:,} triangles; Studio draws at most {limit_t:,}.") + top = max(a, b, c) + if top > mesh.max_index: + mesh.max_index = top + mesh.idx.append(a) + mesh.idx.append(b) + mesh.idx.append(c) + mesh.ntris += 1 + elif tag == t_mesh: + if mesh is not None and cur is not None: + if mesh.nverts == 0 or mesh.ntris == 0: + raise SceneError("INVALID_GEOMETRY", "A mesh has no vertices or no triangles.") + if mesh.max_index >= mesh.nverts: + raise SceneError("INVALID_GEOMETRY", "A triangle refers to a vertex that does not exist.") + cur.mesh = mesh + mesh = None + elif tag == t_component: + if cur is None: + continue + oid = _check_id(elem.get("objectid"), "component") + ref = elem.get(_PROD_PATH) + if ref is not None and not is_root: + raise SceneError("UNRESOLVED_REFERENCE", "A component path is only honoured in the root model part.") + target = _package_path(ref) if ref is not None else part + cur.components.append(Component(oid, target, parse_transform(elem.get("transform"), mf.mm_per_unit))) + if len(cur.components) > L.MAX_NODES: + raise SceneError("LIMIT_EXCEEDED", "An object has more components than Studio will draw.") + elif tag == t_object: + if cur is not None and cur.mesh is not None and cur.components: + raise SceneError("INVALID_ARCHIVE", "An object has both a mesh and components.") + cur = None + elif tag == t_item and is_root: + oid = _check_id(elem.get("objectid"), "build item") + ref = elem.get(_PROD_PATH) + item_part = _package_path(ref) if ref is not None else part + mf.build.append(BuildItem(oid, item_part, parse_transform(elem.get("transform"), mf.mm_per_unit))) + if len(mf.build) > L.MAX_NODES: + raise SceneError("LIMIT_EXCEEDED", "The build has more items than Studio will draw.") + # Every completed element is released, whatever it is (unknown extensions, junk, containers): + # memory stays flat however many elements a model part holds, and the counters above stop it. + _free(elem) + if not saw_root: + raise SceneError("INVALID_ARCHIVE", "A model part is empty.") + return mf + + +_RELS_KNOWN = frozenset({"Relationships", "Relationship"}) +_BAMBU_KNOWN = frozenset({"config", "object", "part", "metadata", "plate", "model_instance", "mesh_stat"}) +_PRUSA_KNOWN = frozenset({"config", "object", "volume", "metadata", "mesh"}) + + +def find_root_model(arc: Archive) -> str: + """The root model part named by ``_rels/.rels`` (default ``3D/3dmodel.model``). + + Every ``.rels`` part is read and an external relationship in any of them is a rejection. + """ + rels = [n for n in arc.names() if n.lower().endswith(".rels")] + if len(rels) > L.MAX_RELS_PARTS: + raise SceneError("LIMIT_EXCEEDED", "The archive has more relationship parts than Studio will read.") + root = None + for name in rels: + stream = arc.open(name, "config") + for event, elem in iter_xml(stream, arc.ctl, known=_RELS_KNOWN, consume=frozenset({"Relationship"}), local=True): + if event != "end" or etree.QName(elem).localname != "Relationship": + continue + if (elem.get("TargetMode") or "").lower() == "external": + raise SceneError("INVALID_ARCHIVE", "The project has an external relationship, which Studio does not follow.") + target = elem.get("Target") + if not target: + raise SceneError("INVALID_ARCHIVE", "A relationship has no target.") + # a relationship Target is relative to the folder that holds the PART the .rels describes + parent = posixpath.dirname(name) + base = posixpath.dirname(parent) if posixpath.basename(parent).casefold() == "_rels" else parent + resolved = resolve_reference(target, base) + if name.casefold() == "_rels/.rels" and elem.get("Type") == _MODEL_REL_TYPE and root is None: + root = resolved + root = root or ROOT_MODEL_DEFAULT + if not arc.has(root): + raise SceneError("UNRESOLVED_REFERENCE", "The project has no root model part.") + return arc.index[root.casefold()][0] + + +#: Stands in for a subtype when two records for one (object, part) disagree; ``role_of`` reads it as unknown. +CONFLICTING_SUBTYPE = "conflicting_records" + + +class SettingsInfo: + """What model_settings.config / Slic3r_PE_model.config say, reduced to what the scene uses.""" + + def __init__(self) -> None: + self.dialect: str | None = None # "bambu" | "prusa" | None + self.part_subtypes: dict[str, dict[str, str | None]] = {} # object id -> {part id: subtype} + self.metadata_objects: set[str] = set() # object ids with ANY part record (even one that is malformed) + self.plates: list[dict] = [] # {"id","ui_number","members":[(oid, inst|None)]} + self.prusa_volumes: dict[str, list[dict]] = {} + + +def parse_settings(arc: Archive, ctl: Control) -> SettingsInfo: + from .assignments import role_of # imported lazily: assignments is a heavy module + + info = SettingsInfo() + if arc.has(MODEL_SETTINGS): + info.dialect = "bambu" + stream = arc.open(arc.index[MODEL_SETTINGS.casefold()][0], "config") + objects = 0 + for event, elem in iter_xml(stream, ctl, known=_BAMBU_KNOWN): + if event != "end": + continue + if elem.tag == "object": + objects += 1 + if objects > 5 * L.MAX_NODES: + raise SceneError("LIMIT_EXCEEDED", "The settings describe more objects than Studio will read.") + oid = elem.get("id") or "" + # a duplicated record may not silently overwrite another: two records for the same + # (object id, part id) that disagree make the role unknown (identical ones are fine) + parts = info.part_subtypes.setdefault(oid, {}) + for p in elem.iterchildren("part"): + info.metadata_objects.add(oid) # the object HAS metadata, whatever this record says + pid = p.get("id") + if pid is None: + continue + subtype = p.get("subtype") + if pid in parts and parts[pid] != subtype: + subtype = CONFLICTING_SUBTYPE + parts[pid] = subtype + _free(elem) + elif elem.tag == "plate": + if len(info.plates) >= 256: + raise SceneError("LIMIT_EXCEEDED", "The settings describe more plates than Studio will read.") + meta = {m.get("key"): m.get("value") for m in elem.iterchildren("metadata")} + number = meta.get("plater_id") + members = [] + for mi in elem.iterchildren("model_instance"): + mm = {m.get("key"): m.get("value") for m in mi.iterchildren("metadata")} + if mm.get("object_id") is not None: + members.append((mm["object_id"], mm.get("instance_id"))) + if len(members) > 10 * L.MAX_NODES: + raise SceneError("LIMIT_EXCEEDED", "A plate lists more objects than Studio will read.") + ui = (int(number) if number is not None and number.isascii() and number.isdigit() + and len(number) <= 6 else None) + info.plates.append({"ui_number": ui, "members": members}) + _free(elem) + elif arc.has(PRUSA_CONFIG): + info.dialect = "prusa" + stream = arc.open(arc.index[PRUSA_CONFIG.casefold()][0], "config") + count = 0 + for event, elem in iter_xml(stream, ctl, known=_PRUSA_KNOWN): + if event != "end" or elem.tag != "object": + continue + count += 1 + if count > L.MAX_OBJECT_DEFINITIONS: + raise SceneError("LIMIT_EXCEEDED", "The settings describe more objects than Studio will read.") + entries = [] + for vol in elem.iterchildren("volume"): + # attributes are read BY NAME: their order in the tag does not matter + first, last = vol.get("firstid"), vol.get("lastid") + valid = (first is not None and last is not None and first.isascii() and last.isascii() + and first.isdigit() and last.isdigit() and len(first) <= 9 and len(last) <= 9) + kind = None + for m in vol.iterchildren("metadata"): + if m.get("key") == "volume_type" and m.get("type") in (None, "volume"): + kind = m.get("value") + entries.append({"range": (int(first), int(last)) if valid else None, "role": role_of(kind)}) + if len(entries) > L.MAX_VOLUMES_PER_MESH: + raise SceneError("LIMIT_EXCEEDED", "An object has more volumes than Studio will read.") + if entries: + info.prusa_volumes.setdefault(elem.get("id") or "", []).extend(entries) + _free(elem) + return info + + +# -------------------------------------------------------------------------------------------- +# Bed +# -------------------------------------------------------------------------------------------- + +def _bed(limitations: list[dict]) -> tuple[dict, dict]: + """(bed object for the scene, rectangle used for findings). Surfaces a fallback, never hides it.""" + from . import plate_placement as pp + + rect = None + try: + rect = pp.parse_printable_area(pp._u1_printable_area()) + except Exception: # noqa: BLE001 - a missing/corrupt template is a limitation, not a crash + rect = None + policy = "u1_template" + if rect is None: + policy = "fallback" + rect = {"min_x": 0.0, "min_y": 0.0, "max_x": 270.0, "max_y": 270.0} + limitations.append({"code": "BED_TEMPLATE_UNAVAILABLE", "target_ids": []}) + height = None + try: + data = json.loads((files("snapstudio_core.data") / "templates" / "u1_base_project_settings.json").read_text("utf-8")) + height = float(data.get("printable_height")) + except Exception: # noqa: BLE001 + height = None + poly = [[rect["min_x"], rect["min_y"]], [rect["max_x"], rect["min_y"]], + [rect["max_x"], rect["max_y"]], [rect["min_x"], rect["max_y"]]] + return ({"polygon_mm": poly, "height_mm": height, "edge_margin_mm": pp.EDGE_MARGIN_MM, "policy": policy}, rect) + + + + +# -------------------------------------------------------------------------------------------- +# Traversal +# -------------------------------------------------------------------------------------------- + +class Node: + __slots__ = ("id", "parent", "resource", "mesh_key", "local", "world", "mirrored", "build_index", + "path", "plate", "placement_known", "printable", "has_non_part", "bounds", + "finding_ids", "children", "top", "repeated", "role_ctx") + + def __init__(self) -> None: + self.parent: Node | None = None + self.children: list[Node] = [] + self.finding_ids: list[str] = [] + self.bounds = None + self.printable = None + self.has_non_part = False + self.mesh_key = None + self.role_ctx = None + + +class TopContext: + """What every node under one build item inherits.""" + + def __init__(self, ordinal: int, build_index, plate: dict, placement_known: bool, repeated: bool): + self.ordinal, self.build_index, self.plate = ordinal, build_index, plate + self.placement_known, self.repeated = placement_known, repeated + self.node: Node | None = None + self.top_part: str | None = None + + +class MeshInfo: + """Volumes and printable-geometry facts for one emitted mesh.""" + + def __init__(self, key: tuple[str, str], data: MeshData): + self.key, self.data = key, data + self.volumes: list[dict] = [] + self.part_verts: list[int] = [] + self.lo = self.hi = None + self.has_part = False + self.has_non_part = False + self.has_unknown = False + + +class Traversal: + def __init__(self, files_by_part: dict[str, ModelFile], settings: SettingsInfo, root_part: str, ctl: Control): + self.files, self.settings, self.root_part, self.ctl = files_by_part, settings, root_part, ctl + self.nodes: list[Node] = [] + self.rendered = 0 + self.mesh_order: list[tuple[str, str]] = [] + self._ids: set[str] = set() + self.mesh_users: dict[tuple[str, str], list[str]] = {} + + def add_tree(self, key: tuple[str, str], transform, ctx: TopContext) -> None: + ctx.top_part = key[0] + self._expand(key, transform, None, [], ctx, ()) + + def _context_role(self, parent: Node | None, via: str | None, inherited: str | None, + ctx: "TopContext | None" = None) -> str | None: + """The role this USE imposes (Bambu/Orca ````), following the traversal context. + + A part record belongs to one (object, component) pair, so a mesh shared by two objects can be a + negative part under one and an ordinary part under the other, and a role given to a component + ASSEMBLY applies to everything beneath it (``inherited`` wins unless it is a plain part). An object + that HAS part records but none for this component gives ``unknown``, never a printable part. + """ + if ctx is not None and self.settings.dialect is not None and ctx.top_part != self.root_part: + # Part records are matched against ROOT-file objects. A build item whose object lives in another + # file (p:path) cannot be associated provably with any record, so every part beneath it is + # unknown (fail closed): never printable, never judged against the bed. + return "unknown" + own = None + if parent is not None and via is not None and self.settings.dialect == "bambu" and parent.resource[0] == self.root_part: + subtypes = self.settings.part_subtypes.get(parent.resource[1]) + if parent.resource[1] in self.settings.metadata_objects and subtypes is not None: + from .assignments import role_of + own = role_of(subtypes[via]) if via in subtypes else "unknown" + if inherited not in (None, "part"): + return inherited + return own if own is not None else inherited + + def _expand(self, key, local, parent: Node | None, path: list[int], ctx: TopContext, + stack: tuple, via: str | None = None, inherited: str | None = None) -> Node: + self.ctl.check() + mf = self.files.get(key[0]) + obj = mf.objects.get(key[1]) if mf else None + if obj is None: + raise SceneError("UNRESOLVED_REFERENCE", "The project refers to an object that is not defined.") + if key in stack: + raise SceneError("INVALID_ARCHIVE", "The project's components refer to themselves.") + if len(path) >= L.MAX_DEPTH: + raise SceneError("LIMIT_EXCEEDED", "The project nests objects deeper than Studio will follow.") + if len(self.nodes) >= L.MAX_NODES: + raise SceneError("LIMIT_EXCEEDED", "The project has more objects than Studio will draw.") + node = Node() + node.parent = parent + node.resource = key + node.path = list(path) + node.local = local if local is not None else IDENTITY16 + node.world = node.local if parent is None else mat_mul(parent.world, node.local) + if not all(math.isfinite(x) for x in node.world): + raise SceneError("INVALID_GEOMETRY", "A combined transform is not finite.") + node.mirrored = mat_det3(node.world) < 0 + node.id = self._node_id(ctx, parent, path) + if node.id in self._ids: # digested ids give 128-bit practical uniqueness; a collision is detected + raise SceneError("INTERNAL", "Two nodes would share an id.") + self._ids.add(node.id) + node.role_ctx = self._context_role(parent, via, inherited, ctx) + node.build_index, node.plate = ctx.build_index, ctx.plate + node.placement_known, node.repeated, node.top = ctx.placement_known, ctx.repeated, ctx + self.nodes.append(node) + if parent is None: + ctx.node = node + else: + parent.children.append(node) + if obj.mesh is not None: + node.mesh_key = key + self.rendered += obj.mesh.ntris + if self.rendered > L.MAX_RENDERED_TRIANGLES: + raise SceneError("LIMIT_EXCEEDED", f"The project has at least {self.rendered:,} triangles counting repeated objects; Studio draws at most {L.MAX_RENDERED_TRIANGLES:,}.") + if key not in self.mesh_users: + self.mesh_users[key] = [] + self.mesh_order.append(key) + self.mesh_users[key].append(node.id) + elif obj.components: + for i, comp in enumerate(obj.components): + self._expand((comp.part, comp.object_id), comp.transform, node, path + [i], ctx, + stack + (key,), via=comp.object_id, inherited=node.role_ctx) + else: + raise SceneError("INVALID_GEOMETRY", "An object has neither a mesh nor components.") + return node + + @staticmethod + def _node_id(ctx: TopContext, parent: Node | None, path: list[int]) -> str: + """``b`` plus ``.c`` per component level. When that would not fit in + ``MAX_ID_LENGTH`` the ancestry is replaced by a deterministic digest of it (``b3.~<32 hex>``), + which is unique per ancestry, so 128 characters always suffice at the deepest allowed nesting.""" + natural = f"b{ctx.ordinal}" + "".join(f".c{i}" for i in path) + if len(natural) <= L.MAX_ID_LENGTH: + return natural + return f"b{ctx.ordinal}.~" + hashlib.sha256(natural.encode("ascii")).hexdigest()[:32] + + # ---- volumes ------------------------------------------------------------------------- + def mesh_data(self, key) -> MeshData: + return self.files[key[0]].objects[key[1]].mesh + + def build_mesh_info(self, key) -> MeshInfo: + data = self.mesh_data(key) + info = MeshInfo(key, data) + total = data.ntris + if (self.settings.dialect == "prusa" and key[0] == self.root_part + and key[1] in self.settings.prusa_volumes): + info.volumes = self._prusa_ranges(self.settings.prusa_volumes[key[1]], total) + else: + info.volumes = [{"id": "v0", "triangle_start": 0, "triangle_count": total, "role": "part", "source": "object"}] + if len(info.volumes) > L.MAX_VOLUMES_PER_MESH: + raise SceneError("LIMIT_EXCEEDED", "A mesh has more volumes than Studio will carry.") + used: set[int] = set() + for vol in info.volumes: + if vol["role"] == "part": + info.has_part = True + s, c = vol["triangle_start"], vol["triangle_count"] + used.update(data.idx[3 * s:3 * (s + c)]) + else: + info.has_non_part = True + if vol["role"] == "unknown": + info.has_unknown = True + if used: + pos = data.pos + lo = [math.inf] * 3 + hi = [-math.inf] * 3 + info.part_verts = sorted(used) + for n, vi in enumerate(info.part_verts): + if n % L.VERTEX_BATCH == 0: + self.ctl.check() + b = 3 * vi + for ax in range(3): + v = pos[b + ax] + if v < lo[ax]: + lo[ax] = v + if v > hi[ax]: + hi[ax] = v + info.lo, info.hi = lo, hi + return info + + @staticmethod + def _prusa_ranges(entries: list[dict], total: int) -> list[dict]: + """Partition ALL triangles into volumes. An unmapped gap is an ``unknown`` volume; ranges that + overlap or leave the mesh make the whole mesh one ``unknown`` volume (never a guess).""" + whole = [{"id": "v0", "triangle_start": 0, "triangle_count": total, "role": "unknown", "source": "prusa_range"}] + if any(e["range"] is None for e in entries): + return whole # a range Studio cannot read: never a printable guess + spans = sorted((e["range"][0], e["range"][1], e["role"]) for e in entries) + out: list[dict] = [] + cursor = 0 + for first, last, role in spans: + if first < cursor or last < first or last >= total: + return whole + if first > cursor: + out.append({"id": f"v{len(out)}", "triangle_start": cursor, "triangle_count": first - cursor, + "role": "unknown", "source": "prusa_range"}) + out.append({"id": f"v{len(out)}", "triangle_start": first, "triangle_count": last - first + 1, + "role": role, "source": "prusa_range"}) + cursor = last + 1 + if cursor < total: + out.append({"id": f"v{len(out)}", "triangle_start": cursor, "triangle_count": total - cursor, + "role": "unknown", "source": "prusa_range"}) + return out or whole + + +def _world_bounds(info: MeshInfo, world: list[float], ctl: Control): + """Exact world-space bounds of the printable (role=part) vertices of one instance.""" + if not info.part_verts: + return None + m = world + # Rows with at most one non-zero entry map each output axis from a single input axis, so the + # transformed local box is exact (no rotation). Anything else transforms the vertices. + rows = [] + simple = True + for i in range(3): + nz = [(j, m[j * 4 + i]) for j in range(3) if abs(m[j * 4 + i]) > 1e-12] + if len(nz) > 1: + simple = False + break + rows.append(nz[0] if nz else None) + if simple: + lo_out = [0.0] * 3 + hi_out = [0.0] * 3 + for i in range(3): + t = m[12 + i] + if rows[i] is None: + lo_out[i] = hi_out[i] = t + else: + j, c = rows[i] + a, b = c * info.lo[j] + t, c * info.hi[j] + t + lo_out[i], hi_out[i] = (a, b) if a <= b else (b, a) + return lo_out, hi_out + pos = info.data.pos + lo_out = [math.inf] * 3 + hi_out = [-math.inf] * 3 + m0, m1, m2, m4, m5, m6, m8, m9, m10, t0, t1, t2 = (m[0], m[1], m[2], m[4], m[5], m[6], m[8], m[9], m[10], m[12], m[13], m[14]) + for n, vi in enumerate(info.part_verts): + if n % L.VERTEX_BATCH == 0: + ctl.check() + b = 3 * vi + x, y, z = pos[b], pos[b + 1], pos[b + 2] + px = m0 * x + m4 * y + m8 * z + t0 + py = m1 * x + m5 * y + m9 * z + t1 + pz = m2 * x + m6 * y + m10 * z + t2 + if px < lo_out[0]: lo_out[0] = px + if px > hi_out[0]: hi_out[0] = px + if py < lo_out[1]: lo_out[1] = py + if py > hi_out[1]: hi_out[1] = py + if pz < lo_out[2]: lo_out[2] = pz + if pz > hi_out[2]: hi_out[2] = pz + return lo_out, hi_out + + +# -------------------------------------------------------------------------------------------- +# Plates +# -------------------------------------------------------------------------------------------- + +def _plate_assignment(settings: SettingsInfo, oid: str, ordinal: int, instances_of_object: int) -> tuple[dict, bool]: + """(plate ref, ambiguous?) for the ``ordinal``-th build item of object ``oid``.""" + if not settings.plates: + return {"state": "unknown", "plate_id": None, "source": None}, False + found: set[str] = set() + for idx, plate in enumerate(settings.plates): + pid = f"p{plate['ui_number']}" if plate["ui_number"] is not None else f"plate-{idx}" + for moid, inst in plate["members"]: + if moid != oid: + continue + if inst is None: + # a record that does not say which instance covers all of them only if there is one + if instances_of_object == 1: + found.add(pid) + else: + return {"state": "ambiguous", "plate_id": None, "source": "plate_config"}, True + elif inst.isascii() and inst.isdigit() and int(inst) == ordinal: + found.add(pid) + if len(found) == 1: + return {"state": "known", "plate_id": next(iter(found)), "source": "plate_config"}, False + if len(found) > 1: + return {"state": "ambiguous", "plate_id": None, "source": "plate_config"}, True + return {"state": "unknown", "plate_id": None, "source": None}, False + + +# -------------------------------------------------------------------------------------------- +# Assembly and encoding +# -------------------------------------------------------------------------------------------- + +class _Limitations: + ORDER = ("UNKNOWN_VOLUME_ROLE", "UNKNOWN_PLACEMENT", "PLATE_MEMBERSHIP_AMBIGUOUS", "PLATE_MEMBERSHIP_UNKNOWN", + "PLATES_UNAVAILABLE", "BED_TEMPLATE_UNAVAILABLE", "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED", + "NON_MM_SOURCE_UNIT", "UNSUPPORTED_UNIT", "MULTI_PLATE_PLACEMENT_UNCHECKED", "NO_BUILD_ITEMS") + + def __init__(self) -> None: + self._by: dict[str, list[str]] = {} + + def has(self, code: str) -> bool: + return code in self._by + + def add(self, code: str, target_ids=()) -> None: + bucket = self._by.setdefault(code, []) + for t in target_ids: + if t not in bucket: + bucket.append(t) + + def render(self) -> list[dict]: + out = [{"code": c, "target_ids": self._by[c][:2000]} for c in self.ORDER if c in self._by] + if len(out) > L.MAX_LIMITATIONS: + raise SceneError("LIMIT_EXCEEDED", "The scene has more limitations than Studio will carry.") + return out + + +def _b64_len(n: int) -> int: + return 4 * ((n + 2) // 3) + + +def _encode_f32(pos: array, ctl: Control) -> str: + step = 3 * L.GEOMETRY_BATCH_TRIANGLES + out = [] + for s in range(0, len(pos), step): + ctl.check() + chunk = array("f", pos[s:s + step]) + if sys.byteorder == "big": + chunk.byteswap() + out.append(base64.b64encode(chunk.tobytes()).decode("ascii")) + return "".join(out) + + +def _encode_u32(idx: array, ctl: Control) -> str: + assert idx.itemsize == 4 + step = 3 * L.GEOMETRY_BATCH_TRIANGLES + out = [] + for s in range(0, len(idx), step): + ctl.check() + chunk = idx[s:s + step] + if sys.byteorder == "big": + chunk = array("I", chunk) + chunk.byteswap() + out.append(base64.b64encode(chunk.tobytes()).decode("ascii")) + return "".join(out) + + +def _rounded(v) -> list[float]: + return [round(float(x), 6) + 0.0 for x in v] + + +def _finish(revision: str, sources: list[dict], bed: dict, rect: dict, trav: Traversal, tops: list[TopContext], + plates: list[dict], plate_count: int, lims: _Limitations, ctl: Control, *, + repeated_tops: list[str]) -> dict: + from . import plate_placement as pp + + ctl.progress("encoding", 0, trav.rendered) + infos = {key: trav.build_mesh_info(key) for key in trav.mesh_order} + + # per-node printable facts, children before parents + for node in reversed(trav.nodes): + if node.mesh_key is not None: + info = infos[node.mesh_key] + if node.role_ctx not in (None, "part"): + # this USE is a modifier / negative part / support helper (or unknown): nothing prints + node.printable = None if node.role_ctx == "unknown" else False + node.has_non_part = True + node.bounds = None + else: + node.printable = True if info.has_part else (None if info.has_unknown else False) + node.has_non_part = info.has_non_part + node.bounds = _world_bounds(info, node.world, ctl) if info.has_part else None + else: + kids = node.children + node.has_non_part = any(k.has_non_part for k in kids) + node.printable = True if any(k.printable is True for k in kids) else ( + None if any(k.printable is None for k in kids) else False) + boxes = [k.bounds for k in kids if k.bounds is not None] + node.bounds = ([min(b[0][i] for b in boxes) for i in range(3)], + [max(b[1][i] for b in boxes) for i in range(3)]) if boxes else None + for node in trav.nodes: + if node.mesh_key is not None and node.role_ctx == "unknown": + lims.add("UNKNOWN_VOLUME_ROLE", [node.id]) + elif node.mesh_key is not None and node.role_ctx in (None, "part") and infos[node.mesh_key].has_unknown: + lims.add("UNKNOWN_VOLUME_ROLE", [node.id]) + + # findings -- only what the geometry plus the file prove + findings: list[dict] = [] + all_mm = all(s["unit"] == "millimeter" for s in sources) and not lims.has("UNSUPPORTED_UNIT") + if not all(s["unit"] == "millimeter" for s in sources): + lims.add("NON_MM_SOURCE_UNIT") + if plate_count > 1: + lims.add("MULTI_PLATE_PLACEMENT_UNCHECKED") + usable_x = rect["max_x"] - rect["min_x"] - 2 * pp.EDGE_MARGIN_MM + usable_y = rect["max_y"] - rect["min_y"] - 2 * pp.EDGE_MARGIN_MM + enabled = bed["policy"] == "u1_template" and plate_count <= 1 + all_mm_or_known = not lims.has("UNSUPPORTED_UNIT") + index_of = {id(n): i for i, n in enumerate(trav.nodes)} + for ctx in tops: + node = ctx.node + if node is None or not enabled or node.bounds is None or node.printable is not True: + continue + if plate_count > 0 and node.plate["state"] != "known": + continue # the project has plates but this instance is provably on none / on two + if ctx.repeated or not ctx.placement_known: + # REPEATED_INSTANCE_PLACEMENT_UNVERIFIED / unknown placement: geometry stays visible, + # but no bed-fit statement of any kind is attached. + continue + lo, hi = node.bounds + pointer = f"/nodes/{index_of[id(node)]}" + if all_mm: + over = pp._overhang({"min": lo, "max": hi}, rect) + if any(v > 0 for v in over.values()): + fid = f"f{len(findings)}" + findings.append({"id": fid, "engine": "scene", "schema": SCHEMA_ID, "pointer": pointer, + "scope": "instance", "kind": "placement", "target_ids": [node.id], + "value": {"code": "PLACEMENT_OUTSIDE_BED", "overhang_mm": over}}) + node.finding_ids.append(fid) + if all_mm_or_known and ((hi[0] - lo[0]) > usable_x + 1e-9 or (hi[1] - lo[1]) > usable_y + 1e-9): + fid = f"f{len(findings)}" + findings.append({"id": fid, "engine": "scene", "schema": SCHEMA_ID, "pointer": pointer, + "scope": "instance", "kind": "size", "target_ids": [node.id], + "value": {"code": "SIZE_EXCEEDS_BED", "overhang_mm": None}}) + node.finding_ids.append(fid) + if len(findings) > L.MAX_FINDINGS: + raise SceneError("LIMIT_EXCEEDED", "The scene has more findings than Studio will carry.") + if repeated_tops: + lims.add("REPEATED_INSTANCE_PLACEMENT_UNVERIFIED", repeated_tops) + + # response preflight: the geometry alone decides this exactly, before any encoding work + geom = 0 + for info in infos.values(): + geom += _b64_len(info.data.nverts * 12) + _b64_len(info.data.ntris * 12) + if geom > L.MAX_RESPONSE_BYTES: + raise SceneError("LIMIT_EXCEEDED", "The geometry alone is larger than the response Studio will send.") + + meshes = [] + done = 0 + for key in trav.mesh_order: + info = infos[key] + d = info.data + meshes.append({ + "key": {"part": key[0], "object_id": key[1]}, + "vertex_count": d.nverts, "triangle_count": d.ntris, + "positions_f32le_base64": _encode_f32(d.pos, ctl), + "indices_u32le_base64": _encode_u32(d.idx, ctl), + "volumes": info.volumes}) + done += d.ntris + ctl.progress("encoding", done, trav.rendered) + + nodes = [] + for node in trav.nodes: + nodes.append({ + "id": node.id, + "parent_id": node.parent.id if node.parent else None, + "resource": {"part": node.resource[0], "object_id": node.resource[1]}, + "mesh_key": ({"part": node.mesh_key[0], "object_id": node.mesh_key[1]} if node.mesh_key else None), + "local_to_parent_mm": _round_matrix(node.local), + "world_mm": _round_matrix(node.world), + "mirrored": node.mirrored, + "role_context": node.role_ctx, + "build_index": node.build_index, + "instance_ref": {"build_index": node.build_index, "component_path": node.path}, + "plate": node.plate, + "placement_state": "known" if node.placement_known else "unknown", + "printable": node.printable, + "has_non_part_volumes": node.has_non_part, + "bounds_mm": ({"min": _rounded(node.bounds[0]), "max": _rounded(node.bounds[1])} if node.bounds else None), + "finding_ids": node.finding_ids}) + limitations = lims.render() + status = "partial" if any(x["code"] in PARTIAL_LIMITATIONS for x in limitations) else "complete" + return { + "schema": SCHEMA_ID, "revision": revision, "status": status, "units": "mm", "axes": "right-handed-z-up", + "sources": sources, "bed": bed, "meshes": meshes, "nodes": nodes, "plates": plates, + "findings": findings, "limitations": limitations, + "counts": {"nodes": len(nodes), "meshes": len(meshes), + "vertices": sum(i.data.nverts for i in infos.values()), + "triangles": sum(i.data.ntris for i in infos.values()), + "rendered_triangles": trav.rendered, "plates": len(plates), + "findings": len(findings), "limitations": len(limitations)}, + "limits": dict(L.LIMITS_ECHO)} + + +def serialize(scene: dict) -> bytes: + """The whole response body, once, compact ASCII JSON, with the cap enforced AS IT IS BUILT. + + Each top-level member (and each element of a list member) is serialized separately and the running + length is checked before it is kept, so an over-cap scene is refused after at most one element past + the cap: the full body (or a UTF-8 copy of it) is never built first. + """ + cap = L.MAX_RESPONSE_BYTES + + def dump(value) -> str: + try: + return json.dumps(value, separators=(",", ":"), allow_nan=False) + except ValueError as exc: + raise SceneError("INVALID_GEOMETRY", "The scene contains a value that is not finite.") from exc + + def over() -> SceneError: + return SceneError("LIMIT_EXCEEDED", "The scene is larger than the response Studio will send.") + + total = 2 # the braces; every other byte is added exactly, before it is kept + members = [] + for key, value in scene.items(): + head = dump(key) + ":" + total += len(head) + (1 if members else 0) # the comma between members + if total > cap: + raise over() + if isinstance(value, list) and value and isinstance(value[0], dict): + items = [] + total += 2 # [ ] + for item in value: + text = dump(item) + total += len(text) + (1 if items else 0) + if total > cap: + raise over() + items.append(text) + members.append(head + "[" + ",".join(items) + "]") + else: + text = dump(value) + total += len(text) + if total > cap: + raise over() + members.append(head + text) + return ("{" + ",".join(members) + "}").encode("ascii") + + +def _sha256_file(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(L.SNAPSHOT_CHUNK_BYTES), b""): + h.update(chunk) + return h.hexdigest() + + +# -------------------------------------------------------------------------------------------- +# Front ends +# -------------------------------------------------------------------------------------------- + +def _build_3mf(path: str, revision: str, ctl: Control) -> dict: + lims = _Limitations() + bed, rect = _bed([]) + if bed["policy"] == "fallback": + lims.add("BED_TEMPLATE_UNAVAILABLE") + arc = Archive(path, ctl) + try: + ctl.progress("parsing", 0, None) + root_part = find_root_model(arc) + budget = Budget() + files_by_part: dict[str, ModelFile] = {} + queue = [root_part] + while queue: + part = queue.pop(0) + if part in files_by_part: + continue + if len(files_by_part) >= L.MAX_MODEL_PARTS: + raise SceneError("LIMIT_EXCEEDED", "The project uses more model parts than Studio will read.") + if not arc.has(part): + raise SceneError("UNRESOLVED_REFERENCE", "The project refers to a model part that is not in the archive.") + canon = arc.index[part.casefold()][0] + mf = parse_model(arc, canon, budget, ctl, canon == root_part) + files_by_part[canon] = mf + ctl.progress("parsing", arc.model_read, arc.model_declared) + for item in mf.build: # a build item's p:path names the FILE its object lives in + if not arc.has(item.part): + raise SceneError("UNRESOLVED_REFERENCE", "A build item refers to a model part that is not in the archive.") + item.part = arc.index[item.part.casefold()][0] + if item.part not in files_by_part: + queue.append(item.part) + for obj in mf.objects.values(): + for comp in obj.components: + if not arc.has(comp.part): + raise SceneError("UNRESOLVED_REFERENCE", "A component refers to a model part that is not in the archive.") + comp.part = arc.index[comp.part.casefold()][0] + if comp.part not in files_by_part: + queue.append(comp.part) + settings = parse_settings(arc, ctl) + root = files_by_part[root_part] + bad_unit_parts = {mf.part for mf in files_by_part.values() if not mf.unit_recognized} + sources = [{"part": mf.part, "unit": mf.unit, "mm_per_unit": mf.mm_per_unit} + for mf in files_by_part.values()] + plates = [] + seen_ui = set() + for idx, plate in enumerate(settings.plates): + ui = plate["ui_number"] + if ui is not None and ui in seen_ui: + continue + seen_ui.add(ui) + plates.append({"id": f"p{ui}" if ui is not None else f"plate-{idx}", "ui_number": ui, "origin_mm": None}) + trav = Traversal(files_by_part, settings, root_part, ctl) + counts: dict[tuple[str, str], int] = {} + for item in root.build: + counts[(item.part, item.object_id)] = counts.get((item.part, item.object_id), 0) + 1 + part_by_bare: dict[str, set[str]] = {} + for item in root.build: + part_by_bare.setdefault(item.object_id, set()).add(item.part) + tops: list[TopContext] = [] + seen_n: dict[tuple[str, str], int] = {} + repeated_tops: list[str] = [] + ambiguous_ids: list[str] = [] + cross_file_ambiguous: list = [] + unknown_plate_ids: list[str] = [] + if root.build: + for bi, item in enumerate(root.build): + ikey = (item.part, item.object_id) + n = seen_n.get(ikey, 0) + seen_n[ikey] = n + 1 + plate, ambiguous = _plate_assignment(settings, item.object_id, n, counts[ikey]) + if len(part_by_bare[item.object_id]) > 1 and settings.plates: + # plate records name a BARE object id: when two build resources in different files share + # it, a record cannot prove membership for either of them + plate, ambiguous = {"state": "ambiguous", "plate_id": None, "source": "plate_config"}, False + cross_file_ambiguous.append(None) + ctx = TopContext(bi, bi, plate, True, counts[ikey] > 1) + trav.add_tree(ikey, item.transform, ctx) + tops.append(ctx) + if ctx.repeated: + repeated_tops.append(ctx.node.id) + if ambiguous: + ambiguous_ids.append(ctx.node.id) + elif plate["state"] == "unknown" and settings.plates: + unknown_plate_ids.append(ctx.node.id) + elif cross_file_ambiguous and plate["state"] == "ambiguous" and ctx.node.id not in unknown_plate_ids: + unknown_plate_ids.append(ctx.node.id) + else: + lims.add("NO_BUILD_ITEMS") + for ordinal, oid in enumerate(root.objects): + ctx = TopContext(ordinal, None, {"state": "unknown", "plate_id": None, "source": None}, False, False) + trav.add_tree((root_part, oid), None, ctx) + tops.append(ctx) + lims.add("UNKNOWN_PLACEMENT", [ctx.node.id]) + if bad_unit_parts: + # target_ids name the NODES that live in the affected part(s); for such a source `unit` and + # `mm_per_unit` in sources[] are the fallback millimetre values (the limitation says so) + # a build item's transform is written in the ROOT model's unit, so when the root is the bad part the + # top node of every item is affected too, even if its mesh lives in another file + lims.add("UNSUPPORTED_UNIT", [n.id for n in trav.nodes + if n.resource[0] in bad_unit_parts or (n.parent is None and root_part in bad_unit_parts)]) + if not settings.plates: + lims.add("PLATES_UNAVAILABLE") + if ambiguous_ids: + lims.add("PLATE_MEMBERSHIP_AMBIGUOUS", ambiguous_ids) + if unknown_plate_ids: + lims.add("PLATE_MEMBERSHIP_UNKNOWN", unknown_plate_ids) + return _finish(revision, sources, bed, rect, trav, tops, plates, len(plates), lims, ctl, + repeated_tops=repeated_tops) + finally: + arc.close() + + +def _build_stl(path: str, revision: str, ctl: Control) -> dict: + import os + import struct + + from .stl_io import StlTooLarge, decode_stl_stream + + lims = _Limitations() + bed, rect = _bed([]) + if bed["policy"] == "fallback": + lims.add("BED_TEMPLATE_UNAVAILABLE") + size = os.path.getsize(path) + ctl.progress("parsing", 0, size) + try: + with open(path, "rb") as fh: + # bounded and incremental: the binary header count is judged BEFORE any triangle is decoded, + # decoding runs in batches with the cancel/deadline hook between them, ASCII is line-streamed + verts, tris = decode_stl_stream(fh, size, max_triangles=L.MAX_PARSED_TRIANGLES, + max_vertices=L.MAX_VERTICES, check=ctl.check, + batch=L.GEOMETRY_BATCH_TRIANGLES) + except StlTooLarge as exc: + raise SceneError("LIMIT_EXCEEDED", f"The model has {exc.triangles:,} {exc.what}; Studio draws at most {exc.limit:,}.") from exc + except (SnapStudioError, ValueError, struct.error) as exc: + raise SceneError("INVALID_GEOMETRY", "This STL file could not be read.") from exc + mesh = MeshData() + for n, v in enumerate(verts): + if n % L.VERTEX_BATCH == 0: + ctl.check() + if not all(math.isfinite(c) and abs(c) <= L.MAX_COORDINATE_MM for c in v): + raise SceneError("INVALID_GEOMETRY", "A vertex coordinate is not usable.") + mesh.pos.extend(v) + for n, t in enumerate(tris): + if n % L.VERTEX_BATCH == 0: + ctl.check() + mesh.idx.extend(t) + mesh.nverts, mesh.ntris, mesh.max_index = len(verts), len(tris), len(verts) - 1 + part = "model.stl" + mf = ModelFile(part) + obj = ObjectDef("0") + obj.mesh = mesh + mf.objects["0"] = obj + trav = Traversal({part: mf}, SettingsInfo(), part, ctl) + ctx = TopContext(0, None, {"state": "unknown", "plate_id": None, "source": None}, False, False) + trav.add_tree((part, "0"), None, ctx) + lims.add("UNKNOWN_PLACEMENT", [ctx.node.id]) + lims.add("PLATES_UNAVAILABLE") + return _finish(revision, [{"part": part, "unit": "millimeter", "mm_per_unit": 1.0}], bed, rect, trav, [ctx], [], 0, + lims, ctl, repeated_tops=[]) + + +def build_scene_dict(path: str, revision: str | None = None, ctl: Control | None = None) -> dict: + """Build the scene/1 object for ``path`` (a private snapshot, when called by a job).""" + ctl = ctl or Control() + low = str(path).lower() + if not (low.endswith(".3mf") or low.endswith(".stl")): + raise SceneError("UNSUPPORTED_FORMAT", "Only .3mf and .stl files can be shown.") + revision = revision or _sha256_file(path) + if low.endswith(".stl"): + return _build_stl(path, revision, ctl) + return _build_3mf(path, revision, ctl) + + +def build_scene(path: str, revision: str | None = None, ctl: Control | None = None) -> bytes: + """The serialized scene (compact UTF-8 JSON), with the whole-body cap enforced.""" + return serialize(build_scene_dict(path, revision, ctl)) diff --git a/backend/snapstudio_core/scene_limits.py b/backend/snapstudio_core/scene_limits.py new file mode 100644 index 00000000..c46b02a5 --- /dev/null +++ b/backend/snapstudio_core/scene_limits.py @@ -0,0 +1,66 @@ +"""Every numeric budget, cap, deadline and TTL behind the scene/1 contract, in one place. + +Nothing else in the scene code carries a literal limit. These are PROVISIONAL budgets (see +docs/testing/scene-contract/README.md for the corpus they were measured against). Each one is +enforced while the work happens, never after, and exceeding one is a clear LIMIT_EXCEEDED, never +a silently shortened scene. +""" +from __future__ import annotations + +MIB = 1024 * 1024 + +# --- what the engine will copy and read ------------------------------------- +SNAPSHOT_MAX_BYTES = 128 * MIB # hard ceiling on the private copy of the source file +SNAPSHOT_CHUNK_BYTES = 1 * MIB # streamed copy / archive read unit (bounded uninterruptible work) +DISK_HEADROOM_BYTES = 64 * MIB # free space required beyond the source size +ARCHIVE_EXPANSION_BYTES = 128 * MIB # total decompressed bytes the reader will pull from one archive +MODEL_XML_BYTES = 32 * MIB # decompressed bytes of all .model parts together +CONFIG_PART_BYTES = 8 * MIB # one metadata/config part (model_settings, Slic3r_PE_model, .rels) +ARCHIVE_MAX_ENTRIES = 20_000 # zip directory entries +MAX_MODEL_PARTS = 256 # distinct .model files parsed for one scene +MAX_RELS_PARTS = 64 # .rels parts examined + +# --- what the scene may contain --------------------------------------------- +MAX_RENDERED_TRIANGLES = 250_000 # counting every repetition of an instanced mesh +MAX_PARSED_TRIANGLES = 250_000 # distinct triangles accepted while parsing +MAX_VERTICES = 300_000 # distinct decoded vertices +MAX_NODES = 2_000 +MAX_OBJECT_DEFINITIONS = 20_000 # elements across all model parts (10x the node budget) +MAX_XML_ELEMENTS = 1_000_000 # every completed XML element, counted while parsing +MAX_PART_NAME_LENGTH = 256 # archive part names carried on nodes/meshes/sources (schema maxLength) +MAX_DEPTH = 64 +MAX_ID_LENGTH = 128 +MAX_FINDINGS = 500 +MAX_LIMITATIONS = 50 +MAX_VOLUMES_PER_MESH = 4_096 +MAX_RESPONSE_BYTES = 8 * MIB # the whole serialized body +MAX_COORDINATE_MM = 1.0e7 # keeps every coordinate representable as float32 + +# --- cooperative interruption (v4 item 2) ----------------------------------- +XML_EVENTS_PER_CHECK = 1_000 # lxml iterparse events between cancel/deadline checks +GEOMETRY_BATCH_TRIANGLES = 5_000 # decode/encode batch; 5_000 * 12 bytes is a multiple of 3 (base64-safe) +VERTEX_BATCH = 15_000 # transformed vertices between checks + +# --- jobs ------------------------------------------------------------------- +JOB_DEADLINE_SECONDS = 60.0 # wall clock from the moment the job becomes running +WEDGE_GRACE_SECONDS = 5.0 # worker still alive this long after cancel/TIMEOUT => WORKER_WEDGED +RESULT_TTL_SECONDS = 120.0 # a terminal job is kept this long +MAX_TERMINAL_JOBS = 8 # ... or until more than this many terminal jobs exist +MAX_REQUEST_ID_LENGTH = 64 +MAX_SESSIONS = 32 # open scene sessions (an idle-expired one that owns no job is pruned first) +SESSION_TTL_SECONDS = 15 * 60 # idle time after which a session without a registered job is pruned +SNAPSHOT_DIR_MIN_AGE_SECONDS = 600 # a dead engine's scene-tmp folder is swept only once it is this old (a live engine's NEVER is) +MAX_PENDING_UNLINKS = 1024 # snapshot files that could not be deleted yet and are retried with back-off + +LIMITS_ECHO = { + "max_archive_bytes": ARCHIVE_EXPANSION_BYTES, + "max_model_xml_bytes": MODEL_XML_BYTES, + "max_rendered_triangles": MAX_RENDERED_TRIANGLES, + "max_vertices": MAX_VERTICES, + "max_nodes": MAX_NODES, + "max_depth": MAX_DEPTH, + "max_response_bytes": MAX_RESPONSE_BYTES, + "max_findings": MAX_FINDINGS, + "max_limitations": MAX_LIMITATIONS, + "max_id_length": MAX_ID_LENGTH, +} diff --git a/backend/snapstudio_core/stl_io.py b/backend/snapstudio_core/stl_io.py index 06928fc5..579e08a7 100644 --- a/backend/snapstudio_core/stl_io.py +++ b/backend/snapstudio_core/stl_io.py @@ -1,4 +1,5 @@ from __future__ import annotations +import re import struct from .errors import SnapStudioError @@ -46,3 +47,132 @@ def parse_stl(data: bytes): if not tris: raise SnapStudioError("STL contained no triangles") return verts, tris + + +# --- bounded, incremental decoder (additive; parse_stl above is unchanged) ----------------------------- + +class StlTooLarge(SnapStudioError): + """The file declares or contains more triangles (or vertices) than the caller allows.""" + + def __init__(self, triangles: int, limit: int, what: str = "triangles") -> None: + self.triangles, self.limit, self.what = triangles, limit, what + super().__init__(f"{triangles} {what}; limit {limit}") + + +_BATCH = 5000 + + +def _decode_binary_batch(raw: bytes, vmap: dict, verts: list, tris: list) -> None: + for rec in struct.iter_unpack("<12fH", raw): + tris.append((_add(vmap, verts, rec[3:6]), _add(vmap, verts, rec[6:9]), _add(vmap, verts, rec[9:12]))) + + +_CHUNK = 1024 * 1024 +_MAX_LINE = 4096 +_LINE_SPLIT = re.compile(rb"\r\n|\n|\r") # \n, \r\n and a lone \r, like parse_stl's splitlines + + +def _plain_text(head: bytes) -> bool: + """Plausibly TEXT: no NUL in the first KB and valid UTF-8 (a multi-byte character cut by the end of the + window is fine). A name such as ``solid Würfel`` is text; random binary is not.""" + first = head[:1024] + if b"\x00" in first: + return False + try: + first.decode("utf-8") + except UnicodeDecodeError as exc: + return exc.reason == "unexpected end of data" and exc.end >= len(first) + return True + + +def _looks_like_ascii_stl(head: bytes) -> bool: + """A "solid"/"facet" start on a file that is plausibly TEXT.""" + if not _plain_text(head): + return False + low = head.lstrip().lower() + return low.startswith(b"solid") or b"facet" in head.lower() + + +def decode_stl_stream(fh, size: int, *, max_triangles: int, max_vertices: int, check=lambda: None, + batch: int = _BATCH): + """Decode an STL from an open binary file without ever holding more than the caller's budget. + + The kind is decided from the 84-byte header AND the file size: a size equal to ``84 + 50 * N`` is + binary; so is a file whose first KB is not plausible text, and for those a declared ``N`` over + ``max_triangles`` is rejected IMMEDIATELY, before any triangle (or any large buffer) exists, whether or + not the size matches. Only plausible ASCII goes down the text path, which reads <= 1 MiB at a time, + refuses any line over 4 KiB, splits on LF, CRLF and a lone CR, keeps a running triangle counter and + calls ``check()`` (the cancel/deadline hook) per chunk. Binary decoding runs in batches of ``batch`` + triangles with ``check()`` between them. Vertices are de-duplicated exactly as ``parse_stl`` does. + Raises ``StlTooLarge`` or ``SnapStudioError``. + """ + head = fh.read(84) + verts: list[tuple] = [] + tris: list[tuple] = [] + vmap: dict = {} + if len(head) >= 84: + (n,) = struct.unpack_from(" max_triangles: + raise StlTooLarge(n, max_triangles) + if not consistent: + raise SnapStudioError("STL size does not match its header") + remaining = n + while remaining: + check() + take = min(batch, remaining) + raw = fh.read(take * 50) + if len(raw) != take * 50: + raise SnapStudioError("STL ended early") + _decode_binary_batch(raw, vmap, verts, tris) + if len(verts) > max_vertices: + raise StlTooLarge(len(verts), max_vertices, "vertices") + remaining -= take + if not tris: + raise SnapStudioError("STL contained no triangles") + return verts, tris + fh.seek(0) + head_for_text = first + else: + fh.seek(0) + head_for_text = head + if not _looks_like_ascii_stl(head_for_text): + raise SnapStudioError("unrecognized STL (not binary-size-consistent or ASCII)") + fh.seek(0) + cur: list[tuple] = [] + carry = b"" + + def feed(line: bytes) -> None: + nonlocal cur + if len(line) > _MAX_LINE: + raise SnapStudioError("STL has a line longer than 4096 bytes") + s = line.strip() + if s.startswith(b"vertex"): + _, x, y, z = s.split()[:4] + cur.append((float(x), float(y), float(z))) + if len(cur) == 3: + tris.append(tuple(_add(vmap, verts, p) for p in cur)) + cur = [] + if len(tris) > max_triangles: + raise StlTooLarge(len(tris), max_triangles) + if len(verts) > max_vertices: + raise StlTooLarge(len(verts), max_vertices, "vertices") + + while True: + check() + chunk = fh.read(_CHUNK) + if not chunk: + break + parts = _LINE_SPLIT.split(carry + chunk) + carry = parts.pop() + if len(carry) > _MAX_LINE: + raise SnapStudioError("STL has a line longer than 4096 bytes") + for line in parts: + feed(line) + if carry: + feed(carry) + if not tris: + raise SnapStudioError("STL contained no triangles") + return verts, tris diff --git a/backend/snapstudio_core/units.py b/backend/snapstudio_core/units.py new file mode 100644 index 00000000..a4413899 --- /dev/null +++ b/backend/snapstudio_core/units.py @@ -0,0 +1,102 @@ +"""One place that knows what a 3MF `unit` means in millimetres. + +A 3MF model part states its unit on the root `model` element; when the attribute is absent the format says +millimetre. Every number in that part (vertex coordinates AND the translation of any transform written in +it) is in that unit. Several Studio readers used to ignore the header, so a 12 inch part was reported as +12 mm. This table and the readers below are the single shared fix: nothing else may keep its own copy of +the factors, and the scene parser calls the SAME ``resolve_unit`` on the SAME root element. + +Three outcomes are kept apart on purpose: + +* **absent** attribute: the specification default, millimetre (``declared`` is False); +* **recognized** value: one of the six defined units; +* **unrecognized** value: NOT silently the same as absent. ``UnitInfo.recognized`` is False. Legacy + dimension readers still fall back to millimetres (they always have), the scene reports a limitation + (``UNSUPPORTED_UNIT``) and withholds fit findings. +""" +from __future__ import annotations + +import io +from typing import NamedTuple + +from lxml import etree + +#: The six units the 3MF core specification defines, in millimetres per unit. +MM_PER_UNIT: dict[str, float] = { + "micron": 0.001, + "millimeter": 1.0, + "centimeter": 10.0, + "inch": 25.4, + "foot": 304.8, + "meter": 1000.0, +} +DEFAULT_UNIT = "millimeter" + + +class UnitInfo(NamedTuple): + name: str # the declared value, or the default when absent + factor: float # millimetres per unit; 1.0 (the documented fallback) when not recognized + recognized: bool # False only for a declared value outside the six units + declared: bool # False when the attribute is absent (the default applies) + + +def resolve_unit(value: str | None) -> UnitInfo: + """Interpret the root element's ``unit`` attribute value (None = absent).""" + if value is None: + return UnitInfo(DEFAULT_UNIT, MM_PER_UNIT[DEFAULT_UNIT], True, False) + factor = MM_PER_UNIT.get(value) + if factor is None: + return UnitInfo(value, 1.0, False, True) + return UnitInfo(value, factor, True, True) + + +def read_unit(raw: bytes | str) -> UnitInfo: + """The unit declared by the REAL root element of a model part, read with the XML parser. + + Only the first start event is consumed, so comments, processing instructions, a DOCTYPE-free prolog + and a long preamble are all irrelevant, and a ```` inside a comment is not seen. A part + that cannot be parsed, or whose root is not ``model``, is NOT a legitimately absent attribute: it reads + as ``recognized=False`` (millimetre factor for the legacy readers, with the flag set); entities are never + resolved and the network is never touched. + """ + data = raw.encode("utf-8") if isinstance(raw, str) else raw + unreadable = UnitInfo(DEFAULT_UNIT, MM_PER_UNIT[DEFAULT_UNIT], False, False) + try: + for _event, elem in etree.iterparse(io.BytesIO(data), events=("start",), resolve_entities=False, + load_dtd=False, no_network=True, huge_tree=False): + if etree.QName(elem).localname != "model": + return unreadable + return resolve_unit(elem.get("unit")) + except (etree.LxmlError, ValueError): + pass + return unreadable + + +def unit_of(raw: bytes | str) -> tuple[str | None, float | None]: + """``(name, mm_per_unit)``; the factor is None for an unrecognized value.""" + info = read_unit(raw) + return info.name, (info.factor if info.recognized else None) + + +def mm_per_unit(raw: bytes | str) -> float: + """Millimetres per unit for a model part; an unrecognized unit reads as millimetres (see UnitInfo).""" + return read_unit(raw).factor + + +def scale_translation(transform, factor: float): + """A 3MF transform with its translation moved into millimetres (or back, with ``1/factor``). + + Accepts the 12-number list form (translation = entries 9..11) and the parsed row form + ``((a,b,c),(d,e,f),(g,h,i),(tx,ty,tz))``. Only the translation carries a length; the 3x3 part is + dimensionless. Factor 1.0 returns the input unchanged, so millimetre behaviour is exactly as before. + """ + if transform is None or factor == 1.0: + return transform + if isinstance(transform[0], (tuple, list)): + tx, ty, tz = transform[3] + return (transform[0], transform[1], transform[2], (tx * factor, ty * factor, tz * factor)) + out = list(transform) + out[9] *= factor + out[10] *= factor + out[11] *= factor + return out diff --git a/backend/tests/fixtures/scene/golden-scene-1.json b/backend/tests/fixtures/scene/golden-scene-1.json new file mode 100644 index 00000000..579dfc85 --- /dev/null +++ b/backend/tests/fixtures/scene/golden-scene-1.json @@ -0,0 +1,166 @@ +{ + "schema": "scene/1", + "revision": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "status": "complete", + "units": "mm", + "axes": "right-handed-z-up", + "sources": [ + { + "part": "3D/3dmodel.model", + "unit": "millimeter", + "mm_per_unit": 1.0 + } + ], + "bed": { + "polygon_mm": [ + [ + 0.5, + 1.0 + ], + [ + 270.5, + 1.0 + ], + [ + 270.5, + 271.0 + ], + [ + 0.5, + 271.0 + ] + ], + "height_mm": 270.05, + "edge_margin_mm": 0.5, + "policy": "u1_template" + }, + "meshes": [ + { + "key": { + "part": "3D/3dmodel.model", + "object_id": "1" + }, + "vertex_count": 8, + "triangle_count": 12, + "positions_f32le_base64": "AAAAAAAAAAAAAAAAAAAgQQAAAAAAAAAAAAAgQQAAIEEAAAAAAAAAAAAAIEEAAAAAAAAAAAAAAAAAACBBAAAgQQAAAAAAACBBAAAgQQAAIEEAACBBAAAAAAAAIEEAACBB", + "indices_u32le_base64": "AAAAAAEAAAACAAAAAAAAAAIAAAADAAAABAAAAAUAAAAGAAAABAAAAAYAAAAHAAAAAAAAAAEAAAAFAAAAAAAAAAUAAAAEAAAAAQAAAAIAAAAGAAAAAQAAAAYAAAAFAAAAAgAAAAMAAAAHAAAAAgAAAAcAAAAGAAAAAwAAAAAAAAAEAAAAAwAAAAQAAAAHAAAA", + "volumes": [ + { + "id": "v0", + "triangle_start": 0, + "triangle_count": 12, + "role": "part", + "source": "object" + } + ] + } + ], + "nodes": [ + { + "id": "b0", + "parent_id": null, + "resource": { + "part": "3D/3dmodel.model", + "object_id": "1" + }, + "mesh_key": { + "part": "3D/3dmodel.model", + "object_id": "1" + }, + "local_to_parent_mm": [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 50.0, + 60.0, + 0.0, + 1.0 + ], + "world_mm": [ + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 50.0, + 60.0, + 0.0, + 1.0 + ], + "mirrored": false, + "role_context": null, + "build_index": 0, + "instance_ref": { + "build_index": 0, + "component_path": [] + }, + "plate": { + "state": "unknown", + "plate_id": null, + "source": null + }, + "placement_state": "known", + "printable": true, + "has_non_part_volumes": false, + "bounds_mm": { + "min": [ + 50.0, + 60.0, + 0.0 + ], + "max": [ + 60.0, + 70.0, + 10.0 + ] + }, + "finding_ids": [] + } + ], + "plates": [], + "findings": [], + "limitations": [ + { + "code": "PLATES_UNAVAILABLE", + "target_ids": [] + } + ], + "counts": { + "nodes": 1, + "meshes": 1, + "vertices": 8, + "triangles": 12, + "rendered_triangles": 12, + "plates": 0, + "findings": 0, + "limitations": 1 + }, + "limits": { + "max_archive_bytes": 134217728, + "max_model_xml_bytes": 33554432, + "max_rendered_triangles": 250000, + "max_vertices": 300000, + "max_nodes": 2000, + "max_depth": 64, + "max_response_bytes": 8388608, + "max_findings": 500, + "max_limitations": 50, + "max_id_length": 128 + } +} diff --git a/backend/tests/fixtures/scene/make_golden.py b/backend/tests/fixtures/scene/make_golden.py new file mode 100644 index 00000000..ddbed05e --- /dev/null +++ b/backend/tests/fixtures/scene/make_golden.py @@ -0,0 +1,53 @@ +"""Writes golden-scene-1.json by hand, independently of snapstudio_core.scene. + +The golden scene is the frozen reference for scene/1: a 10 mm cube, authored in millimetres at +(50, 60, 0) in a plain 3MF with no slicer metadata. It is derived here from first principles (struct + +base64 + the U1 bed numbers), so a test can require the real builder to reproduce it exactly. + +Run: py -3.13 make_golden.py (rewrites golden-scene-1.json next to this file) +""" +import base64 +import json +import struct +from pathlib import Path + +REVISION = "0123456789abcdef" * 4 +VERTS = [(0, 0, 0), (10, 0, 0), (10, 10, 0), (0, 10, 0), (0, 0, 10), (10, 0, 10), (10, 10, 10), (0, 10, 10)] +TRIS = [(0, 1, 2), (0, 2, 3), (4, 5, 6), (4, 6, 7), (0, 1, 5), (0, 5, 4), + (1, 2, 6), (1, 6, 5), (2, 3, 7), (2, 7, 6), (3, 0, 4), (3, 4, 7)] +PART = "3D/3dmodel.model" + + +def b64(fmt, values): + return base64.b64encode(struct.pack("<" + fmt * len(values), *values)).decode("ascii") + + +matrix = [1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 50.0, 60.0, 0.0, 1.0] +scene = { + "schema": "scene/1", "revision": REVISION, "status": "complete", "units": "mm", "axes": "right-handed-z-up", + "sources": [{"part": PART, "unit": "millimeter", "mm_per_unit": 1.0}], + "bed": {"polygon_mm": [[0.5, 1.0], [270.5, 1.0], [270.5, 271.0], [0.5, 271.0]], + "height_mm": 270.05, "edge_margin_mm": 0.5, "policy": "u1_template"}, + "meshes": [{ + "key": {"part": PART, "object_id": "1"}, "vertex_count": 8, "triangle_count": 12, + "positions_f32le_base64": b64("f", [c for v in VERTS for c in v]), + "indices_u32le_base64": b64("I", [i for t in TRIS for i in t]), + "volumes": [{"id": "v0", "triangle_start": 0, "triangle_count": 12, "role": "part", "source": "object"}]}], + "nodes": [{ + "id": "b0", "parent_id": None, "resource": {"part": PART, "object_id": "1"}, + "mesh_key": {"part": PART, "object_id": "1"}, "local_to_parent_mm": matrix, "world_mm": matrix, + "mirrored": False, "role_context": None, "build_index": 0, "instance_ref": {"build_index": 0, "component_path": []}, + "plate": {"state": "unknown", "plate_id": None, "source": None}, "placement_state": "known", + "printable": True, "has_non_part_volumes": False, + "bounds_mm": {"min": [50.0, 60.0, 0.0], "max": [60.0, 70.0, 10.0]}, "finding_ids": []}], + "plates": [], "findings": [], + "limitations": [{"code": "PLATES_UNAVAILABLE", "target_ids": []}], + "counts": {"nodes": 1, "meshes": 1, "vertices": 8, "triangles": 12, "rendered_triangles": 12, + "plates": 0, "findings": 0, "limitations": 1}, + "limits": {"max_archive_bytes": 134217728, "max_model_xml_bytes": 33554432, "max_rendered_triangles": 250000, + "max_vertices": 300000, "max_nodes": 2000, "max_depth": 64, "max_response_bytes": 8388608, + "max_findings": 500, "max_limitations": 50, "max_id_length": 128}} +if __name__ == "__main__": + out = Path(__file__).with_name("golden-scene-1.json") + out.write_text(json.dumps(scene, indent=2) + "\n", encoding="utf-8") + print("wrote", out) diff --git a/backend/tests/scene_fixtures.py b/backend/tests/scene_fixtures.py new file mode 100644 index 00000000..a4c30b64 --- /dev/null +++ b/backend/tests/scene_fixtures.py @@ -0,0 +1,187 @@ +"""Authored 3MF / STL fixtures for the scene/1 tests and benchmark. + +Everything here is synthetic. Archives are written with ZIP_STORED and fixed timestamps so the bytes +are deterministic. Nothing here reads a real user file. +""" +from __future__ import annotations + +import struct +import zipfile +from pathlib import Path + +CORE = "http://schemas.microsoft.com/3dmanufacturing/core/2015/02" +PROD = "http://schemas.microsoft.com/3dmanufacturing/production/2015/06" +REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" +MODEL_REL = "http://schemas.microsoft.com/3dmanufacturing/2013/01/3dmodel" + +CUBE_TRIS = [(0, 1, 2), (0, 2, 3), (4, 5, 6), (4, 6, 7), (0, 1, 5), (0, 5, 4), + (1, 2, 6), (1, 6, 5), (2, 3, 7), (2, 7, 6), (3, 0, 4), (3, 4, 7)] + + +def cube_vertices(size: float = 10.0, origin=(0.0, 0.0, 0.0)): + ox, oy, oz = origin + s = size + return [(ox, oy, oz), (ox + s, oy, oz), (ox + s, oy + s, oz), (ox, oy + s, oz), + (ox, oy, oz + s), (ox + s, oy, oz + s), (ox + s, oy + s, oz + s), (ox, oy + s, oz + s)] + + +def mesh_xml(vertices, triangles) -> str: + v = "".join(f'' for x, y, z in vertices) + t = "".join(f'' for a, b, c in triangles) + return f"{v}{t}" + + +def cube_object(oid, size=10.0, origin=(0.0, 0.0, 0.0)) -> str: + return f'{mesh_xml(cube_vertices(size, origin), CUBE_TRIS)}' + + +def composite_object(oid, comps) -> str: + """comps: [(objectid, path|None, transform|None)]""" + inner = "" + for cid, path, tf in comps: + attrs = f'objectid="{cid}"' + if path is not None: + attrs += f' p:path="{path}"' + if tf: + attrs += f' transform="{tf}"' + inner += f"" + return f'{inner}' + + +def model_xml(objects, items, unit="millimeter", prolog='') -> str: + build = "" + for item in items: # (objectid, transform) or (objectid, transform, p:path) + oid, tf = item[0], item[1] + path = item[2] if len(item) > 2 else None + build += (f'") + return (f'{prolog}' + f"{''.join(objects)}{build}") + + +def sub_model_xml(objects, unit="millimeter") -> str: + return (f'' + f"{''.join(objects)}") + + +def rels_xml(target="/3D/3dmodel.model", extra="") -> str: + return (f'' + f'{extra}') + + +def write_zip(path, entries: dict, order=None) -> Path: + path = Path(path) + with zipfile.ZipFile(path, "w", zipfile.ZIP_STORED) as z: + for name in (order or list(entries)): + data = entries[name] + if isinstance(data, str): + data = data.encode("utf-8") + info = zipfile.ZipInfo(name, date_time=(2020, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_STORED + z.writestr(info, data) + return path + + +def three_mf(path, root_model: str, extra: dict | None = None, rels: str | None = None) -> Path: + entries = {"_rels/.rels": rels or rels_xml(), "3D/3dmodel.model": root_model} + entries.update(extra or {}) + return write_zip(path, entries) + + +IDENT = "1 0 0 0 1 0 0 0 1" + + +def tf(tx=0.0, ty=0.0, tz=0.0, lin=IDENT) -> str: + return f"{lin} {tx} {ty} {tz}" + + +def plain_cube_3mf(path, size=10.0, unit="millimeter", at=(50.0, 60.0, 0.0)) -> Path: + return three_mf(path, model_xml([cube_object("1", size)], [("1", tf(*at))], unit=unit)) + + +# --- Bambu/Orca style ------------------------------------------------------------------------ + +def model_settings_xml(objects: dict, plates: list) -> str: + """objects: {oid: [(part_id, subtype)]}; plates: [(plater_id, [(oid, instance_id|None)])]""" + out = [''] + for oid, parts in objects.items(): + out.append(f'') + for pid, sub in parts: + attr = f' subtype="{sub}"' if sub is not None else "" + out.append(f'') + out.append("") + for plater, members in plates: + out.append(f'') + for oid, inst in members: + inst_xml = f'' if inst is not None else "" + out.append(f'{inst_xml}') + out.append("") + out.append("") + return "".join(out) + + +def bambu_project(path, *, parts, items, plates, roles=None, unit="millimeter", sizes=None, origins=None, + omit_parts=()) -> Path: + """One composite root object `100` made of meshes 1..n in 3D/Objects/object_1.model. + + parts: number of mesh parts; roles: [subtype per part]; items: [(objectid, transform)]. + """ + roles = roles or ["normal_part"] * parts + sizes = sizes or [10.0] * parts + origins = origins or [(i * 20.0, 0.0, 0.0) for i in range(parts)] + sub = sub_model_xml([cube_object(str(i + 1), sizes[i], origins[i]) for i in range(parts)], unit=unit) + root = model_xml( + [composite_object("100", [(str(i + 1), "/3D/Objects/object_1.model", None) for i in range(parts)])], + items, unit=unit) + settings = model_settings_xml({"100": [(str(i + 1), roles[i]) for i in range(parts) if (i + 1) not in omit_parts]}, + plates) + rels3d = (f'' + f'') + return three_mf(path, root, {"3D/Objects/object_1.model": sub, "3D/_rels/3dmodel.model.rels": rels3d, + "Metadata/model_settings.config": settings}) + + +# --- Prusa style ----------------------------------------------------------------------------- + +def prusa_project(path, *, volumes, items=None, unit="millimeter", last_first=False) -> Path: + """One object `1` with 12 triangles per cube volume; volumes: [(first, last, volume_type)].""" + n = max(last for _first, last, _ in volumes) // 12 + 1 if volumes else 1 + verts, tris = [], [] + for k in range(n): + base = len(verts) + verts += cube_vertices(10.0, (k * 20.0, 0.0, 0.0)) + tris += [(a + base, b + base, c + base) for a, b, c in CUBE_TRIS] + obj = f'{mesh_xml(verts, tris)}' + root = model_xml([obj], items or [("1", tf(100, 100, 0))], unit=unit) + attrs = (lambda a, b: f'lastid="{b}" firstid="{a}"') if last_first else (lambda a, b: f'firstid="{a}" lastid="{b}"') + vol = "".join( + f'' + for a, b, vt in volumes) + cfg = f'{vol}' + return three_mf(path, root, {"Metadata/Slic3r_PE_model.config": cfg}) + + +# --- STL ------------------------------------------------------------------------------------- + +def binary_stl(path, size=10.0) -> Path: + verts = cube_vertices(size) + body = b"\x00" * 80 + struct.pack(" Path: + """A single mesh of about `triangles_target` triangles (a flat triangulated grid with shared vertices).""" + side = max(2, int((triangles_target / 2) ** 0.5)) + verts = [(x * 0.1, y * 0.1, 0.0) for y in range(side + 1) for x in range(side + 1)] + tris = [] + w = side + 1 + for y in range(side): + for x in range(side): + a = y * w + x + tris.append((a, a + 1, a + w)) + tris.append((a + 1, a + w + 1, a + w)) + return three_mf(path, model_xml([f'{mesh_xml(verts, tris)}'], + [("1", tf(*at))])) diff --git a/backend/tests/test_scene_api.py b/backend/tests/test_scene_api.py new file mode 100644 index 00000000..2be597b4 --- /dev/null +++ b/backend/tests/test_scene_api.py @@ -0,0 +1,168 @@ +"""The /scene/* HTTP surface: authentication, validation, status codes, pre-serialized body.""" +import http.client +import json +import threading +import time + +import pytest + +from snapstudio_api import scene_jobs as sj +from snapstudio_api.server import build_server +from snapstudio_core import scene_limits as L +from tests import scene_fixtures as fx +from tests.test_scene_contract import assert_valid, sub_validator + + +@pytest.fixture +def server(tmp_path, monkeypatch): + monkeypatch.setenv("SNAPSTUDIO_DATA_DIR", str(tmp_path / "data")) + sj.reset_default_for_tests() + httpd, token = build_server() + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + yield httpd.server_address[1], token + httpd.shutdown() + httpd.server_close() + sj.reset_default_for_tests() + + +def call(port, route, payload, token, *, raw=False): + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=20) + try: + headers = {"Content-Type": "application/json"} + if token: + headers["X-Auth-Token"] = token + conn.request("POST", route, body=json.dumps(payload).encode(), headers=headers) + resp = conn.getresponse() + body = resp.read() + assert resp.getheader("Content-Length") == str(len(body)) + assert resp.getheader("Content-Type") == "application/json" + return resp.status, (body if raw else json.loads(body)) + finally: + conn.close() + + +def poll(port, token, job_id, *states, timeout=15): + end = time.monotonic() + timeout + while time.monotonic() < end: + code, body = call(port, "/scene/status", {"job_id": job_id}, token) + assert code == 200 + if body["state"] in states: + return body + time.sleep(0.02) + raise AssertionError("timeout") + + +def test_every_scene_route_requires_the_token(server, tmp_path): + port, token = server + for route in sj.ROUTES: + assert call(port, route, {}, None)[0] == 401 + assert call(port, route, {}, "wrong-token")[0] == 401 + + +def test_full_flow_returns_the_exact_serialized_scene(server, tmp_path): + port, token = server + path = fx.bambu_project(tmp_path / "p.3mf", parts=2, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])]) + code, started = call(port, "/scene/start", {"path": str(path), "request_id": "req-flow-0001"}, token) + assert code == 200 and not list(sub_validator("job_start").iter_errors(started)) + status = poll(port, token, started["job_id"], "succeeded") + assert not list(sub_validator("job_status").iter_errors(status)) + code, raw = call(port, "/scene/result", {"job_id": started["job_id"], "expected_revision": status["revision"]}, token, raw=True) + assert code == 200 + sc = json.loads(raw) + assert_valid(sc) + assert raw == json.dumps(sc, separators=(",", ":")).encode() # compact, serialized once, no re-dump + assert len(raw) <= L.MAX_RESPONSE_BYTES + + +def test_error_statuses_and_bodies(server, tmp_path): + port, token = server + err = sub_validator("error_response") + cases = [ + ("/scene/start", {"request_id": "req-err-00001"}, 400, "INVALID_REQUEST"), + ("/scene/start", {"path": str(tmp_path / "nope.3mf"), "request_id": "req-err-00001"}, 400, "INVALID_REQUEST"), + ("/scene/start", {"path": "x.3mf", "request_id": "short"}, 400, "INVALID_REQUEST"), + ("/scene/start", {"path": "x.3mf", "request_id": "has spaces in it!"}, 400, "INVALID_REQUEST"), + ("/scene/start", {"path": str(tmp_path / "a.obj"), "request_id": "req-err-00002"}, 422, "UNSUPPORTED_FORMAT"), + ("/scene/status", {}, 400, "INVALID_REQUEST"), + ("/scene/status", {"job_id": "0" * 32}, 404, "EXPIRED"), + ("/scene/result", {"job_id": "0" * 32}, 404, "EXPIRED"), + ("/scene/result", {"job_id": "0" * 32, "expected_revision": "nothex"}, 400, "INVALID_REQUEST"), + ("/scene/cancel", {"job_id": "0" * 32}, 404, "EXPIRED"), + ] + (tmp_path / "a.obj").write_text("v") + for route, payload, status, code in cases: + got, body = call(port, route, payload, token) + assert (got, body["error"]) == (status, code), (route, payload, got, body) + assert not list(err.iter_errors(body)) + + +def test_failed_job_result_is_422_and_cancelled_is_409(server, tmp_path): + port, token = server + bad = tmp_path / "bad.3mf" + bad.write_bytes(b"junk" * 50) + _, started = call(port, "/scene/start", {"path": str(bad), "request_id": "req-bad-00001"}, token) + poll(port, token, started["job_id"], "failed") + code, body = call(port, "/scene/result", {"job_id": started["job_id"]}, token) + assert (code, body["error"]) == (422, "INVALID_ARCHIVE") + ok = fx.plain_cube_3mf(tmp_path / "ok.3mf") + _, second = call(port, "/scene/start", {"path": str(ok), "request_id": "req-ok-000001"}, token) + code, status = call(port, "/scene/cancel", {"job_id": second["job_id"]}, token) + assert code == 200 and status["state"] in ("cancelled", "succeeded") + if status["state"] == "cancelled": + code, body = call(port, "/scene/result", {"job_id": second["job_id"]}, token) + assert (code, body["error"]) == (409, "CANCELLED") + + +def test_request_id_conflict_and_idempotency_over_http(server, tmp_path): + port, token = server + a = fx.plain_cube_3mf(tmp_path / "a.3mf") + b = fx.plain_cube_3mf(tmp_path / "b.3mf", size=11) + _, one = call(port, "/scene/start", {"path": str(a), "request_id": "req-same-0001"}, token) + _, again = call(port, "/scene/start", {"path": str(a), "request_id": "req-same-0001"}, token) + assert one["job_id"] == again["job_id"] + code, body = call(port, "/scene/start", {"path": str(b), "request_id": "req-same-0001"}, token) + assert (code, body["error"], body["reason"]) == (409, "INVALID_REQUEST", "request_id reused for another path") + + +def test_not_ready_and_wedged_over_http(server, tmp_path, monkeypatch): + port, token = server + gate = threading.Event() + started = threading.Event() + + def stuck(path, revision, ctl): + started.set() + gate.wait(10) + return b"{}" + jobs = sj.SceneJobs(builder=stuck, wedge_grace=0.2) + monkeypatch.setattr(sj, "_default", jobs) + path = fx.plain_cube_3mf(tmp_path / "c.3mf") + _, a = call(port, "/scene/start", {"path": str(path), "request_id": "req-wedge-001"}, token) + started.wait(5) + code, body = call(port, "/scene/result", {"job_id": a["job_id"]}, token) + assert (code, body["error"], body["state"]) == (409, "NOT_READY", "running") + call(port, "/scene/cancel", {"job_id": a["job_id"]}, token) + time.sleep(0.4) + code, body = call(port, "/scene/start", {"path": str(path), "request_id": "req-wedge-002"}, token) + assert (code, body["error"]) == (503, "WORKER_WEDGED") + gate.set() + for _ in range(200): + if not jobs.wedged(): + break + time.sleep(0.02) + code, _ = call(port, "/scene/start", {"path": str(path), "request_id": "req-wedge-003"}, token) + assert code == 200 + jobs.close() + + +def test_other_routes_still_behave(server): + port, token = server + assert call(port, "/definitely_not_a_route", {}, token)[0] == 404 + + +@pytest.mark.parametrize("body", [[1, 2], "text", 7, None, [], True]) +def test_a14_a_non_object_json_body_is_a_400_not_a_500(server, body): + port, token = server + for route in sj.ROUTES: + code, resp = call(port, route, body, token) + assert code == 400 and resp["error"] == "INVALID_REQUEST", (route, body, code, resp) diff --git a/backend/tests/test_scene_build.py b/backend/tests/test_scene_build.py new file mode 100644 index 00000000..2a2882e1 --- /dev/null +++ b/backend/tests/test_scene_build.py @@ -0,0 +1,310 @@ +"""scene/1 builder: units, ids, instances, transforms, volumes, plates, STL, bed, comparisons. + +Every scene a test builds is also validated against the committed JSON Schema (real validator) and +the invariants JSON Schema cannot express. +""" +import json +import math +import os +from pathlib import Path + +import pytest + +from snapstudio_core import plate_placement, scene, units +from tests import scene_fixtures as fx +from tests.test_scene_contract import assert_valid + +REV = "ab" * 32 +EXAMPLES = Path(__file__).resolve().parents[2] / "examples" + + +def build(path) -> dict: + sc = scene.build_scene_dict(str(path), REV) + assert_valid(sc) + return sc + + +def node(sc, nid): + return next(n for n in sc["nodes"] if n["id"] == nid) + + +def codes(sc): + return [l["code"] for l in sc["limitations"]] + + +# ------------------------------------------------------------------------------ units + +@pytest.mark.parametrize("unit,factor", sorted(units.MM_PER_UNIT.items())) +def test_all_six_units_convert_to_millimetres(tmp_path, unit, factor): + # a 12-unit cube, built 2 units along x. Both the vertex coordinates and the transform's + # translation are in the model's unit. + path = fx.three_mf(tmp_path / "u.3mf", fx.model_xml([fx.cube_object("1", 12)], [("1", fx.tf(2, 0, 0))], unit=unit)) + sc = build(path) + assert sc["sources"] == [{"part": "3D/3dmodel.model", "unit": unit, "mm_per_unit": factor}] + b = node(sc, "b0")["bounds_mm"] + assert b["min"][0] == pytest.approx(2 * factor, abs=1e-5) + assert b["max"][0] == pytest.approx(14 * factor, abs=1e-5) + assert b["max"][2] == pytest.approx(12 * factor, abs=1e-5) + assert node(sc, "b0")["world_mm"][12] == pytest.approx(2 * factor, rel=1e-8) + if unit != "millimeter": + assert "NON_MM_SOURCE_UNIT" in codes(sc) and sc["status"] == "partial" + assert not [f for f in sc["findings"] if f["kind"] == "placement"] + + +def test_twelve_inch_part_is_304_8_mm(tmp_path): + sc = build(fx.three_mf(tmp_path / "i.3mf", fx.model_xml([fx.cube_object("1", 12)], [("1", None)], unit="inch"))) + b = node(sc, "b0")["bounds_mm"] + assert b["max"][0] - b["min"][0] == pytest.approx(304.8, abs=1e-4) + assert any(f["kind"] == "size" for f in sc["findings"]) # larger than the 270 mm plate + + +def test_unrecognised_unit_is_a_limitation_not_a_silent_millimetre(tmp_path): + # far off the plate: with an unknown unit no fit statement of any kind may be made + path = fx.three_mf(tmp_path / "x.3mf", fx.model_xml([fx.cube_object("1", 400)], [("1", fx.tf(900, 900, 0))], unit="yard")) + sc = build(path) + assert "UNSUPPORTED_UNIT" in codes(sc) and sc["status"] == "partial" + assert sc["findings"] == [] + assert sc["sources"] == [{"part": "3D/3dmodel.model", "unit": "millimeter", "mm_per_unit": 1.0}] # the documented fallback + absent = build(fx.three_mf(tmp_path / "y.3mf", fx.model_xml([fx.cube_object("1", 400)], [("1", fx.tf(900, 900, 0))]).replace(' unit="millimeter"', ""))) + assert "UNSUPPORTED_UNIT" not in codes(absent) and absent["findings"] # absent = millimetre, judged normally + + +def test_sub_model_in_a_different_unit_uses_its_own_unit(tmp_path): + sub = fx.sub_model_xml([fx.cube_object("1", 1)], unit="centimeter") + root = fx.model_xml([fx.composite_object("10", [("1", "/3D/Objects/a.model", None)])], [("10", fx.tf(5, 0, 0))]) + sc = build(fx.three_mf(tmp_path / "m.3mf", root, {"3D/Objects/a.model": sub})) + b = node(sc, "b0")["bounds_mm"] + assert b["min"][0] == pytest.approx(5.0) # root (mm) translation + assert b["max"][0] == pytest.approx(15.0) # 1 cm cube + assert {s["unit"] for s in sc["sources"]} == {"millimeter", "centimeter"} + + +# ------------------------------------------------------------------------------ ids and instances + +def test_cross_file_duplicate_object_ids_stay_distinct(tmp_path): + a = fx.sub_model_xml([fx.cube_object("1", 10)]) + b = fx.sub_model_xml([fx.cube_object("1", 30)]) + root = fx.model_xml([fx.composite_object("9", [("1", "/3D/Objects/a.model", None), + ("1", "/3D/Objects/b.model", fx.tf(100, 0, 0))])], [("9", None)]) + sc = build(fx.three_mf(tmp_path / "d.3mf", root, {"3D/Objects/a.model": a, "3D/Objects/b.model": b})) + keys = {(m["key"]["part"], m["key"]["object_id"]): m for m in sc["meshes"]} + assert set(keys) == {("3D/Objects/a.model", "1"), ("3D/Objects/b.model", "1")} + assert [n["id"] for n in sc["nodes"]] == ["b0", "b0.c0", "b0.c1"] + assert node(sc, "b0.c0")["bounds_mm"]["max"][0] == 10.0 + assert node(sc, "b0.c1")["bounds_mm"]["min"][0] == 100.0 and node(sc, "b0.c1")["bounds_mm"]["max"][0] == 130.0 + assert node(sc, "b0")["bounds_mm"] == {"min": [0.0, 0.0, 0.0], "max": [130.0, 30.0, 30.0]} + + +def test_repeated_instances_share_a_mesh_and_disable_engine_findings(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(100, 100, 0)), ("1", fx.tf(500, 100, 0))]) + sc = build(fx.three_mf(tmp_path / "r.3mf", root)) + assert len(sc["meshes"]) == 1 and [n["id"] for n in sc["nodes"]] == ["b0", "b1"] + assert sc["counts"]["rendered_triangles"] == 24 and sc["counts"]["triangles"] == 12 + assert node(sc, "b0")["bounds_mm"]["min"][0] == 100.0 + assert node(sc, "b1")["bounds_mm"]["min"][0] == 500.0 # geometry stays available + lim = next(l for l in sc["limitations"] if l["code"] == "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED") + assert lim["target_ids"] == ["b0", "b1"] and sc["status"] == "partial" + assert sc["findings"] == [] and all(n["finding_ids"] == [] for n in sc["nodes"]) + # the same cube on its own IS judged: the second instance really is off the plate + single = build(fx.three_mf(tmp_path / "s.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(500, 100, 0))]))) + assert [f["kind"] for f in single["findings"]] == ["placement"] + + +def test_nested_and_mirrored_transforms_compose(tmp_path): + inner = fx.composite_object("5", [("1", None, "-1 0 0 0 1 0 0 0 1 0 0 0")]) # mirror x + mesh = fx.cube_object("1", 10) + outer = fx.composite_object("6", [("5", None, fx.tf(30, 0, 0))]) + root = fx.model_xml([mesh, inner, outer], [("6", fx.tf(0, 50, 0))]) + sc = build(fx.three_mf(tmp_path / "n.3mf", root)) + assert [n["id"] for n in sc["nodes"]] == ["b0", "b0.c0", "b0.c0.c0"] + leaf = node(sc, "b0.c0.c0") + assert leaf["instance_ref"] == {"build_index": 0, "component_path": [0, 0]} + assert leaf["mirrored"] is True and node(sc, "b0")["mirrored"] is False + assert leaf["parent_id"] == "b0.c0" + # mirror -> x in [-10, 0]; +30 -> [20, 30]; y +50 + assert leaf["bounds_mm"] == {"min": [20.0, 50.0, 0.0], "max": [30.0, 60.0, 10.0]} + expected = scene.mat_mul(scene.mat_mul(scene.parse_transform(fx.tf(0, 50, 0), 1.0), + scene.parse_transform(fx.tf(30, 0, 0), 1.0)), + scene.parse_transform("-1 0 0 0 1 0 0 0 1 0 0 0", 1.0)) + assert leaf["world_mm"] == pytest.approx(expected) + + +def test_rotated_instance_bounds_are_tight_not_box_corners(tmp_path): + verts = [(0, 0, 0), (10, 0, 0), (0, 10, 0), (0, 0, 10)] + tris = [(0, 1, 2), (0, 1, 3), (0, 2, 3), (1, 2, 3)] + c = math.cos(math.pi / 4) + lin = f"{c} {c} 0 {-c} {c} 0 0 0 1" # 45 degrees about z + obj = f'{fx.mesh_xml(verts, tris)}' + sc = build(fx.three_mf(tmp_path / "rot.3mf", fx.model_xml([obj], [("1", fx.tf(0, 0, 0, lin))]))) + b = node(sc, "b0")["bounds_mm"] + assert b["max"][1] == pytest.approx(10 * c, abs=1e-4) # a box-corner estimate would give 14.14 + assert b["min"][0] == pytest.approx(-10 * c, abs=1e-4) and b["max"][0] == pytest.approx(10 * c, abs=1e-4) + + +# ------------------------------------------------------------------------------ roles and volumes + +def test_bambu_roles_and_non_part_volumes_do_not_count_for_bounds(tmp_path): + path = fx.bambu_project(tmp_path / "b.3mf", parts=3, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])], + roles=["normal_part", "modifier_part", "negative_part"], + origins=[(0, 0, 0), (500, 0, 0), (900, 0, 0)]) + sc = build(path) + roles = {n["resource"]["object_id"]: n["role_context"] for n in sc["nodes"] if n["mesh_key"]} + assert roles == {"1": "part", "2": "modifier", "3": "negative"} # roles follow the USE (node), not the mesh + assert node(sc, "b0.c0")["printable"] is True + top = node(sc, "b0") + assert top["has_non_part_volumes"] is True and top["printable"] is True + assert top["bounds_mm"] == {"min": [10.0, 10.0, 0.0], "max": [20.0, 20.0, 10.0]} # the modifier at x=500 is ignored + assert node(sc, "b0.c1")["printable"] is False and node(sc, "b0.c1")["bounds_mm"] is None + assert sc["findings"] == [] # fits: modifiers are excluded from fit + + +def test_prusa_ranges_partition_all_triangles(tmp_path): + path = fx.prusa_project(tmp_path / "p.3mf", volumes=[(0, 11, "ModelPart"), (12, 23, "ParameterModifier")]) + sc = build(path) + vols = sc["meshes"][0]["volumes"] + assert [(v["triangle_start"], v["triangle_count"], v["role"], v["source"]) for v in vols] == [ + (0, 12, "part", "prusa_range"), (12, 12, "modifier", "prusa_range")] + assert node(sc, "b0")["has_non_part_volumes"] is True + assert node(sc, "b0")["bounds_mm"]["max"][0] == pytest.approx(100 + 10) # only the first cube counts + + +def test_prusa_gap_and_overlap_never_guess(tmp_path): + gap = build(fx.prusa_project(tmp_path / "g.3mf", volumes=[(0, 5, "ModelPart"), (12, 23, "ModelPart")])) + assert [v["role"] for v in gap["meshes"][0]["volumes"]] == ["part", "unknown", "part"] + assert "UNKNOWN_VOLUME_ROLE" in codes(gap) and gap["status"] == "partial" + overlap = build(fx.prusa_project(tmp_path / "o.3mf", volumes=[(0, 12, "ModelPart"), (10, 23, "ModelPart")])) + assert [v["role"] for v in overlap["meshes"][0]["volumes"]] == ["unknown"] + assert node(overlap, "b0")["printable"] is None and overlap["findings"] == [] + + +def test_unrecognised_bambu_subtype_is_unknown_not_a_part(tmp_path): + sc = build(fx.bambu_project(tmp_path / "u.3mf", parts=1, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])], + roles=["totally_new_part"])) + assert node(sc, "b0.c0")["role_context"] == "unknown" + lim = next(l for l in sc["limitations"] if l["code"] == "UNKNOWN_VOLUME_ROLE") + assert lim["target_ids"] == ["b0.c0"] and node(sc, "b0")["printable"] is None + + +def test_plain_mesh_gets_one_synthetic_part_volume(tmp_path): + sc = build(fx.plain_cube_3mf(tmp_path / "c.3mf")) + assert sc["meshes"][0]["volumes"] == [ + {"id": "v0", "triangle_start": 0, "triangle_count": 12, "role": "part", "source": "object"}] + + +# ------------------------------------------------------------------------------ plates + +def test_two_instances_of_one_resource_on_different_plates(tmp_path): + path = fx.bambu_project(tmp_path / "pl.3mf", parts=1, items=[("100", fx.tf(10, 10, 0)), ("100", fx.tf(20, 10, 0))], + plates=[(1, [("100", 0)]), (2, [("100", 1)])]) + sc = build(path) + assert node(sc, "b0")["plate"] == {"state": "known", "plate_id": "p1", "source": "plate_config"} + assert node(sc, "b1")["plate"] == {"state": "known", "plate_id": "p2", "source": "plate_config"} + assert [p["id"] for p in sc["plates"]] == ["p1", "p2"] and all(p["origin_mm"] is None for p in sc["plates"]) + assert "MULTI_PLATE_PLACEMENT_UNCHECKED" in codes(sc) + assert sc["findings"] == [] + + +def test_membership_ambiguity_is_reported_not_guessed(tmp_path): + both = build(fx.bambu_project(tmp_path / "a.3mf", parts=1, items=[("100", fx.tf(10, 10, 0))], + plates=[(1, [("100", 0)]), (2, [("100", 0)])])) + assert node(both, "b0")["plate"]["state"] == "ambiguous" and node(both, "b0")["plate"]["plate_id"] is None + assert "PLATE_MEMBERSHIP_AMBIGUOUS" in codes(both) + unspecific = build(fx.bambu_project(tmp_path / "b.3mf", parts=1, items=[("100", fx.tf(10, 10, 0)), ("100", fx.tf(40, 10, 0))], + plates=[(1, [("100", None)])])) + assert [node(unspecific, i)["plate"]["state"] for i in ("b0", "b1")] == ["ambiguous", "ambiguous"] + absent = build(fx.bambu_project(tmp_path / "c.3mf", parts=1, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [])])) + assert node(absent, "b0")["plate"]["state"] == "unknown" + + +# ------------------------------------------------------------------------------ STL and no placement + +def test_stl_placement_is_unknown_and_never_judged(tmp_path): + path = fx.binary_stl(tmp_path / "huge.stl", size=900.0) + sc = build(path) + n = node(sc, "b0") + assert n["placement_state"] == "unknown" and n["plate"]["state"] == "unknown" and n["build_index"] is None + assert n["world_mm"] == scene.IDENTITY16 # an identity is a rendering default only + assert sc["findings"] == [] and "UNKNOWN_PLACEMENT" in codes(sc) and sc["status"] == "partial" + assert sc["sources"] == [{"part": "model.stl", "unit": "millimeter", "mm_per_unit": 1.0}] + assert sc["meshes"][0]["volumes"][0]["role"] == "part" + + +def test_model_without_build_items_has_unknown_placement(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], []) + sc = build(fx.three_mf(tmp_path / "nb.3mf", root)) + assert node(sc, "b0")["placement_state"] == "unknown" and node(sc, "b0")["build_index"] is None + assert {"NO_BUILD_ITEMS", "UNKNOWN_PLACEMENT"} <= set(codes(sc)) and sc["findings"] == [] + + +# ------------------------------------------------------------------------------ bed + +@pytest.mark.parametrize("breaker", ["empty", "raises"]) +def test_bed_template_problem_is_surfaced_and_suppresses_fit_findings(tmp_path, monkeypatch, breaker): + def broken(): + if breaker == "raises": + raise FileNotFoundError("template") + return [] + monkeypatch.setattr(plate_placement, "_u1_printable_area", broken) + sc = build(fx.three_mf(tmp_path / "f.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(900, 900, 0))]))) + assert sc["bed"]["policy"] == "fallback" and "BED_TEMPLATE_UNAVAILABLE" in codes(sc) + assert sc["findings"] == [] and sc["status"] == "partial" + + +def test_bed_matches_the_engines_u1_rectangle(): + lims = scene._Limitations() + bed, rect = scene._bed([]) + assert bed["policy"] == "u1_template" and rect == plate_placement.u1_bed_rect() + assert bed["polygon_mm"] == [[0.5, 1.0], [270.5, 1.0], [270.5, 271.0], [0.5, 271.0]] + assert bed["edge_margin_mm"] == plate_placement.EDGE_MARGIN_MM + + +# ------------------------------------------------------------------------------ scene vs the engine + +def _single_plate_cases(tmp_path): + cases = {} + for name, at in {"inside": (50, 60, 0), "past_right": (300, 60, 0), "edge_margin": (270.2, 60, 0), + "past_front": (50, -5, 0)}.items(): + cases[name] = fx.bambu_project(tmp_path / f"{name}.3mf", parts=1, items=[("100", fx.tf(*at))], plates=[(1, [("100", 0)])]) + cases["example_offplate"] = EXAMPLES / "demo_offplate_foreign.3mf" + cases["example_cube"] = EXAMPLES / "sample_cube_SnapmakerU1.3mf" + return cases + + +def test_scene_world_bounds_and_findings_agree_with_plate_placement_assess(tmp_path): + from snapstudio_core import geometry + for name, path in _single_plate_cases(tmp_path).items(): + sc = build(path) + assessed = plate_placement.assess(str(path)) + assert assessed["available"], name + dims = geometry.build_item_dims(str(path)) + assert len(dims) == 1 + top = node(sc, "b0") + assert top["bounds_mm"]["min"] == pytest.approx(list(dims[0]["bounds"]["min"]), abs=1e-4), name + assert top["bounds_mm"]["max"] == pytest.approx(list(dims[0]["bounds"]["max"]), abs=1e-4), name + scene_off = any(f["kind"] == "placement" for f in sc["findings"]) + assert scene_off == bool(assessed["off_plate"]), name + if scene_off: + f = next(f for f in sc["findings"] if f["kind"] == "placement") + assert f["value"]["overhang_mm"] == assessed["items"][0]["overhang_mm"], name + + +def test_repeated_instance_fixture_documents_why_no_finding_is_attached(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(500, 100, 0)), ("1", fx.tf(100, 100, 0))]) + path = fx.three_mf(tmp_path / "rep.3mf", root) + sc = build(path) + assert "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED" in codes(sc) + assert sc["findings"] == [] and not any(n["finding_ids"] for n in sc["nodes"]) + + +# ------------------------------------------------------------------------------ originals are never touched + +@pytest.mark.parametrize("make", ["3mf", "stl", "example"]) +def test_original_file_is_not_modified(tmp_path, make): + path = {"3mf": lambda: fx.bambu_project(tmp_path / "o.3mf", parts=2, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])]), + "stl": lambda: fx.binary_stl(tmp_path / "o.stl"), + "example": lambda: tmp_path.joinpath("e.3mf").write_bytes((EXAMPLES / "demo_offplate_foreign.3mf").read_bytes()) and tmp_path / "e.3mf"}[make]() + before = (Path(path).read_bytes(), os.stat(path).st_mtime_ns) + os.utime(path, ns=(before[1] - 10**9, before[1] - 10**9)) # make a rewrite visible even on coarse clocks + before = (Path(path).read_bytes(), os.stat(path).st_mtime_ns) + scene.build_scene(str(path)) + assert (Path(path).read_bytes(), os.stat(path).st_mtime_ns) == before diff --git a/backend/tests/test_scene_contract.py b/backend/tests/test_scene_contract.py new file mode 100644 index 00000000..79eacaa2 --- /dev/null +++ b/backend/tests/test_scene_contract.py @@ -0,0 +1,150 @@ +"""scene/1 contract: the committed JSON Schema, the golden scene, and the fixture matrix. + +The schema is validated with the real `jsonschema` validator (a test-only extra); only the +invariants JSON Schema cannot express (volumes partition all triangles, index ranges, matrix +finiteness, base64 byte lengths) are checked by the small helper below. +""" +import base64 +import json +import math +import struct +from pathlib import Path + +import jsonschema +import pytest + +from snapstudio_core import scene, scene_limits as L +from tests import scene_fixtures as fx + +FIX = Path(__file__).parent / "fixtures" / "scene" +GOLDEN_REV = "0123456789abcdef" * 4 + + +def validator(): + s = scene.load_schema() + jsonschema.Draft202012Validator.check_schema(s) + return jsonschema.Draft202012Validator(s) + + +def sub_validator(name): + s = scene.load_schema() + return jsonschema.Draft202012Validator({"$schema": s["$schema"], "$defs": s["$defs"], "$ref": f"#/$defs/{name}"}) + + +def check_invariants(sc: dict) -> None: + """What JSON Schema cannot say.""" + keys = set() + for mesh in sc["meshes"]: + key = (mesh["key"]["part"], mesh["key"]["object_id"]) + assert key not in keys + keys.add(key) + pos = base64.b64decode(mesh["positions_f32le_base64"], validate=True) + idx = base64.b64decode(mesh["indices_u32le_base64"], validate=True) + assert len(pos) == mesh["vertex_count"] * 12 + assert len(idx) == mesh["triangle_count"] * 12 + floats = struct.unpack(f"<{mesh['vertex_count'] * 3}f", pos) + assert all(math.isfinite(f) for f in floats) + ints = struct.unpack(f"<{mesh['triangle_count'] * 3}I", idx) + assert all(i < mesh["vertex_count"] for i in ints) + cursor = 0 + for vol in mesh["volumes"]: # partition: no gap, no overlap + assert vol["triangle_start"] == cursor + cursor += vol["triangle_count"] + assert cursor == mesh["triangle_count"] + ids = [n["id"] for n in sc["nodes"]] + assert len(set(ids)) == len(ids) + for n in sc["nodes"]: + assert all(math.isfinite(x) for x in n["world_mm"] + n["local_to_parent_mm"]) + assert n["parent_id"] is None or n["parent_id"] in ids + if n["mesh_key"] is not None: + assert (n["mesh_key"]["part"], n["mesh_key"]["object_id"]) in keys + assert len(n["id"]) <= L.MAX_ID_LENGTH + fids = {f["id"] for f in sc["findings"]} + assert len(fids) == len(sc["findings"]) + for n in sc["nodes"]: + assert set(n["finding_ids"]) <= fids + for f in sc["findings"]: + assert all(t in ids for t in f["target_ids"]) + assert sc["counts"]["nodes"] == len(sc["nodes"]) + assert sc["counts"]["rendered_triangles"] == sum( + next(m for m in sc["meshes"] if (m["key"]["part"], m["key"]["object_id"]) == (n["mesh_key"]["part"], n["mesh_key"]["object_id"]))["triangle_count"] + for n in sc["nodes"] if n["mesh_key"]) + + +def assert_valid(sc: dict) -> None: + errors = sorted(validator().iter_errors(sc), key=lambda e: list(e.path)) + assert not errors, errors[0].message + check_invariants(sc) + + +def test_schema_is_a_valid_closed_schema(): + s = scene.load_schema() + jsonschema.Draft202012Validator.check_schema(s) + assert s["additionalProperties"] is False + # every scene object definition is closed and requires every property it declares + for name in ("source", "bed", "mesh", "volume", "node", "plate", "finding", "limitation", "counts", "limits", + "plate_ref", "instance_ref", "resource_key"): + d = s["$defs"][name] + assert d["additionalProperties"] is False, name + assert set(d["required"]) == set(d["properties"]), name + + +def test_schema_enums_match_the_code(): + d = scene.load_schema()["$defs"] + assert tuple(d["error_code"]["enum"]) == scene.ERROR_CODES + assert "WORKER_WEDGED" in scene.ERROR_CODES and "BUSY" not in scene.ERROR_CODES + assert set(d["limitation_code"]["enum"]) == set(scene._Limitations.ORDER) + assert set(scene.PARTIAL_LIMITATIONS | scene.INFORMATIONAL_LIMITATIONS) == set(scene._Limitations.ORDER) + assert set(d["finding"]["properties"]["kind"]["enum"]) == {"placement", "size", "note"} + from snapstudio_core import units + assert set(d["unit_name"]["enum"]) == set(units.MM_PER_UNIT) + limits = d["limits"]["properties"] + assert set(limits) == set(L.LIMITS_ECHO) + + +def test_golden_scene_validates_and_is_reproduced_exactly(tmp_path): + golden = json.loads((FIX / "golden-scene-1.json").read_text("utf-8")) + assert_valid(golden) + path = fx.plain_cube_3mf(tmp_path / "cube.3mf", size=10.0, at=(50.0, 60.0, 0.0)) + built = scene.build_scene_dict(str(path), GOLDEN_REV) + assert built == golden + assert json.loads(scene.serialize(built)) == golden + + +def test_finding_kind_is_required_and_closed(): + v = sub_validator("finding") + good = {"id": "f0", "engine": "scene", "schema": "scene/1", "pointer": "/nodes/0", "scope": "instance", + "kind": "placement", "target_ids": ["b0"], "value": {"code": "PLACEMENT_OUTSIDE_BED", "overhang_mm": None}} + assert not list(v.iter_errors(good)) + bad = dict(good) + del bad["kind"] + assert list(v.iter_errors(bad)) + assert list(v.iter_errors({**good, "kind": "opinion"})) + assert list(v.iter_errors({**good, "extra": 1})) + + +@pytest.mark.parametrize("name", ["error_response"]) +def test_error_response_accepts_every_code(name): + v = sub_validator(name) + for code in scene.ERROR_CODES: + assert not list(v.iter_errors({"error": code, "message": "x"})) + assert list(v.iter_errors({"error": "BUSY", "message": "x"})) + + +def test_schema_rejects_mutations_of_the_golden(): + golden = json.loads((FIX / "golden-scene-1.json").read_text("utf-8")) + v = validator() + for mutate in ( + lambda s: s.pop("limits"), + lambda s: s.update(extra=1), + lambda s: s["nodes"][0].pop("placement_state"), + lambda s: s["nodes"][0].update(world_mm=[1.0] * 15), + lambda s: s["nodes"][0]["plate"].update(state="maybe"), + lambda s: s["meshes"][0]["volumes"][0].update(role="solid"), + lambda s: s["bed"].update(policy="guess"), + lambda s: s.update(status="done"), + lambda s: s["sources"][0].update(unit="yard"), + ): + broken = json.loads(json.dumps(golden)) + mutate(broken) + assert list(v.iter_errors(broken)) diff --git a/backend/tests/test_scene_hostile.py b/backend/tests/test_scene_hostile.py new file mode 100644 index 00000000..e2518e2a --- /dev/null +++ b/backend/tests/test_scene_hostile.py @@ -0,0 +1,355 @@ +"""scene/1 against hostile and broken input: every case ends in a clear, coded refusal -- never a +crash, a hang, a partial scene or a silently shortened one.""" +import time +import zipfile + +import pytest + +from snapstudio_core import scene, scene_limits as L +from tests import scene_fixtures as fx + +REV = "cd" * 32 + + +def refuse(path, code): + with pytest.raises(scene.SceneError) as e: + scene.build_scene(str(path), REV) + assert e.value.code == code, (e.value.code, e.value.message) + return e.value + + +def model(objects=None, items=None, **kw): + return fx.model_xml(objects or [fx.cube_object("1")], items or [("1", None)], **kw) + + +# ------------------------------------------------------------------------------ format + +def test_wrong_extension_and_not_an_archive(tmp_path): + p = tmp_path / "x.obj" + p.write_bytes(b"v 0 0 0") + refuse(p, "UNSUPPORTED_FORMAT") + q = tmp_path / "x.3mf" + q.write_bytes(b"this is not a zip file at all" * 10) + refuse(q, "INVALID_ARCHIVE") + + +def test_missing_root_model_and_unresolved_references(tmp_path): + refuse(fx.write_zip(tmp_path / "a.3mf", {"_rels/.rels": fx.rels_xml()}), "UNRESOLVED_REFERENCE") + missing_file = model([fx.composite_object("5", [("1", "/3D/Objects/nope.model", None)])], [("5", None)]) + refuse(fx.three_mf(tmp_path / "b.3mf", missing_file), "UNRESOLVED_REFERENCE") + missing_obj = model([fx.composite_object("5", [("77", None, None)])], [("5", None)]) + refuse(fx.three_mf(tmp_path / "c.3mf", missing_obj), "UNRESOLVED_REFERENCE") + missing_item = model([fx.cube_object("1")], [("42", None)]) + refuse(fx.three_mf(tmp_path / "d.3mf", missing_item), "UNRESOLVED_REFERENCE") + + +# ------------------------------------------------------------------------------ archive structure + +@pytest.mark.parametrize("name", ["../evil.model", "/abs.model", "a/../../b.model", "..\\win.model", "C:/x.model", + "a/\x01b"]) +def test_traversal_looking_entry_names_are_rejected(tmp_path, name): + entries = {"_rels/.rels": fx.rels_xml(), "3D/3dmodel.model": model(), name: "x"} + refuse(fx.write_zip(tmp_path / "t.3mf", entries), "INVALID_ARCHIVE") + + +@pytest.mark.parametrize("second", ["3d/3DMODEL.model", "3D\\3dmodel.model", "3D//3dmodel.model", "./3D/3dmodel.model"]) +def test_colliding_normalized_names_are_rejected(tmp_path, second): + import warnings + entries = {"_rels/.rels": fx.rels_xml(), "3D/3dmodel.model": model(), second: model()} + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + path = fx.write_zip(tmp_path / "c.3mf", entries) + refuse(path, "INVALID_ARCHIVE") + + +def test_exact_duplicate_entry_names_are_rejected(tmp_path): + import warnings + path = tmp_path / "dup.3mf" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with zipfile.ZipFile(path, "w", zipfile.ZIP_STORED) as z: + z.writestr("_rels/.rels", fx.rels_xml()) + z.writestr("3D/3dmodel.model", model()) + z.writestr("3D/3dmodel.model", model()) + refuse(path, "INVALID_ARCHIVE") + + +def test_external_relationships_are_rejected_anywhere(tmp_path): + ext = '' + refuse(fx.three_mf(tmp_path / "a.3mf", model(), rels=fx.rels_xml(extra=ext)), "INVALID_ARCHIVE") + rels3d = f'{ext}' + refuse(fx.three_mf(tmp_path / "b.3mf", model(), {"3D/_rels/3dmodel.model.rels": rels3d}), "INVALID_ARCHIVE") + + +def test_references_that_escape_the_package_are_rejected(tmp_path): + refuse(fx.three_mf(tmp_path / "a.3mf", model(), rels=fx.rels_xml(target="../../etc/passwd")), "INVALID_ARCHIVE") + escaping = model([fx.composite_object("5", [("1", "/../../x.model", None)])], [("5", None)]) + refuse(fx.three_mf(tmp_path / "b.3mf", escaping), "INVALID_ARCHIVE") + + +def test_relative_and_rooted_relationship_targets_resolve_but_urls_do_not(tmp_path): + ok = scene.build_scene_dict(str(fx.three_mf(tmp_path / "rel.3mf", model(), rels=fx.rels_xml(target="3D/3dmodel.model"))), REV) + assert ok["counts"]["nodes"] == 1 + for target in ("http://example.invalid/a.model", "file:///C:/x.model", "C:/x.model", "ftp://h/x"): + refuse(fx.three_mf(tmp_path / "url.3mf", model(), rels=fx.rels_xml(target=target)), "INVALID_ARCHIVE") + + +def test_only_the_needed_parts_are_ever_read(tmp_path): + big = "x" * (3 * 1024 * 1024) + path = fx.three_mf(tmp_path / "ok.3mf", model(), {"Metadata/thumbnail.png": big, "Metadata/plate_1.gcode": big, + "Metadata/project_settings.config": big}) + arcs = [] + real = scene.Archive.__init__ + + def spy(self, *a, **k): + real(self, *a, **k) + arcs.append(self) + scene.Archive.__init__ = spy + try: + scene.build_scene(str(path), REV) + finally: + scene.Archive.__init__ = real + assert arcs[0].total_read < 4096 # thumbnails, G-code and project settings were never read + + +# ------------------------------------------------------------------------------ XML hardening + +BILLION = (']>' + "&b;") + + +def test_dtd_and_entity_declarations_are_rejected_before_parsing(tmp_path): + refuse(fx.three_mf(tmp_path / "a.3mf", BILLION), "INVALID_ARCHIVE") + mixed = model(prolog=']>') + refuse(fx.three_mf(tmp_path / "b.3mf", mixed), "INVALID_ARCHIVE") + external = model(prolog='') + refuse(fx.three_mf(tmp_path / "c.3mf", external), "INVALID_ARCHIVE") + # the same refusal for the .rels and the settings parts + refuse(fx.three_mf(tmp_path / "d.3mf", model(), rels=BILLION), "INVALID_ARCHIVE") + settings = ']>' + refuse(fx.three_mf(tmp_path / "e.3mf", model(), {"Metadata/model_settings.config": settings}), "INVALID_ARCHIVE") + + +@pytest.mark.parametrize("encoding", ["utf-16", "utf-16-le", "utf-16-be", "utf-32"]) +def test_utf16_doctype_bomb_is_rejected_by_encoding_before_parsing(tmp_path, encoding): + bomb = BILLION.replace("", f'') + for with_bom in (True, False): + raw = bomb.encode(encoding) + if not with_bom and raw[:2] in (b"\xff\xfe", b"\xfe\xff") or (not with_bom and raw[:4] in (b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff")): + raw = raw[2:] if encoding != "utf-32" else raw[4:] + path = fx.write_zip(tmp_path / f"u-{encoding}-{with_bom}.3mf", + {"_rels/.rels": fx.rels_xml(), "3D/3dmodel.model": raw}) + err = refuse(path, "INVALID_ARCHIVE") + assert "UTF-8" in err.message + + +def test_declared_non_utf8_encoding_is_rejected(tmp_path): + xml = model(prolog='') + refuse(fx.three_mf(tmp_path / "a.3mf", xml), "INVALID_ARCHIVE") + ok = model(prolog='\ufeff') + scene.build_scene(str(fx.three_mf(tmp_path / "b.3mf", ok.encode("utf-8"))), REV) # UTF-8 with BOM is fine + + +def test_parser_level_doctype_check_is_a_second_line_of_defence(tmp_path, monkeypatch): + path = fx.three_mf(tmp_path / "x.3mf", BILLION.replace("", f'')) + original = scene.Archive.open + monkeypatch.setattr(scene.Archive, "open", lambda self, name, kind, scan=True: original(self, name, kind, scan=False)) + err = refuse(path, "INVALID_ARCHIVE") + assert "DTD" in err.message + + +def test_malformed_xml_is_a_clean_refusal(tmp_path): + refuse(fx.three_mf(tmp_path / "a.3mf", ""), "INVALID_ARCHIVE") + refuse(fx.three_mf(tmp_path / "b.3mf", ""), "INVALID_ARCHIVE") + refuse(fx.three_mf(tmp_path / "c.3mf", ""), "INVALID_ARCHIVE") + + +# ------------------------------------------------------------------------------ geometry validation + +def _mesh_model(vertices, triangles, transform=None): + obj = f'{fx.mesh_xml(vertices, triangles)}' + return fx.model_xml([obj], [("1", transform)]) + + +@pytest.mark.parametrize("bad", ["nan", "inf", "-inf", "abc", "", "1e999"]) +def test_non_finite_or_invalid_coordinates(tmp_path, bad): + v = [(0, 0, 0), (1, 0, 0), (0, bad, 0)] + refuse(fx.three_mf(tmp_path / "v.3mf", _mesh_model(v, [(0, 1, 2)])), "INVALID_GEOMETRY") + + +def test_coordinate_out_of_range(tmp_path): + refuse(fx.three_mf(tmp_path / "v.3mf", _mesh_model([(0, 0, 0), (1, 0, 0), (0, 1e30, 0)], [(0, 1, 2)])), "INVALID_GEOMETRY") + + +@pytest.mark.parametrize("tri", [(0, 1, 3), (0, 1, 99999999999), (0, -1, 2)]) +def test_invalid_triangle_indices(tmp_path, tri): + refuse(fx.three_mf(tmp_path / "t.3mf", _mesh_model([(0, 0, 0), (1, 0, 0), (0, 1, 0)], [tri])), "INVALID_GEOMETRY") + + +def test_missing_attributes_and_empty_meshes(tmp_path): + body = ('') + refuse(fx.three_mf(tmp_path / "a.3mf", fx.model_xml([body], [("1", None)])), "INVALID_GEOMETRY") + empty = '' + refuse(fx.three_mf(tmp_path / "b.3mf", fx.model_xml([empty], [("1", None)])), "INVALID_GEOMETRY") + nothing = '' + refuse(fx.three_mf(tmp_path / "c.3mf", fx.model_xml([nothing], [("1", None)])), "INVALID_GEOMETRY") + + +@pytest.mark.parametrize("transform", ["1 0 0 0 1 0 0 0 1 0 0 nan", "1 0 0 0 1 0 0 0 1 0 0 inf", "1 0 0 0 1 0 0 0 1 0 0", + "a b c d e f g h i j k l", "1 0 0 0 1 0 0 0 1 0 0 0 0"]) +def test_invalid_transforms(tmp_path, transform): + refuse(fx.three_mf(tmp_path / "x.3mf", model(items=[("1", transform)])), "INVALID_GEOMETRY") + + +def test_transform_overflow_in_composition_is_refused(tmp_path): + big = "1e200 0 0 0 1e200 0 0 0 1e200 0 0 0" + objs = [fx.cube_object("1"), fx.composite_object("2", [("1", None, big)]), fx.composite_object("3", [("2", None, big)])] + refuse(fx.three_mf(tmp_path / "o.3mf", fx.model_xml(objs, [("3", None)])), "INVALID_GEOMETRY") + + +def test_component_cycles_and_duplicate_ids(tmp_path): + cyc = [fx.composite_object("1", [("2", None, None)]), fx.composite_object("2", [("1", None, None)])] + refuse(fx.three_mf(tmp_path / "c.3mf", fx.model_xml(cyc, [("1", None)])), "INVALID_ARCHIVE") + selfref = [fx.composite_object("1", [("1", None, None)])] + refuse(fx.three_mf(tmp_path / "s.3mf", fx.model_xml(selfref, [("1", None)])), "INVALID_ARCHIVE") + dup = [fx.cube_object("1"), fx.cube_object("1")] + refuse(fx.three_mf(tmp_path / "d.3mf", fx.model_xml(dup, [("1", None)])), "INVALID_ARCHIVE") + both = '' + refuse(fx.three_mf(tmp_path / "b.3mf", fx.model_xml([both], [("1", None)])), "INVALID_ARCHIVE") + + +# ------------------------------------------------------------------------------ budgets (never truncate) + +def test_zip_bomb_is_refused_without_reading_it(tmp_path): + path = tmp_path / "bomb.3mf" + head = f'' + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as z: + z.writestr("_rels/.rels", fx.rels_xml()) + z.writestr("3D/3dmodel.model", head + " " * (40 * 1024 * 1024) + "") + assert path.stat().st_size < 200_000 + t = time.monotonic() + refuse(path, "LIMIT_EXCEEDED") + assert time.monotonic() - t < 2.0 + + +def test_total_expansion_budget_is_metered_while_reading(tmp_path, monkeypatch): + sub = fx.sub_model_xml([fx.cube_object("1")]) + root = model([fx.composite_object("9", [("1", "/3D/Objects/a.model", None)])], [("9", None)]) + path = fx.three_mf(tmp_path / "m.3mf", root, {"3D/Objects/a.model": sub}) + scene.build_scene(str(path), REV) + monkeypatch.setattr(L, "ARCHIVE_EXPANSION_BYTES", 1500) + refuse(path, "LIMIT_EXCEEDED") + monkeypatch.setattr(L, "ARCHIVE_EXPANSION_BYTES", 128 * 1024 * 1024) + monkeypatch.setattr(L, "MODEL_XML_BYTES", 1000) + refuse(path, "LIMIT_EXCEEDED") + + +def test_entry_count_and_part_count_budgets(tmp_path, monkeypatch): + path = fx.three_mf(tmp_path / "e.3mf", model(), {f"x/{i}.txt": "a" for i in range(20)}) + monkeypatch.setattr(L, "ARCHIVE_MAX_ENTRIES", 10) + refuse(path, "LIMIT_EXCEEDED") + + +@pytest.mark.parametrize("limit,value", [("MAX_PARSED_TRIANGLES", 11), ("MAX_VERTICES", 7)]) +def test_triangle_and_vertex_budgets_are_enforced_during_parsing(tmp_path, monkeypatch, limit, value): + path = fx.plain_cube_3mf(tmp_path / "c.3mf") + monkeypatch.setattr(L, limit, value) + refuse(path, "LIMIT_EXCEEDED") + + +def test_refusal_names_the_triangle_count_and_the_limit(tmp_path, monkeypatch): + path = fx.big_grid_3mf(tmp_path / "g.3mf", 400) + monkeypatch.setattr(L, "MAX_PARSED_TRIANGLES", 100) + err = refuse(path, "LIMIT_EXCEEDED") + assert "101" in err.message and "100" in err.message + monkeypatch.undo() + stl = fx.binary_stl(tmp_path / "c.stl") + monkeypatch.setattr(L, "MAX_PARSED_TRIANGLES", 11) + assert "12" in refuse(stl, "LIMIT_EXCEEDED").message + + +def test_rendered_triangles_count_every_repetition(tmp_path, monkeypatch): + path = fx.three_mf(tmp_path / "r.3mf", fx.model_xml([fx.cube_object("1")], [("1", fx.tf(i * 20, 0, 0)) for i in range(5)])) + monkeypatch.setattr(L, "MAX_RENDERED_TRIANGLES", 59) # 5 x 12 = 60 + refuse(path, "LIMIT_EXCEEDED") + monkeypatch.setattr(L, "MAX_RENDERED_TRIANGLES", 60) + assert scene.build_scene_dict(str(path), REV)["counts"]["rendered_triangles"] == 60 + + +def test_node_and_depth_budgets(tmp_path, monkeypatch): + path = fx.three_mf(tmp_path / "n.3mf", fx.model_xml([fx.cube_object("1")], [("1", fx.tf(i * 20, 0, 0)) for i in range(5)])) + monkeypatch.setattr(L, "MAX_NODES", 4) + refuse(path, "LIMIT_EXCEEDED") + chain = [fx.cube_object("0")] + [fx.composite_object(str(i), [(str(i - 1), None, None)]) for i in range(1, 70)] + deep = fx.three_mf(tmp_path / "d.3mf", fx.model_xml(chain, [("69", None)])) + refuse(deep, "LIMIT_EXCEEDED") + monkeypatch.setattr(L, "MAX_NODES", 2000) + shallow = fx.three_mf(tmp_path / "s.3mf", fx.model_xml(chain[:60], [("59", None)])) + assert scene.build_scene_dict(str(shallow), REV)["counts"]["nodes"] == 60 + + +def test_whole_body_cap_applies_to_the_exact_bytes(tmp_path, monkeypatch): + path = fx.plain_cube_3mf(tmp_path / "c.3mf") + size = len(scene.build_scene(str(path), REV)) + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", size) + assert len(scene.build_scene(str(path), REV)) == size + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", size - 1) + refuse(path, "LIMIT_EXCEEDED") + + +def test_geometry_preflight_rejects_before_any_encoding(tmp_path, monkeypatch): + path = fx.plain_cube_3mf(tmp_path / "c.3mf") + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", 100) + monkeypatch.setattr(scene, "_encode_f32", lambda *a: pytest.fail("encoded before the size check")) + refuse(path, "LIMIT_EXCEEDED") + + +def test_stl_budgets(tmp_path, monkeypatch): + stl = fx.binary_stl(tmp_path / "c.stl") + monkeypatch.setattr(L, "MAX_PARSED_TRIANGLES", 11) + refuse(stl, "LIMIT_EXCEEDED") + monkeypatch.undo() + text = tmp_path / "t.stl" + text.write_text("solid x\n" + "facet normal 0 0 0\nouter loop\nvertex 0 0 0\nvertex 1 0 0\nvertex 0 1 0\nendloop\nendfacet\n" * 5 + "endsolid\n") + assert scene.build_scene_dict(str(text), REV)["counts"]["triangles"] == 5 + monkeypatch.setattr(L, "MAX_PARSED_TRIANGLES", 4) + err = refuse(text, "LIMIT_EXCEEDED") # ASCII is line-streamed with a running counter + assert "5" in err.message and "4" in err.message + garbage = tmp_path / "g.stl" + garbage.write_bytes(b"not an stl at all" * 20) + monkeypatch.undo() + refuse(garbage, "INVALID_GEOMETRY") + + +# ------------------------------------------------------------------------------ cooperative interruption + +class CountingControl(scene.Control): + def __init__(self, cancel_at=None): + self.checks, self.cancel_at = 0, cancel_at + + def check(self): + self.checks += 1 + if self.cancel_at is not None and self.checks >= self.cancel_at: + raise scene.SceneCancelled() + + +def test_cancel_is_observed_inside_parsing_and_encoding(tmp_path): + path = fx.big_grid_3mf(tmp_path / "big.3mf", 30_000) + full = CountingControl() + scene.build_scene_dict(str(path), REV, full) + assert full.checks > 100 # checked far more often than once per phase + for at in (3, full.checks // 2, full.checks - 2): + c = CountingControl(cancel_at=at) + with pytest.raises(scene.SceneCancelled): + scene.build_scene_dict(str(path), REV, c) + assert c.checks == at # stops at the first check after the flag + + +def test_deadline_is_a_timeout_not_a_hang(tmp_path): + path = fx.big_grid_3mf(tmp_path / "big.3mf", 30_000) + + class Late(scene.Control): + def check(self): + raise scene.SceneTimeout() + with pytest.raises(scene.SceneTimeout): + scene.build_scene_dict(str(path), REV, Late()) diff --git a/backend/tests/test_scene_jobs.py b/backend/tests/test_scene_jobs.py new file mode 100644 index 00000000..c12bf68a --- /dev/null +++ b/backend/tests/test_scene_jobs.py @@ -0,0 +1,746 @@ +"""Scene jobs: state machine, worker model, snapshots, retention, cancellation, wedge backstop, leaks.""" +import hashlib +import json +import os +import subprocess +import sys +import threading +import time +from pathlib import Path + +import jsonschema +import pytest + +from snapstudio_api import scene_jobs as sj +from snapstudio_core import scene, scene_limits as L +from tests import scene_fixtures as fx +from tests.test_scene_contract import assert_valid, sub_validator + +RID = iter(range(10**6)) + + +def rid() -> str: + return f"req-{next(RID):08d}" + + +def validate(name, obj): + errors = list(sub_validator(name).iter_errors(obj)) + assert not errors, errors[0].message + + +def wait_for(fn, timeout=10.0, step=0.01): + end = time.monotonic() + timeout + while time.monotonic() < end: + value = fn() + if value: + return value + time.sleep(step) + raise AssertionError("timed out") + + +def wait_state(jobs, job_id, *states, timeout=10.0): + return wait_for(lambda: (lambda s: s if s["state"] in states else None)(jobs.status(job_id)), timeout) + + +def fake_builder(gate=None, honor_cancel=True, started=None, body=b'{"ok":true}'): + def builder(path, revision, ctl): + if started is not None: + started.set() + if gate is not None: + while not gate.wait(0.005): + if honor_cancel: + ctl.check() + return body + return builder + + +@pytest.fixture +def cube(tmp_path): + return str(fx.bambu_project(tmp_path / "cube.3mf", parts=2, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])])) + + +@pytest.fixture +def make(tmp_path): + made = [] + + def factory(**kw): + jobs = sj.SceneJobs(snapshot_root=str(tmp_path / "snap"), **kw) + made.append(jobs) + return jobs + yield factory + for jobs in made: + jobs.close() + + +def snapshot_files(jobs): + d = jobs.snapshot_dir() + return [] if d is None or not os.path.isdir(d) else os.listdir(d) + + +# ------------------------------------------------------------------------------ the happy path + +def test_start_status_result_and_snapshot_cleanup(make, cube): + jobs = make() + started = jobs.start(cube, rid()) + validate("job_start", started) + assert started["state"] in ("queued", "running") and started["replaced_job_id"] is None + status = wait_state(jobs, started["job_id"], "succeeded") + validate("job_status", status) + expected = hashlib.sha256(Path(cube).read_bytes()).hexdigest() + assert status["revision"] == expected and status["stage"] is None + code, body = jobs.result(started["job_id"]) + assert code == 200 and isinstance(body, bytes) + sc = json.loads(body) + assert_valid(sc) + assert sc["revision"] == expected and sc["counts"]["nodes"] == 3 + wait_for(lambda: not jobs.worker_alive()) + assert snapshot_files(jobs) == [] # the worker removed its own snapshot + assert body == scene.serialize(sc) # compact bytes, serialized exactly once + + +def test_original_file_bytes_and_mtime_unchanged_after_a_job(make, cube): + jobs = make() + old = os.stat(cube).st_mtime_ns - 5 * 10**9 + os.utime(cube, ns=(old, old)) + before = (Path(cube).read_bytes(), os.stat(cube).st_mtime_ns) + job = jobs.start(cube, rid()) + wait_state(jobs, job["job_id"], "succeeded") + jobs.result(job["job_id"]) + assert (Path(cube).read_bytes(), os.stat(cube).st_mtime_ns) == before + + +def test_result_while_not_ready_and_unknown_job(make, cube): + gate = threading.Event() + jobs = make(builder=fake_builder(gate)) + job = jobs.start(cube, rid()) + wait_state(jobs, job["job_id"], "running") + code, body = jobs.result(job["job_id"]) + assert code == 409 and body["error"] == "NOT_READY" and body["state"] == "running" + validate("error_response", body) + gate.set() + wait_state(jobs, job["job_id"], "succeeded") + with pytest.raises(sj.JobError) as e: + jobs.status("0" * 32) + assert (e.value.http, e.value.code) == (404, "EXPIRED") + + +def test_failed_job_reports_a_coded_error_with_422(make, tmp_path): + bad = tmp_path / "bad.3mf" + bad.write_bytes(b"not a zip" * 20) + jobs = make() + job = jobs.start(str(bad), rid()) + status = wait_state(jobs, job["job_id"], "failed") + assert status["error"]["code"] == "INVALID_ARCHIVE" + validate("job_status", status) + code, body = jobs.result(job["job_id"]) + assert code == 422 and body["error"] == "INVALID_ARCHIVE" + validate("error_response", body) + wait_for(lambda: not jobs.worker_alive()) + assert snapshot_files(jobs) == [] + + +def test_start_validation(make, tmp_path): + jobs = make() + txt = tmp_path / "a.txt" + txt.write_text("x") + with pytest.raises(sj.JobError) as e: + jobs.start(str(txt), rid()) + assert (e.value.http, e.value.code) == (422, "UNSUPPORTED_FORMAT") + with pytest.raises(sj.JobError) as e: + jobs.start(str(tmp_path / "missing.3mf"), rid()) + assert (e.value.http, e.value.code) == (400, "INVALID_REQUEST") + + +# ------------------------------------------------------------------------------ idempotency and request ids + +def test_same_request_id_and_path_returns_the_existing_job(make, cube): + jobs = make() + r = rid() + a = jobs.start(cube, r) + b = jobs.start(cube, r) + assert a["job_id"] == b["job_id"] and b["replaced_job_id"] is None + wait_state(jobs, a["job_id"], "succeeded") + c = jobs.start(cube, r) # still retained + assert c["job_id"] == a["job_id"] and c["state"] == "succeeded" + assert jobs.retained() == 1 + + +def test_request_id_reused_for_another_path_is_a_conflict(make, cube, tmp_path): + other = fx.plain_cube_3mf(tmp_path / "other.3mf") + jobs = make() + r = rid() + jobs.start(cube, r) + with pytest.raises(sj.JobError) as e: + jobs.start(str(other), r) + assert (e.value.http, e.value.code, e.value.extra["reason"]) == (409, "INVALID_REQUEST", "request_id reused for another path") + + +def test_reused_request_id_with_changed_file_is_source_changed(make, cube): + jobs = make() + r = rid() + job = jobs.start(cube, r) + wait_state(jobs, job["job_id"], "succeeded") + Path(cube).write_bytes(Path(cube).read_bytes() + b"\x00") + with pytest.raises(sj.JobError) as e: + jobs.start(cube, r) + assert (e.value.http, e.value.code) == (409, "SOURCE_CHANGED") + fresh = jobs.start(cube, rid()) # a fresh request_id starts a new scene + assert fresh["job_id"] != job["job_id"] + + +def test_expected_revision_mismatch_is_source_changed(make, cube): + jobs = make() + job = jobs.start(cube, rid()) + status = wait_state(jobs, job["job_id"], "succeeded") + assert jobs.result(job["job_id"], status["revision"])[0] == 200 + with pytest.raises(sj.JobError) as e: + jobs.result(job["job_id"], "f" * 64) + assert (e.value.http, e.value.code) == (409, "SOURCE_CHANGED") + + +def test_source_replaced_mid_job_cannot_tear_the_scene(make, cube): + """The snapshot is the single source: replacing the original after the copy changes nothing.""" + original = Path(cube).read_bytes() + seen = {} + gate = threading.Event() + started = threading.Event() + + def builder(path, revision, ctl): + started.set() + while not gate.wait(0.005): + ctl.check() + seen["snapshot"] = Path(path).read_bytes() + seen["revision"] = revision + return b"{}" + jobs = make(builder=builder) + job = jobs.start(cube, rid()) + started.wait(5) + Path(cube).write_bytes(b"replaced with something else entirely") + gate.set() + status = wait_state(jobs, job["job_id"], "succeeded") + assert seen["snapshot"] == original + assert seen["revision"] == status["revision"] == hashlib.sha256(original).hexdigest() + + +def test_file_changing_during_the_copy_is_source_changed(make, cube, monkeypatch): + monkeypatch.setattr(L, "SNAPSHOT_CHUNK_BYTES", 256) + calls = {"n": 0} + real = sj._Control.progress + + def tamper(self, stage, completed, total): + calls["n"] += 1 + if calls["n"] == 2 and stage == "reading": + with open(cube, "ab") as fh: + fh.write(b"appended while copying") + real(self, stage, completed, total) + monkeypatch.setattr(sj._Control, "progress", tamper) + jobs = make() + job = jobs.start(cube, rid()) + status = wait_state(jobs, job["job_id"], "failed") + assert status["error"]["code"] == "SOURCE_CHANGED" + wait_for(lambda: not jobs.worker_alive()) + assert snapshot_files(jobs) == [] + + +# ------------------------------------------------------------------------------ cancel, replace, races + +def test_replace_cancels_the_running_job_and_queues_behind_it(make, cube): + gate_a = threading.Event() + started = threading.Event() + order = [] + + def builder(path, revision, ctl): + order.append(os.path.basename(path)) + started.set() + if len(order) == 1: + while not gate_a.wait(0.005): + ctl.check() + return b"{}" + jobs = make(builder=builder) + a = jobs.start(cube, rid()) + started.wait(5) + b = jobs.start(cube, rid()) + assert b["replaced_job_id"] == a["job_id"] and b["state"] == "queued" + assert jobs.status(a["job_id"])["state"] == "cancelled" + assert jobs.result(a["job_id"])[1]["error"] == "CANCELLED" + wait_state(jobs, b["job_id"], "succeeded") # b ran only after a's thread observed the cancel + assert jobs.status(a["job_id"])["state"] == "cancelled" + assert len(order) == 2 and not gate_a.is_set() + + +def test_at_most_one_queued_job_newest_wins(make, cube): + gate = threading.Event() + started = threading.Event() + jobs = make(builder=fake_builder(gate, honor_cancel=False, started=started)) # holds the worker even when cancelled + a = jobs.start(cube, rid()) + started.wait(5) + b = jobs.start(cube, rid()) + c = jobs.start(cube, rid()) + assert c["replaced_job_id"] == b["job_id"] + assert jobs.status(b["job_id"])["state"] == "cancelled" and jobs.status(c["job_id"])["state"] == "queued" + assert jobs.status(a["job_id"])["state"] == "cancelled" + gate.set() + wait_state(jobs, c["job_id"], "succeeded") + + +def test_cancel_queued_running_and_terminal_is_idempotent(make, cube): + gate = threading.Event() + started = threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + a = jobs.start(cube, rid()) + started.wait(5) + s = jobs.cancel(a["job_id"]) + assert s["state"] == "cancelled" and s["error"]["code"] == "CANCELLED" + assert jobs.cancel(a["job_id"])["state"] == "cancelled" + wait_for(lambda: not jobs.worker_alive()) + b = jobs.start(cube, rid()) + gate.set() + wait_state(jobs, b["job_id"], "succeeded") + assert jobs.cancel(b["job_id"])["state"] == "succeeded" # a late cancel on a terminal job is a no-op + assert jobs.result(b["job_id"])[0] == 200 + + +def test_cancel_wins_the_race_against_a_worker_that_finishes_anyway(make, cube): + gate = threading.Event() + started = threading.Event() + jobs = make(builder=fake_builder(gate, honor_cancel=False, started=started, body=b'{"late":true}')) + a = jobs.start(cube, rid()) + started.wait(5) + jobs.cancel(a["job_id"]) + gate.set() # the worker now returns a result + wait_for(lambda: not jobs.worker_alive()) + assert jobs.status(a["job_id"])["state"] == "cancelled" # first terminal wins; the late result is discarded + assert jobs.result(a["job_id"]) == (409, {"error": "CANCELLED", "message": "That scene was cancelled."}) + assert jobs._jobs[a["job_id"]].result is None + + +def test_cancel_after_completion_does_not_change_the_result(make, cube): + jobs = make(builder=fake_builder()) + a = jobs.start(cube, rid()) + wait_state(jobs, a["job_id"], "succeeded") + jobs.cancel(a["job_id"]) + assert jobs.result(a["job_id"]) == (200, b'{"ok":true}') + + +def test_cancel_is_observed_within_two_seconds_on_a_real_scene(make, tmp_path): + big = fx.big_grid_3mf(tmp_path / "big.3mf", 90_000) + jobs = make() + job = jobs.start(str(big), rid()) + wait_for(lambda: jobs.status(job["job_id"])["state"] == "running" and jobs.status(job["job_id"])["stage"] in ("parsing", "encoding")) + t = time.monotonic() + jobs.cancel(job["job_id"]) + wait_for(lambda: not jobs.worker_alive(), timeout=5) + assert time.monotonic() - t <= 2.0 + assert jobs.status(job["job_id"])["state"] == "cancelled" + assert snapshot_files(jobs) == [] + + +# ------------------------------------------------------------------------------ deadline and wedge + +def test_deadline_is_a_timeout_and_the_clock_starts_when_running(make, cube): + gate = threading.Event() + started = threading.Event() + jobs = make(deadline=0.3, builder=fake_builder(gate, started=started)) + a = jobs.start(cube, rid()) + started.wait(5) + b = jobs.start(cube, rid()) # queued: its clock has not started + assert jobs.status(a["job_id"])["state"] == "cancelled" + wait_state(jobs, b["job_id"], "running") + time.sleep(0.05) + assert jobs.status(b["job_id"])["state"] == "running" + status = wait_state(jobs, b["job_id"], "failed", timeout=3) + assert status["error"]["code"] == "TIMEOUT" + assert jobs.result(b["job_id"])[0] == 422 + wait_for(lambda: not jobs.worker_alive()) + + +def test_worker_that_ignores_cancel_is_wedged_fails_closed_and_recovers(make, cube): + gate = threading.Event() + started = threading.Event() + jobs = make(wedge_grace=0.25, builder=fake_builder(gate, honor_cancel=False, started=started)) + a = jobs.start(cube, rid()) + started.wait(5) + jobs.cancel(a["job_id"]) + assert not jobs.wedged() + b = jobs.start(cube, rid()) # within the grace period: queued behind the cancelled job + assert b["state"] == "queued" + time.sleep(0.4) + assert jobs.wedged() + with pytest.raises(sj.JobError) as e: + jobs.start(cube, rid()) + assert (e.value.http, e.value.code) == (503, "WORKER_WEDGED") + validate("error_response", e.value.body()) + gate.set() # the stuck call finally returns + wait_state(jobs, b["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + assert not jobs.wedged() + assert jobs.start(cube, rid())["state"] in ("queued", "running") # recovered + + +def test_deadline_wedge_reports_timeout_and_rejects_new_starts(make, cube): + gate = threading.Event() + started = threading.Event() + jobs = make(deadline=0.15, wedge_grace=0.15, builder=fake_builder(gate, honor_cancel=False, started=started)) + a = jobs.start(cube, rid()) + started.wait(5) + wait_for(lambda: jobs.status(a["job_id"])["state"] == "failed", timeout=3) + assert jobs.status(a["job_id"])["error"]["code"] == "TIMEOUT" + assert jobs.wedged() + with pytest.raises(sj.JobError) as e: + jobs.start(cube, rid()) + assert e.value.code == "WORKER_WEDGED" + gate.set() + wait_for(lambda: not jobs.worker_alive()) + assert not jobs.wedged() and jobs.status(a["job_id"])["state"] == "failed" + + +# ------------------------------------------------------------------------------ retention + +def test_ttl_and_terminal_cap_with_an_injected_clock(make, cube): + now = [1000.0] + jobs = make(clock=lambda: now[0], builder=fake_builder(), ttl=120.0, max_terminal=8) + ids = [] + for _ in range(12): + j = jobs.start(cube, rid()) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + now[0] += 1.0 + ids.append(j["job_id"]) + assert jobs.retained() == 8 + with pytest.raises(sj.JobError) as e: + jobs.status(ids[0]) # oldest evicted + assert (e.value.http, e.value.code) == (404, "EXPIRED") + assert jobs.status(ids[-1])["state"] == "succeeded" + now[0] += 200.0 + with pytest.raises(sj.JobError): + jobs.status(ids[-1]) # TTL: 120 s after it became terminal + assert jobs.retained() == 0 + + +def test_failed_and_cancelled_jobs_are_retained_like_succeeded_ones(make, cube, tmp_path): + now = [0.0] + gate = threading.Event() + jobs = make(clock=lambda: now[0], builder=fake_builder(gate)) + a = jobs.start(cube, rid()) + wait_state(jobs, a["job_id"], "running") + jobs.cancel(a["job_id"]) + wait_for(lambda: not jobs.worker_alive()) + now[0] += 119.0 + assert jobs.status(a["job_id"])["state"] == "cancelled" + now[0] += 2.0 + with pytest.raises(sj.JobError): + jobs.status(a["job_id"]) + + +def test_eviction_removes_a_snapshot_the_worker_could_not_delete(make, cube, monkeypatch): + now = [0.0] + jobs = make(clock=lambda: now[0], builder=fake_builder(), ttl=10.0) + real_unlink = os.unlink + state = {"fail": True} + + def flaky(path, *a, **k): + if state["fail"] and str(path).endswith(".3mf"): + state["fail"] = False + raise PermissionError("sharing violation") + return real_unlink(path, *a, **k) + monkeypatch.setattr(sj.os, "unlink", flaky) + j = jobs.start(cube, rid()) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + assert len(snapshot_files(jobs)) == 1 # the worker's unlink failed + now[0] += 11.0 + with pytest.raises(sj.JobError): + jobs.status(j["job_id"]) + assert snapshot_files(jobs) == [] # eviction retried it (missing-file tolerant) + + +# ------------------------------------------------------------------------------ snapshot limits + +def test_snapshot_ceiling_and_disk_preflight(make, cube, monkeypatch): + jobs = make() + monkeypatch.setattr(L, "SNAPSHOT_MAX_BYTES", 1000) + a = jobs.start(cube, rid()) + assert wait_state(jobs, a["job_id"], "failed")["error"]["code"] == "LIMIT_EXCEEDED" + wait_for(lambda: not jobs.worker_alive()) + assert snapshot_files(jobs) == [] + monkeypatch.setattr(L, "SNAPSHOT_MAX_BYTES", 128 * 1024 * 1024) + monkeypatch.setattr(sj.shutil, "disk_usage", lambda p: type("U", (), {"free": 10})()) + b = jobs.start(cube, rid()) + status = wait_state(jobs, b["job_id"], "failed") + assert status["error"]["code"] == "INTERNAL" and "disk" in status["error"]["message"] + wait_for(lambda: not jobs.worker_alive()) + assert snapshot_files(jobs) == [] + + +def test_response_cap_surfaces_as_limit_exceeded(make, cube, monkeypatch): + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", 500) + jobs = make() + j = jobs.start(cube, rid()) + assert wait_state(jobs, j["job_id"], "failed")["error"]["code"] == "LIMIT_EXCEEDED" + assert jobs.result(j["job_id"])[0] == 422 + + +# ------------------------------------------------------------------------------ startup sweep + +def test_sweep_removes_only_dead_and_old_directories_and_never_a_live_ones(tmp_path): + root = tmp_path / "scene-tmp" + root.mkdir() + dead = subprocess.Popen([sys.executable, "-c", "pass"]) + dead.wait() + live = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + old = L.SNAPSHOT_DIR_MIN_AGE_SECONDS + 100 + try: + def mk(name, age_s=0): + d = root / name + d.mkdir() + (d / "x.3mf").write_bytes(b"x") + if age_s: + t = time.time() - age_s + os.utime(d, (t, t)) + return d + d_dead_old = mk(f"{dead.pid}-aaaa1111", old) + d_dead_young = mk(f"{dead.pid}-aaaa2222") + d_live = mk(f"{live.pid}-bbbb2222") + d_live_ancient = mk(f"{live.pid}-cccc3333", age_s=10 * 24 * 3600) # a live process's folder, however old + d_own = mk(f"{os.getpid()}-dddd4444", age_s=10 * 24 * 3600) + d_junk = mk("not-a-snapshot-dir", old) + removed = sj.sweep_stale(str(root)) + assert removed == [d_dead_old.name] + for survivor in (d_dead_young, d_live, d_live_ancient, d_own, d_junk): + assert survivor.exists(), survivor.name + finally: + live.kill() + live.wait() + assert sj.sweep_stale(str(tmp_path / "does-not-exist")) == [] + + +def test_sweep_tolerates_a_sharing_violation(tmp_path, monkeypatch): + root = tmp_path / "scene-tmp" + root.mkdir() + dead = subprocess.Popen([sys.executable, "-c", "pass"]) + dead.wait() + d = root / f"{dead.pid}-aaaa1111" + d.mkdir() + t = time.time() - L.SNAPSHOT_DIR_MIN_AGE_SECONDS - 10 + os.utime(d, (t, t)) + real = sj.shutil.rmtree + monkeypatch.setattr(sj.shutil, "rmtree", lambda *a, **k: (_ for _ in ()).throw(PermissionError("in use"))) + assert sj.sweep_stale(str(root)) == [] and d.exists() # no crash; left for the next start + monkeypatch.setattr(sj.shutil, "rmtree", real) + assert sj.sweep_stale(str(root)) == [d.name] + + +def test_posix_liveness_branch_by_monkeypatch(monkeypatch): + monkeypatch.setattr(sj.sys, "platform", "linux") + calls = [] + + def fake_kill(pid, sig): + calls.append((pid, sig)) + if pid == 111: + raise ProcessLookupError() + if pid == 222: + raise PermissionError() + monkeypatch.setattr(sj.os, "kill", fake_kill) + assert sj.pid_alive(111) is False # no such process + assert sj.pid_alive(222) is True # exists, not ours: alive + assert sj.pid_alive(333) is True + assert sj.pid_alive(os.getpid()) is True and sj.pid_alive(0) is False + assert (111, 0) in calls + + +def test_each_registry_uses_its_own_pid_nonce_directory(make, cube): + a, b = make(builder=fake_builder()), make(builder=fake_builder()) + for jobs in (a, b): + wait_state(jobs, jobs.start(cube, rid())["job_id"], "succeeded") + assert a.snapshot_dir() != b.snapshot_dir() + assert os.path.basename(a.snapshot_dir()).startswith(f"{os.getpid()}-") + + +# ------------------------------------------------------------------------------ leak gate + +def test_fifty_start_cancel_cycles_leave_nothing_behind(make, tmp_path): + big = str(fx.big_grid_3mf(tmp_path / "g.3mf", 20_000)) + small = str(fx.plain_cube_3mf(tmp_path / "s.3mf")) + baseline = threading.active_count() + jobs = make() + wedged_ever = False + for i in range(50): + job = jobs.start(big if i % 2 else small, rid()) + if i % 3 == 0: + wait_for(lambda: jobs.status(job["job_id"])["state"] != "queued") + jobs.cancel(job["job_id"]) + wedged_ever = wedged_ever or jobs.wedged() + assert jobs.retained() <= L.MAX_TERMINAL_JOBS + 2 # + the running/queued pair + jobs.close() + wait_for(lambda: not jobs.worker_alive(), timeout=10) + assert jobs.retained() <= L.MAX_TERMINAL_JOBS # retained jobs never exceed the cap once idle + assert all(j.state in sj.TERMINAL for j in jobs._jobs.values()) + assert snapshot_files(jobs) == [] # temp snapshots = 0 + assert all(j.snapshot is None for j in jobs._jobs.values()) + assert threading.active_count() == baseline # thread count back to baseline + assert not wedged_ever and not jobs.wedged() # WORKER_WEDGED never occurred + + +# ------------------------------------------------------------------------------ repair round 1 + +import contextlib +import errno + + +def poke(jobs): + """Any API call runs housekeeping; an unknown id is the cheapest one.""" + with contextlib.suppress(sj.JobError): + jobs.status("0" * 32) + + +def test_a10_a_retained_job_evicted_while_hashing_is_never_handed_back(make, cube, monkeypatch): + now = [0.0] + jobs = make(clock=lambda: now[0], builder=fake_builder(), ttl=120.0) + r = rid() + a = jobs.start(cube, r) + status = wait_state(jobs, a["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + + def hash_while_time_passes(path): + now[0] += 500.0 # the TTL expires during the hash + return status["revision"] + monkeypatch.setattr(jobs, "_current_hash", hash_while_time_passes) + b = jobs.start(cube, r) + assert b["job_id"] != a["job_id"] + assert jobs.status(b["job_id"])["request_id"] == r # the returned job is registered + with pytest.raises(sj.JobError): + jobs.status(a["job_id"]) + + +def test_a11_a_snapshot_that_resists_deletion_is_retried_with_backoff(make, cube, monkeypatch): + now = [0.0] + jobs = make(clock=lambda: now[0], builder=fake_builder(), ttl=10.0) + real = os.unlink + fails = {"n": 0} + + def flaky(path, *a, **k): + if str(path).endswith(".3mf") and fails["n"] < 3: + fails["n"] += 1 + raise PermissionError("sharing violation") + return real(path, *a, **k) + monkeypatch.setattr(sj.os, "unlink", flaky) + j = jobs.start(cube, rid()) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + assert len(snapshot_files(jobs)) == 1 # failure 1: the worker's own unlink + now[0] += 11.0 + poke(jobs) # TTL eviction: failure 2, path goes on the retry list + with pytest.raises(sj.JobError): + jobs.status(j["job_id"]) + assert len(snapshot_files(jobs)) == 1 and len(jobs._pending) == 1 + now[0] += 2.5 + poke(jobs) # first retry: failure 3, longer back-off + assert len(snapshot_files(jobs)) == 1 and jobs._pending[0][1] >= 2 + now[0] += 1.0 + poke(jobs) # not due yet: no attempt + assert fails["n"] == 3 + now[0] += 10.0 + poke(jobs) # now it succeeds and the path is forgotten + assert snapshot_files(jobs) == [] and jobs._pending == [] + + +def test_d2_the_snapshot_directory_is_recreated_for_every_job(make, cube): + jobs = make(builder=fake_builder()) + first = jobs.start(cube, rid()) + wait_state(jobs, first["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + import shutil + shutil.rmtree(jobs.snapshot_dir()) # something removed the per-process folder + second = jobs.start(cube, rid()) + assert wait_state(jobs, second["job_id"], "succeeded")["error"] is None + + +def test_d2_only_a_real_enospc_is_reported_as_a_disk_space_problem(make, cube, monkeypatch): + jobs = make(builder=fake_builder()) + real_open = open + + def failing(code): + def fake(path, mode="r", *a, **k): + if "w" in mode and str(path).endswith(".3mf"): + raise OSError(code, "boom") + return real_open(path, mode, *a, **k) + return fake + monkeypatch.setattr(sj, "open", failing(errno.EACCES), raising=False) + a = jobs.start(cube, rid()) + msg = wait_state(jobs, a["job_id"], "failed")["error"]["message"] + assert "disk space" not in msg and "temporary copy" in msg + wait_for(lambda: not jobs.worker_alive()) + monkeypatch.setattr(sj, "open", failing(errno.ENOSPC), raising=False) + b = jobs.start(cube, rid()) + assert "disk space" in wait_state(jobs, b["job_id"], "failed")["error"]["message"] + + +def test_d6a_a_cancelled_job_whose_worker_is_still_running_is_not_evicted(make, cube): + now = [0.0] + gate, started = threading.Event(), threading.Event() + jobs = make(clock=lambda: now[0], ttl=10.0, max_terminal=0, builder=fake_builder(gate, honor_cancel=False, started=started)) + a = jobs.start(cube, rid()) + started.wait(5) + jobs.cancel(a["job_id"]) + now[0] += 1000.0 + poke(jobs) + assert jobs.status(a["job_id"])["state"] == "cancelled" # worker not done: not evictable, whatever the TTL and cap + gate.set() + wait_for(lambda: not jobs.worker_alive()) + with pytest.raises(sj.JobError) as e: + jobs.status(a["job_id"]) + assert e.value.code == "EXPIRED" + + +def test_d6b_the_deadline_starts_when_the_job_runs_not_when_it_was_created(make, cube): + gate, started = threading.Event(), threading.Event() + calls = [] + + def builder(path, revision, ctl): + calls.append(1) + if len(calls) == 1: + started.set() + gate.wait(5) # the first job holds the worker, ignoring cancel + return b"{}" + ctl.check() + return b'{"second":true}' + jobs = make(deadline=0.4, wedge_grace=30.0, builder=builder) + a = jobs.start(cube, rid()) + started.wait(5) + b = jobs.start(cube, rid()) # queued behind a + time.sleep(0.7) # longer than the deadline, all of it spent queued + gate.set() + assert wait_state(jobs, b["job_id"], "succeeded", "failed")["state"] == "succeeded" + assert jobs.status(a["job_id"])["state"] == "cancelled" + + +def test_d6e_a_size_change_without_an_mtime_change_is_source_changed(make, cube, monkeypatch): + monkeypatch.setattr(L, "SNAPSHOT_CHUNK_BYTES", 256) + st = os.stat(cube) + calls = {"n": 0} + real = sj._Control.progress + + def tamper(self, stage, completed, total): + calls["n"] += 1 + if calls["n"] == 2 and stage == "reading": + with open(cube, "ab") as fh: + fh.write(b"grown") + os.utime(cube, ns=(st.st_atime_ns, st.st_mtime_ns)) # put the mtime back: only the size betrays it + real(self, stage, completed, total) + monkeypatch.setattr(sj._Control, "progress", tamper) + jobs = make() + job = jobs.start(cube, rid()) + assert wait_state(jobs, job["job_id"], "failed")["error"]["code"] == "SOURCE_CHANGED" + wait_for(lambda: not jobs.worker_alive()) + assert snapshot_files(jobs) == [] + + +def test_close_removes_the_per_process_directory_best_effort(make, cube): + jobs = make(builder=fake_builder()) + j = jobs.start(cube, rid()) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + d = jobs.snapshot_dir() + assert os.path.isdir(d) + jobs.close() + assert not os.path.exists(d) diff --git a/backend/tests/test_scene_ordering.py b/backend/tests/test_scene_ordering.py new file mode 100644 index 00000000..eb35d30d --- /dev/null +++ b/backend/tests/test_scene_ordering.py @@ -0,0 +1,876 @@ +"""Session-based start ordering at the engine boundary. + +Invariants (each test name carries its tag): + I1 Delayed starts cannot become valid through bookkeeping eviction. + I2 Cancellation cannot be forgotten while the corresponding start remains admissible. + I3 Idempotency is scoped to (session, request) and ownership is checked consistently on every path. + I4 Capacity exhaustion or session expiry fails EXPLICITLY; nothing old is ever silently revived. + +No sleep decides any outcome: the abandoned start A is held at the pre-registration seam +(`SceneJobs._check_source`, which runs before the registry lock) with a threading.Event, and time is a fake clock. +""" +import os +import threading + +import pytest + +from snapstudio_api import scene_jobs as sj +from snapstudio_core import scene_limits as L +from tests import scene_fixtures as fx +from tests.test_scene_api import call, server # noqa: F401 (the loopback server fixture) +from tests.test_scene_contract import sub_validator +from tests.test_scene_jobs import fake_builder, wait_for + + +def _owner(jobs, job_id): + job = jobs._jobs.get(job_id) + return job.client_id if job is not None else None + + +def st(jobs, job_id): + """status() presenting the job's own session credential (looked up here, in the TEST, not by the engine).""" + return jobs.status(job_id, _owner(jobs, job_id)) + + +def cn(jobs, job_id): + return jobs.cancel(job_id, _owner(jobs, job_id)) + + +def wait_state(jobs, job_id, *states, timeout=10.0): + return wait_for(lambda: (lambda r: r if r["state"] in states else None)(st(jobs, job_id)), timeout) + + +@pytest.fixture +def cube(tmp_path): + return str(fx.plain_cube_3mf(tmp_path / "cube.3mf")) + + +@pytest.fixture +def make(tmp_path): + made = [] + + def factory(**kw): + jobs = sj.SceneJobs(snapshot_root=str(tmp_path / "snap"), builder=kw.pop("builder", fake_builder()), **kw) + made.append(jobs) + return jobs + yield factory + for jobs in made: + jobs.close() + + +def sess(jobs) -> str: + return jobs.open_session()["client_id"] + + +class HeldStart: + """Run jobs.start(...) on a thread that blocks at the pre-registration seam until released.""" + + def __init__(self, jobs, *args, **kwargs): + self.entered, self.release = threading.Event(), threading.Event() + self.result = self.error = None + real = jobs._check_source + + def seam(path): + real(path) + self.entered.set() + assert self.release.wait(10) + jobs._check_source = seam + self.thread = threading.Thread(target=self._run, args=(jobs, args, kwargs), daemon=True) + self.thread.start() + assert self.entered.wait(10) + jobs._check_source = real # later starts are not held + + def _run(self, jobs, args, kwargs): + try: + self.result = jobs.start(*args, **kwargs) + except sj.JobError as exc: + self.error = exc + + def finish(self): + self.release.set() + self.thread.join(10) + assert not self.thread.is_alive() + return self + + +def refused(jobs, code, *args, http=None): + with pytest.raises(sj.JobError) as e: + jobs.start(*args) + assert e.value.code == code, (e.value.code, e.value.message) + if http: + assert e.value.http == http + assert not list(sub_validator("error_response").iter_errors(e.value.body())) + return e.value + + +# ------------------------------------------------------------------ the session + +def test_i4_open_session_shape_and_expiry_ttl(make): + jobs = make() + out = jobs.open_session() + assert set(out) == {"client_id", "ttl_s"} and len(out["client_id"]) == 24 and out["ttl_s"] == L.SESSION_TTL_SECONDS + assert not list(sub_validator("session").iter_errors(out)) + assert sess(jobs) != out["client_id"] + + +def test_the_per_session_state_is_constant_size(make, cube): + jobs = make() + s = sess(jobs) + for seq in range(1, 6): + jobs.start(cube, f"req-{seq:08d}", s, seq) + jobs.cancel_request(s, f"req-x-{seq:06d}", seq) + assert set(vars(jobs._sessions[s])) == {"client_id", "last_touch", "watermark", "dead_through", "current"} + assert not hasattr(jobs, "_tombstones") and not hasattr(jobs, "_clients") + + +# ------------------------------------------------------------------ I1: a delayed start never becomes valid + +@pytest.mark.parametrize("wanted", ["running", "completed", "cancelled", "evicted"]) +def test_i1_a_delayed_abandoned_start_is_refused_whatever_became_of_the_wanted_one(make, cube, wanted): + now = [0.0] + gate, started = threading.Event(), threading.Event() + builder = fake_builder(gate, started=started) if wanted == "running" else fake_builder() + jobs = make(clock=lambda: now[0], builder=builder, ttl=10.0) + s = sess(jobs) + a = HeldStart(jobs, cube, "req-A-000001", s, 1) # abandoned, delayed in flight + b = jobs.start(cube, "req-B-000002", s, 2) # the wanted start gets in first + if wanted == "running": + assert started.wait(10) and st(jobs, b["job_id"])["state"] == "running" + else: + wait_state(jobs, b["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + if wanted == "cancelled": + jobs.cancel_request(s, "req-B-000002", 2) + if wanted == "evicted": + now[0] += 60.0 + with pytest.raises(sj.JobError): + st(jobs, b["job_id"]) # B's job is gone from the registry + a.finish() + assert a.error is not None and a.error.code in ("STALE_START", "CANCELLED_BEFORE_START") + if wanted == "cancelled": + assert a.error.code == "CANCELLED_BEFORE_START" + elif wanted == "evicted": + assert a.error.code == "STALE_START" # eviction did not make the late start valid + if wanted == "running": + assert st(jobs, b["job_id"])["state"] == "running" # B was neither cancelled nor replaced + gate.set() + wait_state(jobs, b["job_id"], "succeeded") + elif wanted == "completed": + assert st(jobs, b["job_id"])["state"] == "succeeded" + + +def test_i1_astra_repro_a_repeat_of_b_at_a_higher_seq_then_the_delayed_a(make, cube): + jobs = make() + s = sess(jobs) + b = jobs.start(cube, "req-B-000001", s, 1) + wait_state(jobs, b["job_id"], "succeeded") + a = HeldStart(jobs, cube, "req-A-000002", s, 2) + err = refused(jobs, "INVALID_REQUEST", cube, "req-B-000001", s, 3, http=400) # B repeated as seq 3 + assert err.extra["reason"] == "request id reused" + a.finish() + assert a.error.code == "STALE_START" # the watermark had already moved to 3 + assert st(jobs, b["job_id"])["state"] == "succeeded" + refused(jobs, "STALE_START", cube, "req-B-000001", s, 3) # the refused attempt is not an idempotent repeat + ok = jobs.start(cube, "req-B-000004", s, 4) # recovery: fresh request_id, higher seq + assert wait_state(jobs, ok["job_id"], "succeeded")["state"] == "succeeded" + + +def test_i1_a_stale_start_cannot_replace_a_running_wanted_start(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + s = sess(jobs) + a = HeldStart(jobs, cube, "req-A-000001", s, 1) + b = jobs.start(cube, "req-B-000002", s, 2) + started.wait(10) + a.finish() + assert a.error.code == "STALE_START" and st(jobs, b["job_id"])["state"] == "running" + gate.set() + + +def test_i1_an_exact_retry_of_an_evicted_start_is_stale_and_replaces_nothing(make, cube): + now = [0.0] + gate, started = threading.Event(), threading.Event() + calls = [] + + def builder(path, revision, ctl): + calls.append(1) + if len(calls) == 1: + return b"{}" + started.set() + while not gate.wait(0.005): + ctl.check() + return b"{}" + jobs = make(clock=lambda: now[0], ttl=10.0, builder=builder) + s, t = sess(jobs), sess(jobs) + first = jobs.start(cube, "req-E-000001", s, 1) + wait_state(jobs, first["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + other = jobs.start(cube, "req-O-000001", t, 1) # someone else's job is running + assert started.wait(10) + now[0] += 60.0 + with pytest.raises(sj.JobError): + st(jobs, first["job_id"]) + err = refused(jobs, "STALE_START", cube, "req-E-000001", s, 1, http=409) + assert st(jobs, other["job_id"])["state"] == "running" # and it did not replace the running job + gate.set() + wait_state(jobs, other["job_id"], "succeeded") + assert err + + +def test_i1_idempotent_repeat_while_registered_returns_the_same_job(make, cube): + jobs = make() + s = sess(jobs) + b = jobs.start(cube, "req-B-000002", s, 2) + again = jobs.start(cube, "req-B-000002", s, 2) + assert again["job_id"] == b["job_id"] and again["replaced_job_id"] is None + + +def test_i1_equal_seq_with_another_request_or_another_path_is_invalid(make, cube, tmp_path): + jobs = make() + s = sess(jobs) + jobs.start(cube, "req-B-000002", s, 2) + assert refused(jobs, "INVALID_REQUEST", cube, "req-X-000003", s, 2, http=400).extra["reason"] == "seq reused for another request" + other = str(fx.plain_cube_3mf(tmp_path / "other.3mf", size=11)) + refused(jobs, "INVALID_REQUEST", other, "req-B-000002", s, 2, http=400) + + +def test_i1_a_higher_seq_never_silently_returns_an_old_job(make, cube): + jobs = make() + s = sess(jobs) + j = jobs.start(cube, "req-R-000001", s, 1) + refused(jobs, "INVALID_REQUEST", cube, "req-R-000001", s, 2, http=400) + assert st(jobs, j["job_id"])["request_id"] == "req-R-000001" + + +def test_i1_stale_and_cancel_are_decided_before_source_changed(make, cube, monkeypatch): + for during, expected in (("newer", "STALE_START"), ("cancel", "CANCELLED_BEFORE_START"), ("expiry", "SESSION_EXPIRED")): + now = [0.0] + jobs = make(clock=lambda: now[0], session_ttl=100.0, ttl=10**9, max_terminal=100) + s = sess(jobs) + a = jobs.start(cube, "req-A-000001", s, 1) + wait_state(jobs, a["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + inside, release = threading.Event(), threading.Event() + + def hashing(path): # the file HAS changed, yet the answer is decided first + inside.set() + assert release.wait(10) + return "f" * 64 + monkeypatch.setattr(jobs, "_current_hash", hashing) + out = {} + + def repeat(): + try: + out["r"] = jobs.start(cube, "req-A-000001", s, 1) + except sj.JobError as exc: + out["e"] = exc + t = threading.Thread(target=repeat, daemon=True) + t.start() + assert inside.wait(10) + if during == "newer": + jobs.start(cube, "req-N-000002", s, 2) + elif during == "cancel": + jobs.cancel_request(s, "req-A-000001", 1) + else: + now[0] += 500.0 + jobs._jobs.pop(a["job_id"]) + jobs._by_request.pop((s, "req-A-000001")) # nothing registered: the session may now expire + jobs._housekeeping() + release.set() + t.join(10) + assert out["e"].code == expected, (during, out) + + +def test_without_any_newer_start_a_changed_file_is_still_source_changed(make, cube): + jobs = make() + s = sess(jobs) + a = jobs.start(cube, "req-A-000001", s, 1) + wait_state(jobs, a["job_id"], "succeeded") + with open(cube, "ab") as fh: + fh.write(b"\0") + refused(jobs, "SOURCE_CHANGED", cube, "req-A-000001", s, 1) + + +# ------------------------------------------------------------------ I2: cancellation is not forgotten + +def test_i2_a_cancel_that_arrives_before_the_start_refuses_it_and_every_lower_seq(make, cube): + jobs = make() + s = sess(jobs) + out = jobs.cancel_request(s, "req-A-000001", 3) # the client moved on from attempts 1..3 + assert out["job_id"] is None and out["state"] == "cancelled" and out["error"]["code"] == "CANCELLED" + assert not list(sub_validator("job_status").iter_errors(out)) + for seq, rid_ in ((3, "req-A-000001"), (2, "req-late-0002"), (1, "req-late-0001")): + refused(jobs, "CANCELLED_BEFORE_START", cube, rid_, s, seq, http=409) + assert jobs.retained() == 0 # nothing was registered + fresh = jobs.start(cube, "req-A-000004", s, 4) # a fresh request_id with a higher seq works + assert wait_state(jobs, fresh["job_id"], "succeeded")["state"] == "succeeded" + + +def test_i2_a_cancelled_start_stays_refused_after_its_job_is_evicted_and_after_unrelated_activity(make, cube): + now = [0.0] + jobs = make(clock=lambda: now[0], ttl=1.0) + s = sess(jobs) + a = jobs.start(cube, "req-A-000001", s, 1) + jobs.cancel_request(s, "req-A-000001", 1) + wait_for(lambda: not jobs.worker_alive()) + now[0] += 100.0 + with pytest.raises(sj.JobError): + st(jobs, a["job_id"]) # the cancelled job is long gone + for i in range(200): # a great deal of unrelated cancel traffic + jobs.cancel_request(s, f"req-noise-{i:05d}", 1) + refused(jobs, "CANCELLED_BEFORE_START", cube, "req-A-000001", s, 1) + assert len(vars(jobs._sessions[s])) == 5 # and the memory is still O(1) + + +def test_i2_cancelling_a_never_touches_b(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + s = sess(jobs) + a = jobs.start(cube, "req-A-000001", s, 1) + started.wait(10) + b = jobs.start(cube, "req-B-000002", s, 2) # replaces A as today + assert st(jobs, a["job_id"])["state"] == "cancelled" + jobs.cancel_request(s, "req-A-000001", 1) # cancelling A... + assert st(jobs, b["job_id"])["state"] in ("queued", "running") # ...never touches B (exact request match) + jobs.cancel_request(s, "req-C-nonexistent", 1) + assert st(jobs, b["job_id"])["state"] in ("queued", "running") + out = jobs.cancel_request(s, "req-B-000002", 2) + assert out["job_id"] == b["job_id"] and out["state"] == "cancelled" + gate.set() + + +def test_i2_cancel_by_job_id_is_unchanged(make, cube): + jobs = make() + s = sess(jobs) + j = jobs.start(cube, "req-J-000001", s, 1) + wait_state(jobs, j["job_id"], "succeeded") + assert cn(jobs, j["job_id"])["state"] == "succeeded" + with pytest.raises(sj.JobError) as e: + cn(jobs, "0" * 32) + assert e.value.code == "EXPIRED" + + +# ------------------------------------------------------------------ I3: (session, request) scoping and ownership + +def test_i3_the_same_request_id_in_two_sessions_gives_each_its_own_job_and_cancel(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + s, t = sess(jobs), sess(jobs) + a = jobs.start(cube, "req-same-0001", s, 1) + started.wait(10) + b = jobs.start(cube, "req-same-0001", t, 1) # same request_id and path, another session + assert a["job_id"] != b["job_id"] and b["replaced_job_id"] is None # H1: s's job id is not revealed to t + assert st(jobs, a["job_id"])["state"] == "cancelled" # (the newest start still replaces it, as before) + assert jobs.start(cube, "req-same-0001", t, 1)["job_id"] == b["job_id"] # idempotent inside its own scope + out = jobs.cancel_request(s, "req-same-0001", 1) + assert out["job_id"] == a["job_id"] + assert st(jobs, b["job_id"])["state"] in ("queued", "running") # t's job untouched: no leak across sessions + assert jobs.cancel_request(t, "req-same-0001", 1)["job_id"] == b["job_id"] + gate.set() + + +def test_i3_a_session_cannot_cancel_or_see_another_sessions_job_by_request(make, cube): + jobs = make() + s, t = sess(jobs), sess(jobs) + j = jobs.start(cube, "req-own-00001", s, 1) + wait_state(jobs, j["job_id"], "succeeded") + out = jobs.cancel_request(t, "req-own-00001", 1) + assert out["job_id"] is None + assert st(jobs, j["job_id"])["state"] == "succeeded" + assert jobs.start(cube, "req-own-00001", s, 1)["job_id"] == j["job_id"] + + +def test_i3_the_legacy_namespace_is_separate_and_unchanged(make, cube): + jobs = make() + s = sess(jobs) + legacy = jobs.start(cube, "req-ns-000001") + ordered = jobs.start(cube, "req-ns-000001", s, 1) + assert legacy["job_id"] != ordered["job_id"] + assert jobs.start(cube, "req-ns-000001")["job_id"] == legacy["job_id"] + + +def test_i3_many_sessions_start_concurrently_and_independently(make, cube): + jobs = make() + ids = [sess(jobs) for _ in range(8)] + barrier = threading.Barrier(len(ids)) + results, errors = {}, [] + + def go(i, cid): + barrier.wait(10) + try: + results[i] = jobs.start(cube, "req-same-0001", cid, 1) # every session reuses seq 1 AND the request_id + except sj.JobError as exc: + errors.append(exc) + threads = [threading.Thread(target=go, args=(i, c), daemon=True) for i, c in enumerate(ids)] + for t in threads: + t.start() + for t in threads: + t.join(10) + assert not errors and len(results) == len(ids) + assert len({r["job_id"] for r in results.values()}) == len(ids) + + +# ------------------------------------------------------------------ I4: expiry and capacity fail explicitly + +def test_i4_an_expired_session_refuses_start_and_cancel_and_a_late_delayed_start(make, cube): + now = [0.0] + jobs = make(clock=lambda: now[0], session_ttl=100.0) + s = sess(jobs) + late = HeldStart(jobs, cube, "req-A-000001", s, 1) # in flight when the session ends + now[0] += 500.0 + jobs.open_session() # any call prunes the idle-expired session + assert s not in jobs._sessions + late.finish() + assert late.error is not None and (late.error.http, late.error.code) == (409, "SESSION_EXPIRED") + assert s not in jobs._sessions and jobs.retained() == 0 # refused, never re-created, nothing registered + refused(jobs, "SESSION_EXPIRED", cube, "req-B-000002", s, 2) + with pytest.raises(sj.JobError) as e: + jobs.cancel_request(s, "req-B-000002", 2) + assert e.value.code == "SESSION_EXPIRED" + assert s not in jobs._sessions + + +@pytest.mark.parametrize("how", ["refused_start", "cancel", "status"]) +def test_i4_every_start_cancel_and_status_touches_the_session(make, cube, how): + """Each call keeps alive a session that would otherwise be idle-expired (the job it owned is already gone).""" + now = [0.0] + jobs = make(clock=lambda: now[0], session_ttl=100.0, ttl=100.0) + s = sess(jobs) + j = jobs.start(cube, "req-A-000002", s, 2) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + now[0] += 90.0 + if how == "status": + st(jobs, j["job_id"]) + elif how == "cancel": + jobs.cancel_request(s, "req-zzz-000001", 1) + else: + refused(jobs, "STALE_START", cube, "req-old-000001", s, 1) # a refused start still touches the session + now[0] += 90.0 # 180 s since creation, 90 s since the touch + jobs.open_session() # evicts the expired job (the session owned it until now) + jobs.open_session() # now the session is judged on its idle time alone + assert s in jobs._sessions + + +def test_i4_session_limit_is_explicit_when_nothing_is_prunable(make, cube): + jobs = make(max_sessions=3) + ids = [sess(jobs) for _ in range(3)] + with pytest.raises(sj.JobError) as e: + jobs.open_session() + assert (e.value.http, e.value.code) == (503, "SESSION_LIMIT") + assert not list(sub_validator("error_response").iter_errors(e.value.body())) + assert set(jobs._sessions) == set(ids) # a live session is never evicted to make room + + +def test_i4_an_idle_expired_session_without_a_job_is_pruned_to_make_room(make): + now = [0.0] + jobs = make(clock=lambda: now[0], max_sessions=2, session_ttl=100.0) + first, second = sess(jobs), sess(jobs) + now[0] += 50.0 + jobs._sessions[second].last_touch = now[0] # the second one stays active + now[0] += 60.0 # the first is idle 110 s (> ttl), the second 60 s + third = sess(jobs) + assert first not in jobs._sessions and second in jobs._sessions and third in jobs._sessions + + +def test_i4_a_session_that_owns_a_registered_job_is_never_pruned(make, cube): + now = [0.0] + jobs = make(clock=lambda: now[0], max_sessions=2, session_ttl=100.0, ttl=10**9, max_terminal=100) + s = sess(jobs) + j = jobs.start(cube, "req-A-000001", s, 1) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + other = sess(jobs) + now[0] += 10**6 # idle far beyond the ttl + with pytest.raises(sj.JobError) as e: + jobs.open_session() # `other` is prunable, `s` is not -> room for one + jobs.open_session() + assert e.value.code == "SESSION_LIMIT" + assert s in jobs._sessions and other not in jobs._sessions + + +def test_i4_an_engine_restart_drops_every_session_explicitly(make, cube): + old = make() + s = sess(old) + restarted = make() # a new SceneJobs is a new engine process + refused(restarted, "SESSION_EXPIRED", cube, "req-A-000001", s, 1, http=409) + with pytest.raises(sj.JobError) as e: + restarted.cancel_request(s, "req-A-000001", 1) + assert e.value.code == "SESSION_EXPIRED" + assert restarted.retained() == 0 and not restarted._sessions + + +def test_i4_an_unknown_session_id_is_not_created_by_a_start(make, cube): + jobs = make() + refused(jobs, "SESSION_EXPIRED", cube, "req-A-000001", "never-issued-session-id", 1) + assert not jobs._sessions + + +# ------------------------------------------------------------------ the legacy path + +def test_legacy_starts_without_the_fields_still_work(make, cube): + jobs = make() + a = jobs.start(cube, "req-L-000001") + wait_state(jobs, a["job_id"], "succeeded") + b = jobs.start(cube, "req-L-000002") + wait_state(jobs, b["job_id"], "succeeded") + s = sess(jobs) + c = jobs.start(cube, "req-O-000003", s, 7) + wait_state(jobs, c["job_id"], "succeeded") + d = jobs.start(cube, "req-L-000004") + assert wait_state(jobs, d["job_id"], "succeeded")["state"] == "succeeded" + + +# ------------------------------------------------------------------ validation + +@pytest.mark.parametrize("extra", [ + {"client_id": None, "seq": None}, {"client_id": "x" * 24, "seq": None}, {"client_id": None, "seq": 1}, + {"client_id": "x" * 24 + "\n", "seq": 1}, {"seq": None}, {"client_id": None}, {"seq": 1}, {"client_id": "x" * 24}, + {"client_id": "x" * 24, "seq": 0}, {"client_id": "x" * 24, "seq": -1}, {"client_id": "x" * 24, "seq": True}, + {"client_id": "x" * 24, "seq": 1.0}, {"client_id": "x" * 24, "seq": "1"}, {"client_id": "x" * 24, "seq": 2 ** 53}, + {"client_id": "short", "seq": 1}, {"client_id": 12345678, "seq": 1}, +]) +def test_validation_holes_are_closed_on_start(make, cube, extra): + jobs = make() + status, body = sj.handle("/scene/start", dict({"path": cube, "request_id": "req-V-000001"}, **extra), jobs) + assert (status, body["error"]) == (400, "INVALID_REQUEST"), (extra, status, body) + assert jobs.retained() == 0 and not jobs._sessions + + +def test_validation_holes_are_closed_on_cancel_and_ids(make, cube): + jobs = make() + s = sess(jobs) + bad = [ + {"client_id": s, "request_id": "req-V-000001"}, # seq missing + {"client_id": s, "request_id": "req-V-000001\n", "seq": 1}, {"client_id": s + "\n", "request_id": "req-V-000001", "seq": 1}, + {"client_id": s, "request_id": "req-V-000001", "seq": None}, {"client_id": None, "request_id": "req-V-000001", "seq": 1}, + {"client_id": s, "request_id": "req-V-000001", "seq": True}, {"client_id": s, "request_id": "req-V-000001", "seq": 1.5}, + {"job_id": "0" * 32, "client_id": s, "request_id": "req-V-000001", "seq": 1}, + {"request_id": "req-V-000001", "seq": 1}, + ] + for body in bad: + status, resp = sj.handle("/scene/cancel", body, jobs) + assert (status, resp["error"]) == (400, "INVALID_REQUEST"), (body, status, resp) + status, resp = sj.handle("/scene/result", {"job_id": "0" * 32, "expected_revision": "a" * 64 + "\n"}, jobs) + assert status == 400 + assert jobs._sessions[s].dead_through == 0 # no malformed cancel changed any state + + +def test_seq_upper_bound_is_two_to_the_53_minus_1(make, cube): + jobs = make() + s = sess(jobs) + status, _ = sj.handle("/scene/start", {"path": cube, "request_id": "req-S-000001", "client_id": s, "seq": 2 ** 53 - 1}, jobs) + assert status == 200 + + +# ------------------------------------------------------------------ the real HTTP route + +def test_http_session_start_stale_cancel_and_expiry_shapes(server, tmp_path): # noqa: F811 + port, token = server + path = str(fx.plain_cube_3mf(tmp_path / "h.3mf")) + code, opened = call(port, "/scene/session", {}, token) + assert code == 200 and set(opened) == {"client_id", "ttl_s"} and not list(sub_validator("session").iter_errors(opened)) + cid = opened["client_id"] + code, b = call(port, "/scene/start", {"path": path, "request_id": "req-B-http0002", "client_id": cid, "seq": 2}, token) + assert code == 200 and not list(sub_validator("job_start").iter_errors(b)) + code, body = call(port, "/scene/start", {"path": path, "request_id": "req-A-http0001", "client_id": cid, "seq": 1}, token) + assert (code, body["error"]) == (409, "STALE_START") and not list(sub_validator("error_response").iter_errors(body)) + code, status = call(port, "/scene/status", {"job_id": b["job_id"], "client_id": cid}, token) + assert code == 200 and status["state"] != "cancelled" + code, c = call(port, "/scene/cancel", {"client_id": cid, "request_id": "req-C-http0003", "seq": 3}, token) + assert code == 200 and c["job_id"] is None and c["state"] == "cancelled" and not list(sub_validator("job_status").iter_errors(c)) + code, body = call(port, "/scene/start", {"path": path, "request_id": "req-C-http0003", "client_id": cid, "seq": 3}, token) + assert (code, body["error"]) == (409, "CANCELLED_BEFORE_START") + code, body = call(port, "/scene/start", {"path": path, "request_id": "req-D-http0004", "client_id": "never-issued-session-id", "seq": 1}, token) + assert (code, body["error"]) == (409, "SESSION_EXPIRED") + code, body = call(port, "/scene/cancel", {"client_id": "never-issued-session-id", "request_id": "req-D-http0004", "seq": 1}, token) + assert (code, body["error"]) == (409, "SESSION_EXPIRED") + code, legacy = call(port, "/scene/start", {"path": path, "request_id": "req-L-http0005"}, token) + assert code == 200 + + +def test_http_session_limit_is_503(server): # noqa: F811 + port, token = server + codes = [call(port, "/scene/session", {}, token)[0] for _ in range(L.MAX_SESSIONS + 1)] + assert codes[:L.MAX_SESSIONS] == [200] * L.MAX_SESSIONS and codes[-1] == 503 + code, body = call(port, "/scene/session", {}, token) + assert (code, body["error"]) == (503, "SESSION_LIMIT") + + +def test_http_routes_reject_non_object_bodies_and_need_the_token(server): # noqa: F811 + port, token = server + for route in sj.ROUTES: + assert call(port, route, None, token)[0] == 400 + assert call(port, route, {}, None)[0] == 401 + + +# ================================================================== round: job-id ownership, post-admission failures + +def _unknown_body(jobs, call): + with pytest.raises(sj.JobError) as e: + call() + return (e.value.http, e.value.body()) + + +def test_h1_job_id_operations_are_session_owned_and_refused_exactly_like_unknown_ids(make, cube): + jobs = make() + a, b = sess(jobs), sess(jobs) + j = jobs.start(cube, "req-own-00001", a, 1) + wait_state(jobs, j["job_id"], "succeeded") + unknown = "0" * 32 + for call_for in (lambda jid, cid: jobs.status(jid, cid), lambda jid, cid: jobs.result(jid, None, cid), + lambda jid, cid: jobs.cancel(jid, cid)): + baseline = _unknown_body(jobs, lambda: call_for(unknown, b)) + assert baseline[0] == 404 and baseline[1]["error"] == "EXPIRED" + assert _unknown_body(jobs, lambda: call_for(j["job_id"], b)) == baseline # another session + assert _unknown_body(jobs, lambda: call_for(j["job_id"], None)) == baseline # no credential at all + assert _unknown_body(jobs, lambda: call_for(j["job_id"], "x" * 24)) == baseline # a never-issued id + assert jobs.status(j["job_id"], a)["state"] == "succeeded" # the owner still can + assert jobs.result(j["job_id"], None, a)[0] == 200 and jobs.cancel(j["job_id"], a)["state"] == "succeeded" + + +def test_h1_a_foreign_cancel_by_job_id_does_not_cancel_anything(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + a, b = sess(jobs), sess(jobs) + j = jobs.start(cube, "req-own-00001", a, 1) + started.wait(10) + with pytest.raises(sj.JobError): + jobs.cancel(j["job_id"], b) + assert jobs.status(j["job_id"], a)["state"] == "running" + gate.set() + + +def test_h1_replaced_job_id_never_leaks_across_sessions(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, honor_cancel=True, started=started)) + a, b = sess(jobs), sess(jobs) + ja = jobs.start(cube, "req-a-000001", a, 1) + started.wait(10) + jb = jobs.start(cube, "req-b-000001", b, 1) # another session replaces A's running job + assert jb["replaced_job_id"] is None # ...and learns nothing about it + assert jobs.status(ja["job_id"], a)["state"] == "cancelled" + jb2 = jobs.start(cube, "req-b-000002", b, 2) # the same session replacing its own job + assert jb2["replaced_job_id"] == jb["job_id"] + legacy = jobs.start(cube, "req-l-000001") # a legacy caller replacing a session job + assert legacy["replaced_job_id"] is None + legacy2 = jobs.start(cube, "req-l-000002") # legacy replacing legacy: as before + assert legacy2["replaced_job_id"] == legacy["job_id"] + sj_after_legacy = jobs.start(cube, "req-b-000003", b, 3) # a session replacing a legacy job + assert sj_after_legacy["replaced_job_id"] is None + gate.set() + + +def test_h1_legacy_jobs_keep_todays_behaviour_for_job_id_operations(make, cube): + jobs = make() + j = jobs.start(cube, "req-l-000001") + wait_state(jobs, j["job_id"], "succeeded") + assert jobs.status(j["job_id"])["state"] == "succeeded" + assert jobs.status(j["job_id"], sess(jobs))["state"] == "succeeded" # no owner: any caller, as today + assert jobs.result(j["job_id"])[0] == 200 and jobs.cancel(j["job_id"])["state"] == "succeeded" + + +def test_h1_http_routes_carry_the_credential(server, tmp_path): # noqa: F811 + port, token = server + path = str(fx.plain_cube_3mf(tmp_path / "h1.3mf")) + _, a = call(port, "/scene/session", {}, token) + _, b = call(port, "/scene/session", {}, token) + _, job = call(port, "/scene/start", {"path": path, "request_id": "req-h1-000001", "client_id": a["client_id"], "seq": 1}, token) + for route in ("/scene/status", "/scene/result", "/scene/cancel"): + missing = call(port, route, {"job_id": job["job_id"]}, token) + foreign = call(port, route, {"job_id": job["job_id"], "client_id": b["client_id"]}, token) + unknown = call(port, route, {"job_id": "0" * 32, "client_id": b["client_id"]}, token) + assert missing == foreign == unknown and missing[0] == 404 and missing[1]["error"] == "EXPIRED" + bad = call(port, route, {"job_id": job["job_id"], "client_id": None}, token) + assert bad[0] == 400 + code, ok = call(port, "/scene/status", {"job_id": job["job_id"], "client_id": a["client_id"]}, token) + assert code == 200 and ok["job_id"] == job["job_id"] + code, both = call(port, "/scene/cancel", {"job_id": job["job_id"], "client_id": a["client_id"], "request_id": "req-h1-000001"}, token) + assert code == 400 + + +def test_h1_job_id_cancel_and_result_touch_the_owner_session(make, cube): + for how in ("cancel_job", "result"): + now = [0.0] + jobs = make(clock=lambda: now[0], session_ttl=100.0, ttl=100.0) + s = sess(jobs) + j = jobs.start(cube, "req-A-000002", s, 2) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + now[0] += 90.0 + (jobs.cancel(j["job_id"], s) if how == "cancel_job" else jobs.result(j["job_id"], None, s)) + now[0] += 90.0 + jobs.open_session() + jobs.open_session() + assert s in jobs._sessions, how + + +def _wedge_then_recover(make, cube): + """A cancelled running job whose worker ignores the cancel, past the grace period: the engine is wedged.""" + now = [0.0] + gate, started = threading.Event(), threading.Event() + jobs = make(clock=lambda: now[0], wedge_grace=10.0, builder=fake_builder(gate, honor_cancel=False, started=started)) + s = sess(jobs) + late = HeldStart(jobs, cube, "req-late-0002", s, 2) # attempt 2 is delayed in flight + first = jobs.start(cube, "req-one-00001", s, 1) + assert started.wait(10) + jobs.cancel_request(s, "req-one-00001", 1) # seq 1 running AND cancelled... + now[0] += 100.0 + assert jobs.wedged() # ...and the worker is stuck past the grace + return jobs, s, gate, late, first + + +def test_h2_a_wedged_rejection_still_advances_the_watermark(make, cube): + jobs, s, gate, late, _ = _wedge_then_recover(make, cube) + refused(jobs, "WORKER_WEDGED", cube, "req-three-0003", s, 3, http=503) + gate.set() # the stuck call returns: the worker recovers + wait_for(lambda: not jobs.worker_alive()) + late.finish() # the delayed seq 2 finally arrives + assert late.error is not None and late.error.code == "STALE_START" + assert jobs.retained() == 1 # only the seq-1 job: seq 2 registered nothing + refused(jobs, "STALE_START", cube, "req-three-0003", s, 3) # an exact retry of the refused attempt is stale + ok = jobs.start(cube, "req-four-0004", s, 4) # the documented recovery: fresh request_id, higher seq + assert ok["state"] in ("queued", "running") + + +def test_h2_unsupported_format_and_missing_file_also_advance_the_watermark(make, cube, tmp_path): + jobs = make() + s = sess(jobs) + late = HeldStart(jobs, cube, "req-late-0002", s, 2) + text = tmp_path / "notes.txt" + text.write_text("x") + refused(jobs, "UNSUPPORTED_FORMAT", str(text), "req-three-0003", s, 3, http=422) + late.finish() + assert late.error.code == "STALE_START" + late2 = HeldStart(jobs, cube, "req-late-0004", s, 4) + refused(jobs, "INVALID_REQUEST", str(tmp_path / "missing.3mf"), "req-five-0005", s, 5, http=400) + late2.finish() + assert late2.error.code == "STALE_START" + refused(jobs, "STALE_START", str(text), "req-old-00001", s, 1) # staleness is reported first and moves nothing + assert jobs._sessions[s].watermark == 5 + assert jobs.retained() == 0 + + +# ------------------------------------------------------------------ Astra optionals + +def test_a_delayed_cancel_for_an_old_attempt_cannot_cancel_a_newer_job_that_reused_the_request_id(make, cube): + now = [0.0] + gate, started = threading.Event(), threading.Event() + calls = [] + + def builder(path, revision, ctl): + calls.append(1) + if len(calls) == 1: + return b"{}" + started.set() + while not gate.wait(0.005): + ctl.check() + return b"{}" + jobs = make(clock=lambda: now[0], ttl=10.0, builder=builder) + s = sess(jobs) + old = jobs.start(cube, "req-R-000001", s, 1) + wait_state(jobs, old["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + now[0] += 60.0 + with pytest.raises(sj.JobError): + st(jobs, old["job_id"]) # the old job was evicted + new = jobs.start(cube, "req-R-000001", s, 5) # the request id is reused at a higher seq + assert started.wait(10) and st(jobs, new["job_id"])["state"] == "running" + out = jobs.cancel_request(s, "req-R-000001", 1) # the delayed cancel for the OLD attempt + assert out["job_id"] is None + assert st(jobs, new["job_id"])["state"] == "running" # the newer job is untouched + assert jobs.cancel_request(s, "req-R-000001", 5)["job_id"] == new["job_id"] + gate.set() + + +def test_an_expired_session_whose_last_job_was_just_evicted_does_not_cause_session_limit(make, cube): + now = [0.0] + jobs = make(clock=lambda: now[0], max_sessions=1, session_ttl=50.0, ttl=50.0) + s = sess(jobs) + j = jobs.start(cube, "req-A-000001", s, 1) + wait_state(jobs, j["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + now[0] += 500.0 # the job AND the session expire together + fresh = jobs.open_session() # one call: evict the job, then prune the session + assert fresh["client_id"] != s and s not in jobs._sessions + + +# ================================================================== round: exact retries and exact-attempt cancels + +def test_b1_a_failed_precheck_on_an_exact_retry_keeps_the_original_job_reference(make, cube, tmp_path): + jobs = make() + s = sess(jobs) + b = jobs.start(cube, "req-B-000002", s, 2) + wait_state(jobs, b["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + hidden = cube + ".away" + os.replace(cube, hidden) # the file vanishes: the retry's precheck fails + refused(jobs, "INVALID_REQUEST", cube, "req-B-000002", s, 2, http=400) + assert jobs._sessions[s].current[3] == b["job_id"] # the retained job's reference was NOT cleared + os.replace(hidden, cube) + again = jobs.start(cube, "req-B-000002", s, 2) # a later exact retry still gets the job + assert again["job_id"] == b["job_id"] + + +def test_b1_concurrent_exact_retries_one_hashing_one_failing_its_precheck(make, cube): + jobs = make() + s = sess(jobs) + b = jobs.start(cube, "req-B-000002", s, 2) + status = wait_state(jobs, b["job_id"], "succeeded") + wait_for(lambda: not jobs.worker_alive()) + inside, release = threading.Event(), threading.Event() + real_hash = jobs._current_hash + + def slow(path): # retry R1 is mid-hash, outside the lock + inside.set() + assert release.wait(10) + return status["revision"] + jobs._current_hash = slow + out = {} + + def retry(): + try: + out["r"] = jobs.start(cube, "req-B-000002", s, 2) + except sj.JobError as exc: + out["e"] = exc + t = threading.Thread(target=retry, daemon=True) + t.start() + assert inside.wait(10) + hidden = cube + ".away" + os.replace(cube, hidden) + refused(jobs, "INVALID_REQUEST", cube, "req-B-000002", s, 2, http=400) # R2 fails its precheck meanwhile + os.replace(hidden, cube) + release.set() + t.join(10) + jobs._current_hash = real_hash + assert "e" not in out, out.get("e") and out["e"].code + assert out["r"]["job_id"] == b["job_id"] # R1 still gets the job that still exists + + +def test_b2_a_cancel_at_a_higher_seq_does_not_cancel_the_older_job_of_the_same_request(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + s = sess(jobs) + five = jobs.start(cube, "req-R-000001", s, 5) + started.wait(10) + out = jobs.cancel_request(s, "req-R-000001", 6) # a cancel for seq 6, not for this attempt + assert out["job_id"] is None + assert st(jobs, five["job_id"])["state"] == "running" # the seq-5 job is untouched + assert jobs._sessions[s].dead_through == 6 # but seq 6 and everything below stays refused + refused(jobs, "CANCELLED_BEFORE_START", cube, "req-R-000002", s, 6) + gate.set() + + +def test_b2_only_the_exact_attempt_is_cancelled(make, cube): + gate, started = threading.Event(), threading.Event() + jobs = make(builder=fake_builder(gate, started=started)) + s = sess(jobs) + five = jobs.start(cube, "req-R-000001", s, 5) + started.wait(10) + assert jobs.cancel_request(s, "req-R-000001", 4)["job_id"] is None # an older attempt's cancel: no match + assert st(jobs, five["job_id"])["state"] == "running" + assert jobs._sessions[s].dead_through == 4 + out = jobs.cancel_request(s, "req-R-000001", 5) # the exact attempt + assert out["job_id"] == five["job_id"] and out["state"] == "cancelled" + assert jobs._sessions[s].dead_through == 5 + gate.set() diff --git a/backend/tests/test_scene_repair.py b/backend/tests/test_scene_repair.py new file mode 100644 index 00000000..7b32b79e --- /dev/null +++ b/backend/tests/test_scene_repair.py @@ -0,0 +1,690 @@ +"""Repair round 1: each test here fails without the corresponding fix (finding id in the docstring).""" +import hashlib +import json +import re +import struct +import time +import tracemalloc + +import pytest + +from snapstudio_core import scene, scene_limits as L, units +from tests import scene_fixtures as fx +from tests.test_scene_build import build, codes, node +from tests.test_scene_contract import assert_valid, sub_validator + +REV = "ef" * 32 + + +def refuse(path, code): + with pytest.raises(scene.SceneError) as e: + scene.build_scene(str(path), REV) + assert e.value.code == code, (e.value.code, e.value.message) + return e.value + + +# ------------------------------------------------------------------ A1: build items with p:path + +def test_a1_build_item_p_path_uses_the_referenced_file_not_the_root(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(5, 5, 0), "/3D/Objects/ext.model")]) + ext = fx.sub_model_xml([fx.cube_object("1", 30)]) + sc = build(fx.three_mf(tmp_path / "p.3mf", root, {"3D/Objects/ext.model": ext})) + n = node(sc, "b0") + assert n["resource"] == {"part": "3D/Objects/ext.model", "object_id": "1"} + assert n["bounds_mm"] == {"min": [5.0, 5.0, 0.0], "max": [35.0, 35.0, 30.0]} # the 30 mm cube + assert [m["key"]["part"] for m in sc["meshes"]] == ["3D/Objects/ext.model"] # the root cube is not substituted + + +def test_a1_unresolvable_p_path_is_an_error_not_a_fallback_to_the_root(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", None, "/3D/Objects/missing.model")]) + refuse(fx.three_mf(tmp_path / "a.3mf", root), "UNRESOLVED_REFERENCE") + ext = fx.sub_model_xml([fx.cube_object("7", 30)]) # file exists, object 1 does not + root2 = fx.model_xml([fx.cube_object("1", 10)], [("1", None, "/3D/Objects/ext.model")]) + refuse(fx.three_mf(tmp_path / "b.3mf", root2, {"3D/Objects/ext.model": ext}), "UNRESOLVED_REFERENCE") + + +def test_a1_same_object_id_in_two_files_is_not_a_repeated_instance(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(5, 5, 0), "/3D/Objects/ext.model"), ("1", fx.tf(100, 5, 0))]) + ext = fx.sub_model_xml([fx.cube_object("1", 30)]) + sc = build(fx.three_mf(tmp_path / "r.3mf", root, {"3D/Objects/ext.model": ext})) + assert "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED" not in codes(sc) + assert node(sc, "b0")["resource"]["part"] == "3D/Objects/ext.model" and node(sc, "b1")["resource"]["part"] == "3D/3dmodel.model" + twice = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(5, 5, 0), "/3D/Objects/ext.model"), ("1", fx.tf(100, 5, 0), "/3D/Objects/ext.model")]) + assert "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED" in codes(build(fx.three_mf(tmp_path / "t.3mf", twice, {"3D/Objects/ext.model": ext}))) + + +# ------------------------------------------------------------------ A2: the unit header, read from the real root + +CORE = fx.CORE + + +def _prefixed(xml: str) -> str: + xml = re.sub(r"<(/?)(model|resources|object|mesh|vertices|vertex|triangles|triangle|components|component|build|item)\b", + r"<\1m:\2", xml) + return xml.replace(f'xmlns="{CORE}"', f'xmlns:m="{CORE}"') + + +UNIT_DOCS = { + "prefixed": lambda: _prefixed(fx.model_xml([fx.cube_object("1", 12)], [("1", None)], unit="inch")), + "long_comment": lambda: fx.model_xml([fx.cube_object("1", 12)], [("1", None)], unit="inch", + prolog=''), + "pi_and_comments": lambda: fx.model_xml([fx.cube_object("1", 12)], [("1", None)], unit="inch", + prolog=''), + "decoy_comment": lambda: fx.model_xml([fx.cube_object("1", 12)], [("1", None)], unit="millimeter", + prolog=''), + "absent": lambda: fx.model_xml([fx.cube_object("1", 12)], [("1", None)]).replace(' unit="millimeter"', ""), +} +EXPECT = {"prefixed": "inch", "long_comment": "inch", "pi_and_comments": "inch", "decoy_comment": "millimeter", "absent": "millimeter"} + + +@pytest.mark.parametrize("name", sorted(UNIT_DOCS)) +def test_a2_the_unit_comes_from_the_real_root_element(tmp_path, name): + xml = UNIT_DOCS[name]() + want = EXPECT[name] + assert units.read_unit(xml.encode()).name == want + assert units.mm_per_unit(xml.encode()) == units.MM_PER_UNIT[want] + sc = build(fx.three_mf(tmp_path / f"{name}.3mf", xml)) + assert sc["sources"][0]["unit"] == want # the scene parser and the shared helper agree + b = node(sc, "b0")["bounds_mm"] + assert b["max"][0] == pytest.approx(12 * units.MM_PER_UNIT[want], abs=1e-4) + + +def test_a2_the_shared_unit_reader_sees_the_unit_through_a_long_preamble(): + # The size consumers on main stay behind geometry_basis (not checked for non-mm); only the scene + # contract and this helper normalise units, so the helper is asserted directly. + def raw(xml): + return xml if isinstance(xml, bytes) else xml.encode("utf-8") + assert units.unit_of(raw(UNIT_DOCS["long_comment"]())) == ("inch", 25.4) + assert units.unit_of(raw(UNIT_DOCS["decoy_comment"]())) == ("millimeter", 1.0) + + +def test_a2_an_unrecognised_unit_is_not_the_same_as_an_absent_one(): + absent = units.read_unit(b'') + yard = units.read_unit(b'') + assert absent.recognized and not absent.declared and absent.factor == 1.0 + assert not yard.recognized and yard.declared and yard.name == "yard" and yard.factor == 1.0 # legacy fallback, flagged + assert units.unit_of(b'') == ("yard", None) + + +# ------------------------------------------------------------------ A4 / D4: metadata present but unmatched is never a printable part + +def test_a4_prusa_attribute_order_does_not_matter(tmp_path): + vols = [(0, 11, "ModelPart"), (12, 23, "ParameterModifier")] + normal = build(fx.prusa_project(tmp_path / "a.3mf", volumes=vols)) + swapped = build(fx.prusa_project(tmp_path / "b.3mf", volumes=vols, last_first=True)) + assert swapped["meshes"][0]["volumes"] == normal["meshes"][0]["volumes"] + assert [v["role"] for v in swapped["meshes"][0]["volumes"]] == ["part", "modifier"] + assert node(swapped, "b0")["has_non_part_volumes"] is True + + +def test_a4_bambu_part_missing_from_the_settings_is_unknown_not_printable(tmp_path): + path = fx.bambu_project(tmp_path / "m.3mf", parts=2, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])], + origins=[(0, 0, 0), (500, 0, 0)], omit_parts=(2,)) + sc = build(path) + missing = node(sc, "b0.c1") + assert missing["role_context"] == "unknown" and missing["printable"] is None and missing["bounds_mm"] is None + lim = next(l for l in sc["limitations"] if l["code"] == "UNKNOWN_VOLUME_ROLE") + assert lim["target_ids"] == ["b0.c1"] and sc["status"] == "partial" + assert node(sc, "b0")["bounds_mm"] == {"min": [10.0, 10.0, 0.0], "max": [20.0, 20.0, 10.0]} # the unknown part does not count + assert node(sc, "b0.c1")["finding_ids"] == [] and sc["findings"] == [] + + +def test_a4_an_object_with_no_metadata_at_all_keeps_the_synthetic_part(tmp_path): + sc = build(fx.plain_cube_3mf(tmp_path / "p.3mf")) + assert sc["meshes"][0]["volumes"][0]["role"] == "part" and node(sc, "b0")["role_context"] is None + + +# ------------------------------------------------------------------ A5: roles follow the traversal context + +def _shared_project(tmp_path, roles): + sub = fx.sub_model_xml([fx.cube_object("1", 10)]) + comp = [("1", "/3D/Objects/o.model", None)] + root = fx.model_xml([fx.composite_object("100", comp), fx.composite_object("101", comp)], + [("100", fx.tf(10, 10, 0)), ("101", fx.tf(60, 10, 0))]) + settings = fx.model_settings_xml({"100": [("1", roles[0])], "101": [("1", roles[1])]}, [(1, [("100", 0), ("101", 0)])]) + return fx.three_mf(tmp_path / "s.3mf", root, {"3D/Objects/o.model": sub, "Metadata/model_settings.config": settings}) + + +def test_a5_a_shared_mesh_keeps_each_use_s_role(tmp_path): + sc = build(_shared_project(tmp_path, ("negative_part", "normal_part"))) + assert len(sc["meshes"]) == 1 # one mesh, two uses + neg, part = node(sc, "b0.c0"), node(sc, "b1.c0") + assert (neg["role_context"], neg["printable"], neg["bounds_mm"]) == ("negative", False, None) + assert (part["role_context"], part["printable"]) == ("part", True) + assert part["bounds_mm"]["min"] == [60.0, 10.0, 0.0] + flipped = build(_shared_project(tmp_path, ("normal_part", "modifier_part"))) + assert (node(flipped, "b0.c0")["role_context"], node(flipped, "b1.c0")["role_context"]) == ("part", "modifier") + + +def test_a5_a_negative_part_over_an_assembly_applies_to_everything_beneath(tmp_path): + sub = fx.sub_model_xml([fx.cube_object("1", 10), fx.composite_object("2", [("1", None, None), ("1", None, fx.tf(20, 0, 0))])]) + root = fx.model_xml([fx.composite_object("100", [("2", "/3D/Objects/o.model", None)])], [("100", fx.tf(10, 10, 0))]) + settings = fx.model_settings_xml({"100": [("2", "negative_part")]}, [(1, [("100", 0)])]) + sc = build(fx.three_mf(tmp_path / "a.3mf", root, {"3D/Objects/o.model": sub, "Metadata/model_settings.config": settings})) + assert [n["id"] for n in sc["nodes"]] == ["b0", "b0.c0", "b0.c0.c0", "b0.c0.c1"] + for nid in ("b0.c0", "b0.c0.c0", "b0.c0.c1"): + n = node(sc, nid) + assert n["role_context"] == "negative" and n["printable"] is False and n["bounds_mm"] is None and n["has_non_part_volumes"] + assert node(sc, "b0")["printable"] is False and node(sc, "b0")["bounds_mm"] is None + assert sc["findings"] == [] + + +# ------------------------------------------------------------------ A7: the response cap never allocates the whole body first + +def test_a7_part_names_longer_than_the_schema_are_refused_before_they_are_placed(tmp_path): + name = "3D/Objects/" + "n" * 300 + ".model" + sub = fx.sub_model_xml([fx.cube_object("1", 10)]) + root = fx.model_xml([fx.composite_object("9", [("1", "/" + name, fx.tf(i % 50, i // 50, 0)) for i in range(2000)])], [("9", None)]) + path = fx.three_mf(tmp_path / "long.3mf", root, {name: sub}) + tracemalloc.start() + err = refuse(path, "LIMIT_EXCEEDED") + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + assert "part name" in err.message and peak < 50 * 1024 * 1024 + + +def test_a7_part_names_at_the_limit_still_yield_a_schema_valid_scene(tmp_path): + name = "3D/Objects/" + "n" * (L.MAX_PART_NAME_LENGTH - len("3D/Objects/.model")) + ".model" + assert len(name) == L.MAX_PART_NAME_LENGTH + sub = fx.sub_model_xml([fx.cube_object("1", 10)]) + root = fx.model_xml([fx.composite_object("9", [("1", "/" + name, fx.tf(i % 50, i // 50, 0)) for i in range(600)])], [("9", None)]) + sc = build(fx.three_mf(tmp_path / "ok.3mf", root, {name: sub})) + assert sc["counts"]["nodes"] == 601 and len(scene.serialize(sc)) <= L.MAX_RESPONSE_BYTES + + +def test_a7_serializer_checks_the_cap_while_building_and_never_dumps_the_whole_body(tmp_path, monkeypatch): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(i % 20 * 12, i // 20 * 12, 0)) for i in range(300)]) + sc = scene.build_scene_dict(str(fx.three_mf(tmp_path / "many.3mf", root)), REV) + full = len(json.dumps(sc, separators=(",", ":"))) + assert full > 100_000 + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", 30_000) + seen = [] + real = json.dumps + + def spy(value, *a, **k): + text = real(value, *a, **k) + seen.append(len(text)) + return text + monkeypatch.setattr(scene.json, "dumps", spy) + with pytest.raises(scene.SceneError) as e: + scene.serialize(sc) + assert e.value.code == "LIMIT_EXCEEDED" + assert max(seen) < 30_000 and sum(seen) < 30_000 + 5_000 # stopped one element past the cap + monkeypatch.undo() + size = len(scene.serialize(sc)) + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", size) + assert len(scene.serialize(sc)) == size # the accounting is exact + monkeypatch.setattr(L, "MAX_RESPONSE_BYTES", size - 1) + with pytest.raises(scene.SceneError): + scene.serialize(sc) + + +# ------------------------------------------------------------------ A8 / D3: parse-time memory bounds cover every element + +def test_a8_hundreds_of_thousands_of_unused_objects_are_rejected_early(tmp_path): + objects = "".join(f'' for i in range(2, 300_002)) + root = fx.model_xml([fx.cube_object("1", 10)], [("1", None)]).replace("", objects + "") + t = time.monotonic() + err = refuse(fx.three_mf(tmp_path / "objs.3mf", root), "LIMIT_EXCEEDED") + assert "objects" in err.message and time.monotonic() - t < 10 + + +def test_d3_junk_elements_are_capped_and_memory_stays_bounded(tmp_path): + junk = "" * 2_000_000 # 8 MB of unknown elements + root = fx.model_xml([fx.cube_object("1", 10)], [("1", None)]).replace("", junk + "") + path = fx.three_mf(tmp_path / "junk.3mf", root) + tracemalloc.start() + err = refuse(path, "LIMIT_EXCEEDED") + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + assert "elements" in err.message and peak <= 300 * 1024 * 1024 + + +# ------------------------------------------------------------------ A9: STL is decoded incrementally + +def _stl_with_header(path, n): + path.write_bytes(b"\0" * 80 + struct.pack("', + f'', +]) +def test_a12_external_relationship_is_rejected_whatever_the_namespace(tmp_path, rels): + path = fx.three_mf(tmp_path / "e.3mf", fx.model_xml([fx.cube_object("1")], [("1", None)]), + {"3D/_rels/3dmodel.model.rels": '' + rels}) + err = refuse(path, "INVALID_ARCHIVE") + assert "external" in err.message + + +def test_d6_the_byte_prescan_rejects_a_doctype_in_a_config_read_without_an_xml_parser(tmp_path): + cfg = ']>' + path = fx.three_mf(tmp_path / "c.3mf", fx.model_xml([fx.cube_object("1")], [("1", None)]), {"Metadata/Slic3r_PE_model.config": cfg}) + arc = scene.Archive(str(path), scene.Control()) + try: + with pytest.raises(scene.SceneError) as e: + arc.read_small("Metadata/Slic3r_PE_model.config") + assert e.value.code == "INVALID_ARCHIVE" + finally: + arc.close() + + +# ------------------------------------------------------------------ A13: ids fit by construction + +def test_a13_ids_fit_128_characters_at_the_deepest_allowed_nesting(tmp_path): + chain = [fx.cube_object("0")] + [fx.composite_object(str(i), [(str(i - 1), None, None)]) for i in range(1, L.MAX_DEPTH)] + path = fx.three_mf(tmp_path / "deep.3mf", fx.model_xml(chain, [(str(L.MAX_DEPTH - 1), None)])) + sc = build(path) + ids = [n["id"] for n in sc["nodes"]] + assert len(ids) == L.MAX_DEPTH == len(set(ids)) and max(map(len, ids)) <= 128 + assert any("~" in i for i in ids) # long ancestries are digested, not truncated + assert [n["id"] for n in build(path)["nodes"]] == ids # deterministic + parents = {n["id"]: n["parent_id"] for n in sc["nodes"]} + assert all(p is None or p in parents for p in parents.values()) + assert node(sc, ids[-1])["instance_ref"]["component_path"] == [0] * (L.MAX_DEPTH - 1) + assert hashlib.sha256(ids[-1].encode()).hexdigest() # (ids are plain strings) + + +# ------------------------------------------------------------------ D5: schema closure and plate state + +def test_d5_error_response_is_closed_and_a_known_plate_needs_an_id(): + err = sub_validator("error_response") + assert not list(err.iter_errors({"error": "EXPIRED", "message": "x"})) + assert list(err.iter_errors({"error": "EXPIRED", "message": "x", "extra": 1})) + plate = sub_validator("plate_ref") + assert not list(plate.iter_errors({"state": "known", "plate_id": "p1", "source": "plate_config"})) + assert list(plate.iter_errors({"state": "known", "plate_id": None, "source": "plate_config"})) + assert not list(plate.iter_errors({"state": "unknown", "plate_id": None, "source": None})) + + +def test_d5_a_node_on_no_plate_of_a_project_with_plates_gets_no_fit_finding_and_is_partial(tmp_path): + path = fx.bambu_project(tmp_path / "np.3mf", parts=1, items=[("100", fx.tf(900, 900, 0))], plates=[(1, [])]) + sc = build(path) + assert node(sc, "b0")["plate"]["state"] == "unknown" + assert sc["findings"] == [] and sc["status"] == "partial" and "PLATE_MEMBERSHIP_UNKNOWN" in codes(sc) + on_plate = build(fx.bambu_project(tmp_path / "op.3mf", parts=1, items=[("100", fx.tf(900, 900, 0))], plates=[(1, [("100", 0)])])) + assert [f["kind"] for f in on_plate["findings"]] == ["placement"] # with membership proven, the finding returns + + +# ================================================================== repair round 2 + +import os + + +def _config(tmp_path, name, settings_xml, model=None, extra=None): + root = model or fx.model_xml([fx.cube_object("1", 10)], [("1", None)]) + return fx.three_mf(tmp_path / name, root, dict({"Metadata/model_settings.config": settings_xml}, **(extra or {}))) + + +# ------------------------------------------------------------------ B1: STL kind and bounds + +def test_b1_a_large_solid_payload_with_no_newlines_is_rejected_without_allocating_it(tmp_path): + path = tmp_path / "bomb.stl" + with open(path, "wb") as fh: + fh.write(b"solid ") + for _ in range(64): + fh.write(b"a" * (1024 * 1024)) # 64 MiB, not one newline + tracemalloc.start() + t = time.monotonic() + err = refuse(path, "INVALID_GEOMETRY") + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + assert time.monotonic() - t < 5 and peak < 20 * 1024 * 1024, (peak, err.message) + + +def test_b1_a_header_over_the_budget_with_trailing_bytes_is_rejected_before_any_decoding(tmp_path, monkeypatch): + from snapstudio_core import stl_io + n = L.MAX_PARSED_TRIANGLES + 1 + path = tmp_path / "trail.stl" + path.write_bytes(b"solid mislabelled binary".ljust(80, b" ") + struct.pack("" + JUNK + "") + elif which == "prusa": + path = fx.three_mf(tmp_path / "p.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", None)]), + {"Metadata/Slic3r_PE_model.config": "" + JUNK + ""}) + else: + path = fx.three_mf(tmp_path / "r.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", None)]), + rels=f'' + JUNK + "") + tracemalloc.start() + err = refuse(path, "LIMIT_EXCEEDED") + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + assert "elements" in err.message and peak <= 300 * 1024 * 1024 + + +def test_b2_unrecognised_elements_are_released_as_they_complete(tmp_path, monkeypatch): + calls = [] + real = scene._free + monkeypatch.setattr(scene, "_free", lambda e: (calls.append(e.tag), real(e))[1]) + junk = "" * 3000 + scene.build_scene_dict(str(_config(tmp_path, "j.3mf", "" + junk + "")), REV) + assert sum(1 for t in calls if t == "x") >= 3000 + calls.clear() + root = fx.model_xml([fx.cube_object("1", 10)], [("1", None)]).replace("", junk + "") + scene.build_scene_dict(str(fx.three_mf(tmp_path / "m.3mf", root)), REV) + assert sum(1 for t in calls if t.endswith("}x") or t == "x") >= 3000 + + +# ------------------------------------------------------------------ B3: p:path rules + +def test_b3_a_relative_build_item_path_is_refused(tmp_path): + ext = fx.sub_model_xml([fx.cube_object("1", 30)]) + root = fx.model_xml([], [("1", None, "3D/Objects/ext.model")]) + refuse(fx.three_mf(tmp_path / "a.3mf", root, {"3D/Objects/ext.model": ext}), "UNRESOLVED_REFERENCE") + ok = fx.model_xml([], [("1", None, "/3D/Objects/ext.model")]) + assert scene.build_scene_dict(str(fx.three_mf(tmp_path / "b.3mf", ok, {"3D/Objects/ext.model": ext})), REV)["counts"]["nodes"] == 1 + + +def test_b3_a_relative_component_path_is_refused(tmp_path): + ext = fx.sub_model_xml([fx.cube_object("1", 30)]) + root = fx.model_xml([fx.composite_object("9", [("1", "3D/Objects/ext.model", None)])], [("9", None)]) + refuse(fx.three_mf(tmp_path / "a.3mf", root, {"3D/Objects/ext.model": ext}), "UNRESOLVED_REFERENCE") + + +def test_b3_component_paths_are_honoured_only_from_the_root_model(tmp_path): + leaf = fx.sub_model_xml([fx.cube_object("1", 10)]) + mid = fx.sub_model_xml([fx.composite_object("2", [("1", "/3D/Objects/leaf.model", None)])]) + root = fx.model_xml([fx.composite_object("9", [("2", "/3D/Objects/mid.model", None)])], [("9", None)]) + err = refuse(fx.three_mf(tmp_path / "n.3mf", root, {"3D/Objects/mid.model": mid, "3D/Objects/leaf.model": leaf}), + "UNRESOLVED_REFERENCE") + assert "root model" in err.message + + +# ------------------------------------------------------------------ B4: duplicate records never silently overwrite + +def _two_part_settings(records): + return ('' + records + + '' + '') + + +def _bambu_one_part(tmp_path, name, settings): + sub = fx.sub_model_xml([fx.cube_object("1", 10)]) + root = fx.model_xml([fx.composite_object("100", [("1", "/3D/Objects/o.model", None)])], [("100", fx.tf(10, 10, 0))]) + return fx.three_mf(tmp_path / name, root, {"3D/Objects/o.model": sub, "Metadata/model_settings.config": settings}) + + +def test_b4_duplicate_records_that_disagree_make_the_role_unknown(tmp_path): + neg = '' + part = '' + both = '' + for name, records in (("objects", neg + part), ("parts", both)): + sc = build(_bambu_one_part(tmp_path, name + ".3mf", _two_part_settings(records))) + n = node(sc, "b0.c0") + assert n["role_context"] == "unknown" and n["printable"] is None + assert sc["status"] == "partial" and "UNKNOWN_VOLUME_ROLE" in codes(sc) and sc["findings"] == [] + same = build(_bambu_one_part(tmp_path, "same.3mf", _two_part_settings(neg + neg))) + assert node(same, "b0.c0")["role_context"] == "negative" # identical duplicates are fine + + +# ------------------------------------------------------------------ B5: build objects in another file fail closed + +def test_b5_a_negative_part_assembly_in_another_file_is_unknown_and_never_judged(tmp_path): + a = fx.sub_model_xml([fx.cube_object("1", 10), fx.composite_object("2", [("1", None, None)])]) + root = fx.model_xml([], [("2", fx.tf(900, 100, 0), "/3D/Objects/a.model")]) + settings = fx.model_settings_xml({"2": [("1", "negative_part")]}, [(1, [("2", 0)])]) + sc = build(fx.three_mf(tmp_path / "x.3mf", root, {"3D/Objects/a.model": a, "Metadata/model_settings.config": settings})) + assert [n["role_context"] for n in sc["nodes"]] == ["unknown", "unknown"] + assert all(n["printable"] is None for n in sc["nodes"]) + assert sc["status"] == "partial" and "UNKNOWN_VOLUME_ROLE" in codes(sc) + assert sc["findings"] == [] + + +def test_b5_without_any_settings_a_file_qualified_build_object_is_still_ordinary(tmp_path): + root = fx.model_xml([], [("1", fx.tf(5, 5, 0), "/3D/Objects/a.model")]) + a = fx.sub_model_xml([fx.cube_object("1", 10)]) + sc = build(fx.three_mf(tmp_path / "plain.3mf", root, {"3D/Objects/a.model": a})) + assert node(sc, "b0")["printable"] is True and node(sc, "b0")["role_context"] is None + + +# ------------------------------------------------------------------ B6: plate matching is file-qualified + +def test_b6_a_bare_object_id_shared_across_files_cannot_prove_plate_membership(tmp_path): + ext = fx.sub_model_xml([fx.cube_object("1", 10)]) + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(900, 10, 0)), ("1", fx.tf(5, 5, 0), "/3D/Objects/ext.model")]) + settings = fx.model_settings_xml({}, [(1, [("1", 0)])]) + sc = build(fx.three_mf(tmp_path / "s.3mf", root, {"3D/Objects/ext.model": ext, "Metadata/model_settings.config": settings})) + assert [node(sc, "b0")["plate"]["state"], node(sc, "b1")["plate"]["state"]] == ["ambiguous", "ambiguous"] + assert all(node(sc, i)["plate"]["plate_id"] is None for i in ("b0", "b1")) + lim = next(l for l in sc["limitations"] if l["code"] == "PLATE_MEMBERSHIP_UNKNOWN") + assert set(lim["target_ids"]) == {"b0", "b1"} + assert sc["findings"] == [] and sc["status"] == "partial" # the root cube at X=900 is NOT reported + + +# ------------------------------------------------------------------ B7: unsupported unit names its nodes + +def test_b7_unsupported_unit_lists_the_affected_nodes(tmp_path): + sub = fx.sub_model_xml([fx.cube_object("1", 10)], unit="yard") + root = fx.model_xml([fx.composite_object("9", [("1", "/3D/Objects/y.model", None)]), fx.cube_object("2", 10)], + [("9", None), ("2", fx.tf(50, 0, 0))]) + sc = build(fx.three_mf(tmp_path / "u.3mf", root, {"3D/Objects/y.model": sub})) + lim = next(l for l in sc["limitations"] if l["code"] == "UNSUPPORTED_UNIT") + assert lim["target_ids"] == ["b0.c0"] # only the nodes living in the yard part + assert {s["part"]: s["unit"] for s in sc["sources"]}["3D/Objects/y.model"] == "millimeter" # the documented fallback + description = scene.load_schema()["$defs"]["source"]["description"] + assert "UNSUPPORTED_UNIT" in description and "fallback" in description.lower() + + +# ------------------------------------------------------------------ B9: id collisions are detected + +def test_b9_a_node_id_collision_is_an_internal_error(tmp_path, monkeypatch): + monkeypatch.setattr(scene.Traversal, "_node_id", staticmethod(lambda ctx, parent, path: "same")) + root = fx.model_xml([fx.cube_object("1", 10)], [("1", None), ("1", fx.tf(20, 0, 0))]) + refuse(fx.three_mf(tmp_path / "c.3mf", root), "INTERNAL") + + +# ------------------------------------------------------------------ B10: surviving mutants + +def test_b10_an_assembly_role_is_inherited_under_a_root_file_parent_and_the_outer_role_wins(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10), fx.composite_object("50", [("1", None, None)]), + fx.composite_object("100", [("50", None, None)])], [("100", fx.tf(10, 10, 0))]) + settings = fx.model_settings_xml({"100": [("50", "negative_part")], "50": [("1", "normal_part")]}, [(1, [("100", 0)])]) + sc = build(fx.three_mf(tmp_path / "r.3mf", root, {"Metadata/model_settings.config": settings})) + assert [node(sc, i)["role_context"] for i in ("b0.c0", "b0.c0.c0")] == ["negative", "negative"] + assert node(sc, "b0.c0.c0")["printable"] is False and sc["findings"] == [] + + +def test_b10_an_unreadable_prusa_range_makes_the_whole_mesh_unknown_and_partial(tmp_path): + cfg = ('' + '' + '') + sc = build(fx.three_mf(tmp_path / "p.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(900, 900, 0))]), + {"Metadata/Slic3r_PE_model.config": cfg})) + assert [v["role"] for v in sc["meshes"][0]["volumes"]] == ["unknown"] + assert node(sc, "b0")["printable"] is None and sc["status"] == "partial" and sc["findings"] == [] + + +def test_b10_binary_stl_is_decoded_in_bounded_batches_with_the_hook_between_them(tmp_path, monkeypatch): + from snapstudio_core import stl_io + path = _stl_with_header(tmp_path / "b.stl", 12_000) + sizes, checks = [], [] + real = stl_io._decode_binary_batch + monkeypatch.setattr(stl_io, "_decode_binary_batch", lambda raw, *a: (sizes.append(len(raw) // 50), real(raw, *a))[1]) + with open(path, "rb") as fh: + stl_io.decode_stl_stream(fh, os.path.getsize(path), max_triangles=250_000, max_vertices=300_000, + check=lambda: checks.append(1)) + assert sizes == [5000, 5000, 2000] and len(checks) >= 3 + + +# ================================================================== repair round 3 + +def _bambu_part_at_900(tmp_path, name, object_xml, plate_extra=""): + settings = ('' + object_xml + + '' + '' + plate_extra + '') + sub = fx.sub_model_xml([fx.cube_object("1", 10)]) + root = fx.model_xml([fx.composite_object("100", [("1", "/3D/Objects/o.model", None)])], [("100", fx.tf(900, 10, 0))]) + return fx.three_mf(tmp_path / name, root, {"3D/Objects/o.model": sub, "Metadata/model_settings.config": settings}) + + +@pytest.mark.parametrize("name,obj", [ + ("after", ''), + ("between", ''), + ("leading", ''), +]) +def test_r3_1_an_unknown_element_next_to_a_part_record_never_changes_its_role(tmp_path, name, obj): + sc = build(_bambu_part_at_900(tmp_path, f"{name}.3mf", obj)) + n = node(sc, "b0.c0") + assert n["role_context"] == "negative" and n["printable"] is False + assert sc["findings"] == [] # a negative part is never judged against the bed + + +def test_r3_1_an_unknown_element_after_a_model_instance_keeps_plate_membership(tmp_path): + obj = '' + sc = build(_bambu_part_at_900(tmp_path, "pl.3mf", obj, plate_extra="")) + assert node(sc, "b0")["plate"] == {"state": "known", "plate_id": "p1", "source": "plate_config"} + assert [f["kind"] for f in sc["findings"]] == ["placement"] # and the part at X=900 is judged, as before + + +@pytest.mark.parametrize("junk", ["", "", ""]) +def test_r3_1_an_unknown_element_after_a_prusa_volume_never_drops_it(tmp_path, junk): + verts, tris = [], [] + for k in range(2): + base = len(verts) + verts += fx.cube_vertices(10.0, (k * 20.0, 0.0, 0.0)) + tris += [(a + base, b + base, c + base) for a, b, c in fx.CUBE_TRIS] + obj = f'{fx.mesh_xml(verts, tris)}' + vol = lambda a, b, vt: f'' + cfg = ('' + vol(0, 11, "ModelPart") + junk + vol(12, 23, "NegativeVolume") + junk + "" + junk + "") + sc = build(fx.three_mf(tmp_path / "p.3mf", fx.model_xml([obj], [("1", fx.tf(900, 100, 0))]), + {"Metadata/Slic3r_PE_model.config": cfg})) + assert [v["role"] for v in sc["meshes"][0]["volumes"]] == ["part", "negative"] + assert node(sc, "b0")["bounds_mm"]["max"][0] == pytest.approx(910.0) # the negative cube (to 930) does not count + + +# ------------------------------------------------------------------ R3-2: every completed element is released + +def test_r3_2_relationships_and_namespaced_settings_elements_are_released(tmp_path, monkeypatch): + seen = [] + real = scene._free + monkeypatch.setattr(scene, "_free", lambda e: (seen.append(e.tag), real(e))[1]) + rels = f'' + '' * 50 + "" + cfg = '' + "" * 50 + "" + path = fx.three_mf(tmp_path / "r.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", None)]), + {"3D/_rels/x.rels": rels, "Metadata/model_settings.config": cfg}) + scene.build_scene_dict(str(path), REV) + assert seen.count(f"{{{fx.REL_NS}}}Relationship") >= 50 + assert seen.count("{urn:x}object") >= 50 + + +def test_r3_2_hundreds_of_thousands_of_small_records_stay_within_a_memory_gate(tmp_path): + rels = f'' + '' * 300_000 + "" + assert len(rels) < L.CONFIG_PART_BYTES + cfg = '' + "" * 880_000 + "" + assert len(cfg) < L.CONFIG_PART_BYTES + path = fx.three_mf(tmp_path / "big.3mf", fx.model_xml([fx.cube_object("1", 10)], [("1", None)]), + {"3D/_rels/x.rels": rels, "Metadata/model_settings.config": cfg}) + tracemalloc.start() + sc = scene.build_scene_dict(str(path), REV) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + assert sc["counts"]["nodes"] == 1 and peak <= 300 * 1024 * 1024 + + +# ------------------------------------------------------------------ R3-3: a present-but-empty p:path is an error + +def test_r3_3_an_empty_build_item_path_is_not_treated_as_absent(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10)], [("1", None, "")]) # object 1 EXISTS in this model + refuse(fx.three_mf(tmp_path / "i.3mf", root), "UNRESOLVED_REFERENCE") + + +def test_r3_3_an_empty_component_path_is_not_treated_as_absent(tmp_path): + root = fx.model_xml([fx.cube_object("1", 10), fx.composite_object("9", [("1", "", None)])], [("9", None)]) + refuse(fx.three_mf(tmp_path / "c.3mf", root), "UNRESOLVED_REFERENCE") + + +# ------------------------------------------------------------------ R3-4: a part record without an id still means "has metadata" + +def test_r3_4_a_part_record_without_an_id_makes_the_object_have_metadata(tmp_path): + sc = build(_bambu_part_at_900(tmp_path, "noid.3mf", '')) + n = node(sc, "b0.c0") + assert n["role_context"] == "unknown" and n["printable"] is None + assert sc["status"] == "partial" and "UNKNOWN_VOLUME_ROLE" in codes(sc) and sc["findings"] == [] + plain = build(_bambu_part_at_900(tmp_path, "nometa.3mf", '')) + assert node(plain, "b0.c0")["role_context"] is None # an object with no part records is unchanged + + +# ------------------------------------------------------------------ R3-5: the root unit reaches external build items + +def test_r3_5_a_bad_root_unit_names_the_external_build_item_nodes(tmp_path): + ext = fx.sub_model_xml([fx.cube_object("1", 10)]) + root = fx.model_xml([], [("1", fx.tf(5, 5, 0), "/3D/Objects/e.model")], unit="yard") + sc = build(fx.three_mf(tmp_path / "y.3mf", root, {"3D/Objects/e.model": ext})) + lim = next(l for l in sc["limitations"] if l["code"] == "UNSUPPORTED_UNIT") + assert lim["target_ids"] == ["b0"] # its translation is in the unrecognized unit + assert sc["findings"] == [] + + +# ------------------------------------------------------------------ R3-6: a non-ASCII solid name is text + +def test_r3_6_an_ascii_stl_with_a_non_ascii_name_loads(tmp_path): + facet = "facet normal 0 0 0\nouter loop\nvertex 0 0 0\nvertex 1 0 0\nvertex 0 1 0\nendloop\nendfacet\n" + path = tmp_path / "wuerfel.stl" + path.write_bytes(("solid Würfel_größe\n" + facet * 3 + "endsolid Würfel_größe\n").encode("utf-8")) + assert scene.build_scene_dict(str(path), REV)["counts"]["triangles"] == 3 + binary = tmp_path / "bin.stl" # high bytes that are NOT UTF-8 stay binary + binary.write_bytes(bytes(range(128, 256)) * 3 + struct.pack("') == ("inch", 25.4) + assert units.unit_of(b"") == ("foot", 304.8) + assert units.unit_of(b"") == ("millimeter", 1.0) # absent attribute = default + assert units.unit_of(b'') == ("yard", None) # unknown is surfaced, not guessed + assert units.mm_per_unit(b'') == 1.0 # legacy readers keep treating it as mm + info = units.read_unit(b'') # not a element + assert (info.recognized, info.declared, info.factor) == (False, False, 1.0) # unreadable is not "absent" + assert units.read_unit(b"this is not xml").recognized is False + assert units.read_unit(b"").recognized is False and units.read_unit(b"").recognized is True + assert units.scale_translation([1, 0, 0, 0, 1, 0, 0, 0, 1, 2, 3, 4], 25.4)[9:] == [2 * 25.4, 3 * 25.4, 4 * 25.4] + same = [1, 0, 0, 0, 1, 0, 0, 0, 1, 2, 3, 4] + assert units.scale_translation(same, 1.0) is same diff --git a/backend/tools/scene_benchmark.py b/backend/tools/scene_benchmark.py new file mode 100644 index 00000000..fa66c50a --- /dev/null +++ b/backend/tools/scene_benchmark.py @@ -0,0 +1,303 @@ +"""Benchmark and budget-coverage report for scene/1. + + py -3.13 backend/tools/scene_benchmark.py --out docs/testing/scene-contract/results.json \ + [--real-world DIR] [--local-root DIR ...] [--local-sample 5] + +Measures, per corpus item: whether it fits the budgets, parse+encode time (cold and warm best-of-N), +peak extra memory (tracemalloc peak AND a sampled working-set delta), response size and counts. Then the +hard gates (100k-triangle fixture, cancel latency, 50-cycle leak check) and the coverage fraction. + +Privacy: items from outside the repository (``--local-root``) are recorded ONLY as anonymized ids +``P-01..P-05`` with byte size and triangle count. Names, paths and contents are never written. Nothing +here modifies any input; every file is only read. +""" +from __future__ import annotations + +import argparse +import json +import os +import platform +import statistics +import sys +import tempfile +import threading +import time +import tracemalloc +from pathlib import Path + +HERE = Path(__file__).resolve().parent +BACKEND = HERE.parent +REPO = BACKEND.parent +sys.path.insert(0, str(BACKEND)) + +from snapstudio_core import scene, scene_limits as L # noqa: E402 +from tests import scene_fixtures as fx # noqa: E402 + +MIB = 1024 * 1024 + + +def rss_bytes() -> int: + if sys.platform == "win32": + import ctypes + from ctypes import wintypes + + class PMC(ctypes.Structure): + _fields_ = [("cb", wintypes.DWORD), ("PageFaultCount", wintypes.DWORD), + ("PeakWorkingSetSize", ctypes.c_size_t), ("WorkingSetSize", ctypes.c_size_t), + ("QuotaPeakPagedPoolUsage", ctypes.c_size_t), ("QuotaPagedPoolUsage", ctypes.c_size_t), + ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t), ("QuotaNonPagedPoolUsage", ctypes.c_size_t), + ("PagefileUsage", ctypes.c_size_t), ("PeakPagefileUsage", ctypes.c_size_t)] + k32 = ctypes.WinDLL("kernel32") + psapi = ctypes.WinDLL("psapi") + k32.GetCurrentProcess.restype = wintypes.HANDLE + psapi.GetProcessMemoryInfo.argtypes = [wintypes.HANDLE, ctypes.POINTER(PMC), wintypes.DWORD] + counters = PMC() + counters.cb = ctypes.sizeof(PMC) + psapi.GetProcessMemoryInfo(k32.GetCurrentProcess(), ctypes.byref(counters), counters.cb) + return int(counters.WorkingSetSize) + with open("/proc/self/statm") as fh: + return int(fh.read().split()[1]) * os.sysconf("SC_PAGE_SIZE") + + +def machine() -> dict: + total = None + try: + if sys.platform == "win32": + import ctypes + + class MS(ctypes.Structure): + _fields_ = [("l", ctypes.c_ulong), ("m", ctypes.c_ulong), ("tp", ctypes.c_ulonglong), ("ap", ctypes.c_ulonglong), + ("tpf", ctypes.c_ulonglong), ("apf", ctypes.c_ulonglong), ("tv", ctypes.c_ulonglong), + ("av", ctypes.c_ulonglong), ("ae", ctypes.c_ulonglong)] + ms = MS() + ms.l = ctypes.sizeof(MS) + ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(ms)) + total = ms.tp + else: + total = os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") + except Exception: # noqa: BLE001 + pass + return {"platform": platform.platform(), "python": platform.python_version(), "cpu_count": os.cpu_count(), + "processor": platform.processor() or None, "ram_gib": round(total / 1024 ** 3, 1) if total else None} + + +class Sampler(threading.Thread): + """Samples the working set while a measurement runs.""" + + def __init__(self) -> None: + super().__init__(daemon=True) + self.stop = threading.Event() + self.peak = 0 + + def run(self) -> None: + while not self.stop.is_set(): + self.peak = max(self.peak, rss_bytes()) + time.sleep(0.005) + + +LIFTED = {"MAX_PARSED_TRIANGLES": 2_000_000, "MAX_VERTICES": 2_000_000, "MAX_RENDERED_TRIANGLES": 2_000_000, + "MAX_RESPONSE_BYTES": 256 * MIB, "MODEL_XML_BYTES": 512 * MIB, "ARCHIVE_EXPANSION_BYTES": 1024 * MIB, + "MAX_NODES": 50_000} + + +def lifted(path: str) -> dict: + """Evidence for a budget decision: what the refused file would cost if the budgets were raised + (to 2M triangles / 256 MiB). Reports what it measures; never ships those numbers as limits.""" + saved = {k: getattr(L, k) for k in LIFTED} + try: + for k, v in LIFTED.items(): + setattr(L, k, v) + t = time.perf_counter() + try: + sc = scene.build_scene_dict(path, "0" * 64) + body = scene.serialize(sc) + except scene.SceneError as exc: + return {"still_refused": exc.code, "message": exc.message} + return {"triangles": sc["counts"]["triangles"], "rendered_triangles": sc["counts"]["rendered_triangles"], + "vertices": sc["counts"]["vertices"], "nodes": sc["counts"]["nodes"], "response_bytes": len(body), + "time_s": round(time.perf_counter() - t, 2)} + finally: + for k, v in saved.items(): + setattr(L, k, v) + + +def measure(path: str, warm_runs: int) -> dict: + size = os.path.getsize(path) + rec: dict = {"bytes": size} + ceiling = size > L.SNAPSHOT_MAX_BYTES + rec["job_gate"] = "SNAPSHOT_CEILING" if ceiling else "ok" + try: + t = time.perf_counter() + body = scene.build_scene(path, "0" * 64) + cold = time.perf_counter() - t + except scene.SceneError as exc: + rec.update(result="refused", code=exc.code, message=exc.message, fits_budgets=False) + rec["lifted"] = lifted(path) + return rec + sc = json.loads(body) + rec.update(result="ok", code=None, status=sc["status"], fits_budgets=not ceiling, + triangles=sc["counts"]["triangles"], rendered_triangles=sc["counts"]["rendered_triangles"], + vertices=sc["counts"]["vertices"], nodes=sc["counts"]["nodes"], meshes=sc["counts"]["meshes"], + response_bytes=len(body), limitations=[l["code"] for l in sc["limitations"]], + cold_s=round(cold, 3)) + times = [] + for _ in range(warm_runs): + t = time.perf_counter() + scene.build_scene(path, "0" * 64) + times.append(time.perf_counter() - t) + rec["warm_best_s"] = round(min(times), 3) + rec["warm_median_s"] = round(statistics.median(times), 3) + base = rss_bytes() + sampler = Sampler() + sampler.start() + tracemalloc.start() + scene.build_scene(path, "0" * 64) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + sampler.stop.set() + sampler.join() + rec["tracemalloc_peak_mib"] = round(peak / MIB, 1) + rec["rss_delta_mib"] = round(max(0, sampler.peak - base) / MIB, 1) + return rec + + +def corpus(tmp: Path, args) -> list[tuple[str, str, str]]: + """(id, class, path). Synthetic fixtures are authored here; nothing private is named.""" + items: list[tuple[str, str, str]] = [] + items.append(("S-cube", "synthetic", str(fx.plain_cube_3mf(tmp / "s-cube.3mf")))) + items.append(("S-bambu-3parts", "synthetic", str(fx.bambu_project( + tmp / "s-bambu.3mf", parts=3, items=[("100", fx.tf(10, 10, 0))], plates=[(1, [("100", 0)])], + roles=["normal_part", "modifier_part", "negative_part"])))) + items.append(("S-prusa-volumes", "synthetic", str(fx.prusa_project( + tmp / "s-prusa.3mf", volumes=[(0, 11, "ModelPart"), (12, 23, "ParameterModifier")])))) + items.append(("S-stl", "synthetic", str(fx.binary_stl(tmp / "s.stl")))) + for n in (10_000, 50_000, 100_000): + items.append((f"S-grid-{n // 1000}k", "synthetic", str(fx.big_grid_3mf(tmp / f"s-grid-{n}.3mf", n)))) + root = fx.model_xml([fx.cube_object("1", 10)], [("1", fx.tf(i * 12.0 % 250, (i // 20) * 12.0, 0)) for i in range(300)]) + items.append(("S-instances-300", "synthetic", str(fx.three_mf(tmp / "s-inst.3mf", root)))) + items.append(("S-over-budget", "synthetic-over-budget", str(fx.big_grid_3mf(tmp / "s-over.3mf", 280_000)))) + for p in sorted((REPO / "examples").iterdir()): + if p.suffix.lower() in (".3mf", ".stl") and ".orig." not in p.name: + items.append((f"E-{p.stem}", "repo-example", str(p))) + rw = Path(args.real_world) if args.real_world else BACKEND / "tests" / "fixtures" / "real-world" + if rw.is_dir(): + for p in sorted(rw.glob("*.3mf")): + if ".orig." not in p.name: + items.append((f"R-{p.stem}", "real-world", str(p))) + pool = sorted((p for d in args.local_root or [] for p in Path(d).rglob("*") + if p.suffix.lower() in (".3mf", ".stl") and p.is_file()), + key=lambda p: (p.stat().st_size, str(p))) + if pool: # evenly spaced by size across ALL local roots; only the anonymized id is ever recorded + picks = [pool[int(i * (len(pool) - 1) / max(1, args.local_sample - 1))] for i in range(args.local_sample)] + for n, p in enumerate(dict.fromkeys(picks), 1): + items.append((f"P-{n:02d}", "local-anonymized", str(p))) + return items + + +def survey(roots: list[str]) -> dict: + """Size-only survey of local projects (no names): how many are over the snapshot ceiling.""" + sizes = [] + for d in roots: + for p in Path(d).rglob("*"): + if p.suffix.lower() == ".3mf" and p.is_file(): + sizes.append(p.stat().st_size) + if not sizes: + return {} + over = sum(1 for s in sizes if s > L.SNAPSHOT_MAX_BYTES) + sizes.sort() + return {"files": len(sizes), "over_snapshot_ceiling": over, "fraction_within_ceiling": round(1 - over / len(sizes), 3), + "median_mib": round(statistics.median(sizes) / MIB, 1), "p90_mib": round(sizes[int(0.9 * (len(sizes) - 1))] / MIB, 1)} + + +def gates(tmp: Path) -> dict: + sys.path.insert(0, str(BACKEND)) + from snapstudio_api import scene_jobs as sj + + out: dict = {} + big = fx.big_grid_3mf(tmp / "gate-100k.3mf", 100_000) + m = measure(str(big), 3) + out["fixture_100k_triangles"] = {"parse_encode_warm_s": m.get("warm_best_s"), "cold_s": m.get("cold_s"), + "extra_peak_mib_tracemalloc": m.get("tracemalloc_peak_mib"), + "extra_peak_mib_rss": m.get("rss_delta_mib"), "response_bytes": m.get("response_bytes"), + "gate_time_le_10s": (m.get("warm_best_s") or 1e9) <= 10, + "gate_mem_le_500mib": max(m.get("tracemalloc_peak_mib") or 0, m.get("rss_delta_mib") or 0) <= 500, + "target_time_le_3s": (m.get("warm_best_s") or 1e9) <= 3, + "target_mem_le_300mib": max(m.get("tracemalloc_peak_mib") or 0, m.get("rss_delta_mib") or 0) <= 300, + "gate_response_le_8mib": (m.get("response_bytes") or 1e9) <= L.MAX_RESPONSE_BYTES} + jobs = sj.SceneJobs(snapshot_root=str(tmp / "snap")) + latencies = [] + for _ in range(5): + job = jobs.start(str(big), f"bench-cancel-{time.monotonic_ns()}") + end = time.monotonic() + 10 + while time.monotonic() < end and jobs.status(job["job_id"])["stage"] not in ("parsing", "encoding"): + time.sleep(0.005) + time.sleep(0.15) + t = time.perf_counter() + jobs.cancel(job["job_id"]) + while jobs.worker_alive(): + time.sleep(0.002) + latencies.append(time.perf_counter() - t) + out["cancel_latency_s"] = {"max": round(max(latencies), 3), "median": round(statistics.median(latencies), 3), + "gate_le_2s": max(latencies) <= 2.0} + small = str(fx.plain_cube_3mf(tmp / "leak-small.3mf")) + mid = str(fx.big_grid_3mf(tmp / "leak-mid.3mf", 20_000)) + base_threads = threading.active_count() + wedged = False + leak = sj.SceneJobs(snapshot_root=str(tmp / "snap2")) + for i in range(50): + job = leak.start(mid if i % 2 else small, f"bench-leak-{i:04d}-{time.monotonic_ns()}") + if i % 3 == 0: + time.sleep(0.01) + leak.cancel(job["job_id"]) + wedged = wedged or leak.wedged() + leak.close() + end = time.monotonic() + 10 + while leak.worker_alive() and time.monotonic() < end: + time.sleep(0.01) + snaps = os.listdir(leak.snapshot_dir()) if leak.snapshot_dir() and os.path.isdir(leak.snapshot_dir()) else [] + out["leak_50_cycles"] = {"retained_jobs": leak.retained(), "cap": L.MAX_TERMINAL_JOBS, "temp_snapshots": len(snaps), + "threads_before": base_threads, "threads_after": threading.active_count(), + "worker_wedged_seen": wedged, + "gate_pass": leak.retained() <= L.MAX_TERMINAL_JOBS and not snaps + and threading.active_count() == base_threads and not wedged} + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--out", required=True) + ap.add_argument("--real-world", help="folder holding the fetched real-world 3MF fixtures") + ap.add_argument("--local-root", action="append", help="folder of local projects (anonymized ids only)") + ap.add_argument("--local-sample", type=int, default=5) + ap.add_argument("--warm-runs", type=int, default=3) + args = ap.parse_args() + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + rows = [] + for ident, klass, path in corpus(tmp, args): + rec = measure(path, args.warm_runs) + rec.update(id=ident, **{"class": klass}) + rows.append(rec) + print(f"{ident:24s} {klass:22s} {rec.get('result'):8s} {rec.get('code') or '':16s} " + f"{rec.get('warm_best_s', '-')!s:>7} s {rec.get('response_bytes', '-')!s:>9} B", flush=True) + results = {"schema": "scene-benchmark/1", "machine": machine(), "budgets": dict(L.LIMITS_ECHO), + "snapshot_ceiling_bytes": L.SNAPSHOT_MAX_BYTES, "items": rows} + within = [r for r in rows if r["class"] in ("repo-example", "real-world", "local-anonymized", "synthetic")] + results["coverage"] = { + "by_class": {k: {"items": sum(1 for r in rows if r["class"] == k), + "fit": sum(1 for r in rows if r["class"] == k and r.get("fits_budgets"))} + for k in sorted({r["class"] for r in rows})}, + "fraction_fitting_excluding_deliberate_over_budget": round( + sum(1 for r in within if r.get("fits_budgets")) / max(1, len(within)), 3)} + if args.local_root: + results["local_survey_3mf_sizes_only"] = survey(args.local_root) + results["gates"] = gates(tmp) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + print("wrote", out) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/testing/scene-contract/README.md b/docs/testing/scene-contract/README.md new file mode 100644 index 00000000..76566254 --- /dev/null +++ b/docs/testing/scene-contract/README.md @@ -0,0 +1,338 @@ +# scene/1: the backend scene contract (PR 3a) + +The engine can now describe a project's geometry, placement and engine findings as one bounded, +machine-checked document (`scene/1`), built by a cancellable background job. This is the backend half +only. There is no viewer and no desktop change in this PR. + +* Contract: `backend/snapstudio_core/data/scene-1.schema.json` (JSON Schema 2020-12, frozen: every object + closed, every field required, unknown is `null`). +* Golden scene: `backend/tests/fixtures/scene/golden-scene-1.json`, written by hand and independently of the + builder by `make_golden.py` next to it. A test requires the real builder to reproduce it exactly. +* Builder: `backend/snapstudio_core/scene.py`. Every numeric budget lives in + `backend/snapstudio_core/scene_limits.py` and nowhere else. +* Jobs and routes: `backend/snapstudio_api/scene_jobs.py`; `POST /scene/start | status | result | cancel` + (existing `X-Auth-Token`; the result body is serialized once and sent by the new `_send_bytes`). +* Units fix: `backend/snapstudio_core/units.py`, applied to every dimension source (below). +* Measurements: `results.json` in this folder, produced by `backend/tools/scene_benchmark.py`. + +Not covered: any viewer, the Tauri window, WebKitGTK, a screen reader, a real printer, and Linux (see +"What was not run"). + +## What a scene is, and is not + +A scene is a **consistent snapshot of the file as it was copied for the job**. The job copies the source once +into a private file, hashes it (the `revision` is the SHA-256 of those bytes) and parses only the copy, so +replacing the original while a job runs cannot tear the scene. `SOURCE_CHANGED` applies only to (a) a file that +changed while it was being copied, (b) a reused `request_id`+path whose file now hashes differently from that +job's revision, and (c) an `expected_revision` that differs from the job's. It does **not** apply to edits made +after the snapshot was taken. The original file is only ever read (a test compares its bytes and mtime before and +after; so does the job test). + +Scenes say what the file proves and no more: + +* **Placement.** STL files, models with no build item and unresolved placement are `placement_state: "unknown"`. + Their `world_mm` is an identity used only so something can be drawn. No bed-fit or size statement of any kind is + attached to an unknown-placement node. +* **Plates.** Membership is per instance, from `model_settings.config` (`plate: known | ambiguous | unknown`). + Plate origins are never invented (`origin_mm` is always `null`). A multi-plate project gets + `MULTI_PLATE_PLACEMENT_UNCHECKED` and no placement findings, because the grid spacing is not in the file. +* **Volumes and roles.** Every mesh partitions *all* its triangles into volumes (`part`, `modifier`, `negative`, + `support_enforcer`, `support_blocker`, `unknown`). The synthetic whole-mesh `part` volume + (`source: "object"`) is used ONLY for an object that has no volume metadata at all. Prusa volume ranges are read + by attribute name (the order of `firstid`/`lastid` does not matter); an unmapped gap, an overlap or an unreadable + range is `unknown` (`UNKNOWN_VOLUME_ROLE`), never a guess. Bambu/Orca part records are **per use**, so the role + lives on the node (`role_context`), not on the shared mesh: a mesh can be a negative part under one object and an + ordinary part under another, and a role given to a component assembly applies to everything beneath it. Decision + (differs from `placement.read_objects`, which defaults to `normal_part`): an object that HAS part records but none + for one of its components gives that component `role_context: "unknown"`, `UNKNOWN_VOLUME_ROLE` and status + `partial`, and no fit finding. A node's bounds, `printable` and every fit statement use only role=part geometry; + non-part geometry is carried (`has_non_part_volumes`) and excluded from fit. +* **Production-extension build items.** A build item's `p:path` names the model FILE its object lives in. Node + identity, repeated-instance counting and lookup use the pair (part path, object id); an unresolvable `p:path` is + `UNRESOLVED_REFERENCE`, never a fall back to the root file. Per the Production Extension a `p:path` must be an + absolute package path (leading `/`), and component `p:path` references are honoured only from the root model + part; relative paths, and component paths in any other part, are `UNRESOLVED_REFERENCE`. +* **Fail closed across files and duplicates.** Where the file cannot prove an association the answer is + unknown, not a guess: (1) when the project has slicer settings, every part beneath a build object that lives in + another file gets `role_context: "unknown"` (the part records are matched against root-file objects), with + `UNKNOWN_VOLUME_ROLE`, status `partial` and no fit finding; (2) plate records name a bare object id, so when two + build resources in different files share an id, neither can be proven on a plate (`plate.state: "ambiguous"`, + `PLATE_MEMBERSHIP_UNKNOWN`, no finding); (3) two settings records for the same (object id, part id) that disagree + make the role unknown, identical duplicates are fine; (4) `UNSUPPORTED_UNIT` lists, in `target_ids`, the nodes + living in the affected part(s), and `sources[].unit`/`mm_per_unit` are then the fallback millimetre values. +* **Bounded parsing everywhere.** Model parts, the Bambu settings, the Prusa config and every `.rels` are parsed + with a counter on completed elements (1,000,000 each) and every completed element is released as soon as its data + is consumed (the element itself only, never its siblings: records a parent reads later, such as a negative + ``, are never lost to an unknown neighbour), so a part under its byte cap that is made of millions of tiny + elements is refused early. A part record without an `id` still counts as "this object has part metadata" (the + components it cannot cover are `unknown`). A present-but-empty `p:path` is `UNRESOLVED_REFERENCE`, not absent. + When the ROOT model's unit is unrecognized, the top node of every build item is listed in the `UNSUPPORTED_UNIT` + `target_ids`, since each item's translation is written in that unit. STL files: the kind is + decided from the header AND the size; a binary-looking header over the triangle budget is refused before any + decoding or large allocation whether or not the size matches; ASCII is read in 1 MiB chunks with a 4 KiB line cap, + split on LF, CRLF and a lone CR. +* **Bed.** The U1 rectangle comes from Studio's own template (`x 0.5..270.5, y 1..271`, 0.5 mm inward margin, + the same numbers `plate_placement` uses). If the template yields no usable rectangle, `bed.policy` is + `"fallback"`, `BED_TEMPLATE_UNAVAILABLE` is reported and **all fit findings are suppressed**. +* **Findings** carry `kind` (`placement | size | note`), `scope` (`project | plate | instance`), the engine + that produced them (`scene`), a JSON pointer into the scene and target node ids. Scene-produced + out-of-bounds and size statements are `placement` and `size`; the viewer copy rule (PR 3b) is that the word + "fits" is never derived from a scene. +* **Repeated instances.** Several build items of one object id get their own nodes and their own geometric + world bounds, but `REPEATED_INSTANCE_PLACEMENT_UNVERIFIED` is reported and no finding is attached, because the + engine's existing placement answer for repeated items is wrong on `main` (see "Known limitations"). +* **Non-millimetre sources.** Coordinates are converted to millimetres, and `sources[]` records each part's + unit. Because the contract was written before the conversion was shared, placement findings are still + disabled for any non-mm source (`NON_MM_SOURCE_UNIT`); size findings are not. A declared unit outside the six + defined ones is NOT treated like an absent attribute: the scene reports `UNSUPPORTED_UNIT`, falls back to + millimetres in `sources[]` (documented) and makes no fit statement. +* **Plates.** If a project has plates but an instance is on none, it is `PLATE_MEMBERSHIP_UNKNOWN`, status + `partial`, with no fit finding. + +## Units fix (a correction of wrong claims, not a new feature) + +Several readers ignored the 3MF `` header, so a 12 inch part was reported as 12 mm and +every size-based bed-fit statement for a non-millimetre project was wrong. One shared helper +(`snapstudio_core/units.py`: the six-unit `mm_per_unit` table, a header reader and a translation scaler) is now +applied to **every** dimension source. The header is read from the REAL root `model` element with the XML parser +(first start event; namespace prefixes, comments, processing instructions and a long preamble are irrelevant, and +a `` inside a comment is not seen), by the same `resolve_unit` function the scene parser calls. +Absent means millimetre; an unrecognized value is a distinct, flagged outcome (`UnitInfo.recognized`), with the +legacy readers keeping their old millimetre fallback: + +| Source | Change | +|---|---| +| `intelligence.project_info` (feeds DesignHealth, `service.insights`, strategy signals, `bed_fit`, `validation_report`, `canonical`, `layout`, `mm_doctor`, `toolhead_fit`) | vertex coordinates scaled per model part by that part's own unit | +| `geometry.build_item_dims` (feeds `layout`, `scale_doctor`, `plate_placement.assess`) | vertices scaled per part; component and build-item **translations** scaled too (a translation is a length in the model's unit) | +| `geometry.load_mesh` (feeds `scale_doctor.preview`, `mesh_diagnostics`) | vertices scaled per part | +| `placement.read_objects` | mesh points and transform translations scaled (through the shared helper); its repeated-instance behaviour is untouched | +| `plate_placement.prepare_placed_copy` (the "move onto the plate" fix) | the offset it writes is computed in millimetres and is now converted back to the ROOT model's unit before it is added to a build-item translation. A move that does not achieve its goal no longer leaves its copy on disk | + +Behaviour for millimetre files is unchanged: the scale factor is exactly 1.0 and the code skips the multiplication. +Evidence: outputs of `project_info`, `build_item_dims`, `load_mesh`, `read_objects`, `plate_placement.assess` and +`scale_doctor.scale_options` were hashed over all 50 fixtures and examples in the tree on `main` (b01f7d0) and on +this branch and compared: **identical** (sha256 `73a018e5...69ef` both sides). `tests/test_units_consumers.py` pins every consumer +named in the plan (`canonical`, `layout`, `validation_report`, `bed_fit` via `service.bed_fit`, strategy signals, +DesignHealth data path via `service.insights`, Scale Doctor, `plate_placement.assess`, `read_objects`) for all six +units, and asserts that a 12 inch part reads 304.8 mm everywhere. + +What this does **not** do (decision, plan v4 item 1): the size-based consumers still answer "does the model's +size fit the bed". They are not placement-aware. A small object translated off the plate, or separated instances +whose union is large, is still judged by size only. The scene's placement findings are the placement-aware +source. No consumer copy was edited here; qualifying those claims "by size" is PR 3c. + +## Jobs + +One worker thread, at most one queued job (newest wins; a running job is marked cancelled at once and the +replacement waits, queued, until the cancelled thread observes the flag and exits). States: +`queued | running | succeeded | failed | cancelled`. Terminal states are immutable and the first terminal wins +(a worker finishing after a cancel discards its result). 60 s deadline from the moment the job starts +running. A terminal job is kept 120 s or until more than 8 terminal jobs exist. An evicted or unknown id answers +404 `EXPIRED`; there is no stored "expired" state. + +| Route | Answers | +|---|---| +| `/scene/session {}` | 200 `{client_id, ttl_s}` (server-issued 24-character id); 503 `SESSION_LIMIT` | +| `/scene/start {path, request_id, client_id?, seq?}` | 200 job; 400 `INVALID_REQUEST` (also: only one of `client_id`/`seq`, null or malformed values, `seq` reused for another request, `request_id` reused at a higher `seq`); 422 `UNSUPPORTED_FORMAT`; 409 `INVALID_REQUEST` (request_id reused for another path), `SOURCE_CHANGED`, `STALE_START`, `CANCELLED_BEFORE_START` or `SESSION_EXPIRED`; 503 `WORKER_WEDGED` | +| `/scene/status {job_id, client_id?}` | 200 `{state, stage, completed, total, error, revision}`; 404 `EXPIRED`. A job started in a session needs that session's `client_id`; a missing or different one answers exactly like an unknown id (404 `EXPIRED`), so existence is never confirmed | +| `/scene/result {job_id, client_id?, expected_revision?}` | 200 scene; 422 `{error: }` for a failed job; 409 `CANCELLED` / `NOT_READY` / `SOURCE_CHANGED`; 404 `EXPIRED` | +| `/scene/cancel {job_id, client_id?}` or `{client_id, request_id, seq}` | 200 status (idempotent; a late cancel on a terminal job is a no-op; for a session job `client_id` is required, as for status and result). By request: 409 `SESSION_EXPIRED` for an unknown session; otherwise `dead_through` rises to `seq` and the job of that EXACT attempt (client_id, request_id and `seq` all equal) is cancelled, if any, so a delayed cancel for an old attempt cannot cancel a newer job that reused the request id, and a cancel for a higher `seq` does not cancel an older job; that `seq` and every lower one stay refused for the life of the session; with no such job the body is a `job_status` with `job_id: null`, `state: "cancelled"` | + +Error codes: `INVALID_REQUEST UNSUPPORTED_FORMAT INVALID_ARCHIVE INVALID_GEOMETRY UNRESOLVED_REFERENCE +LIMIT_EXCEEDED SOURCE_CHANGED CANCELLED TIMEOUT EXPIRED NOT_READY WORKER_WEDGED INTERNAL STALE_START +CANCELLED_BEFORE_START SESSION_EXPIRED SESSION_LIMIT` (`BUSY` is not an error code: a new start replaces the client's own earlier work). + +### Start ordering by session + +A client that aborts or times out a start cannot stop that request from still reaching the engine, possibly AFTER +the start it actually wants. The engine therefore orders a client's starts itself. The design is a **session** with +O(1) state, not a set of remembered requests. Four invariants, each pinned by tests named `test_i1_...` to +`test_i4_...`: + +* **I1: delayed starts cannot become valid through bookkeeping eviction.** A start is judged against the + session's watermark, not against whether its job is still retained, so an evicted job does not make an old + start admissible again. +* **I2: cancellation cannot be forgotten while the start remains admissible.** A cancel raises the session's + `dead_through` mark, so that start and every lower `seq` stay refused for as long as the session lives. +* **I3: idempotency is scoped to (session, request) and ownership is checked consistently on every path.** Jobs + are keyed by `(client_id, request_id)`; two sessions may use the same request_id and path and each gets, and + can cancel, its own job; cancel by request matches both ids exactly. +* **I4: capacity exhaustion or session expiry fails explicitly; nothing old is silently revived.** + +**Protocol.** `POST /scene/session {}` returns `{client_id, ttl_s}`; the engine chooses the id (24 characters of +`[A-Za-z0-9_-]`). Starts and by-request cancels carry that `client_id` and a `seq`, an integer from 1 to 2**53-1 that +the client treats as a strictly increasing generation counter for the session (the client only ever abandons lower +seqs when it moves to a higher one). Both keys or neither: null or half-present pairs are `400 INVALID_REQUEST`, +never a legacy start; ids must match in full (a trailing newline is rejected). With neither key the start behaves +exactly as before, in its own request_id-only namespace. + +**State per session** (nothing else): `watermark` (highest admitted seq), `dead_through` (highest cancelled seq), +`current` = (seq, request_id, path, job_id) of the entry at the watermark. + +**Rules, in order, under the registry lock that also registers jobs** (one admission function for every +successful start): unknown or expired `client_id` -> 409 `SESSION_EXPIRED` (a late start from an expired session +is refused, never re-created); `seq <= dead_through` -> 409 `CANCELLED_BEFORE_START`; `seq < watermark` -> 409 +`STALE_START`; `seq == watermark` with the same request_id and path -> the registered job (idempotent), or 409 +`STALE_START` if that job is no longer registered (an exact retry of an already-processed start is refused); equal +with anything else -> 400 `INVALID_REQUEST`; `seq > watermark` -> register (replacing the older job as before) and +set the watermark, except that a request_id already registered for the session is 400 `INVALID_REQUEST` +("request id reused": the old job is never handed back), and that attempt still raises the watermark so an older +attempt still in flight is refused. An idempotent repeat that was still hashing the file is re-judged by the same +rules BEFORE `SOURCE_CHANGED` is reported. After any refusal the client retries with a FRESH `request_id` and a +HIGHER `seq`. + +**Ownership (I3).** Every job records the session that started it. Status, result and cancel-by-job_id on a session +job need that session's `client_id` (404 `EXPIRED`, identical to an unknown id, otherwise), and `replaced_job_id` +in a start response names a replaced job only when it belonged to the same session (or both are legacy); in every +other case it is `null`. Legacy jobs keep today's behaviour. + +**An exact retry never loses its job.** A retry of the entry at the watermark (same `seq`, request_id and path) that is +rejected before registration (for example the file has since vanished) leaves the session's reference to that +entry's job untouched, so later and concurrent exact retries still get the job for as long as it is registered. + +**After an admitted start that is then rejected** (`WORKER_WEDGED`, an unsupported format, a missing file) the +session's watermark has already advanced to that `seq`, so a delayed older attempt cannot be accepted behind it. +The client retries with a fresh `request_id` and a higher `seq`; an exact retry is `STALE_START`. + +**Sessions.** At most 32 (`MAX_SESSIONS`), idle TTL 15 minutes (`SESSION_TTL_SECONDS`), touched by every start, +cancel (by request or by job_id), status and result. Sessions are pruned after expired jobs are evicted in the same pass. Only idle-expired sessions that own no registered job are pruned; a live session is never +evicted. At capacity with nothing prunable, opening a session is 503 `SESSION_LIMIT`. + +**Limits, stated plainly.** Ordering is provided PER SESSION. It is not provided across sessions: a start from +another session replaces the previous job exactly as before. An engine restart drops every session, so the first +start from an old `client_id` answers `SESSION_EXPIRED` and the client must open a new session. Session expiry +likewise ends that session's ordering (explicitly: `SESSION_EXPIRED`, never a silent revival). + +## Budgets (provisional) and the coverage decision + +| Budget | Value | +|---|---| +| Snapshot copy ceiling / archive expansion | 128 MiB / 128 MiB | +| Model XML (all `.model` parts) | 32 MiB | +| Rendered triangles (every repetition counted) / distinct parsed triangles | 250,000 / 250,000 | +| Distinct vertices | 300,000 | +| Nodes / depth | 2,000 / 64 | +| Whole serialized response | 8 MiB (exact bytes; geometry pre-check refuses before encoding) | +| Ids | 128 characters, by construction (a long ancestry is replaced by a deterministic digest) | +| Part names carried on nodes / `` definitions / XML elements | 256 characters / 20,000 / 1,000,000, all counted while parsing | +| Findings / limitations | 500 / 50 | + +Exceeding any of these is `LIMIT_EXCEEDED`, never a shortened scene. + +### Corpus and results + +Qualifications: the four real-world fixtures are upstream files fetched by `fetch_real_world.py` and are absent from +this worktree (the benchmark read them from the main checkout); all five local samples are STL files (89% of the 11,604 local .stl/.3mf files are STL), so they exercise the STL path and say little about 3MF project structure; they +were chosen by size quantile, not at random; the fixture +called "100k" is 99,458 triangles; timings are one machine, one run. + +Machine: Windows 11, Intel64 Family 6 Model 183 (32 logical CPUs), 63.8 GiB RAM, Python 3.13.14. Numbers are one +run on a quiet machine; warm = best of 3. Corpus: authored synthetic fixtures, the repo examples, the four +real-world fixtures (fetched by `fetch_real_world.py`, not committed) and five local projects chosen by size +quantile and recorded **only** as `P-01..P-05` (names and contents are never written). All 19 repo examples +fit and are summarized in `results.json`; four representative ones are shown here. + +| id | class | size | result | triangles (rendered) | nodes | response | parse+encode warm | tracemalloc peak | RSS delta | +|---|---|---|---|---|---|---|---|---|---| +| S-cube | synthetic | 1.5 KiB | ok (complete) | 12 (12) | 1 | 2.0 KiB | 0.001 s | 0.1 MiB | 0.1 MiB | +| S-bambu-3parts | synthetic | 4.7 KiB | ok (complete) | 36 (36) | 4 | 5.0 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| S-prusa-volumes | synthetic | 2.6 KiB | ok (complete) | 24 (24) | 1 | 2.4 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| S-stl | synthetic | 0.7 KiB | ok (partial) | 12 (12) | 1 | 2.0 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| S-grid-10k | synthetic | 603.3 KiB | ok (complete) | 9,800 (9,800) | 1 | 233.6 KiB | 0.036 s | 1.3 MiB | 0.1 MiB | +| S-grid-50k | synthetic | 3.11 MiB | ok (complete) | 49,928 (49,928) | 1 | 1.15 MiB | 0.176 s | 5.6 MiB | 3.8 MiB | +| S-grid-100k | synthetic | 6.22 MiB | ok (complete) | 99,458 (99,458) | 1 | 2.28 MiB | 0.405 s | 10.9 MiB | 3.5 MiB | +| S-instances-300 | synthetic | 19.9 KiB | ok (partial) | 12 (3,600) | 300 | 187.3 KiB | 0.008 s | 1.8 MiB | 0.0 MiB | +| S-over-budget | synthetic (deliberately over) | 17.83 MiB | **refused `LIMIT_EXCEEDED`** | 279,752 | - | 6.42 MiB if budgets raised | 1.15 s if raised | - | - | +| E-demo_offplate_foreign | repo example | 9.7 KiB | ok (complete) | 12 (12) | 2 | 2.9 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| E-demo_u1_showcase | repo example | 1.7 KiB | ok (partial) | 12 (12) | 1 | 2.0 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| E-sample_cube | repo example | 0.7 KiB | ok (partial) | 12 (12) | 1 | 2.0 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| E-sample_cube_SnapmakerU1 | repo example | 9.7 KiB | ok (complete) | 12 (12) | 2 | 2.7 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| R-bambu-pa-pattern | real-world | 22.5 KiB | ok (complete) | 12 (12) | 2 | 2.8 KiB | 0.003 s | 0.1 MiB | 0.0 MiB | +| R-orca-badge | real-world | 797.9 KiB | ok (complete) | 72,586 (72,586) | 42 | 1.70 MiB | 0.274 s | 8.2 MiB | 3.0 MiB | +| R-orca-pa-line-dual | real-world | 29.0 KiB | ok (complete) | 24 (192) | 24 | 18.5 KiB | 0.002 s | 0.2 MiB | 0.0 MiB | +| R-prusa-seam-test | real-world | 2.40 MiB | ok (complete) | 225,154 (225,154) | 1 | 5.15 MiB | 0.87 s | 25.0 MiB | 30.2 MiB | +| P-01 | local | 0.5 KiB | ok (partial) | 8 (8) | 1 | 1.9 KiB | 0.001 s | 0.1 MiB | 0.0 MiB | +| P-02 | local | 963.6 KiB | ok (partial) | 19,733 (19,733) | 1 | 464.3 KiB | 0.02 s | 4.5 MiB | 3.2 MiB | +| P-03 | local | 3.79 MiB | ok (partial) | 79,518 (79,518) | 1 | 1.82 MiB | 0.097 s | 18.4 MiB | 24.9 MiB | +| P-04 | local | 15.60 MiB | **refused `LIMIT_EXCEEDED`** | 327,141 | - | 7.49 MiB if budgets raised | 0.43 s if raised | - | - | +| P-05 | local | 1454.44 MiB | **refused `LIMIT_EXCEEDED`** (over the 128 MiB snapshot ceiling) | - | - | - | - | - | - | + +Coverage (fraction of the corpus that fits, excluding the one deliberately over-budget fixture): **0.944** +(repo examples 19/19, synthetic 8/8, real-world **4/4**, local **3/5**). A size-only survey of the 1,230 local `.3mf` +files under the maintainer's model folders found 126 (**10.2%**) larger than the 128 MiB snapshot ceiling (median 31.2 +MiB, p90 130.1 MiB); no names or paths are recorded. Some projects also exceed the triangle budget: the local `P-04` +has 327,141 triangles and `P-05` is over the ceiling (30.5 million triangles). + +### Budget decision (accepted) + +The first run, at the originally specified 100,000 triangles, fit only 3 of the 4 real-world fixtures: +`R-prusa-seam-test` has 225,154 triangles. Evidence: raised runs took about 1.2 s, and the response is 5.15 MiB. +The 8 MiB whole-body wire cap is the real wall, not time or memory. Decision: **`MAX_RENDERED_TRIANGLES` and +`MAX_PARSED_TRIANGLES` are now 250,000** (`scene_limits.py`, the only place they live); the 300,000 decoded-vertex +budget already covered the fixture (112,569 vertices) and is unchanged, as are the 8 MiB cap and every other gate. +Models over the budget stay refused with a message that names the count and the limit, for example +"The model has 327,141 triangles; Studio draws at most 250,000." (for a 3MF the count is "at least N", reported the +moment the budget is passed). `P-04` is such a model; its scene would be about 7.5 MiB. Chunked or binary mesh +transfer, or decimation, for larger models is a documented follow-up and is not part of this PR. The 128 MiB source +ceiling is unchanged; 10.2% of the surveyed local `.3mf` files exceed it and are refused. + +### Gates + +| Gate | Result | +|---|---| +| Whole-body response cap enforced on the exact bytes | pass (tests; largest measured scene 5.15 MiB) | +| Node / triangle / vertex / depth budgets enforced during traversal | pass (tests follow the constants) | +| Cancel observed within 2 s | pass: max 0.005 s over 5 cancels on the 100k fixture (and a test on a 90k scene) | +| 50 start/cancel cycles: retained <= cap, temp snapshots 0, threads at baseline | pass: retained 8 of 8, snapshots 0, threads 1 to 1 | +| `WORKER_WEDGED` never occurs in the 50-cycle run | pass (its fail-closed path is tested separately) | +| Repo examples fit | pass (19/19) | +| Four real-world fixtures fit | pass (4/4) after the budget decision above | +| 100k-triangle fixture: parse+encode <= 10 s, extra peak memory <= 500 MB | pass: 0.358 s, 11.0 MiB (tracemalloc) / 6.1 MiB (RSS) | +| Targets (not gates): <= 3 s and <= 300 MB on the 100k fixture | met | + +## Reproduce + +```text +cd backend +py -3.13 -m pip install -e ".[test]" +py -3.13 -m pytest -q tests/test_scene_contract.py tests/test_scene_build.py tests/test_scene_hostile.py \ + tests/test_scene_jobs.py tests/test_scene_api.py tests/test_units_consumers.py +py -3.13 tools/scene_benchmark.py --out ../docs/testing/scene-contract/results.json \ + --real-world tests/fixtures/real-world [--local-root ...] +``` + +## Known limitations (carry into the PR description) + +1. **Size-only consumers are not placement-aware.** DesignHealth, `bed_fit`, `validation_report` and the service + bed-fit path answer "does the SIZE fit". A small object translated off the plate is not detected by them, and + separated instances inflate the union. The scene's placement findings are the placement-aware source. Tracked + as issue #91; the copy change qualifying those claims "by size" is the planned PR 3c. +2. **Repeated build items of one object id are mis-placed by `plate_placement.assess` on `main`** + (`placement.py` keeps the last transform: a cube at X=500 and X=100 reports "inside"). Not fixed here (tracked as issue #93). The scene reports `REPEATED_INSTANCE_PLACEMENT_UNVERIFIED` and attaches no finding to those nodes. +3. **Plates.** Non-Bambu projects (Prusa, plain 3MF, STL) have no plate records (`PLATES_UNAVAILABLE`). Grid origins + of multi-plate projects are unknown, so multi-plate projects get no placement findings. +4. **Stuck native calls** cannot be interrupted in-process (fail-closed `WORKER_WEDGED` backstop, no multiprocessing). +5. **STL** is decoded incrementally: a binary header count over the budget is refused before any triangle is + decoded, decoding runs in batches of 5,000 triangles with the cancel hook between them, and ASCII is line-streamed + with a running counter. +6. **Budgets are provisional.** Models over 250,000 triangles (or over the 128 MiB source ceiling) are refused with `LIMIT_EXCEEDED`; chunked/binary transfer or decimation is a follow-up. +7. `jsonschema` is a test-only extra (`pip install -e ".[test]"`); the engine never imports it. + +## Deviations from the written plan + +* **Node ids.** `b.c.c...` does not fit 128 characters at 64 levels, so beyond 128 the ancestry is + replaced by a deterministic digest (`b3.~<32 hex>`); the 128-character contract is kept, not widened. The digest + (128 bits of SHA-256) gives practical collision resistance, not mathematical uniqueness; the builder asserts + uniqueness while building and fails with `INTERNAL` if two node ids ever collide. +* **Freeze order.** The JSON Schema was frozen first, but the golden scene and the fixture matrix were authored + after the first draft of the traversal, not before it. The golden is written independently of the builder and + the builder is held to it. +* **Size findings** are only emitted for nodes whose placement is known (plan v2: no bed-relative statement for + unknown placement), single-plate, template bed, non-repeated instances. +* **No `request_validation.py` change was needed**; the route validation reuses its existing helpers from + `scene_jobs.py`. + +## What was not run + +Linux (the POSIX branch of the pid-liveness check is exercised only by monkeypatch; the Linux CI job), the frozen sidecar, the packaged app, any +viewer, a real printer and any network. diff --git a/docs/testing/scene-contract/results.json b/docs/testing/scene-contract/results.json new file mode 100644 index 00000000..9629d874 --- /dev/null +++ b/docs/testing/scene-contract/results.json @@ -0,0 +1,915 @@ +{ + "schema": "scene-benchmark/1", + "machine": { + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.13.14", + "cpu_count": 32, + "processor": "Intel64 Family 6 Model 183 Stepping 1, GenuineIntel", + "ram_gib": 63.8 + }, + "budgets": { + "max_archive_bytes": 134217728, + "max_model_xml_bytes": 33554432, + "max_rendered_triangles": 250000, + "max_vertices": 300000, + "max_nodes": 2000, + "max_depth": 64, + "max_response_bytes": 8388608, + "max_findings": 500, + "max_limitations": 50, + "max_id_length": 128 + }, + "snapshot_ceiling_bytes": 134217728, + "items": [ + { + "bytes": 1553, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 1, + "meshes": 1, + "response_bytes": 2027, + "limitations": [ + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.005, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.1, + "id": "S-cube", + "class": "synthetic" + }, + { + "bytes": 4778, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 36, + "rendered_triangles": 36, + "vertices": 24, + "nodes": 4, + "meshes": 3, + "response_bytes": 5084, + "limitations": [], + "cold_s": 0.008, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "S-bambu-3parts", + "class": "synthetic" + }, + { + "bytes": 2690, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 24, + "rendered_triangles": 24, + "vertices": 16, + "nodes": 1, + "meshes": 1, + "response_bytes": 2454, + "limitations": [ + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "S-prusa-volumes", + "class": "synthetic" + }, + { + "bytes": 684, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 1, + "meshes": 1, + "response_bytes": 2049, + "limitations": [ + "UNKNOWN_PLACEMENT", + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.008, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "S-stl", + "class": "synthetic" + }, + { + "bytes": 617746, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 9800, + "rendered_triangles": 9800, + "vertices": 5041, + "nodes": 1, + "meshes": 1, + "response_bytes": 239176, + "limitations": [ + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.046, + "warm_best_s": 0.036, + "warm_median_s": 0.037, + "tracemalloc_peak_mib": 1.3, + "rss_delta_mib": 0.1, + "id": "S-grid-10k", + "class": "synthetic" + }, + { + "bytes": 3256096, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 49928, + "rendered_triangles": 49928, + "vertices": 25281, + "nodes": 1, + "meshes": 1, + "response_bytes": 1205070, + "limitations": [ + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.215, + "warm_best_s": 0.176, + "warm_median_s": 0.178, + "tracemalloc_peak_mib": 5.6, + "rss_delta_mib": 3.8, + "id": "S-grid-50k", + "class": "synthetic" + }, + { + "bytes": 6523679, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 99458, + "rendered_triangles": 99458, + "vertices": 50176, + "nodes": 1, + "meshes": 1, + "response_bytes": 2395870, + "limitations": [ + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.59, + "warm_best_s": 0.405, + "warm_median_s": 0.411, + "tracemalloc_peak_mib": 10.9, + "rss_delta_mib": 3.5, + "id": "S-grid-100k", + "class": "synthetic" + }, + { + "bytes": 20350, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 3600, + "vertices": 8, + "nodes": 300, + "meshes": 1, + "response_bytes": 191745, + "limitations": [ + "PLATES_UNAVAILABLE", + "REPEATED_INSTANCE_PLACEMENT_UNVERIFIED" + ], + "cold_s": 0.019, + "warm_best_s": 0.008, + "warm_median_s": 0.011, + "tracemalloc_peak_mib": 1.8, + "rss_delta_mib": 0.0, + "id": "S-instances-300", + "class": "synthetic" + }, + { + "bytes": 18696925, + "job_gate": "ok", + "result": "refused", + "code": "LIMIT_EXCEEDED", + "message": "The project has at least 250,001 triangles; Studio draws at most 250,000.", + "fits_budgets": false, + "lifted": { + "triangles": 279752, + "rendered_triangles": 279752, + "vertices": 140625, + "nodes": 1, + "response_bytes": 6727764, + "time_s": 1.15 + }, + "id": "S-over-budget", + "class": "synthetic-over-budget" + }, + { + "bytes": 9899, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2974, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.002, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-demo_offplate_foreign", + "class": "repo-example" + }, + { + "bytes": 1790, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 1, + "meshes": 1, + "response_bytes": 2086, + "limitations": [ + "PLATE_MEMBERSHIP_UNKNOWN" + ], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.002, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-demo_u1_showcase", + "class": "repo-example" + }, + { + "bytes": 684, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 1, + "meshes": 1, + "response_bytes": 2049, + "limitations": [ + "UNKNOWN_PLACEMENT", + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.001, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube", + "class": "repo-example" + }, + { + "bytes": 9667, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.002, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_foreign", + "class": "repo-example" + }, + { + "bytes": 10157, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.001, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_foreign_SnapmakerU1", + "class": "repo-example" + }, + { + "bytes": 9951, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_scaled_100_U1", + "class": "repo-example" + }, + { + "bytes": 9950, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_SnapmakerU1", + "class": "repo-example" + }, + { + "bytes": 10155, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_SnapmakerU1_SnapmakerU1", + "class": "repo-example" + }, + { + "bytes": 9951, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.001, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.001, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_10", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.002, + "warm_best_s": 0.001, + "warm_median_s": 0.002, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_2", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.004, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_3", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_4", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_5", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.1, + "id": "E-sample_cube_U1_SnapmakerU1_6", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_7", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "E-sample_cube_U1_SnapmakerU1_8", + "class": "repo-example" + }, + { + "bytes": 10156, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2740, + "limitations": [], + "cold_s": 0.004, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.1, + "id": "E-sample_cube_U1_SnapmakerU1_9", + "class": "repo-example" + }, + { + "bytes": 23059, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 12, + "rendered_triangles": 12, + "vertices": 8, + "nodes": 2, + "meshes": 1, + "response_bytes": 2875, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.003, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "R-bambu-pa-pattern", + "class": "real-world" + }, + { + "bytes": 817091, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 72586, + "rendered_triangles": 72586, + "vertices": 36353, + "nodes": 42, + "meshes": 39, + "response_bytes": 1786167, + "limitations": [], + "cold_s": 0.318, + "warm_best_s": 0.274, + "warm_median_s": 0.277, + "tracemalloc_peak_mib": 8.2, + "rss_delta_mib": 3.0, + "id": "R-orca-badge", + "class": "real-world" + }, + { + "bytes": 29649, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 24, + "rendered_triangles": 192, + "vertices": 16, + "nodes": 24, + "meshes": 2, + "response_bytes": 18971, + "limitations": [], + "cold_s": 0.003, + "warm_best_s": 0.002, + "warm_median_s": 0.003, + "tracemalloc_peak_mib": 0.2, + "rss_delta_mib": 0.0, + "id": "R-orca-pa-line-dual", + "class": "real-world" + }, + { + "bytes": 2514963, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "complete", + "fits_budgets": true, + "triangles": 225154, + "rendered_triangles": 225154, + "vertices": 112569, + "nodes": 1, + "meshes": 1, + "response_bytes": 5405324, + "limitations": [ + "PLATES_UNAVAILABLE" + ], + "cold_s": 1.037, + "warm_best_s": 0.87, + "warm_median_s": 0.875, + "tracemalloc_peak_mib": 25.0, + "rss_delta_mib": 30.2, + "id": "R-prusa-seam-test", + "class": "real-world" + }, + { + "bytes": 484, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 8, + "rendered_triangles": 8, + "vertices": 6, + "nodes": 1, + "meshes": 1, + "response_bytes": 1988, + "limitations": [ + "UNKNOWN_PLACEMENT", + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.001, + "warm_best_s": 0.001, + "warm_median_s": 0.001, + "tracemalloc_peak_mib": 0.1, + "rss_delta_mib": 0.0, + "id": "P-01", + "class": "local-anonymized" + }, + { + "bytes": 986734, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 19733, + "rendered_triangles": 19733, + "vertices": 9869, + "nodes": 1, + "meshes": 1, + "response_bytes": 475424, + "limitations": [ + "UNKNOWN_PLACEMENT", + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.021, + "warm_best_s": 0.02, + "warm_median_s": 0.021, + "tracemalloc_peak_mib": 4.5, + "rss_delta_mib": 3.2, + "id": "P-02", + "class": "local-anonymized" + }, + { + "bytes": 3975984, + "job_gate": "ok", + "result": "ok", + "code": null, + "status": "partial", + "fits_budgets": true, + "triangles": 79518, + "rendered_triangles": 79518, + "vertices": 39546, + "nodes": 1, + "meshes": 1, + "response_bytes": 1906815, + "limitations": [ + "UNKNOWN_PLACEMENT", + "PLATES_UNAVAILABLE" + ], + "cold_s": 0.097, + "warm_best_s": 0.097, + "warm_median_s": 0.097, + "tracemalloc_peak_mib": 18.4, + "rss_delta_mib": 24.9, + "id": "P-03", + "class": "local-anonymized" + }, + { + "bytes": 16357134, + "job_gate": "ok", + "result": "refused", + "code": "LIMIT_EXCEEDED", + "message": "The model has 327,141 triangles; Studio draws at most 250,000.", + "fits_budgets": false, + "lifted": { + "triangles": 327141, + "rendered_triangles": 327141, + "vertices": 163521, + "nodes": 1, + "response_bytes": 7852385, + "time_s": 0.43 + }, + "id": "P-04", + "class": "local-anonymized" + }, + { + "bytes": 1525086434, + "job_gate": "SNAPSHOT_CEILING", + "result": "refused", + "code": "LIMIT_EXCEEDED", + "message": "The model has 30,501,727 triangles; Studio draws at most 250,000.", + "fits_budgets": false, + "lifted": { + "still_refused": "LIMIT_EXCEEDED", + "message": "The model has 30,501,727 triangles; Studio draws at most 2,000,000." + }, + "id": "P-05", + "class": "local-anonymized" + } + ], + "coverage": { + "by_class": { + "local-anonymized": { + "items": 5, + "fit": 3 + }, + "real-world": { + "items": 4, + "fit": 4 + }, + "repo-example": { + "items": 19, + "fit": 19 + }, + "synthetic": { + "items": 8, + "fit": 8 + }, + "synthetic-over-budget": { + "items": 1, + "fit": 0 + } + }, + "fraction_fitting_excluding_deliberate_over_budget": 0.944 + }, + "local_survey_3mf_sizes_only": { + "files": 1230, + "over_snapshot_ceiling": 126, + "fraction_within_ceiling": 0.898, + "median_mib": 31.2, + "p90_mib": 130.1 + }, + "gates": { + "fixture_100k_triangles": { + "parse_encode_warm_s": 0.358, + "cold_s": 0.417, + "extra_peak_mib_tracemalloc": 11.0, + "extra_peak_mib_rss": 6.1, + "response_bytes": 2395870, + "gate_time_le_10s": true, + "gate_mem_le_500mib": true, + "target_time_le_3s": true, + "target_mem_le_300mib": true, + "gate_response_le_8mib": true + }, + "cancel_latency_s": { + "max": 0.005, + "median": 0.002, + "gate_le_2s": true + }, + "leak_50_cycles": { + "retained_jobs": 8, + "cap": 8, + "temp_snapshots": 0, + "threads_before": 1, + "threads_after": 1, + "worker_wedged_seen": false, + "gate_pass": true + } + } +}