From 88300dc727221731b4634584e0b138cf4e14d879 Mon Sep 17 00:00:00 2001 From: Gang Tao Date: Wed, 11 Nov 2026 13:13:17 -0700 Subject: [PATCH 1/2] feat(corpus): Export / Import corpus from the Manage tab (#102) CLI-only export/import (#29) needed kubectl exec + kubectl cp in production. Admins now get two Manage-tab buttons backed by: - GET /api/corpus/export -> one .tar.gz of the CLI bundle layout (FileResponse streamed from scratch, cleaned after send) - POST /api/corpus/import -> multipart upload + replace flag; the archive is validated in full before anything is written (flat regular files only, manifest present, nothing unlisted), then loaded as a JobManager job shown in the ingest panel with per-stream row counts. 409 while any UI job is active; 413 over TPK_IMPORT_MAX_MB. Both are admin-only: a corpus:manage role may be scoped to a subset of entries but the bundle is the whole graph. transfer: pack_bundle/unpack_bundle + BadBundle; export blanks the dev-mode `path` of GitHub-sourced entries (meaningless on another host). JobManager: `kind` field, submit_import, has_active, finished_at in a finally. k8s: `transfer` emptyDir (4Gi) + TPK_TRANSFER_DIR in all three manifests. Docs + README config rows. Co-Authored-By: Claude Fable 5.1 --- README.md | 5 +- deploy/k8s/README.md | 4 + deploy/k8s/allinone.yaml | 11 +++ deploy/k8s/app-only.yaml | 10 +++ deploy/k8s/enterprise.yaml | 10 +++ docs/FEATURES.md | 3 +- src/tpk/api.py | 157 ++++++++++++++++++++++++++++++--- src/tpk/transfer.py | 86 +++++++++++++++++- tests/test_api.py | 158 +++++++++++++++++++++++++++++++++ tests/test_jobmanager.py | 51 +++++++++++ tests/test_transfer.py | 107 +++++++++++++++++++++++ web/src/App.tsx | 2 +- web/src/Manage.tsx | 173 ++++++++++++++++++++++++++++++++++--- web/src/app.css | 1 + 14 files changed, 750 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 788613f..495dc86 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,8 @@ export the matching env var — whichever suits your deployment. Secrets are | `TPK_AGENT_MODEL` | `[agent].model` | per-provider | Chat model override | | `TPK_AGENT_REASONING_EFFORT` | `[agent].reasoning_effort` | unset | OpenAI reasoning effort (e.g. `low`) | | `TPK_CHECKOUT_DIR` | `[server].checkout_dir` | `~/.tpk/checkouts` | GitHub checkout cache | +| `TPK_TRANSFER_DIR` | `[server].transfer_dir` | system temp dir | Scratch space for corpus export/import through the UI (#102) | +| `TPK_IMPORT_MAX_MB` | `[server].import_max_mb` | `2048` | Upload cap for `POST /api/corpus/import` (`0` = unlimited) | | `TPK_SESSION_TTL` | `[server].session_ttl` | `86400` | Login session lifetime (seconds) | | `TPK_CHAT_AUDIT` | `[server].chat_audit` | `true` | Chat Q&A auditing (`false`/`0` disables) | | `TPK_DAILY_TOKEN_LIMIT` | `[server].daily_token_limit` | `500000` | Global fallback per-user daily token budget for non-admins (`0` = unlimited; a per-user or role `daily_token_limit` wins) | @@ -125,7 +127,8 @@ the connecting user needs `CREATE DATABASE` (first run) plus read/write on `tpk` **Upgrading an existing deployment:** streams created before this change live in `default` and are **not** moved automatically — after upgrading, tpk looks in `tpk` and finds an empty graph. Either re-ingest (`tpk ingest`), or carry data -over with `tpk export` (old version) → `tpk import` (new version). Point +over with `tpk export` (old version) → `tpk import` (new version), or from the +Manage tab as an admin (**Export corpus** → **Import corpus**, #102). Point `TIMEPLUS_DATABASE=default` to keep using the old location instead. ## Docker compose (quickest start) diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index 1ff1f48..f82c70a 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -327,6 +327,10 @@ spec: - **Backups.** Snapshot the DB data PVC (`/var/lib/proton` or `/var/lib/timeplusd`), or use `tpk export` to write a portable bundle you can `tpk import` elsewhere. + The same bundle is one click away in the Manage tab for an admin (**Export + corpus** downloads a `.tar.gz`, **Import corpus** uploads one); it is staged + in the `transfer` emptyDir (`TPK_TRANSFER_DIR=/opt/tpk/transfer`, 4Gi + `sizeLimit` — raise it if a packed bundle is bigger). - **All-in-one disk guard (dev tuning).** The `timeplus/tpk` image bakes in the dev override `deploy/timeplusd-dev/small-segments.yaml`, which — besides disabling nativelog preallocation — raises proton's disk-usage write guard to diff --git a/deploy/k8s/allinone.yaml b/deploy/k8s/allinone.yaml index 9a96030..8b09905 100644 --- a/deploy/k8s/allinone.yaml +++ b/deploy/k8s/allinone.yaml @@ -90,6 +90,11 @@ spec: secretKeyRef: name: tpk-secrets key: TIMEPLUS_PASSWORD + # Scratch space for corpus export/import through the UI (#102): + # an emptyDir sized for a packed bundle, so a big download/upload + # never fills the container's writable layer. + - name: TPK_TRANSFER_DIR + value: /opt/tpk/transfer # Optional chat-agent overrides (defaults inferred from the keys below). # - name: TPK_AGENT_PROVIDER # value: "anthropic" @@ -137,6 +142,12 @@ spec: mountPath: /var/lib/proton - name: checkouts # cached github repo checkouts mountPath: /opt/tpk/.checkouts + - name: transfer # export/import scratch (#102) + mountPath: /opt/tpk/transfer + volumes: + - name: transfer + emptyDir: + sizeLimit: 4Gi volumeClaimTemplates: - metadata: name: data diff --git a/deploy/k8s/app-only.yaml b/deploy/k8s/app-only.yaml index d2440b0..ecfd951 100644 --- a/deploy/k8s/app-only.yaml +++ b/deploy/k8s/app-only.yaml @@ -98,6 +98,11 @@ spec: # The DB is already up; a short connect-retry covers transient blips. - name: TPK_DB_WAIT_SECONDS value: "120" + # Scratch space for corpus export/import through the UI (#102): + # an emptyDir sized for a packed bundle, so a big download/upload + # never fills the container's writable layer. + - name: TPK_TRANSFER_DIR + value: /opt/tpk/transfer # Optional chat-agent overrides: # - name: TPK_AGENT_PROVIDER # value: "anthropic" @@ -133,10 +138,15 @@ spec: volumeMounts: - name: checkouts mountPath: /opt/tpk/.checkouts + - name: transfer # export/import scratch (#102) + mountPath: /opt/tpk/transfer volumes: - name: checkouts persistentVolumeClaim: claimName: tpk-checkouts + - name: transfer + emptyDir: + sizeLimit: 4Gi --- # Access service for the chat agent + web UI — target from Ingress / port-forward. apiVersion: v1 diff --git a/deploy/k8s/enterprise.yaml b/deploy/k8s/enterprise.yaml index a0d855b..b1064bb 100644 --- a/deploy/k8s/enterprise.yaml +++ b/deploy/k8s/enterprise.yaml @@ -253,6 +253,11 @@ spec: # app CrashLoopBackOff-ing on a slow first boot (cold image/PVC). - name: TPK_DB_WAIT_SECONDS value: "300" + # Scratch space for corpus export/import through the UI (#102): + # an emptyDir sized for a packed bundle, so a big download/upload + # never fills the container's writable layer. + - name: TPK_TRANSFER_DIR + value: /opt/tpk/transfer # Optional chat-agent overrides: # - name: TPK_AGENT_PROVIDER # value: "anthropic" @@ -288,10 +293,15 @@ spec: volumeMounts: - name: checkouts mountPath: /opt/tpk/.checkouts + - name: transfer # export/import scratch (#102) + mountPath: /opt/tpk/transfer volumes: - name: checkouts persistentVolumeClaim: claimName: tpk-checkouts + - name: transfer + emptyDir: + sizeLimit: 4Gi --- # Access service for the chat agent + web UI — target from Ingress / port-forward. apiVersion: v1 diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 7ae031a..428ba30 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -108,7 +108,8 @@ Manage what the graph indexes at runtime, without editing files or restarting. - **Background jobs.** Add / reindex operations run on a background worker; job status is pollable. - **Manage tab** in the web UI drives all of it (add, toggle, reindex, delete), and shows the **ingest history** — every run, whether started from the UI, `tpk ingest` or `kubectl exec`: status (`running` / `ok` / `failed` with the error / `stale` for a run that never wrote its end row), nodes, edges, commit, duration. Backed by the append-only `kg_ingest_log` stream via `GET /api/ingest-log`. - **Release-upgrade workflow:** add the new ref → ingest → verify → flip enabled; a legacy migration re-keys old bare-name rows on first ingest. -- **Export / import (issue #29).** `tpk export` dumps the ingested graph + corpus registry to a portable bundle (proton `FORMAT Parquet`, one file per stream + manifest); `tpk import` loads it into a new environment — no re-ingest, no LLM calls. Idempotent upsert by default (`--replace` to reset); auth streams excluded. +- **Export / import (issue #29).** `tpk export` dumps the ingested graph + corpus registry to a portable bundle (proton `FORMAT Parquet`, one file per stream + manifest); `tpk import` loads it into a new environment — no re-ingest, no LLM calls. Idempotent upsert by default (`--replace` to reset); auth streams excluded. The bundle carries each entry's registry row (source, ref, visibility, extraction mode, description) and the ingest history; a dev-mode local `path` is blanked for GitHub-sourced entries, since it is meaningless on another host. +- **Export / import from the Manage tab (issue #102).** Admins get **Export corpus** (one `.tar.gz` download of the same bundle, `GET /api/corpus/export`) and **Import corpus** (upload the archive, optional *replace*; `POST /api/corpus/import`). The archive is validated in full before anything is written — flat regular files only, a manifest, nothing the manifest doesn't list — then loaded as a job shown in the ingest panel with per-stream row counts. An import is refused (409) while any UI-started ingest or import job is active, so it never drops a stream such a job is writing into (a `tpk ingest` run from the CLI or `kubectl exec` is a separate process the server cannot see — don't import during one). Admin-only rather than `corpus:manage`, because a manager role may be scoped to a subset of entries and the bundle is the whole graph. Scratch space: `TPK_TRANSFER_DIR` (an `emptyDir` in the k8s manifests); upload cap `TPK_IMPORT_MAX_MB` (default 2048). ## 7. Authentication & role-based access (issue #8) diff --git a/src/tpk/api.py b/src/tpk/api.py index eea30b4..188e8c5 100644 --- a/src/tpk/api.py +++ b/src/tpk/api.py @@ -1,19 +1,24 @@ -"""Management API: corpus CRUD + background ingest jobs.""" +"""Management API: corpus CRUD + background ingest jobs + corpus +export/import transport (#102).""" import queue import re +import shutil +import tempfile import threading import uuid from datetime import datetime, timezone from pathlib import Path -from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile +from fastapi.responses import FileResponse from pydantic import BaseModel +from starlette.background import BackgroundTask import tpk.auth as auth_mod -from tpk import corpus, db +from tpk import corpus, db, transfer from tpk.auth import AuthLayer, User -from tpk.config import RepoConfig, Settings, config_path, entry_key, load_llm +from tpk.config import RepoConfig, Settings, config_path, entry_key, load_llm, setting from tpk import ingest as ingest_mod from tpk.ingest import ingest_repo @@ -139,8 +144,33 @@ def _validate(body: AddRepo) -> RepoConfig: ) +def transfer_dir() -> Path: + """Scratch space for export bundles and uploaded import archives (#102): + env > [server].transfer_dir > the system temp dir. In Kubernetes point it + at an emptyDir sized for the bundle (see deploy/k8s).""" + return Path(setting("TPK_TRANSFER_DIR", "server", "transfer_dir", + tempfile.gettempdir())).expanduser() + + +def import_max_bytes() -> int: + """Upload cap for `POST /api/corpus/import`, in MB (0 = unlimited).""" + mb = setting("TPK_IMPORT_MAX_MB", "server", "import_max_mb", 2048, cast=int) + return int(mb) * 1024 * 1024 + + +IMPORT_ENTRY_KEY = "corpus import" + + class JobManager: - """One worker thread; jobs use their own DB client (never the KG's).""" + """One worker thread; jobs use their own DB client (never the KG's). + + Two job kinds share the queue: `ingest` (one corpus entry) and `import` + (a whole bundle, #102). Running them on the same single worker is what + serialises an import against ingests already queued; the API refuses to + *submit* an import while anything is active (409), so an import never + drops a stream a UI-started ingest is writing into. (A `tpk ingest` + run from the CLI or `kubectl exec` is a separate process this manager + cannot see -- don't import while one of those is running.)""" def __init__(self, prefix: str = ""): self.prefix = prefix @@ -160,10 +190,10 @@ def _ensure_worker(self): threading.Thread(target=self._run, daemon=True).start() self._started = True - def submit(self, cfg: RepoConfig) -> str: + def _submit(self, kind: str, key: str, payload) -> str: job_id = uuid.uuid4().hex[:12] rec = { - "id": job_id, "entry_key": entry_key(cfg), "status": "queued", + "id": job_id, "kind": kind, "entry_key": key, "status": "queued", "phase": None, "message": "", "nodes": 0, "edges": 0, "error": None, "submitted_at": datetime.now(timezone.utc).isoformat(), "updated_at": None, @@ -171,10 +201,22 @@ def submit(self, cfg: RepoConfig) -> str: } with self._jobs_lock: self.jobs[job_id] = rec - self._q.put((job_id, cfg)) + self._q.put((job_id, kind, payload)) self._ensure_worker() return job_id + def submit(self, cfg: RepoConfig) -> str: + return self._submit("ingest", entry_key(cfg), cfg) + + def submit_import(self, bundle_dir, replace: bool = False) -> str: + """Load an already-unpacked bundle directory; the worker deletes the + directory when done, whatever the outcome.""" + return self._submit("import", IMPORT_ENTRY_KEY, (Path(bundle_dir), replace)) + + def has_active(self) -> bool: + with self._jobs_lock: + return any(j["status"] in ("queued", "running") for j in self.jobs.values()) + def snapshot(self) -> list[dict]: with self._jobs_lock: return list(self.jobs.values()) @@ -183,13 +225,35 @@ def get(self, job_id: str) -> dict | None: with self._jobs_lock: return self.jobs.get(job_id) + def _run_import(self, rec: dict, bundle_dir: Path, replace: bool): + try: + with self._jobs_lock: + rec["phase"] = "import" + rec["message"] = "replacing graph streams" if replace else "loading bundle" + rec["updated_at"] = datetime.now(timezone.utc).isoformat() + client = db.get_client(Settings.from_env()) + manifest = transfer.import_bundle(client, bundle_dir, prefix=self.prefix, + replace=replace) + streams = {name: info["rows"] for name, info in manifest["streams"].items()} + with self._jobs_lock: + rec["streams"] = streams + rec["nodes"] = streams.get("kg_nodes", 0) + rec["edges"] = streams.get("kg_edges", 0) + rec["status"] = "ok" + finally: + shutil.rmtree(bundle_dir, ignore_errors=True) + def _run(self): while True: - job_id, cfg = self._q.get() + job_id, kind, payload = self._q.get() with self._jobs_lock: rec = self.jobs[job_id] rec["status"] = "running" try: + if kind == "import": + self._run_import(rec, *payload) + continue + cfg = payload client = db.get_client(Settings.from_env()) llm = load_llm(REPOS_TOML) if REPOS_TOML.exists() else None backend = None if (llm is None or llm.backend == "auto") else llm.backend @@ -220,8 +284,9 @@ def _on_progress(p, _rec=rec): with self._jobs_lock: rec["status"] = "failed" rec["error"] = f"{type(exc).__name__}: {exc}" - with self._jobs_lock: - rec["finished_at"] = datetime.now(timezone.utc).isoformat() + finally: + with self._jobs_lock: + rec["finished_at"] = datetime.now(timezone.utc).isoformat() def create_api_router(prefix: str = "", auth=None) -> APIRouter: @@ -378,6 +443,76 @@ def api_ingest_log(limit: int = Query(50, ge=1, le=200), entry: str | None = Non for runs that have not written their end row.""" return {"runs": ingest_mod.list_runs(_client(), prefix=prefix, limit=limit, entry=entry)} + # -- corpus export / import transport (#102) --------------------------- + # Admin-only, not corpus:manage: a non-admin manager is role-scoped to a + # subset of entries, but a bundle is the WHOLE graph (every internal + # entry included) and an import rewrites all of it. + + @router.get("/corpus/export") + def export_corpus(format: str = Query("Parquet"), actor: User = Depends(auth.require_admin)): + """Dump the graph + corpus registry and stream it back as one + `.tar.gz` -- the CLI bundle layout, so `tpk import` reads it unpacked.""" + if format not in transfer.FORMATS: + raise HTTPException(422, "format must be Parquet|Native") + root = transfer_dir() + root.mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix="tpk-export-", dir=root)) + try: + transfer.export_bundle(_client(), work / "bundle", prefix=prefix, fmt=format) + stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + name = f"tpk-corpus-{stamp}{transfer.ARCHIVE_SUFFIX}" + archive = transfer.pack_bundle(work / "bundle", work / name) + shutil.rmtree(work / "bundle", ignore_errors=True) + except Exception: + shutil.rmtree(work, ignore_errors=True) + raise + # FileResponse streams from disk; the scratch dir goes once the body + # has been sent. + return FileResponse(archive, media_type="application/gzip", filename=name, + background=BackgroundTask(shutil.rmtree, work, ignore_errors=True)) + + @router.post("/corpus/import") + def import_corpus(file: UploadFile = File(...), replace: bool = Form(False), + actor: User = Depends(auth.require_admin)): + """Upload a bundle archive and load it as a background job (poll + `/api/jobs/{id}`). The archive is validated in full before anything + is written; a load is refused while any job is queued or running.""" + if jobs.has_active(): + raise HTTPException(409, "an ingest or import job is already running; retry when it finishes") + root = transfer_dir() + root.mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix="tpk-import-", dir=root)) + try: + limit = import_max_bytes() + # Dot-prefixed so no archive member (those are rejected) can + # name it; the bundle files unpack beside it into `work`. + archive = work / ".upload.tar.gz" + size = 0 + with open(archive, "wb") as fh: + while chunk := file.file.read(1024 * 1024): + size += len(chunk) + if limit and size > limit: + raise HTTPException(413, f"archive exceeds the {limit // (1024 * 1024)} MB upload limit (TPK_IMPORT_MAX_MB)") + fh.write(chunk) + try: + manifest = transfer.unpack_bundle(archive, work) + except transfer.BadBundle as exc: + raise HTTPException(400, f"invalid bundle: {exc}") + archive.unlink() + except Exception: + shutil.rmtree(work, ignore_errors=True) + raise + # Re-check under the submit: has_active() above and the submit are + # not atomic, but the single worker still serialises whatever lands + # in the queue, so the race only costs the 409 courtesy, not safety. + if jobs.has_active(): + shutil.rmtree(work, ignore_errors=True) + raise HTTPException(409, "an ingest or import job is already running; retry when it finishes") + job_id = jobs.submit_import(work, replace=replace) + return {"job_id": job_id, + "streams": {k: v["rows"] for k, v in manifest["streams"].items()}, + "format": manifest.get("format"), "created_at": manifest.get("created_at")} + @router.get("/jobs") def list_jobs(actor: User = Depends(auth.require_cap(auth_mod.CAP_CORPUS_VIEW))): return sorted(jobs.snapshot(), key=lambda j: j["submitted_at"], reverse=True)[:50] diff --git a/src/tpk/transfer.py b/src/tpk/transfer.py index 1c83b7e..aecf1f5 100644 --- a/src/tpk/transfer.py +++ b/src/tpk/transfer.py @@ -5,11 +5,15 @@ through the client's raw stream/insert — the engine encodes/decodes the files, so there is no Python-side type juggling and no extra dependency. -A bundle is a directory: one file per stream plus a `manifest.json`. +A bundle is a directory: one file per stream plus a `manifest.json`. For +transport through the web UI (#102) a bundle is packed into a single flat +`.tar.gz` (`pack_bundle` / `unpack_bundle`); the on-disk layout is the same +either way, so `tpk import` reads an unpacked archive unchanged. """ import json import shutil +import tarfile from datetime import datetime, timezone from pathlib import Path @@ -23,6 +27,16 @@ MANIFEST_NAME = "manifest.json" BUNDLE_VERSION = 1 _EXT = {"Parquet": "parquet", "Native": "native"} +FORMATS = tuple(_EXT) +ARCHIVE_SUFFIX = ".tar.gz" + +# A `path` in `kg_repos` is a checkout on the EXPORTING host's filesystem and +# means nothing after a cross-environment import (#102). When the entry also +# names a GitHub source the target can re-fetch from, the bundle carries an +# empty path so the Manage tab doesn't show a dead local path. +_EXPORT_EXPR = { + ("kg_repos", "path"): "if(github != '', '', path) AS path", +} def _stream_columns(client, stream: str, prefix: str = "") -> list[str]: @@ -64,8 +78,9 @@ def export_bundle(client, out_dir, streams=None, prefix: str = "", # raw_stream returns the engine's serialized bytes as a file-like # object; copy it out in chunks so a 300k-row stream never lands # wholly in memory. + select = [_EXPORT_EXPR.get((stream, c), c) for c in cols] src = client.raw_stream( - f"SELECT {', '.join(cols)} FROM table({db.qualified(stream, prefix)})", fmt=fmt) + f"SELECT {', '.join(select)} FROM table({db.qualified(stream, prefix)})", fmt=fmt) with open(out / fname, "wb") as fh: shutil.copyfileobj(src, fh) manifest["streams"][stream] = { @@ -127,3 +142,70 @@ def import_bundle(client, in_dir, prefix: str = "", replace: bool = False) -> di column_names=info["columns"], insert_block=fh, fmt=fmt) return manifest + + +# -- single-file transport (#102) ------------------------------------------- + +def pack_bundle(bundle_dir, archive_path) -> Path: + """Tar+gzip a bundle directory into one flat archive (manifest first, + then exactly the files the manifest lists). Returns the archive path.""" + src = Path(bundle_dir) + manifest = load_manifest(src) + out = Path(archive_path) + with tarfile.open(out, "w:gz") as tar: + tar.add(src / MANIFEST_NAME, arcname=MANIFEST_NAME) + for info in manifest["streams"].values(): + tar.add(src / info["file"], arcname=info["file"]) + return out + + +class BadBundle(ValueError): + """The archive is not a bundle this version will load.""" + + +def _check_member(m: tarfile.TarInfo): + # Flat archive of regular files only: no directories, links, devices, + # nested paths or `..` -- every member lands directly in the dest dir. + if not m.isfile(): + raise BadBundle(f"archive member {m.name!r} is not a regular file") + name = m.name + if name != Path(name).name or name in (".", "..") or name.startswith("."): + raise BadBundle(f"archive member {m.name!r} has an unsafe name") + + +def unpack_bundle(archive_path, dest_dir, max_members: int = 64) -> dict: + """Extract a `pack_bundle` archive into `dest_dir`, validating every + member BEFORE writing anything: flat regular files only, a readable + `manifest.json`, and no file the manifest doesn't list. Returns the + manifest. Raises `BadBundle` on anything else.""" + dest = Path(dest_dir) + dest.mkdir(parents=True, exist_ok=True) + try: + tar = tarfile.open(archive_path, "r:gz") + except (tarfile.TarError, OSError, EOFError) as exc: + raise BadBundle(f"not a gzipped tar archive: {exc}") from exc + with tar: + members = tar.getmembers() + if len(members) > max_members: + raise BadBundle(f"archive has {len(members)} members (max {max_members})") + for m in members: + _check_member(m) + names = [m.name for m in members] + if MANIFEST_NAME not in names: + raise BadBundle(f"archive has no {MANIFEST_NAME}") + raw = tar.extractfile(MANIFEST_NAME).read() + try: + manifest = json.loads(raw) + listed = {info["file"] for info in manifest["streams"].values()} + except (ValueError, KeyError, TypeError, AttributeError) as exc: + raise BadBundle(f"malformed {MANIFEST_NAME}: {exc}") from exc + extra = sorted(set(names) - listed - {MANIFEST_NAME}) + if extra: + raise BadBundle(f"archive contains files the manifest does not list: {extra}") + missing = sorted(listed - set(names)) + if missing: + raise BadBundle(f"manifest lists files missing from the archive: {missing}") + for m in members: + with tar.extractfile(m) as src, open(dest / m.name, "wb") as fh: + shutil.copyfileobj(src, fh) + return manifest diff --git a/tests/test_api.py b/tests/test_api.py index 4ff3f2e..9b2c0f5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -204,3 +204,161 @@ def test_last_admin_protection(tp, monkeypatch, admin_hdr): assert c.post("/api/users/update", headers=admin_hdr, json={"username": "root", "disabled": True}).status_code == 400 assert c.post("/api/roles", headers=admin_hdr, json={"name": "other", "entry_keys": []}).status_code == 200 assert c.post("/api/users/update", headers=admin_hdr, json={"username": "root", "role": "other"}).status_code == 400 + + +# -- corpus export / import transport (#102) --------------------------------- + +def _seed_graph(client, prefix): + from datetime import datetime, timezone + from tpk import corpus + from tpk.config import RepoConfig + from tpk.model import EDGE_COLUMNS, NODE_COLUMNS, Edge, Node, edge_rows, node_rows + now = datetime.now(timezone.utc) + nodes = [Node(f"n{i}", "demo@v1", "function", f"fn{i}", f"demo::fn{i}", "src/x.cpp", + i, i + 5, f"summary {i}", "c0", "public") for i in range(3)] + edges = [Edge("n0", "n1", "calls", "EXTRACTED", "demo@v1")] + client.insert(f"{prefix}kg_nodes", node_rows(nodes, now), column_names=NODE_COLUMNS) + client.insert(f"{prefix}kg_edges", edge_rows(edges, now), column_names=EDGE_COLUMNS) + corpus.upsert_entry(client, RepoConfig(name="demo", ref="v1", github="org/demo"), prefix=prefix) + _eventually(lambda: int(client.query( + f"SELECT count() FROM table({prefix}kg_nodes)").result_rows[0][0]) == 3) + + +def _transfer_env(monkeypatch, tmp_path): + monkeypatch.setenv("TPK_TRANSFER_DIR", str(tmp_path / "scratch")) + + +def test_export_download_is_a_valid_bundle_archive(tp, monkeypatch, admin_hdr, tmp_path): + import io + import json + import tarfile + client, prefix = tp + _transfer_env(monkeypatch, tmp_path) + _seed_graph(client, prefix) + c, _ = _client(tp, monkeypatch) + + r = c.get("/api/corpus/export", headers=admin_hdr) + assert r.status_code == 200, r.text + assert r.headers["content-type"] == "application/gzip" + assert "tpk-corpus-" in r.headers["content-disposition"] + assert r.headers["content-disposition"].endswith('.tar.gz"') + with tarfile.open(fileobj=io.BytesIO(r.content), mode="r:gz") as tar: + names = tar.getnames() + manifest = json.load(tar.extractfile("manifest.json")) + assert manifest["streams"]["kg_nodes"]["rows"] == 3 + assert manifest["streams"]["kg_edges"]["rows"] == 1 + assert manifest["streams"]["kg_repos"]["rows"] == 1 + assert set(names) == {"manifest.json", *(i["file"] for i in manifest["streams"].values())} + # scratch dir cleaned once the body was sent + assert list((tmp_path / "scratch").iterdir()) == [] + assert c.get("/api/corpus/export?format=CSV", headers=admin_hdr).status_code == 422 + + +def test_import_upload_runs_as_job(tp, monkeypatch, admin_hdr, tmp_path): + client, prefix = tp + _transfer_env(monkeypatch, tmp_path) + _seed_graph(client, prefix) + c, _ = _client(tp, monkeypatch) + archive = c.get("/api/corpus/export", headers=admin_hdr).content + + # wipe the graph, then load it back through the upload endpoint + client.command(f"DROP STREAM IF EXISTS {prefix}kg_nodes") + from tpk import db + db.ensure_schema(client, prefix) + _eventually(lambda: int(client.query( + f"SELECT count() FROM table({prefix}kg_nodes)").result_rows[0][0]) == 0) + + r = c.post("/api/corpus/import", headers=admin_hdr, data={"replace": "true"}, + files={"file": ("corpus.tar.gz", archive, "application/gzip")}) + assert r.status_code == 200, r.text + body = r.json() + assert body["streams"]["kg_nodes"] == 3 + job_id = body["job_id"] + rec = _eventually(lambda: (lambda j: j if j["status"] in ("ok", "failed") else None)( + c.get(f"/api/jobs/{job_id}", headers=admin_hdr).json()), timeout=30) + assert rec and rec["status"] == "ok", rec + assert rec["kind"] == "import" and rec["streams"]["kg_nodes"] == 3 + assert _eventually(lambda: int(client.query( + f"SELECT count() FROM table({prefix}kg_nodes)").result_rows[0][0]) == 3) + # the unpacked bundle is gone after the job + assert _eventually(lambda: list((tmp_path / "scratch").iterdir()) == []) + + +def test_import_rejects_bad_archive_before_writing(tp, monkeypatch, admin_hdr, tmp_path): + import io + import json + import tarfile + client, prefix = tp + _transfer_env(monkeypatch, tmp_path) + c, _ = _client(tp, monkeypatch) + + r = c.post("/api/corpus/import", headers=admin_hdr, + files={"file": ("x.tar.gz", b"not an archive", "application/gzip")}) + assert r.status_code == 400 and "invalid bundle" in r.text + + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tar: + m = json.dumps({"streams": {"kg_nodes": {"file": "../kg_nodes.parquet", "columns": [], "rows": 0}}}).encode() + for name, data in [("manifest.json", m), ("../kg_nodes.parquet", b"x")]: + ti = tarfile.TarInfo(name); ti.size = len(data) + tar.addfile(ti, io.BytesIO(data)) + r = c.post("/api/corpus/import", headers=admin_hdr, + files={"file": ("x.tar.gz", buf.getvalue(), "application/gzip")}) + assert r.status_code == 400 and "unsafe name" in r.text + assert c.get("/api/jobs", headers=admin_hdr).json() == [] + assert list((tmp_path / "scratch").iterdir()) == [] + assert not (tmp_path / "kg_nodes.parquet").exists() + + +def test_import_refused_while_a_job_is_active(tp, monkeypatch, admin_hdr, tmp_path): + import threading + client, prefix = tp + _transfer_env(monkeypatch, tmp_path) + _seed_graph(client, prefix) + c, _ = _client(tp, monkeypatch) + archive = c.get("/api/corpus/export", headers=admin_hdr).content + + import tpk.api as api_mod + from tpk.ingest import IngestResult + gate = threading.Event() + + def slow_ingest(client_, cfg, prefix="", **kw): + gate.wait(10) + from tpk.config import entry_key + return IngestResult(entry_key(cfg), "j", 1, 0, "ok") + monkeypatch.setattr(api_mod, "ingest_repo", slow_ingest) + + job = c.post("/api/repos/reindex", headers=admin_hdr, json={"name": "demo", "ref": "v1"}).json()["job_id"] + _eventually(lambda: c.get(f"/api/jobs/{job}", headers=admin_hdr).json()["status"] == "running") + r = c.post("/api/corpus/import", headers=admin_hdr, + files={"file": ("corpus.tar.gz", archive, "application/gzip")}) + assert r.status_code == 409 + gate.set() + _eventually(lambda: c.get(f"/api/jobs/{job}", headers=admin_hdr).json()["status"] == "ok") + assert c.post("/api/corpus/import", headers=admin_hdr, + files={"file": ("corpus.tar.gz", archive, "application/gzip")}).status_code == 200 + + +def test_import_upload_size_cap(tp, monkeypatch, admin_hdr, tmp_path): + _transfer_env(monkeypatch, tmp_path) + monkeypatch.setenv("TPK_IMPORT_MAX_MB", "1") + c, _ = _client(tp, monkeypatch) + r = c.post("/api/corpus/import", headers=admin_hdr, + files={"file": ("big.tar.gz", b"\\0" * (2 * 1024 * 1024), "application/gzip")}) + assert r.status_code == 413 and "1 MB" in r.text + assert list((tmp_path / "scratch").iterdir()) == [] + + +def test_export_import_are_admin_only(tp, monkeypatch, admin_hdr): + client, prefix = tp + c, _ = _client(tp, monkeypatch) + # a full corpus:manage role is still not enough -- the bundle is the whole graph + auth.upsert_role(client, auth.Role("curator", [], capabilities=[auth.CAP_CORPUS_MANAGE]), prefix=prefix) + auth.upsert_user(client, auth.User("cur", auth.hash_password("password-1"), "curator"), prefix=prefix) + _eventually(lambda: c.post("/auth/login", json={"username": "cur", "password": "password-1"}).status_code == 200) + t = c.post("/auth/login", json={"username": "cur", "password": "password-1"}).json()["token"] + hdr = {"Authorization": f"Bearer {t}"} + assert c.get("/api/corpus/export").status_code == 401 + assert c.get("/api/corpus/export", headers=hdr).status_code == 403 + assert c.post("/api/corpus/import", headers=hdr, + files={"file": ("x.tar.gz", b"x", "application/gzip")}).status_code == 403 diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index a559f41..3f0befe 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -53,3 +53,54 @@ def fake_ingest(client, cfg, prefix="", backend=None, model=None, # snapshot() (what /api/jobs returns) exposes the same rec. snap = next(j for j in mgr.snapshot() if j["id"] == job_id) assert snap["phase"] == "done" and "updated_at" in snap + + +def test_jobmanager_import_job_loads_bundle_and_cleans_up(monkeypatch, tmp_path): + """#102: an import job runs transfer.import_bundle on the single worker, + records per-stream rows, and deletes the unpacked bundle dir afterwards.""" + import tpk.api as api_mod + from tpk.api import IMPORT_ENTRY_KEY, JobManager + + monkeypatch.setattr(api_mod.db, "get_client", lambda *a, **k: object()) + seen = {} + + def fake_import(client, in_dir, prefix="", replace=False): + seen["in_dir"], seen["replace"], seen["prefix"] = in_dir, replace, prefix + return {"format": "Parquet", "streams": {"kg_nodes": {"rows": 7}, "kg_edges": {"rows": 4}}} + + monkeypatch.setattr(api_mod.transfer, "import_bundle", fake_import) + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "manifest.json").write_text("{}") + + mgr = JobManager(prefix="p_") + job_id = mgr.submit_import(bundle, replace=True) + rec = _wait_done(mgr, job_id) + assert rec["status"] == "ok" + assert rec["kind"] == "import" and rec["entry_key"] == IMPORT_ENTRY_KEY + assert rec["streams"] == {"kg_nodes": 7, "kg_edges": 4} + assert (rec["nodes"], rec["edges"]) == (7, 4) + assert rec["finished_at"] is not None + assert seen == {"in_dir": bundle, "replace": True, "prefix": "p_"} + assert not bundle.exists() + assert mgr.has_active() is False + + +def test_jobmanager_import_failure_is_recorded_and_cleaned(monkeypatch, tmp_path): + import tpk.api as api_mod + from tpk.api import JobManager + + monkeypatch.setattr(api_mod.db, "get_client", lambda *a, **k: object()) + + def boom(client, in_dir, prefix="", replace=False): + raise ValueError("kg_nodes: target stream is missing column(s) ['x']") + + monkeypatch.setattr(api_mod.transfer, "import_bundle", boom) + bundle = tmp_path / "bundle" + bundle.mkdir() + mgr = JobManager() + rec = _wait_done(mgr, mgr.submit_import(bundle)) + assert rec["status"] == "failed" + assert "missing column" in rec["error"] + assert rec["finished_at"] is not None + assert not bundle.exists() diff --git a/tests/test_transfer.py b/tests/test_transfer.py index f333e27..851430c 100644 --- a/tests/test_transfer.py +++ b/tests/test_transfer.py @@ -1,7 +1,11 @@ """Corpus export/import round-trip (issue #29).""" +import io +import json +import tarfile import uuid from datetime import datetime, timezone +from pathlib import Path import pytest @@ -174,3 +178,106 @@ def test_schema_mismatch_rejected(tp, tmp_path): assert _count(client, dst, "kg_edges") == 0 finally: db.drop_schema(client, dst) + + +# -- single-file transport + path stripping (#102) -------------------------- + +def _write_bundle(tmp_path, files=None, manifest=None): + """A fake on-disk bundle for archive tests (no DB needed).""" + b = tmp_path / "b" + b.mkdir() + files = files if files is not None else {"kg_nodes.parquet": b"nodes", "kg_edges.parquet": b"edges"} + for name, data in files.items(): + (b / name).write_bytes(data) + if manifest is None: + manifest = {"bundle_version": 1, "format": "Parquet", "created_at": "x", + "streams": {n.split(".")[0]: {"file": n, "columns": [], "rows": 0} for n in files}} + (b / transfer.MANIFEST_NAME).write_text(json.dumps(manifest)) + return b + + +def test_pack_unpack_roundtrip(tmp_path): + b = _write_bundle(tmp_path) + archive = transfer.pack_bundle(b, tmp_path / "c.tar.gz") + out = tmp_path / "out" + manifest = transfer.unpack_bundle(archive, out) + assert set(manifest["streams"]) == {"kg_nodes", "kg_edges"} + assert (out / "kg_nodes.parquet").read_bytes() == b"nodes" + assert (out / "kg_edges.parquet").read_bytes() == b"edges" + assert json.loads((out / transfer.MANIFEST_NAME).read_text()) == manifest + + +def _tar(path, members): + """members: list of (TarInfo, bytes|None).""" + with tarfile.open(path, "w:gz") as tar: + for info, data in members: + if data is None: + tar.addfile(info) + else: + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return path + + +def _file(name, data): + ti = tarfile.TarInfo(name) + return ti, data + + +def test_unpack_rejects_path_traversal(tmp_path): + manifest = json.dumps({"streams": {"x": {"file": "../evil", "columns": [], "rows": 0}}}).encode() + archive = _tar(tmp_path / "t.tar.gz", [_file("manifest.json", manifest), _file("../evil", b"x")]) + with pytest.raises(transfer.BadBundle, match="unsafe name"): + transfer.unpack_bundle(archive, tmp_path / "out") + assert not (tmp_path / "evil").exists() + assert list((tmp_path / "out").iterdir()) == [] + + +def test_unpack_rejects_symlink_and_dirs(tmp_path): + manifest = json.dumps({"streams": {}}).encode() + link = tarfile.TarInfo("kg_nodes.parquet") + link.type = tarfile.SYMTYPE + link.linkname = "/etc/passwd" + archive = _tar(tmp_path / "t.tar.gz", [_file("manifest.json", manifest), (link, None)]) + with pytest.raises(transfer.BadBundle, match="not a regular file"): + transfer.unpack_bundle(archive, tmp_path / "out") + + +def test_unpack_rejects_unlisted_and_missing_files(tmp_path): + manifest = json.dumps({"streams": {"kg_nodes": {"file": "kg_nodes.parquet", "columns": [], "rows": 0}}}).encode() + archive = _tar(tmp_path / "extra.tar.gz", [_file("manifest.json", manifest), + _file("kg_nodes.parquet", b"n"), _file("stray.bin", b"s")]) + with pytest.raises(transfer.BadBundle, match="does not list"): + transfer.unpack_bundle(archive, tmp_path / "out1") + assert list((tmp_path / "out1").iterdir()) == [] + archive = _tar(tmp_path / "missing.tar.gz", [_file("manifest.json", manifest)]) + with pytest.raises(transfer.BadBundle, match="missing from the archive"): + transfer.unpack_bundle(archive, tmp_path / "out2") + + +def test_unpack_rejects_no_manifest_and_garbage(tmp_path): + archive = _tar(tmp_path / "t.tar.gz", [_file("kg_nodes.parquet", b"n")]) + with pytest.raises(transfer.BadBundle, match="no manifest"): + transfer.unpack_bundle(archive, tmp_path / "out") + garbage = tmp_path / "g.tar.gz" + garbage.write_bytes(b"this is not an archive") + with pytest.raises(transfer.BadBundle, match="not a gzipped tar"): + transfer.unpack_bundle(garbage, tmp_path / "out2") + + +def test_export_strips_local_path_when_github_set(tp, tmp_path): + client, prefix = tp + corpus.upsert_entry(client, RepoConfig(name="gh", ref="v1", github="org/gh", path=Path("/home/me/gh")), + prefix=prefix) + corpus.upsert_entry(client, RepoConfig(name="local", ref="", path=Path("/home/me/local")), prefix=prefix) + _wait_count(client, prefix, "kg_repos", 2) + manifest = transfer.export_bundle(client, tmp_path / "bundle", prefix=prefix, streams=["kg_repos"]) + assert manifest["streams"]["kg_repos"]["rows"] == 2 + + client.command(f"DROP STREAM IF EXISTS {prefix}kg_repos") + db.ensure_schema(client, prefix) + transfer.import_bundle(client, tmp_path / "bundle", prefix=prefix) + _wait_count(client, prefix, "kg_repos", 2) + entries = {e.name: e for e in corpus.list_entries(client, prefix=prefix)} + assert entries["gh"].path is None # github entry: path dropped + assert entries["local"].path == Path("/home/me/local") # dev-mode entry keeps its path diff --git a/web/src/App.tsx b/web/src/App.tsx index 482b989..ec17105 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -172,7 +172,7 @@ export default function App() { ) : view === "tokens" ? ( ) : view === "manage" ? ( - + ) : view === "users" ? ( ) : null} diff --git a/web/src/Manage.tsx b/web/src/Manage.tsx index aaa96d0..4895e4a 100644 --- a/web/src/Manage.tsx +++ b/web/src/Manage.tsx @@ -14,6 +14,10 @@ type Job = { phase?: string | null; message?: string; updated_at?: string | null; + // "ingest" (one entry) or "import" (a whole bundle, #102); import jobs + // report per-stream row counts instead of an ingest-log row. + kind?: string; + streams?: Record; }; // One ingest run from the persistent log (#15): UI, CLI and kubectl runs alike. @@ -66,6 +70,7 @@ function formatRelative(iso: string): string { const PHASE_LABEL: Record = { fetch: "fetching…", extract: "extracting…", parse: "parsing…", upsert: "saving…", + import: "importing…", }; const STALL_MS = 60_000; @@ -81,10 +86,19 @@ function elapsedLabel(sinceIso: string, now: number): string { return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m${String(s % 60).padStart(2, "0")}s`; } -export default function Manage({ capabilities }: { capabilities: string[] }) { +// Pull a download filename out of `Content-Disposition: attachment; +// filename="x"`; falls back to a generic name if the header is absent. +function downloadName(resp: Response, fallback: string): string { + const m = /filename="?([^";]+)"?/.exec(resp.headers.get("content-disposition") ?? ""); + return m ? m[1] : fallback; +} + +export default function Manage({ capabilities, isAdmin }: { capabilities: string[]; isAdmin?: boolean }) { // corpus:manage gates every mutation; with only corpus:view the screen is - // read-only. + // read-only. Export/import of the WHOLE graph is admin-only (#102): a + // corpus:manage role may be scoped to a subset of entries. const canManage = hasCap(capabilities, CAP.corpusManage); + const canTransfer = !!isAdmin; const [repos, setRepos] = useState([]); const [jobs, setJobs] = useState([]); const [runs, setRuns] = useState([]); @@ -100,15 +114,69 @@ export default function Manage({ capabilities }: { capabilities: string[] }) { return () => clearInterval(t); }, []); + const [importOpen, setImportOpen] = useState(false); + const [importFile, setImportFile] = useState(null); + const [importReplace, setImportReplace] = useState(false); + const [importBusy, setImportBusy] = useState(false); + const [exporting, setExporting] = useState(false); + function openAdd() { setForm({ ...EMPTY_FORM }); setError(""); setAddOpen(true); } function closeAdd() { setAddOpen(false); setError(""); } + function openImport() { setImportFile(null); setImportReplace(false); setError(""); setImportOpen(true); } + function closeImport() { if (importBusy) return; setImportOpen(false); setError(""); } useEffect(() => { - if (!addOpen) return; - const onKey = (e: KeyboardEvent) => { if (e.key === "Escape") { setAddOpen(false); setError(""); } }; + if (!addOpen && !importOpen) return; + const onKey = (e: KeyboardEvent) => { + if (e.key !== "Escape") return; + if (addOpen) { setAddOpen(false); setError(""); } + if (importOpen && !importBusy) { setImportOpen(false); setError(""); } + }; window.addEventListener("keydown", onKey); return () => window.removeEventListener("keydown", onKey); - }, [addOpen]); + }, [addOpen, importOpen, importBusy]); + + // Export: the download needs the bearer token, so it goes through + // apiFetch as a blob rather than a plain . + async function exportCorpus() { + setExporting(true); + setError(""); + try { + const resp = await apiFetch("/api/corpus/export"); + if (!resp.ok) throw new Error(`${resp.status}: ${await resp.text()}`); + const url = URL.createObjectURL(await resp.blob()); + const a = document.createElement("a"); + a.href = url; + a.download = downloadName(resp, "tpk-corpus.tar.gz"); + document.body.appendChild(a); + a.click(); + a.remove(); + URL.revokeObjectURL(url); + } catch (e) { + setError(String(e)); + } finally { + setExporting(false); + } + } + + async function importCorpus() { + if (!importFile) { setError("Choose a bundle archive first"); return; } + setImportBusy(true); + setError(""); + try { + const body = new FormData(); + body.append("file", importFile); + body.append("replace", importReplace ? "true" : "false"); + const resp = await apiFetch("/api/corpus/import", { method: "POST", body }); + if (!resp.ok) throw new Error(`${resp.status}: ${await resp.text()}`); + setImportOpen(false); + await refresh(); + } catch (e) { + setError(String(e)); + } finally { + setImportBusy(false); + } + } async function refresh() { try { @@ -144,6 +212,10 @@ export default function Manage({ capabilities }: { capabilities: string[] }) { } const activeJobs = jobs.filter((j) => j.status === "queued" || j.status === "running"); + // Finished import jobs have no ingest-log row (the bundle carries its own + // provenance), so they are shown from the job list until the server + // restarts. + const importResults = jobs.filter((j) => j.kind === "import" && (j.status === "ok" || j.status === "failed")); // History comes from the persistent log. A run the JobManager is driving // right now is shown in the live block above (with phase + progress), so it // is dropped from the history list until it has an end row. @@ -154,7 +226,7 @@ export default function Manage({ capabilities }: { capabilities: string[] }) {
{/* Add-flow errors render inside the modal; keep this page-level banner for the table's toggle/reindex/delete errors only. */} - {!addOpen && error &&
{error}
} + {!addOpen && !importOpen && error &&
{error}
}
@@ -164,11 +236,26 @@ export default function Manage({ capabilities }: { capabilities: string[] }) {
- {canManage && ( - - )} +
+ {canTransfer && ( + <> + + + + )} + {canManage && ( + + )} +
@@ -322,8 +409,21 @@ export default function Manage({ capabilities }: { capabilities: string[] }) {
)} - {history.length > 0 ? ( + {history.length > 0 || importResults.length > 0 ? (
+ {importResults.map((j) => ( +
+
{j.entry_key}
+ {j.status === "ok" ? ( +
+ ok · {Object.entries(j.streams ?? {}).map(([s, n]) => `${s} ${n}`).join(" · ")} +
+ ) : ( +
failed · {j.error || "unknown error"}
+ )} +
{formatRelative(j.finished_at ?? j.submitted_at)}
+
+ ))} {history.map((r) => (
{r.entry_key}
@@ -350,6 +450,55 @@ export default function Manage({ capabilities }: { capabilities: string[] }) {
+ {importOpen && ( +
+
e.stopPropagation()}> +
+
Import corpus
+
+ +
+
{ e.preventDefault(); importCorpus(); }}> +
+ + setImportFile(e.target.files?.[0] ?? null)} /> +
+ A .tar.gz from Export corpus or tpk export: graph, corpus + entries and ingest history. Users and roles are not included. Loading runs as a job in the + panel below; without replace it upserts into the current graph. +
+
+
+ + {importReplace && ( +
+ Drops and recreates every graph stream (nodes, edges, corpus entries, ingest history) before + loading — anything not in the bundle is gone. Roles that reference removed entries keep + dangling keys. +
+ )} +
+ + {error &&
{error}
} + +
+ + +
+
+
+
+ )} + {addOpen && (
e.stopPropagation()}> diff --git a/web/src/app.css b/web/src/app.css index 9886a9a..f6051fe 100644 --- a/web/src/app.css +++ b/web/src/app.css @@ -689,6 +689,7 @@ h2 { font-size: 14px; font-weight: 600; margin-top: 24px; color: var(--gray-200) .tk-manage-title { font-size: 18px; font-weight: 600; line-height: 1.4; color: var(--gray-200); } .tk-manage-subtitle { font-size: 12px; color: var(--gray-500); } .tk-manage-header-spacer { flex: 1; } +.tk-manage-header-actions { display: flex; gap: 8px; align-items: center; } .tk-manage-error { color: var(--red-500); font-size: 12px; } /* Users/Roles tab bar */ From 3bc7f03defc538b7768e42d15fe11fd7b6ba5cff Mon Sep 17 00:00:00 2001 From: Gang Tao Date: Thu, 12 Nov 2026 11:25:14 -0700 Subject: [PATCH 2/2] fix(transfer): carry _tp_time for the ingest log so run order survives import Found in the docker-compose end-to-end run: every imported kg_ingest_log row landed on one insert timestamp, so ingest.latest_status (arg_max by _tp_time) could not tell `ok` from `started` -- an imported entry showed "started" and would flip to "stale" after the stale window, and run durations collapsed to 0. _tp_time is insertable on append-only streams (verified on timeplusd 3.2.15), so export/import now keep it for TIME_ORDERED_STREAMS (kg_ingest_log only; keyed streams upsert by PK and need no order). Old bundles without the column still load; new bundles need this version. Co-Authored-By: Claude Fable 5.1 --- docs/FEATURES.md | 2 +- src/tpk/transfer.py | 17 ++++++++++++++++- tests/test_transfer.py | 24 ++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 428ba30..17a728b 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -108,7 +108,7 @@ Manage what the graph indexes at runtime, without editing files or restarting. - **Background jobs.** Add / reindex operations run on a background worker; job status is pollable. - **Manage tab** in the web UI drives all of it (add, toggle, reindex, delete), and shows the **ingest history** — every run, whether started from the UI, `tpk ingest` or `kubectl exec`: status (`running` / `ok` / `failed` with the error / `stale` for a run that never wrote its end row), nodes, edges, commit, duration. Backed by the append-only `kg_ingest_log` stream via `GET /api/ingest-log`. - **Release-upgrade workflow:** add the new ref → ingest → verify → flip enabled; a legacy migration re-keys old bare-name rows on first ingest. -- **Export / import (issue #29).** `tpk export` dumps the ingested graph + corpus registry to a portable bundle (proton `FORMAT Parquet`, one file per stream + manifest); `tpk import` loads it into a new environment — no re-ingest, no LLM calls. Idempotent upsert by default (`--replace` to reset); auth streams excluded. The bundle carries each entry's registry row (source, ref, visibility, extraction mode, description) and the ingest history; a dev-mode local `path` is blanked for GitHub-sourced entries, since it is meaningless on another host. +- **Export / import (issue #29).** `tpk export` dumps the ingested graph + corpus registry to a portable bundle (proton `FORMAT Parquet`, one file per stream + manifest); `tpk import` loads it into a new environment — no re-ingest, no LLM calls. Idempotent upsert by default (`--replace` to reset); auth streams excluded. The bundle carries each entry's registry row (source, ref, visibility, extraction mode, description) and the ingest history; a dev-mode local `path` is blanked for GitHub-sourced entries, since it is meaningless on another host. The append-only ingest log is exported with its `_tp_time`, so run order (and therefore each entry's `ok`/`started`/`stale` status and run durations) survives an import; a bundle written by an older `tpk` still loads, but one written by this version needs this version or newer to import. - **Export / import from the Manage tab (issue #102).** Admins get **Export corpus** (one `.tar.gz` download of the same bundle, `GET /api/corpus/export`) and **Import corpus** (upload the archive, optional *replace*; `POST /api/corpus/import`). The archive is validated in full before anything is written — flat regular files only, a manifest, nothing the manifest doesn't list — then loaded as a job shown in the ingest panel with per-stream row counts. An import is refused (409) while any UI-started ingest or import job is active, so it never drops a stream such a job is writing into (a `tpk ingest` run from the CLI or `kubectl exec` is a separate process the server cannot see — don't import during one). Admin-only rather than `corpus:manage`, because a manager role may be scoped to a subset of entries and the bundle is the whole graph. Scratch space: `TPK_TRANSFER_DIR` (an `emptyDir` in the k8s manifests); upload cap `TPK_IMPORT_MAX_MB` (default 2048). ## 7. Authentication & role-based access (issue #8) diff --git a/src/tpk/transfer.py b/src/tpk/transfer.py index aecf1f5..e413a50 100644 --- a/src/tpk/transfer.py +++ b/src/tpk/transfer.py @@ -39,14 +39,29 @@ } +# Append-only streams have no key: their row ORDER is the data (ingest.py +# picks the newest `kg_ingest_log` row per entry by `_tp_time`). Insert time +# is the only ordering, so the bundle must carry `_tp_time` for these, or +# every imported row lands on one timestamp and `started`/`ok` become +# indistinguishable (an imported entry then shows "started", later "stale"). +# Keyed streams are upserted by primary key and need no such column. +TIME_ORDERED_STREAMS = frozenset({"kg_ingest_log"}) + + def _stream_columns(client, stream: str, prefix: str = "") -> list[str]: """Real, insertable columns of a stream — excludes proton's internal - `_tp_*` pseudo-columns and any ALIAS/MATERIALIZED columns.""" + `_tp_*` pseudo-columns (except `_tp_time` on time-ordered streams, which + is insertable and carries the row order) and any ALIAS/MATERIALIZED + columns.""" rows = client.query(f"DESCRIBE {db.qualified(stream, prefix)}").result_rows + keep_time = stream in TIME_ORDERED_STREAMS cols = [] for row in rows: name = row[0] default_type = row[2] if len(row) > 2 else "" + if name == "_tp_time" and keep_time: + cols.append(name) + continue if name.startswith("_tp") or default_type in ("ALIAS", "MATERIALIZED"): continue cols.append(name) diff --git a/tests/test_transfer.py b/tests/test_transfer.py index 851430c..1412b00 100644 --- a/tests/test_transfer.py +++ b/tests/test_transfer.py @@ -281,3 +281,27 @@ def test_export_strips_local_path_when_github_set(tp, tmp_path): entries = {e.name: e for e in corpus.list_entries(client, prefix=prefix)} assert entries["gh"].path is None # github entry: path dropped assert entries["local"].path == Path("/home/me/local") # dev-mode entry keeps its path + + +def test_ingest_log_order_survives_import(tp, tmp_path): + """A `started` row older than its `ok` row must still read as `ok` after a + round trip: the bundle carries `_tp_time` for the append-only log, so the + newest-row-per-entry logic in ingest.latest_status keeps working.""" + from tpk import ingest as ingest_mod + client, prefix = tp + cols = ["repo", "run_id", "nodes", "edges", "git_sha", "status", "_tp_time"] + client.insert(f"{prefix}kg_ingest_log", + [["demo@v1", "r1", 0, 0, "abc", "started", datetime(2026, 1, 1, 10, 0, tzinfo=timezone.utc)], + ["demo@v1", "r1", 3, 2, "abc", "ok", datetime(2026, 1, 1, 10, 5, tzinfo=timezone.utc)]], + column_names=cols) + _wait_count(client, prefix, "kg_ingest_log", 2) + assert ingest_mod.latest_status(client, prefix=prefix)["demo@v1"][1] == "ok" + + manifest = transfer.export_bundle(client, tmp_path / "b", prefix=prefix, streams=["kg_ingest_log"]) + assert "_tp_time" in manifest["streams"]["kg_ingest_log"]["columns"] + + transfer.import_bundle(client, tmp_path / "b", prefix=prefix, replace=True) + _wait_count(client, prefix, "kg_ingest_log", 2) + sha, status, t = ingest_mod.latest_status(client, prefix=prefix)["demo@v1"] + assert status == "ok" + assert t.replace(tzinfo=timezone.utc) == datetime(2026, 1, 1, 10, 5, tzinfo=timezone.utc)