diff --git a/README.md b/README.md index 495dc86..674e4c6 100644 --- a/README.md +++ b/README.md @@ -548,6 +548,10 @@ serialization (the engine encodes/decodes; no extra dependency): tpk import corpus-dump/ # load into this environment (upsert) tpk import corpus-dump/ --replace # reset the target streams first (drop stale rows) + tpk export --out one/ -e proton@v1.6.0 -e docs@main # only these entries (#104) + tpk import corpus-dump/ -e docs@main # only this entry out of a bundle + tpk import corpus-dump/ -e docs@main --replace # ...purging its existing rows first + The bundle covers the expensive/portable streams — `kg_nodes`, `kg_edges`, `kg_repos`, `kg_ingest_log` — plus a `manifest.json` (format, per-stream columns + row counts, `created_at`). **Auth streams @@ -569,6 +573,16 @@ all exist in the target deployment is rejected up front (schema drift between versions), before anything is written. Bundle files stream to/from disk, so a 300k-node graph never lands wholly in memory. +`--entry KEY` / `-e KEY` (repeatable, #104) limits either direction to the +named corpus entries (`name@ref`, or bare `name` for a local-path entry). +The manifest always lists the entries a bundle holds (`entries`) and whether +it was filtered (`partial`). A partial import stages the bundle in throwaway +streams and copies only the matching rows, so the target's other entries are +never touched; with `--entry`, `--replace` purges just those entries' +registry row, nodes and edges before loading (the append-only ingest log keeps +its history). Unknown keys are rejected before anything is written, on both +sides. + Streams under `TPK_STREAM_PREFIX` are exported/imported when that env var is set, matching the rest of the CLI. diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 17a728b..bbc84df 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -110,6 +110,7 @@ Manage what the graph indexes at runtime, without editing files or restarting. - **Release-upgrade workflow:** add the new ref → ingest → verify → flip enabled; a legacy migration re-keys old bare-name rows on first ingest. - **Export / import (issue #29).** `tpk export` dumps the ingested graph + corpus registry to a portable bundle (proton `FORMAT Parquet`, one file per stream + manifest); `tpk import` loads it into a new environment — no re-ingest, no LLM calls. Idempotent upsert by default (`--replace` to reset); auth streams excluded. The bundle carries each entry's registry row (source, ref, visibility, extraction mode, description) and the ingest history; a dev-mode local `path` is blanked for GitHub-sourced entries, since it is meaningless on another host. The append-only ingest log is exported with its `_tp_time`, so run order (and therefore each entry's `ok`/`started`/`stale` status and run durations) survives an import; a bundle written by an older `tpk` still loads, but one written by this version needs this version or newer to import. - **Export / import from the Manage tab (issue #102).** Admins get **Export corpus** (one `.tar.gz` download of the same bundle, `GET /api/corpus/export`) and **Import corpus** (upload the archive, optional *replace*; `POST /api/corpus/import`). The archive is validated in full before anything is written — flat regular files only, a manifest, nothing the manifest doesn't list — then loaded as a job shown in the ingest panel with per-stream row counts. An import is refused (409) while any UI-started ingest or import job is active, so it never drops a stream such a job is writing into (a `tpk ingest` run from the CLI or `kubectl exec` is a separate process the server cannot see — don't import during one). Admin-only rather than `corpus:manage`, because a manager role may be scoped to a subset of entries and the bundle is the whole graph. Scratch space: `TPK_TRANSFER_DIR` (an `emptyDir` in the k8s manifests); upload cap `TPK_IMPORT_MAX_MB` (default 2048). +- **Export / import only selected entries (issue #104).** Both directions take an entry filter, end to end: `tpk export -e KEY` / `tpk import -e KEY` (repeatable), `GET /api/corpus/export?entries=KEY&…`, a repeated `entries` form field on `POST /api/corpus/import`, and pickers in the Manage tab. **Export corpus** now opens a modal (*Whole corpus* / *Selected entries*); the **Import corpus** modal reads `manifest.json` out of the chosen `.tar.gz` in the browser (it is the first tar member; `DecompressionStream` plus a tar-header parse, no second upload) and offers the bundle's entries as a pick list, falling back to whole-bundle import for bundles or browsers without one. The manifest carries `entries` and `partial`. A partial import stages the bundle under a throwaway stream prefix and copies the selected rows engine-side (`INSERT … SELECT … WHERE has(keys, repo)`), so the other entries in the target are never touched; the targets are not written until every staged stream is readable. With a selection, *replace* means **replace those entries** (purge their registry row, nodes and edges first, as delete-with-purge does; ingest history is append-only and kept), and the UI labels it that way without the red whole-graph warning. Unknown keys get a 422 (import can only check them against bundles that list `entries`; older bundles import whole). Still admin-only. ## 7. Authentication & role-based access (issue #8) diff --git a/src/tpk/api.py b/src/tpk/api.py index 188e8c5..6db9896 100644 --- a/src/tpk/api.py +++ b/src/tpk/api.py @@ -208,10 +208,14 @@ def _submit(self, kind: str, key: str, payload) -> str: def submit(self, cfg: RepoConfig) -> str: return self._submit("ingest", entry_key(cfg), cfg) - def submit_import(self, bundle_dir, replace: bool = False) -> str: + def submit_import(self, bundle_dir, replace: bool = False, entries=None) -> str: """Load an already-unpacked bundle directory; the worker deletes the - directory when done, whatever the outcome.""" - return self._submit("import", IMPORT_ENTRY_KEY, (Path(bundle_dir), replace)) + directory when done, whatever the outcome. `entries` (#104) restricts + the load to those entry keys.""" + job_id = self._submit("import", IMPORT_ENTRY_KEY, (Path(bundle_dir), replace, entries)) + with self._jobs_lock: + self.jobs[job_id]["entries"] = list(entries) if entries is not None else None + return job_id def has_active(self) -> bool: with self._jobs_lock: @@ -225,15 +229,19 @@ def get(self, job_id: str) -> dict | None: with self._jobs_lock: return self.jobs.get(job_id) - def _run_import(self, rec: dict, bundle_dir: Path, replace: bool): + def _run_import(self, rec: dict, bundle_dir: Path, replace: bool, entries=None): try: with self._jobs_lock: rec["phase"] = "import" - rec["message"] = "replacing graph streams" if replace else "loading bundle" + if entries is not None: + rec["message"] = (f"{'replacing' if replace else 'loading'} " + f"{len(entries)} selected entr{'y' if len(entries) == 1 else 'ies'}") + else: + rec["message"] = "replacing graph streams" if replace else "loading bundle" rec["updated_at"] = datetime.now(timezone.utc).isoformat() client = db.get_client(Settings.from_env()) manifest = transfer.import_bundle(client, bundle_dir, prefix=self.prefix, - replace=replace) + replace=replace, entries=entries) streams = {name: info["rows"] for name, info in manifest["streams"].items()} with self._jobs_lock: rec["streams"] = streams @@ -449,18 +457,29 @@ def api_ingest_log(limit: int = Query(50, ge=1, le=200), entry: str | None = Non # entry included) and an import rewrites all of it. @router.get("/corpus/export") - def export_corpus(format: str = Query("Parquet"), actor: User = Depends(auth.require_admin)): + def export_corpus(format: str = Query("Parquet"), + entries: list[str] = Query(default=[]), + actor: User = Depends(auth.require_admin)): """Dump the graph + corpus registry and stream it back as one - `.tar.gz` -- the CLI bundle layout, so `tpk import` reads it unpacked.""" + `.tar.gz` -- the CLI bundle layout, so `tpk import` reads it unpacked. + `entries` (repeatable, #104) limits the bundle to those entry keys.""" if format not in transfer.FORMATS: raise HTTPException(422, "format must be Parquet|Native") + client = _client() + selected = sorted(set(entries)) if entries else None + if selected: + unknown = sorted(set(selected) - set(transfer.list_entry_keys(client, prefix=prefix))) + if unknown: + raise HTTPException(422, f"unknown corpus entries: {', '.join(unknown)}") root = transfer_dir() root.mkdir(parents=True, exist_ok=True) work = Path(tempfile.mkdtemp(prefix="tpk-export-", dir=root)) try: - transfer.export_bundle(_client(), work / "bundle", prefix=prefix, fmt=format) + transfer.export_bundle(client, work / "bundle", prefix=prefix, fmt=format, + entries=selected) stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") - name = f"tpk-corpus-{stamp}{transfer.ARCHIVE_SUFFIX}" + scope = "-partial" if selected is not None else "" + name = f"tpk-corpus-{stamp}{scope}{transfer.ARCHIVE_SUFFIX}" archive = transfer.pack_bundle(work / "bundle", work / name) shutil.rmtree(work / "bundle", ignore_errors=True) except Exception: @@ -473,10 +492,14 @@ def export_corpus(format: str = Query("Parquet"), actor: User = Depends(auth.req @router.post("/corpus/import") def import_corpus(file: UploadFile = File(...), replace: bool = Form(False), + entries: list[str] = Form(default=[]), actor: User = Depends(auth.require_admin)): """Upload a bundle archive and load it as a background job (poll `/api/jobs/{id}`). The archive is validated in full before anything - is written; a load is refused while any job is queued or running.""" + is written; a load is refused while any job is queued or running. + `entries` (repeatable form field, #104) loads only those entry keys; + with `replace`, only those entries' existing rows are purged first.""" + selected = sorted(set(entries)) if entries else None if jobs.has_active(): raise HTTPException(409, "an ingest or import job is already running; retry when it finishes") root = transfer_dir() @@ -499,6 +522,11 @@ def import_corpus(file: UploadFile = File(...), replace: bool = Form(False), except transfer.BadBundle as exc: raise HTTPException(400, f"invalid bundle: {exc}") archive.unlink() + listed = manifest.get("entries") + if selected and listed is not None: + unknown = sorted(set(selected) - set(listed)) + if unknown: + raise HTTPException(422, f"bundle does not contain: {', '.join(unknown)}") except Exception: shutil.rmtree(work, ignore_errors=True) raise @@ -508,9 +536,11 @@ def import_corpus(file: UploadFile = File(...), replace: bool = Form(False), if jobs.has_active(): shutil.rmtree(work, ignore_errors=True) raise HTTPException(409, "an ingest or import job is already running; retry when it finishes") - job_id = jobs.submit_import(work, replace=replace) + job_id = jobs.submit_import(work, replace=replace, entries=selected) return {"job_id": job_id, "streams": {k: v["rows"] for k, v in manifest["streams"].items()}, + "entries": selected, "bundle_entries": manifest.get("entries"), + "partial": manifest.get("partial", False), "format": manifest.get("format"), "created_at": manifest.get("created_at")} @router.get("/jobs") diff --git a/src/tpk/cli.py b/src/tpk/cli.py index b9e6d8e..e0dc523 100644 --- a/src/tpk/cli.py +++ b/src/tpk/cli.py @@ -149,15 +149,28 @@ def status(): def export( out: Path = typer.Option(..., "--out", "-o", help="Output bundle directory"), fmt: str = typer.Option("Parquet", "--format", help="Bundle format: Parquet or Native"), + entry: list[str] = typer.Option( + None, "--entry", "-e", + help="Only this corpus entry (name@ref, or name for a local-path entry); repeatable. " + "Default: the whole corpus.", + ), ): """Export the ingested graph + corpus registry to a portable bundle.""" from tpk import transfer client = db.get_client(Settings.from_env()) prefix = setting("TPK_STREAM_PREFIX", "db", "stream_prefix", "") - manifest = transfer.export_bundle(client, out, prefix=prefix, fmt=fmt) + entries = list(entry) if entry else None + if entries: + known = set(transfer.list_entry_keys(client, prefix=prefix)) + unknown = sorted(set(entries) - known) + if unknown: + raise typer.BadParameter(f"unknown corpus entries: {', '.join(unknown)}") + manifest = transfer.export_bundle(client, out, prefix=prefix, fmt=fmt, entries=entries) for stream, info in manifest["streams"].items(): typer.echo(f"{stream:16s} {info['rows']:>9} rows -> {info['file']}") + scope = "partial" if manifest.get("partial") else "whole corpus" + typer.echo(f"entries ({scope}): {', '.join(manifest['entries']) or '(none)'}") typer.echo(f"bundle written to {out}") @@ -165,7 +178,13 @@ def export( def import_bundle( src: Path = typer.Argument(..., help="Bundle directory to load"), replace: bool = typer.Option( - False, "--replace", help="Reset target streams before loading (else upsert)" + False, "--replace", + help="Reset target streams before loading (else upsert). With --entry: purge just " + "those entries' existing rows first.", + ), + entry: list[str] = typer.Option( + None, "--entry", "-e", + help="Only load this entry from the bundle (name@ref); repeatable. Default: the whole bundle.", ), ): """Load a corpus bundle into this environment — no re-ingest.""" @@ -173,9 +192,20 @@ def import_bundle( client = db.get_client(Settings.from_env()) prefix = setting("TPK_STREAM_PREFIX", "db", "stream_prefix", "") - manifest = transfer.import_bundle(client, src, prefix=prefix, replace=replace) + entries = list(entry) if entry else None + if entries: + listed = transfer.load_manifest(src).get("entries") + if listed is not None: + unknown = sorted(set(entries) - set(listed)) + if unknown: + raise typer.BadParameter( + f"bundle does not contain: {', '.join(unknown)} (it has: {', '.join(listed) or 'none'})") + manifest = transfer.import_bundle(client, src, prefix=prefix, replace=replace, entries=entries) for stream, info in manifest["streams"].items(): - typer.echo(f"{stream:16s} loaded {info['rows']:>9} rows") + typer.echo(f"{stream:16s} loaded {info['rows']:>9} rows" + + (" (filtered)" if entries else "")) + if entries: + typer.echo(f"entries loaded: {', '.join(entries)}") typer.echo("import complete") diff --git a/src/tpk/db.py b/src/tpk/db.py index deeb7f5..c729d70 100644 --- a/src/tpk/db.py +++ b/src/tpk/db.py @@ -144,12 +144,13 @@ def _add_column_if_missing(client, stream: str, column: str, col_type: str) -> N pass -def ensure_schema(client, prefix: str = "") -> None: - # All tpk streams live under a dedicated database (default `tpk`, #58); - # create it first so the qualified DDL below resolves. CREATE DATABASE is a - # global statement (works from the default-database session get_client uses) - # and is a no-op if it already exists, on both timeplusd and proton. - client.command(f"CREATE DATABASE IF NOT EXISTS {database()}") +GRAPH_STREAMS = ("kg_nodes", "kg_edges", "kg_ingest_log", "kg_repos") + + +def ensure_graph_streams(client, prefix: str = "") -> None: + """DDL for the four graph/corpus streams only (`GRAPH_STREAMS`). Split out + of `ensure_schema` so a corpus import can stage a bundle under a + throwaway prefix (#104) without also creating the auth streams.""" # Keyed (upsertable, latest-state) streams -- MUTABLE on timeplusd, # versioned_kv + `deleted` tombstone on proton (see _keyed_stream). client.command(_keyed_stream(prefix, "kg_nodes", [ @@ -181,6 +182,24 @@ def ensure_schema(client, prefix: str = "") -> None: "enabled bool", "visibility string", "extraction string", "description string", "updated_at datetime64(3, 'UTC')", ], pk="name, ref")) + + +def drop_graph_streams(client, prefix: str) -> None: + """Drop the four graph streams under `prefix` (staging cleanup, #104). + Refuses an empty prefix, like `drop_schema`.""" + if not prefix: + raise ValueError("refusing to drop unprefixed (production) streams") + for name in GRAPH_STREAMS: + client.command(f"DROP STREAM IF EXISTS {qualified(name, prefix)}") + + +def ensure_schema(client, prefix: str = "") -> None: + # All tpk streams live under a dedicated database (default `tpk`, #58); + # create it first so the qualified DDL below resolves. CREATE DATABASE is a + # global statement (works from the default-database session get_client uses) + # and is a no-op if it already exists, on both timeplusd and proton. + client.command(f"CREATE DATABASE IF NOT EXISTS {database()}") + ensure_graph_streams(client, prefix) client.command(_keyed_stream(prefix, "kg_users", [ "username string", "password_hash string", "role string", "must_change_password bool", "disabled bool", diff --git a/src/tpk/transfer.py b/src/tpk/transfer.py index e413a50..52bbc00 100644 --- a/src/tpk/transfer.py +++ b/src/tpk/transfer.py @@ -9,11 +9,19 @@ transport through the web UI (#102) a bundle is packed into a single flat `.tar.gz` (`pack_bundle` / `unpack_bundle`); the on-disk layout is the same either way, so `tpk import` reads an unpacked archive unchanged. + +Both directions take an optional entry filter (#104): a list of entry keys +(`name@ref`, or bare `name` for local-path entries). Every graph stream +carries the key (`repo` on nodes/edges/ingest_log, `name`+`ref` on +kg_repos), so a partial export is a WHERE on each stream's dump and a partial +import stages the bundle in throwaway streams and copies the matching rows. """ import json import shutil import tarfile +import time +import uuid from datetime import datetime, timezone from pathlib import Path @@ -38,6 +46,36 @@ ("kg_repos", "path"): "if(github != '', '', path) AS path", } +# SQL for a row's entry key, per stream -- the same identity +# `config.entry_key` computes in Python (`name@ref`, or `name` alone when the +# entry has no ref). +_KEY_EXPR = {"kg_repos": "if(ref != '', concat(name, '@', ref), name)"} + + +def _key_expr(stream: str) -> str: + return _KEY_EXPR.get(stream, "repo") + + +def _entry_filter(stream: str, entries) -> tuple[str, dict]: + """(`WHERE ...` clause, query parameters) selecting rows of `entries`; + ('', {}) when `entries` is None (no filter).""" + if entries is None: + return "", {} + keys = list(entries) + if not keys: + # `has([], x)` has no element type to infer; an empty selection is + # simply no rows. + return " WHERE 0", {} + return f" WHERE has(%(entries)s, {_key_expr(stream)})", {"entries": keys} + + +def list_entry_keys(client, prefix: str = "") -> list[str]: + """Every entry key the corpus registry currently holds, sorted.""" + rows = client.query( + f"SELECT {_key_expr('kg_repos')} FROM {db.latest(db.qualified('kg_repos', prefix))}" + ).result_rows + return sorted(r[0] for r in rows) + # Append-only streams have no key: their row ORDER is the data (ingest.py # picks the newest `kg_ingest_log` row per entry by `_tp_time`). Insert time @@ -68,23 +106,33 @@ def _stream_columns(client, stream: str, prefix: str = "") -> list[str]: return cols -def _row_count(client, stream: str, prefix: str = "") -> int: - r = client.query(f"SELECT count() FROM table({db.qualified(stream, prefix)})").result_rows +def _row_count(client, stream: str, prefix: str = "", entries=None) -> int: + where, params = _entry_filter(stream, entries) + r = client.query(f"SELECT count() FROM table({db.qualified(stream, prefix)}){where}", + parameters=params).result_rows return int(r[0][0]) if r else 0 def export_bundle(client, out_dir, streams=None, prefix: str = "", - fmt: str = "Parquet") -> dict: + fmt: str = "Parquet", entries=None) -> dict: """Stream each graph/corpus stream to `/.` in the - engine's `fmt` serialization and write a manifest. Returns the manifest.""" + engine's `fmt` serialization and write a manifest. Returns the manifest. + + `entries` (#104): keep only rows of these entry keys. The manifest always + records which entries the bundle holds (`entries`, from the registry + rows it carries) and whether it was filtered (`partial`).""" streams = list(streams) if streams is not None else list(GRAPH_STREAMS) ext = _EXT.get(fmt, fmt.lower()) out = Path(out_dir) out.mkdir(parents=True, exist_ok=True) + if entries is not None: + entries = sorted(set(entries)) manifest = { "bundle_version": BUNDLE_VERSION, "format": fmt, "created_at": datetime.now(timezone.utc).isoformat(), + "partial": entries is not None, + "entries": [], "streams": {}, } for stream in streams: @@ -94,15 +142,20 @@ def export_bundle(client, out_dir, streams=None, prefix: str = "", # object; copy it out in chunks so a 300k-row stream never lands # wholly in memory. select = [_EXPORT_EXPR.get((stream, c), c) for c in cols] + where, params = _entry_filter(stream, entries) src = client.raw_stream( - f"SELECT {', '.join(select)} FROM table({db.qualified(stream, prefix)})", fmt=fmt) + f"SELECT {', '.join(select)} FROM table({db.qualified(stream, prefix)}){where}", + parameters=params, fmt=fmt) with open(out / fname, "wb") as fh: shutil.copyfileobj(src, fh) manifest["streams"][stream] = { "file": fname, "columns": cols, - "rows": _row_count(client, stream, prefix=prefix), + "rows": _row_count(client, stream, prefix=prefix, entries=entries), } + if "kg_repos" in streams: + keys = list_entry_keys(client, prefix=prefix) + manifest["entries"] = [k for k in keys if entries is None or k in entries] (out / MANIFEST_NAME).write_text(json.dumps(manifest, indent=2)) return manifest @@ -111,16 +164,54 @@ def load_manifest(in_dir) -> dict: return json.loads((Path(in_dir) / MANIFEST_NAME).read_text()) -def import_bundle(client, in_dir, prefix: str = "", replace: bool = False) -> dict: +def purge_entries(client, entries, prefix: str = "") -> None: + """Delete the registry row, nodes and edges of each entry key (what the + Manage tab's delete-with-purge does, for a list of keys). The append-only + ingest log keeps its history rows: there is no DELETE on it.""" + keys = list(entries) + if not keys: + return + params = {"entries": keys} + db.delete(client, db.qualified("kg_repos", prefix), + f"has(%(entries)s, {_key_expr('kg_repos')})", params, ("name", "ref")) + db.delete(client, db.qualified("kg_nodes", prefix), "has(%(entries)s, repo)", params, ("id",)) + db.delete(client, db.qualified("kg_edges", prefix), "has(%(entries)s, repo)", params, + ("src", "dst", "rel")) + + +def _wait_rows(client, stream: str, prefix: str, want: int, timeout: float = 60.0) -> None: + """Block until `table(stream)` shows at least `want` rows. Keyed streams + have a sub-second write->read lag and the append-only log a ~2s one; a + staged load must be fully visible before it is copied out.""" + deadline = time.monotonic() + timeout + while True: + have = _row_count(client, stream, prefix=prefix) + if have >= want: + return + if time.monotonic() >= deadline: + raise TimeoutError( + f"{stream}: staged {have} of {want} bundle rows became visible within {timeout:.0f}s") + time.sleep(0.2) + + +def import_bundle(client, in_dir, prefix: str = "", replace: bool = False, + entries=None) -> dict: """Load a bundle into the current environment. Runs `ensure_schema` first (so a fresh env gets the streams). Returns the manifest. Idempotency: `kg_nodes`/`kg_edges`/`kg_repos` are mutable, primary-keyed streams, so a plain load is an idempotent upsert. `kg_ingest_log` is - append-only (no key), so a plain re-import APPENDS its provenance rows — + append-only (no key), so a plain re-import APPENDS its provenance rows -- harmless for `tpk status` (which takes arg_max per repo) but not deduped. Use `replace` to reset every target stream to exactly the bundle's rows. + `entries` (#104): load only rows of these entry keys. The bundle files + are then loaded into staging streams under a throwaway prefix and the + matching rows copied into the targets engine-side, so no Python-side + Parquet decoding is needed. With `entries`, `replace` means "replace + THOSE entries": their existing registry row, nodes and edges are purged + first; the other entries, and all streams, are left alone. + Rejects a bundle whose columns aren't all present in the target stream (schema drift between the exporting and importing versions) before writing anything.""" @@ -139,14 +230,23 @@ def import_bundle(client, in_dir, prefix: str = "", replace: bool = False) -> di missing = [c for c in info["columns"] if c not in target_cols] if missing: raise ValueError( - f"{stream}: target stream is missing column(s) {missing} — " + f"{stream}: target stream is missing column(s) {missing} -- " f"schema mismatch between the bundle and this deployment") + if entries is not None: + loaded = _import_entries(client, src, manifest, prefix, replace, sorted(set(entries))) + # Report what was actually copied, not the bundle's totals: a job or + # CLI run that loaded one entry of a ten-entry bundle should say so. + out = dict(manifest, loaded_entries=sorted(set(entries))) + out["streams"] = {stream: dict(info, rows=loaded[stream]) + for stream, info in manifest["streams"].items()} + return out + for stream, info in manifest["streams"].items(): if replace: # `replace` clears stale rows not present in the bundle. Drop + # recreate (synchronous DDL) rather than TRUNCATE, whose async - # apply on a mutable stream can race — and wipe — the load that + # apply on a mutable stream can race -- and wipe -- the load that # follows. Done per stream, immediately before its load, so a # mid-run failure only affects the stream in flight rather than # leaving every earlier-dropped stream empty. @@ -159,6 +259,44 @@ def import_bundle(client, in_dir, prefix: str = "", replace: bool = False) -> di return manifest +def _import_entries(client, src: Path, manifest: dict, prefix: str, replace: bool, + entries: list[str]) -> dict[str, int]: + """Partial load (#104): stage every bundled stream under a throwaway + prefix, wait for the staged rows to be readable, then copy the selected + entries' rows into the targets. The targets are not touched until all + staging succeeded, so a failed stage leaves the graph as it was. Returns + the number of rows copied per stream.""" + fmt = manifest.get("format", "Parquet") + stage = f"{prefix}imp{uuid.uuid4().hex[:8]}_" + db.ensure_graph_streams(client, stage) + loaded: dict[str, int] = {} + try: + for stream, info in manifest["streams"].items(): + with open(src / info["file"], "rb") as fh: + client.raw_insert(table=db.qualified(stream, stage), + column_names=info["columns"], + insert_block=fh, fmt=fmt) + _wait_rows(client, stream, stage, info["rows"]) + if replace: + purge_entries(client, entries, prefix=prefix) + for stream, info in manifest["streams"].items(): + cols = ", ".join(info["columns"]) + # Keyed staging streams read through latest() (hides proton + # tombstones); the append-only log reads table() so `_tp_time` + # comes through with the row. + source = (f"table({db.qualified(stream, stage)})" if stream in TIME_ORDERED_STREAMS + else db.latest(db.qualified(stream, stage))) + where, params = _entry_filter(stream, entries) + client.command( + f"INSERT INTO {db.qualified(stream, prefix)} ({cols}) " + f"SELECT {cols} FROM {source}{where}", parameters=params) + loaded[stream] = int(client.query( + f"SELECT count() FROM {source}{where}", parameters=params).result_rows[0][0]) + finally: + db.drop_graph_streams(client, stage) + return loaded + + # -- single-file transport (#102) ------------------------------------------- def pack_bundle(bundle_dir, archive_path) -> Path: diff --git a/tests/test_api.py b/tests/test_api.py index 9b2c0f5..53a8ff7 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -362,3 +362,92 @@ def test_export_import_are_admin_only(tp, monkeypatch, admin_hdr): assert c.get("/api/corpus/export", headers=hdr).status_code == 403 assert c.post("/api/corpus/import", headers=hdr, files={"file": ("x.tar.gz", b"x", "application/gzip")}).status_code == 403 + + +# -- entry-filtered export / import (#104) ------------------------------------ + +def _seed_second_entry(client, prefix): + from datetime import datetime, timezone + from tpk import corpus + from tpk.config import RepoConfig + from tpk.model import NODE_COLUMNS, Node, node_rows + now = datetime.now(timezone.utc) + nodes = [Node(f"m{i}", "other@v2", "function", f"g{i}", f"other::g{i}", "src/y.cpp", + i, i + 2, f"other {i}", "c1", "public") for i in range(2)] + client.insert(f"{prefix}kg_nodes", node_rows(nodes, now), column_names=NODE_COLUMNS) + corpus.upsert_entry(client, RepoConfig(name="other", ref="v2", github="org/other"), prefix=prefix) + _eventually(lambda: int(client.query( + f"SELECT count() FROM table({prefix}kg_nodes)").result_rows[0][0]) == 5) + + +def _manifest_of(archive: bytes) -> dict: + import io + import json + import tarfile + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tar: + return json.load(tar.extractfile("manifest.json")) + + +def _node_repos(client, prefix): + from tpk import db + rows = client.query(f"SELECT DISTINCT repo FROM {db.latest(prefix + 'kg_nodes')}").result_rows + return sorted(r[0] for r in rows) + + +def test_export_selected_entries(tp, monkeypatch, admin_hdr, tmp_path): + client, prefix = tp + _transfer_env(monkeypatch, tmp_path) + _seed_graph(client, prefix) + _seed_second_entry(client, prefix) + c, _ = _client(tp, monkeypatch) + + r = c.get("/api/corpus/export", headers=admin_hdr, params={"entries": ["other@v2"]}) + assert r.status_code == 200, r.text + assert "-partial.tar.gz" in r.headers["content-disposition"] + m = _manifest_of(r.content) + assert m["partial"] is True and m["entries"] == ["other@v2"] + assert m["streams"]["kg_nodes"]["rows"] == 2 + assert m["streams"]["kg_repos"]["rows"] == 1 + + full = _manifest_of(c.get("/api/corpus/export", headers=admin_hdr).content) + assert full["partial"] is False and full["entries"] == ["demo@v1", "other@v2"] + + r = c.get("/api/corpus/export", headers=admin_hdr, params={"entries": ["nope@v0", "other@v2"]}) + assert r.status_code == 422 and "nope@v0" in r.text + + +def test_import_selected_entries(tp, monkeypatch, admin_hdr, tmp_path): + client, prefix = tp + _transfer_env(monkeypatch, tmp_path) + _seed_graph(client, prefix) + _seed_second_entry(client, prefix) + c, _ = _client(tp, monkeypatch) + archive = c.get("/api/corpus/export", headers=admin_hdr).content + + # a key the bundle does not hold is refused before any job starts + r = c.post("/api/corpus/import", headers=admin_hdr, data={"entries": ["ghost@v9"]}, + files={"file": ("corpus.tar.gz", archive, "application/gzip")}) + assert r.status_code == 422 and "ghost@v9" in r.text + assert list((tmp_path / "scratch").iterdir()) == [] + + # drop everything, load only other@v2 back + from tpk import db + for s in ("kg_nodes", "kg_edges", "kg_repos"): + client.command(f"DROP STREAM IF EXISTS {prefix}{s}") + db.ensure_schema(client, prefix) + _eventually(lambda: _node_repos(client, prefix) == []) + + r = c.post("/api/corpus/import", headers=admin_hdr, data={"entries": ["other@v2"]}, + files={"file": ("corpus.tar.gz", archive, "application/gzip")}) + assert r.status_code == 200, r.text + body = r.json() + assert body["entries"] == ["other@v2"] and body["bundle_entries"] == ["demo@v1", "other@v2"] + job_id = body["job_id"] + rec = _eventually(lambda: (lambda j: j if j["status"] in ("ok", "failed") else None)( + c.get(f"/api/jobs/{job_id}", headers=admin_hdr).json()), timeout=90) + assert rec and rec["status"] == "ok", rec + assert rec["entries"] == ["other@v2"] + assert _eventually(lambda: _node_repos(client, prefix) == ["other@v2"]) + repos = c.get("/api/repos", headers=admin_hdr).json() + assert [e["name"] for e in repos] == ["other"] + assert _eventually(lambda: list((tmp_path / "scratch").iterdir()) == []) diff --git a/tests/test_cli.py b/tests/test_cli.py index 882fa19..aad312d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -72,3 +72,57 @@ def test_ingest_reads_corpus_store_and_matches_name_or_key(tp, monkeypatch, tmp_ "--repo", "storeonly@v1"]) assert res.exit_code == 0, res.output assert calls == ["storeonly"] + + +@requires_timeplus +def test_export_import_entry_option(tp, monkeypatch, tmp_path): + """`tpk export -e KEY` / `tpk import -e KEY` (#104): a partial bundle + round-trips only the named entry, and unknown keys are rejected.""" + import uuid + from datetime import datetime, timezone + + from tpk import corpus, db, transfer + from tpk.config import RepoConfig + from tpk.model import NODE_COLUMNS, Node, node_rows + + client, prefix = tp + now = datetime.now(timezone.utc) + for key, name, ref in (("a@v1", "a", "v1"), ("b@v2", "b", "v2")): + client.insert(f"{prefix}kg_nodes", + node_rows([Node(f"{name}-n", key, "function", "f", f"{name}::f", "x.py", + 1, 2, "", "c0", "public")], now), + column_names=NODE_COLUMNS) + corpus.upsert_entry(client, RepoConfig(name=name, ref=ref, github=f"o/{name}"), prefix=prefix) + monkeypatch.setattr(cli_mod.db, "get_client", lambda s: client) + monkeypatch.setenv("TPK_STREAM_PREFIX", prefix) + runner = CliRunner() + + res = runner.invoke(app, ["export", "-o", str(tmp_path / "b"), "-e", "zzz@v0"]) + assert res.exit_code != 0 and "unknown corpus entries: zzz@v0" in res.output + + res = runner.invoke(app, ["export", "-o", str(tmp_path / "b"), "-e", "a@v1"]) + assert res.exit_code == 0, res.output + assert "entries (partial): a@v1" in res.output + assert transfer.load_manifest(tmp_path / "b")["entries"] == ["a@v1"] + + res = runner.invoke(app, ["import", str(tmp_path / "b"), "-e", "b@v2"]) + assert res.exit_code != 0 and "bundle does not contain: b@v2" in res.output + + target = f"test_{uuid.uuid4().hex[:6]}_" + db.ensure_schema(client, target) + try: + monkeypatch.setenv("TPK_STREAM_PREFIX", target) + res = runner.invoke(app, ["import", str(tmp_path / "b"), "-e", "a@v1"]) + assert res.exit_code == 0, res.output + assert "entries loaded: a@v1" in res.output + import time + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + rows = client.query( + f"SELECT DISTINCT repo FROM {db.latest(target + 'kg_nodes')}").result_rows + if rows: + break + time.sleep(0.2) + assert sorted(r[0] for r in rows) == ["a@v1"] + finally: + db.drop_schema(client, target) diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index 3f0befe..faf7478 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -64,8 +64,9 @@ def test_jobmanager_import_job_loads_bundle_and_cleans_up(monkeypatch, tmp_path) monkeypatch.setattr(api_mod.db, "get_client", lambda *a, **k: object()) seen = {} - def fake_import(client, in_dir, prefix="", replace=False): + def fake_import(client, in_dir, prefix="", replace=False, entries=None): seen["in_dir"], seen["replace"], seen["prefix"] = in_dir, replace, prefix + seen["entries"] = entries return {"format": "Parquet", "streams": {"kg_nodes": {"rows": 7}, "kg_edges": {"rows": 4}}} monkeypatch.setattr(api_mod.transfer, "import_bundle", fake_import) @@ -81,18 +82,44 @@ def fake_import(client, in_dir, prefix="", replace=False): assert rec["streams"] == {"kg_nodes": 7, "kg_edges": 4} assert (rec["nodes"], rec["edges"]) == (7, 4) assert rec["finished_at"] is not None - assert seen == {"in_dir": bundle, "replace": True, "prefix": "p_"} + assert seen == {"in_dir": bundle, "replace": True, "prefix": "p_", "entries": None} + assert rec["entries"] is None assert not bundle.exists() assert mgr.has_active() is False +def test_jobmanager_import_job_passes_selected_entries(monkeypatch, tmp_path): + """#104: an entry-filtered import hands the keys to transfer.import_bundle + and records them on the job.""" + import tpk.api as api_mod + from tpk.api import JobManager + + monkeypatch.setattr(api_mod.db, "get_client", lambda *a, **k: object()) + seen = {} + + def fake_import(client, in_dir, prefix="", replace=False, entries=None): + seen["replace"], seen["entries"] = replace, entries + return {"format": "Parquet", "streams": {"kg_nodes": {"rows": 2}}} + + monkeypatch.setattr(api_mod.transfer, "import_bundle", fake_import) + bundle = tmp_path / "bundle" + bundle.mkdir() + mgr = JobManager() + rec = _wait_done(mgr, mgr.submit_import(bundle, replace=True, entries=["a@v1", "b@v2"])) + assert rec["status"] == "ok" + assert seen == {"replace": True, "entries": ["a@v1", "b@v2"]} + assert rec["entries"] == ["a@v1", "b@v2"] + assert rec["message"] == "replacing 2 selected entries" + assert not bundle.exists() + + def test_jobmanager_import_failure_is_recorded_and_cleaned(monkeypatch, tmp_path): import tpk.api as api_mod from tpk.api import JobManager monkeypatch.setattr(api_mod.db, "get_client", lambda *a, **k: object()) - def boom(client, in_dir, prefix="", replace=False): + def boom(client, in_dir, prefix="", replace=False, entries=None): raise ValueError("kg_nodes: target stream is missing column(s) ['x']") monkeypatch.setattr(api_mod.transfer, "import_bundle", boom) diff --git a/tests/test_transfer.py b/tests/test_transfer.py index 1412b00..76240c9 100644 --- a/tests/test_transfer.py +++ b/tests/test_transfer.py @@ -305,3 +305,168 @@ def test_ingest_log_order_survives_import(tp, tmp_path): sha, status, t = ingest_mod.latest_status(client, prefix=prefix)["demo@v1"] assert status == "ok" assert t.replace(tzinfo=timezone.utc) == datetime(2026, 1, 1, 10, 5, tzinfo=timezone.utc) + + +# -- entry-filtered export / import (#104) ------------------------------------ + +def _seed_other(client, prefix): + """A second entry, `other@v2`, beside `_seed`'s `demo@v1`.""" + now = datetime.now(timezone.utc) + nodes = [Node(f"m{i}", "other@v2", "function", f"g{i}", f"other::g{i}", + "src/y.cpp", i, i + 2, f"other {i}", "c1", "public") for i in range(2)] + edges = [Edge("m0", "m1", "calls", "EXTRACTED", "other@v2")] + client.insert(f"{prefix}kg_nodes", node_rows(nodes, now), column_names=NODE_COLUMNS) + client.insert(f"{prefix}kg_edges", edge_rows(edges, now), column_names=EDGE_COLUMNS) + corpus.upsert_entry(client, RepoConfig(name="other", ref="v2", github="org/other"), + prefix=prefix) + client.insert(f"{prefix}kg_ingest_log", + [["other@v2", "run-9", 2, 1, "def (v2)", "ok"]], + column_names=["repo", "run_id", "nodes", "edges", "git_sha", "status"]) + _wait_count(client, prefix, "kg_ingest_log", 2) + + +def _latest_count(client, prefix, stream): + """Live rows of a keyed stream (hides proton tombstones, unlike `_count`).""" + return int(client.query(f"SELECT count() FROM {db.latest(prefix + stream)}").result_rows[0][0]) + + +def _wait_latest(client, prefix, stream, want, timeout=10.0): + import time + deadline = time.monotonic() + timeout + while _latest_count(client, prefix, stream) != want and time.monotonic() < deadline: + time.sleep(0.1) + return _latest_count(client, prefix, stream) + + +def _keys(client, prefix, stream, col): + """Distinct values of `col` in the latest state of a keyed stream.""" + rows = client.query(f"SELECT DISTINCT {col} FROM {db.latest(prefix + stream)}").result_rows + return sorted(r[0] for r in rows) + + +def _log_repos(client, prefix): + rows = client.query(f"SELECT DISTINCT repo FROM table({prefix}kg_ingest_log)").result_rows + return sorted(r[0] for r in rows) + + +def _fresh_prefix(client): + prefix = f"test_{uuid.uuid4().hex[:6]}_" + db.ensure_schema(client, prefix) + return prefix + + +def test_full_export_manifest_lists_entries(tp, tmp_path): + client, prefix = tp + _seed(client, prefix) + _seed_other(client, prefix) + manifest = transfer.export_bundle(client, tmp_path / "b", prefix=prefix) + assert manifest["partial"] is False + assert manifest["entries"] == ["demo@v1", "other@v2"] + assert transfer.list_entry_keys(client, prefix=prefix) == ["demo@v1", "other@v2"] + + +def test_export_entries_filters_every_stream(tp, tmp_path): + client, prefix = tp + _seed(client, prefix) + _seed_other(client, prefix) + + manifest = transfer.export_bundle(client, tmp_path / "b", prefix=prefix, entries=["demo@v1"]) + assert manifest["partial"] is True + assert manifest["entries"] == ["demo@v1"] + rows = {s: i["rows"] for s, i in manifest["streams"].items()} + assert rows == {"kg_nodes": 3, "kg_edges": 2, "kg_repos": 1, "kg_ingest_log": 1} + + target = _fresh_prefix(client) + try: + transfer.import_bundle(client, tmp_path / "b", prefix=target) + assert _wait_count(client, target, "kg_nodes", 3) == 3 + assert _keys(client, target, "kg_nodes", "repo") == ["demo@v1"] + assert _keys(client, target, "kg_edges", "repo") == ["demo@v1"] + assert _keys(client, target, "kg_repos", "name") == ["demo"] + _wait_count(client, target, "kg_ingest_log", 1) + assert _log_repos(client, target) == ["demo@v1"] + finally: + db.drop_schema(client, target) + + +def test_export_empty_selection_is_an_empty_bundle(tp, tmp_path): + client, prefix = tp + _seed(client, prefix) + manifest = transfer.export_bundle(client, tmp_path / "b", prefix=prefix, entries=[]) + assert manifest["entries"] == [] + assert all(i["rows"] == 0 for i in manifest["streams"].values()) + + +def test_import_entries_loads_only_selected(tp, tmp_path): + client, prefix = tp + _seed(client, prefix) + _seed_other(client, prefix) + transfer.export_bundle(client, tmp_path / "b", prefix=prefix) + + target = _fresh_prefix(client) + try: + result = transfer.import_bundle(client, tmp_path / "b", prefix=target, entries=["other@v2"]) + # the returned manifest reports the rows actually copied, not the bundle totals + assert result["loaded_entries"] == ["other@v2"] + assert {s: i["rows"] for s, i in result["streams"].items()} == { + "kg_nodes": 2, "kg_edges": 1, "kg_repos": 1, "kg_ingest_log": 1} + assert _wait_count(client, target, "kg_nodes", 2) == 2 + assert _keys(client, target, "kg_nodes", "repo") == ["other@v2"] + assert _keys(client, target, "kg_edges", "repo") == ["other@v2"] + assert _keys(client, target, "kg_repos", "name") == ["other"] + _wait_count(client, target, "kg_ingest_log", 1) + assert _log_repos(client, target) == ["other@v2"] + # the staged copy came with its row time, so status is still readable + status = client.query( + f"SELECT arg_max(status, _tp_time) FROM table({target}kg_ingest_log)").result_rows[0][0] + assert status == "ok" + finally: + db.drop_schema(client, target) + # staging streams are gone + leftovers = client.query( + "SELECT name FROM system.tables WHERE name LIKE %(p)s", + parameters={"p": f"{target}imp%"}).result_rows + assert leftovers == [] + + +def test_import_entries_replace_purges_only_those_entries(tp, tmp_path): + client, prefix = tp + _seed(client, prefix) + transfer.export_bundle(client, tmp_path / "b", prefix=prefix, entries=["demo@v1"]) + + target = _fresh_prefix(client) + try: + # target already has a stale demo@v1 node and an unrelated entry + now = datetime.now(timezone.utc) + stale = Node("n_stale", "demo@v1", "function", "old", "demo::old", "src/x.cpp", + 1, 2, "stale", "c0", "public") + client.insert(f"{target}kg_nodes", node_rows([stale], now), column_names=NODE_COLUMNS) + _seed_other(client, target) + corpus.upsert_entry(client, RepoConfig(name="demo", ref="v1", github="org/demo", + description="old row"), prefix=target) + _wait_latest(client, target, "kg_nodes", 3) + + transfer.import_bundle(client, tmp_path / "b", prefix=target, + entries=["demo@v1"], replace=True) + assert _wait_latest(client, target, "kg_nodes", 5) == 5 + ids = _keys(client, target, "kg_nodes", "id") + assert "n_stale" not in ids and {"n0", "n1", "n2", "m0", "m1"} <= set(ids) + assert _keys(client, target, "kg_edges", "repo") == ["demo@v1", "other@v2"] + assert _keys(client, target, "kg_repos", "name") == ["demo", "other"] + finally: + db.drop_schema(client, target) + + +def test_import_entries_unknown_key_loads_nothing(tp, tmp_path): + client, prefix = tp + _seed(client, prefix) + transfer.export_bundle(client, tmp_path / "b", prefix=prefix) + target = _fresh_prefix(client) + try: + transfer.import_bundle(client, tmp_path / "b", prefix=target, entries=["nope@v0"]) + import time + time.sleep(1.0) + assert _count(client, target, "kg_nodes") == 0 + assert _count(client, target, "kg_repos") == 0 + finally: + db.drop_schema(client, target) diff --git a/web/src/Manage.tsx b/web/src/Manage.tsx index 4895e4a..aff69c9 100644 --- a/web/src/Manage.tsx +++ b/web/src/Manage.tsx @@ -1,5 +1,6 @@ import { useEffect, useRef, useState } from "react"; import { apiFetch } from "./api"; +import { peekManifest, type BundleManifest } from "./bundle"; import { CAP, hasCap } from "./capabilities"; type Repo = { @@ -18,6 +19,8 @@ type Job = { // report per-stream row counts instead of an ingest-log row. kind?: string; streams?: Record; + // import jobs: the entry keys the load was limited to (null = whole bundle), #104 + entries?: string[] | null; }; // One ingest run from the persistent log (#15): UI, CLI and kubectl runs alike. @@ -93,6 +96,37 @@ function downloadName(resp: Response, fallback: string): string { return m ? m[1] : fallback; } +// Checkbox list of entry keys with select-all / none, shared by the Export +// and Import modals (#104). +function EntryPicker({ keys, selected, onChange, disabled }: { + keys: string[]; selected: Set; onChange: (next: Set) => void; disabled?: boolean; +}) { + function toggle(key: string, on: boolean) { + const next = new Set(selected); + if (on) next.add(key); else next.delete(key); + onChange(next); + } + return ( +
+
+ {selected.size} of {keys.length} selected + + + +
+
+ {keys.map((k) => ( + + ))} +
+
+ ); +} + export default function Manage({ capabilities, isAdmin }: { capabilities: string[]; isAdmin?: boolean }) { // corpus:manage gates every mutation; with only corpus:view the screen is // read-only. Export/import of the WHOLE graph is admin-only (#102): a @@ -118,31 +152,59 @@ export default function Manage({ capabilities, isAdmin }: { capabilities: string const [importFile, setImportFile] = useState(null); const [importReplace, setImportReplace] = useState(false); const [importBusy, setImportBusy] = useState(false); + // Entry scope for import (#104): the bundle's manifest is read client-side + // from the chosen archive; "selected" is only offered when it lists entries. + const [importManifest, setImportManifest] = useState(null); + const [importPeek, setImportPeek] = useState<"idle" | "reading" | "ok" | "none">("idle"); + const [importScope, setImportScope] = useState<"all" | "selected">("all"); + const [importSel, setImportSel] = useState>(new Set()); const [exporting, setExporting] = useState(false); + const [exportOpen, setExportOpen] = useState(false); + const [exportScope, setExportScope] = useState<"all" | "selected">("all"); + const [exportSel, setExportSel] = useState>(new Set()); function openAdd() { setForm({ ...EMPTY_FORM }); setError(""); setAddOpen(true); } function closeAdd() { setAddOpen(false); setError(""); } - function openImport() { setImportFile(null); setImportReplace(false); setError(""); setImportOpen(true); } + function openImport() { + setImportFile(null); setImportReplace(false); setImportManifest(null); setImportPeek("idle"); + setImportScope("all"); setImportSel(new Set()); setError(""); setImportOpen(true); + } function closeImport() { if (importBusy) return; setImportOpen(false); setError(""); } + function openExport() { setExportScope("all"); setExportSel(new Set()); setError(""); setExportOpen(true); } + function closeExport() { if (exporting) return; setExportOpen(false); setError(""); } + + async function chooseImportFile(f: File | null) { + setImportFile(f); setImportManifest(null); setImportScope("all"); setImportSel(new Set()); + if (!f) { setImportPeek("idle"); return; } + setImportPeek("reading"); + const m = await peekManifest(f); + setImportManifest(m); + setImportPeek(m && Array.isArray(m.entries) ? "ok" : "none"); + } + const bundleEntries: string[] = importPeek === "ok" ? (importManifest?.entries ?? []) : []; useEffect(() => { - if (!addOpen && !importOpen) return; + if (!addOpen && !importOpen && !exportOpen) return; const onKey = (e: KeyboardEvent) => { if (e.key !== "Escape") return; if (addOpen) { setAddOpen(false); setError(""); } if (importOpen && !importBusy) { setImportOpen(false); setError(""); } + if (exportOpen && !exporting) { setExportOpen(false); setError(""); } }; window.addEventListener("keydown", onKey); return () => window.removeEventListener("keydown", onKey); - }, [addOpen, importOpen, importBusy]); + }, [addOpen, importOpen, importBusy, exportOpen, exporting]); // Export: the download needs the bearer token, so it goes through // apiFetch as a blob rather than a plain . async function exportCorpus() { + const selected = exportScope === "selected" ? [...exportSel] : null; + if (selected && selected.length === 0) { setError("Pick at least one entry"); return; } setExporting(true); setError(""); try { - const resp = await apiFetch("/api/corpus/export"); + const qs = selected ? "?" + selected.map((k) => `entries=${encodeURIComponent(k)}`).join("&") : ""; + const resp = await apiFetch("/api/corpus/export" + qs); if (!resp.ok) throw new Error(`${resp.status}: ${await resp.text()}`); const url = URL.createObjectURL(await resp.blob()); const a = document.createElement("a"); @@ -152,6 +214,7 @@ export default function Manage({ capabilities, isAdmin }: { capabilities: string a.click(); a.remove(); URL.revokeObjectURL(url); + setExportOpen(false); } catch (e) { setError(String(e)); } finally { @@ -167,6 +230,10 @@ export default function Manage({ capabilities, isAdmin }: { capabilities: string const body = new FormData(); body.append("file", importFile); body.append("replace", importReplace ? "true" : "false"); + if (importScope === "selected") { + if (importSel.size === 0) throw new Error("Pick at least one entry"); + for (const k of importSel) body.append("entries", k); + } const resp = await apiFetch("/api/corpus/import", { method: "POST", body }); if (!resp.ok) throw new Error(`${resp.status}: ${await resp.text()}`); setImportOpen(false); @@ -226,7 +293,7 @@ export default function Manage({ capabilities, isAdmin }: { capabilities: string
{/* Add-flow errors render inside the modal; keep this page-level banner for the table's toggle/reindex/delete errors only. */} - {!addOpen && !importOpen && error &&
{error}
} + {!addOpen && !importOpen && !exportOpen && error &&
{error}
}
@@ -239,7 +306,7 @@ export default function Manage({ capabilities, isAdmin }: { capabilities: string
{canTransfer && ( <> - @@ -413,10 +480,11 @@ export default function Manage({ capabilities, isAdmin }: { capabilities: string
{importResults.map((j) => (
-
{j.entry_key}
+
{j.entry_key}{j.entries ? ` (${j.entries.length} ${j.entries.length === 1 ? "entry" : "entries"})` : ""}
{j.status === "ok" ? (
ok · {Object.entries(j.streams ?? {}).map(([s, n]) => `${s} ${n}`).join(" · ")} + {j.entries ? ` · ${j.entries.join(", ")}` : ""}
) : (
failed · {j.error || "unknown error"}
@@ -450,6 +518,50 @@ export default function Manage({ capabilities, isAdmin }: { capabilities: string
+ {exportOpen && ( +
+
e.stopPropagation()}> +
+
Export corpus
+
+ +
+
{ e.preventDefault(); exportCorpus(); }}> +
+ + + + {exportScope === "selected" && ( + r.entry_key)} selected={exportSel} onChange={setExportSel} disabled={exporting} /> + )} +
+ One .tar.gz with the graph, corpus entries and ingest history of the chosen scope. + Users and roles are not included. tpk import and Import corpus read it. +
+
+ + {error &&
{error}
} + +
+ + +
+
+
+
+ )} + {importOpen && (
e.stopPropagation()}> @@ -463,20 +575,58 @@ export default function Manage({ capabilities, isAdmin }: { capabilities: string setImportFile(e.target.files?.[0] ?? null)} /> + onChange={(e) => chooseImportFile(e.target.files?.[0] ?? null)} />
A .tar.gz from Export corpus or tpk export: graph, corpus entries and ingest history. Users and roles are not included. Loading runs as a job in the panel below; without replace it upserts into the current graph.
+ {importPeek === "reading" &&
Reading bundle…
} + {importPeek === "ok" && importManifest && ( +
+ Bundle{importManifest.partial ? " (partial export)" : ""} + {importManifest.created_at ? ` from ${new Date(importManifest.created_at).toLocaleString()}` : ""}: + {" "}{bundleEntries.length} {bundleEntries.length === 1 ? "entry" : "entries"} + {importManifest.streams?.kg_nodes ? `, ${importManifest.streams.kg_nodes.rows} nodes` : ""}. +
+ )} + {importPeek === "none" && ( +
+ Could not read an entry list from this archive (older bundle or browser); it will be imported whole. +
+ )}
+ {importPeek === "ok" && ( +
+ + + + {importScope === "selected" && ( + + )} +
+ )}
- {importReplace && ( + {importReplace && importScope === "selected" && ( +
+ Deletes the selected entries' current nodes, edges and registry rows before loading, so stale + nodes from an earlier ingest do not linger. Other entries are untouched; ingest history is kept. +
+ )} + {importReplace && importScope !== "selected" && (
Drops and recreates every graph stream (nodes, edges, corpus entries, ingest history) before loading — anything not in the bundle is gone. Roles that reference removed entries keep @@ -489,8 +639,8 @@ export default function Manage({ capabilities, isAdmin }: { capabilities: string
-
diff --git a/web/src/app.css b/web/src/app.css index f6051fe..b1ce7cc 100644 --- a/web/src/app.css +++ b/web/src/app.css @@ -690,6 +690,16 @@ h2 { font-size: 14px; font-weight: 600; margin-top: 24px; color: var(--gray-200) .tk-manage-subtitle { font-size: 12px; color: var(--gray-500); } .tk-manage-header-spacer { flex: 1; } .tk-manage-header-actions { display: flex; gap: 8px; align-items: center; } + +/* Entry pick list in the Export / Import modals (#104) */ +.tk-radio { display: block; font-size: 13px; margin: 4px 0; } +.tk-entry-picker { margin: 8px 0; border: 1px solid var(--gray-700); border-radius: 4px; } +.tk-entry-picker-bar { display: flex; gap: 10px; align-items: center; padding: 6px 8px; font-size: 12px; color: var(--gray-500); border-bottom: 1px solid var(--gray-700); } +.tk-entry-picker-bar-spacer { flex: 1; } +.tk-entry-picker-list { max-height: 220px; overflow-y: auto; padding: 4px 8px; } +.tk-entry-picker-item { display: block; font-size: 13px; padding: 3px 0; } +.tk-link-btn { background: none; border: none; padding: 0; color: var(--pink-400); cursor: pointer; font-size: 12px; } +.tk-link-btn:disabled { background: none; opacity: .5; cursor: default; } .tk-manage-error { color: var(--red-500); font-size: 12px; } /* Users/Roles tab bar */ diff --git a/web/src/bundle.ts b/web/src/bundle.ts new file mode 100644 index 0000000..87dc6e1 --- /dev/null +++ b/web/src/bundle.ts @@ -0,0 +1,76 @@ +// Read `manifest.json` out of a corpus bundle archive (.tar.gz) in the +// browser, so the Import modal can offer the bundle's entries as a pick list +// (#104) without a second upload or a server-side staged upload. +// +// `pack_bundle` writes the manifest as the FIRST tar member, so only the +// head of the archive needs inflating: the gzip stream is read through +// DecompressionStream just far enough to reach the manifest's bytes, then +// cancelled. A tar member is a 512-byte header (name at 0..100, size in +// octal at 124..136, type flag at 156) followed by the content padded to a +// 512-byte boundary; PAX extended headers (type x/g) are skipped. + +export type BundleManifest = { + bundle_version?: number; + format?: string; + created_at?: string; + partial?: boolean; + entries?: string[]; + streams?: Record; +}; + +const BLOCK = 512; +const MANIFEST = "manifest.json"; +const MAX_PEEK = 4 * 1024 * 1024; // give up if the manifest isn't within 4 MB +const MAX_MEMBERS = 8; + +function field(bytes: Uint8Array, off: number, len: number): string { + let end = off; + while (end < off + len && bytes[end] !== 0) end++; + return new TextDecoder().decode(bytes.subarray(off, end)); +} + +function octal(bytes: Uint8Array, off: number, len: number): number { + const s = field(bytes, off, len).trim(); + return s ? parseInt(s, 8) : 0; +} + +export async function peekManifest(file: File): Promise { + if (typeof DecompressionStream === "undefined" || typeof file.stream !== "function") return null; + const reader = file.stream().pipeThrough(new DecompressionStream("gzip")).getReader(); + let buf = new Uint8Array(0); + async function ensure(n: number): Promise { + while (buf.length < n) { + if (n > MAX_PEEK) return false; + const { value, done } = await reader.read(); + if (done) return false; + const next = new Uint8Array(buf.length + value.length); + next.set(buf); + next.set(value, buf.length); + buf = next; + } + return true; + } + try { + let off = 0; + for (let i = 0; i < MAX_MEMBERS; i++) { + if (!(await ensure(off + BLOCK))) return null; + const hdr = buf.subarray(off, off + BLOCK); + if (hdr.every((b) => b === 0)) return null; // end-of-archive marker + const name = field(hdr, 0, 100); + const size = octal(hdr, 124, 12); + const type = String.fromCharCode(hdr[156]); + const dataStart = off + BLOCK; + if (name === MANIFEST && (type === "0" || type === "\0")) { + if (!(await ensure(dataStart + size))) return null; + const parsed = JSON.parse(new TextDecoder().decode(buf.subarray(dataStart, dataStart + size))); + return parsed && typeof parsed === "object" && parsed.streams ? (parsed as BundleManifest) : null; + } + off = dataStart + Math.ceil(size / BLOCK) * BLOCK; + } + return null; + } catch { + return null; // not gzip, not tar, not JSON: the server will say so on upload + } finally { + reader.cancel().catch(() => undefined); + } +}