From ae10bf04b9e6e3dabb9198b85cc702ff6c632491 Mon Sep 17 00:00:00 2001 From: Krokz Date: Wed, 23 Sep 2026 17:22:11 +0300 Subject: [PATCH 1/3] feat(schema): adopt a versioned memory preset through define_schema --- src/grag/api/main.py | 2 +- src/grag/client.py | 16 +++- src/grag/core/mutate.py | 8 ++ src/grag/core/presets.py | 148 ++++++++++++++++++++++++++++++++++ src/grag/core/schema.py | 9 ++- src/grag/core/types.py | 21 +++++ src/grag/mcp_server/server.py | 15 ++-- src/grag/retrieval/search.py | 19 +++++ tests/test_api.py | 2 +- tests/test_presets.py | 139 +++++++++++++++++++++++++++++++ 10 files changed, 367 insertions(+), 12 deletions(-) create mode 100644 src/grag/core/presets.py create mode 100644 tests/test_presets.py diff --git a/src/grag/api/main.py b/src/grag/api/main.py index 38d95f0..6cd6545 100644 --- a/src/grag/api/main.py +++ b/src/grag/api/main.py @@ -362,7 +362,7 @@ def health() -> dict: "status": ("shutting_down" if registry.closing or (shutdown and shutdown["state"] != "open") else "reopen_required" if runtime and runtime["writer_state"] != "ready" else "ok"), "version": grag.__version__, - "capabilities": {"ingestion_scope": 2, "snapshot_format": 2, "passive_diagnostics": 1}, + "capabilities": {"ingestion_scope": 2, "snapshot_format": 2, "passive_diagnostics": 1, "memory_preset": 1}, "database_id": identity, "server_id": database_identity(server_target), "pid": os.getpid(), diff --git a/src/grag/client.py b/src/grag/client.py index e19a294..0e89cdb 100644 --- a/src/grag/client.py +++ b/src/grag/client.py @@ -16,7 +16,7 @@ from grag.config import GragConfig from grag.core.errors import ConfigurationError, GragError from grag.core.limits import validate_request -from grag.core.types import UpsertNodesRequest +from grag.core.types import DefineSchemaRequest, UpsertNodesRequest _ROUTES = { "describe_schema": ("GET", "/api/schema"), @@ -159,6 +159,12 @@ def call(self, operation: str, req: BaseModel | None = None) -> dict: "The running owner does not support the current ingestion scope policy.", hint="Restart that server with the updated grag installation, then retry. No ingestion request was sent.", ) + preset = isinstance(req, DefineSchemaRequest) and req.preset is not None + if preset and self.capabilities.get("memory_preset", 0) < 1: + raise ConfigurationError( + "The running owner does not support schema presets.", + hint="Restart that server with the updated grag installation, then retry. No schema request was sent.", + ) method, path = _ROUTES[operation] payload = req.model_dump(mode="json") if req is not None else {} if isinstance(req, UpsertNodesRequest): @@ -168,11 +174,17 @@ def call(self, operation: str, req: BaseModel | None = None) -> dict: for node, item in zip(req.nodes, payload["nodes"], strict=True): if node.evidence is not None: item["evidence"] = node.evidence.model_dump(mode="json", exclude_unset=True) - return self._request( + result = self._request( method, path, **({"json": payload} if req is not None else {}), ) + if preset and not result.get("preset"): + raise GragError( + "The server returned no preset report.", + hint="Inspect the schema with describe_schema before retrying; the server may predate presets.", + ) + return result def __exit__( self, diff --git a/src/grag/core/mutate.py b/src/grag/core/mutate.py index c696902..a24dd10 100644 --- a/src/grag/core/mutate.py +++ b/src/grag/core/mutate.py @@ -274,6 +274,14 @@ def _similar_error(name: str, existing: str, kind: str) -> SchemaError: def define_schema( engine: Engine, config: GragConfig, req: DefineSchemaRequest, *, detail: SchemaDetail = "full", ) -> SchemaDocument: + if req.preset is not None: + from grag.core.presets import adopt_preset + from grag.core.schema import build_schema_document + + report = adopt_preset(engine, req.preset, allow_similar=req.allow_similar) + doc = build_schema_document(engine, config, detail=detail) + doc.preset = report + return doc # Validate every name before the first DDL, and publish tables + registry # together. Joining an ingest transaction does not imply a savepoint. with engine.atomic_writes(): diff --git a/src/grag/core/presets.py b/src/grag/core/presets.py new file mode 100644 index 0000000..c57d32e --- /dev/null +++ b/src/grag/core/presets.py @@ -0,0 +1,148 @@ +"""Opt-in, versioned schema presets — currently the memory preset. + +grag needs no preset: agents can define their own labels. The memory preset is a +shared starting point for decisions, insights, tasks and open questions. Adoption +is additive and idempotent. It creates missing tables and adds missing properties. +It never drops, renames or retypes anything: a different key, an incompatible +property type, or a clashing or near-duplicate table is reported and left as it +is. The adopted version is recorded in ``_grag_meta``, so a later grag knows which +migrations a database still needs. +""" + +from __future__ import annotations + +from collections.abc import Callable + +from grag.core.engine import Engine +from grag.core.errors import SchemaError +from grag.core.types import META_TABLE, NodeTableSpec, PresetReport, PropertySpec + +_META_KV = "_grag_meta" # the engine's key/value table (version stamps) + +_MEMORY_LABELS = ("Decision", "Insight", "Task", "Question") +_MEMORY_PROPS = ("title", "body", "status", "scope") + +# Current table specs per preset. Every adoption ensures these, so additive +# changes in a new version need no migration step. +PRESETS: dict[str, tuple[int, list[NodeTableSpec]]] = { + "memory": (1, [ + NodeTableSpec(name=label, properties=[PropertySpec(name=p) for p in _MEMORY_PROPS]) + for label in _MEMORY_LABELS + ]), +} + +# Non-additive changes (rename, split, backfill): preset -> {target version: step}. +# A step runs when adoption moves a database past that version. Steps must be +# idempotent: export/import does not carry _grag_meta, so a restored database +# re-adopts from no recorded version. +MIGRATIONS: dict[str, dict[int, Callable[[Engine, PresetReport], None]]] = {"memory": {}} + + +def adopt_preset(engine: Engine, name: str, *, allow_similar: bool = False) -> PresetReport: + """Bring the database to the preset's current version; report what changed.""" + version, specs = PRESETS[name] + key = f"preset.{name}.version" + if engine.in_write_transaction: + raise SchemaError("Preset adoption cannot join another write transaction.", + hint="Adopt the preset as its own define_schema call.") + with engine.atomic_writes(): + previous = _recorded_version(engine, key) + if previous is not None and previous > version: + raise SchemaError( + f"This database records {name} preset version {previous}; this grag supports up to {version}.", + hint="Upgrade gragdb (pip install -U gragdb) before adopting. Nothing was changed.", + ) + report = PresetReport(name=name, version=version, previous_version=previous) + for target in sorted(MIGRATIONS[name]): + if (previous or 0) < target <= version: + MIGRATIONS[name][target](engine, report) + altered = _ensure_tables(engine, specs, report, allow_similar=allow_similar) + _record_version(engine, key, version) + # DROP/CREATE_FTS_INDEX run only in auto-transaction mode, after the commit. + from grag.retrieval.search import refresh_fts_index + + for table in altered: + refresh_fts_index(engine, table) + return report + + +def _recorded_version(engine: Engine, key: str) -> int | None: + from grag.core.mutate import _table_index + + if _META_KV not in _table_index(engine): + return None + rows = engine.execute_write(f"MATCH (m:{_META_KV} {{key: $k}}) RETURN m.value", {"k": key}).rows + if not rows or rows[0][0] is None: + return None + try: + return int(rows[0][0]) + except ValueError: + raise SchemaError(f"Unreadable preset version {rows[0][0]!r} under '{key}'.", + hint="Inspect _grag_meta; nothing was changed.") from None + + +def _record_version(engine: Engine, key: str, version: int) -> None: + from grag.core.mutate import _table_index + + if _META_KV not in _table_index(engine): + engine.execute_write(f"CREATE NODE TABLE {_META_KV}(key STRING PRIMARY KEY, value STRING)") + engine.execute_write(f"MERGE (m:{_META_KV} {{key: $k}}) ON CREATE SET m.value = $v ON MATCH SET m.value = $v", + {"k": key, "v": str(version)}) + + +def _ensure_tables(engine: Engine, specs: list[NodeTableSpec], report: PresetReport, *, + allow_similar: bool) -> list[str]: + """Create missing tables and add missing properties. Returns existing tables + that gained STRING properties (their FTS index needs the new columns).""" + from grag.core.mutate import ( + _META_DDL, + _merge_meta, + _node_ddl, + _pk_of, + _table_columns, + _table_index, + similar_table, + ) + + tables = _table_index(engine) + if META_TABLE not in tables: + engine.execute_write(_META_DDL) + tables[META_TABLE] = "NODE" + altered: list[str] = [] + for spec in specs: + kind = tables.get(spec.name) + if kind == "REL": + report.conflicts.append(f"{spec.name} exists as a relationship table; left unchanged.") + continue + if kind is None: + twin = None if allow_similar else similar_table( + spec.name, [n for n, k in tables.items() if k == "NODE"]) + if twin is not None: + report.conflicts.append( + f"{spec.name} not created: similar table {twin} exists. Reuse {twin}, " + "or adopt with allow_similar=true to create both.") + continue + engine.execute_write(_node_ddl(spec)) + tables[spec.name] = "NODE" + _merge_meta(engine, name=spec.name, kind="node", pk=spec.primary_key, + searchable=spec.searchable, from_label="", to_label="") + report.created.append(spec.name) + continue + columns = _table_columns(engine, spec.name) + pk = _pk_of(engine, spec.name) + if pk != spec.primary_key or columns.get(pk or "") != "STRING": + report.conflicts.append( + f"{spec.name} key is {pk}:{columns.get(pk or '')}; the preset expects " + f"{spec.primary_key}:STRING. Left unchanged.") + for prop in spec.properties: + declared = columns.get(prop.name) + if declared is None: + engine.execute_write( + f"ALTER TABLE {spec.name} ADD {prop.name} {prop.type} DEFAULT CAST(NULL AS {prop.type})") + report.added.append(f"{spec.name}.{prop.name}") + if prop.type == "STRING" and spec.name not in altered: + altered.append(spec.name) + elif declared.upper() != prop.type: + report.conflicts.append( + f"{spec.name}.{prop.name} is {declared}; the preset expects {prop.type}. Left unchanged.") + return altered diff --git a/src/grag/core/schema.py b/src/grag/core/schema.py index 4229dd5..255a31b 100644 --- a/src/grag/core/schema.py +++ b/src/grag/core/schema.py @@ -214,7 +214,7 @@ def _build_schema_document(engine: Engine, detail: SchemaDetail) -> SchemaDocume detail=detail, ) encoded = json.dumps( - doc.model_dump(exclude={"freshness", "text", "schema_revision", "unchanged"}), + doc.model_dump(exclude={"freshness", "text", "schema_revision", "unchanged", "preset"}), sort_keys=True, ensure_ascii=False, separators=(",", ":"), ) doc.schema_revision = hashlib.sha256(("grag-schema-v1:" + encoded).encode()).hexdigest() @@ -223,10 +223,13 @@ def _build_schema_document(engine: Engine, detail: SchemaDetail) -> SchemaDocume def schema_text(doc: SchemaDocument) -> str: """Shared MCP and REST text presentation, including all control metadata.""" - footer = json.dumps({ + control: dict = { "schema_revision": doc.schema_revision, "detail": doc.detail, "unchanged": doc.unchanged, "freshness": doc.freshness.model_dump(), - }, separators=(",", ":")) + } + if doc.preset is not None: + control["preset"] = doc.preset.model_dump() + footer = json.dumps(control, separators=(",", ":")) return f"{doc.text}\n\n---\n{footer}" if doc.text else footer diff --git a/src/grag/core/types.py b/src/grag/core/types.py index 8f12126..265dff5 100644 --- a/src/grag/core/types.py +++ b/src/grag/core/types.py @@ -175,6 +175,15 @@ class DefineSchemaRequest(BaseModel): # to reuse it — the main way agent-built graphs fragment. Set True to # create it anyway. allow_similar: bool = False + # Adopt a versioned built-in schema instead of listing tables (see + # grag.core.presets). Additive and idempotent; conflicts are reported. + preset: Literal["memory"] | None = None + + @model_validator(mode="after") + def _preset_alone(self) -> DefineSchemaRequest: + if self.preset is not None and (self.node_tables or self.rel_tables): + raise ValueError("preset cannot be combined with node_tables or rel_tables; adopt it in a separate call") + return self class PropertyDoc(BaseModel): @@ -219,6 +228,17 @@ class FreshnessReport(BaseModel): SchemaDetail = Literal["compact", "full"] +class PresetReport(BaseModel): + """What one preset adoption did. Conflicts were reported, not changed.""" + + name: str + version: int + previous_version: int | None = None + created: list[str] = Field(default_factory=list) # table names + added: list[str] = Field(default_factory=list) # "Label.property" + conflicts: list[str] = Field(default_factory=list) + + class SchemaDocument(BaseModel): """Full schema introspection. `text` is the prompt-shaped rendering an LLM anchors on before writing Cypher — keep it compact.""" @@ -230,6 +250,7 @@ class SchemaDocument(BaseModel): schema_revision: str = "" unchanged: bool = False freshness: FreshnessReport = Field(default_factory=FreshnessReport) + preset: PresetReport | None = None # only on define_schema with a preset # --- mutation ------------------------------------------------------------------- diff --git a/src/grag/mcp_server/server.py b/src/grag/mcp_server/server.py index 23e1fab..001eee5 100644 --- a/src/grag/mcp_server/server.py +++ b/src/grag/mcp_server/server.py @@ -312,21 +312,24 @@ def describe_schema( @_return_errors def define_schema( service: GragService, - node_tables: Sequence[NodeTableSpec | dict], - rel_tables: Sequence[RelTableSpec | dict], + node_tables: Sequence[NodeTableSpec | dict] = (), + rel_tables: Sequence[RelTableSpec | dict] = (), if_not_exists: bool = True, allow_similar: bool = False, + preset: Literal["memory"] | None = None, ) -> str: """Create/reuse node and directed relationship tables; inspect existing schema first. Node keys default to STRING id. Declare property types and relationship endpoints. Names must be unquoted ASCII identifiers; near-duplicates require allow_similar=true. if_not_exists - preserves existing tables. Returns compact schema. + preserves existing tables. preset="memory" alone adds Decision/Insight/Task/Question + additively and reports conflicts. Returns compact schema. """ req = DefineSchemaRequest( node_tables=[NodeTableSpec.model_validate(t) for t in node_tables], rel_tables=[RelTableSpec.model_validate(t) for t in rel_tables], if_not_exists=if_not_exists, allow_similar=allow_similar, + preset=preset, ) doc = service.define_schema(req, detail="compact") return schema_text(doc) @@ -611,10 +614,11 @@ def describe_schema_tool( @server.tool(name="define_schema", structured_output=False, description=_doc(define_schema)) @_mcp_result def define_schema_tool( - node_tables: list[NodeTableSpec], - rel_tables: list[RelTableSpec], + node_tables: list[NodeTableSpec] = [], # noqa: B006 - schema default, never mutated + rel_tables: list[RelTableSpec] = [], # noqa: B006 if_not_exists: bool = True, allow_similar: bool = False, + preset: Literal["memory"] | None = None, ctx: Context | None = None, ) -> str: return define_schema( @@ -623,6 +627,7 @@ def define_schema_tool( rel_tables, if_not_exists, allow_similar, + preset, ) @server.tool(name="upsert_nodes", structured_output=False, description=_doc(upsert_nodes)) diff --git a/src/grag/retrieval/search.py b/src/grag/retrieval/search.py index 007aaad..9586bfc 100644 --- a/src/grag/retrieval/search.py +++ b/src/grag/retrieval/search.py @@ -290,6 +290,25 @@ def _ensure_fts_index(engine: Engine, table: str, index: str, cols: list[str]) - ensured.add(index) +def refresh_fts_index(engine: Engine, table: str) -> None: + """Rebuild an existing FTS index whose columns no longer match the table's + STRING properties, e.g. after ALTER ADD. The DROP/CREATE procedures need + auto-transaction mode, so call this after the schema change commits. A + crash between them leaves no index, which the next search recreates.""" + index = fts_index_name(table) + with engine.serialized_writes(): + rows = engine.execute_write("CALL SHOW_INDEXES() RETURN *").rows + current = next((list(row[3]) for row in rows if row[0] == table and row[1] == index), None) + cols = string_props(engine, table) + if current is None or sorted(current) == sorted(cols): + return + _ensure_extension(engine, "FTS") + _FTS_INDEXES.setdefault(engine, set()).discard(index) + engine.execute_write(f"CALL DROP_FTS_INDEX('{_ident(table)}', '{index}')") + if cols: + _ensure_fts_index(engine, table, index, cols) + + # --------------------------------------------------------------------------- # reciprocal rank fusion # --------------------------------------------------------------------------- diff --git a/tests/test_api.py b/tests/test_api.py index 0af797a..34d8dc2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -97,7 +97,7 @@ def test_health(client): assert res.json() == { "status": "ok", "version": grag.__version__, - "capabilities": {"ingestion_scope": 2, "snapshot_format": 2, "passive_diagnostics": 1}, + "capabilities": {"ingestion_scope": 2, "snapshot_format": 2, "passive_diagnostics": 1, "memory_preset": 1}, "database_id": database_identity(client.app.state.service.config.db_path), "server_id": database_identity(client.app.state.service.config.db_path), "pid": os.getpid(), diff --git a/tests/test_presets.py b/tests/test_presets.py new file mode 100644 index 0000000..11be207 --- /dev/null +++ b/tests/test_presets.py @@ -0,0 +1,139 @@ +"""M18: the opt-in memory preset is versioned, additive and reports conflicts.""" +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from grag.config import GragConfig +from grag.core import presets +from grag.core.engine import Engine +from grag.core.errors import SchemaError +from grag.core.mutate import define_schema, upsert_nodes +from grag.core.schema import build_schema_document, schema_text +from grag.core.types import ( + DefineSchemaRequest, + NodeTableSpec, + PropertySpec, + RelTableSpec, + SearchRequest, + UpsertNode, + UpsertNodesRequest, +) +from grag.retrieval.search import search_knowledge + +PRESET = DefineSchemaRequest(preset="memory") + + +def _columns(engine, table): + return {row[1]: row[2] for row in engine.execute(f"CALL TABLE_INFO('{table}') RETURN *").rows} + + +def _version(engine): + return engine.execute("MATCH (m:_grag_meta {key: 'preset.memory.version'}) RETURN m.value").rows + + +def _search(engine, query): + return [s.node.id for s in search_knowledge( + engine, engine.config, SearchRequest(query=query, labels=["Decision"], hops=0)).seeds] + + +def test_fresh_adoption_creates_tables_records_version_and_repeats_cleanly(engine): + doc = define_schema(engine, engine.config, PRESET) + assert doc.preset.created == ["Decision", "Insight", "Task", "Question"] + assert doc.preset.previous_version is None and not doc.preset.conflicts + for label in doc.preset.created: + assert _columns(engine, label).items() >= {"id": "STRING", "title": "STRING", "body": "STRING", + "status": "STRING", "scope": "STRING"}.items() + assert _version(engine) == [["1"]] + again = define_schema(engine, engine.config, PRESET) + assert again.preset.previous_version == 1 + assert not (again.preset.created or again.preset.added or again.preset.conflicts) + # The report is not part of the schema view or its revision. + plain = build_schema_document(engine, engine.config) + assert plain.preset is None and plain.schema_revision == again.schema_revision + assert '"preset"' in schema_text(again) and '"preset"' not in schema_text(plain) + + +def test_existing_table_gains_properties_and_its_search_index_covers_them(engine, tmp_path): + define_schema(engine, engine.config, DefineSchemaRequest( + node_tables=[NodeTableSpec(name="Decision", properties=[PropertySpec(name="body")])])) + upsert_nodes(engine, engine.config, UpsertNodesRequest(nodes=[UpsertNode( + label="Decision", key="cache", properties={"body": "alpha body"})])) + assert _search(engine, "alpha") == ["Decision:cache"] # builds the index over (body) + + report = define_schema(engine, engine.config, PRESET).preset + assert report.added == ["Decision.title", "Decision.status", "Decision.scope"] + assert report.created == ["Insight", "Task", "Question"] + result = upsert_nodes(engine, engine.config, UpsertNodesRequest(nodes=[UpsertNode( + label="Decision", key="cache", properties={"title": "zebra heading"})])) + assert not result.warnings + assert _search(engine, "zebra") == ["Decision:cache"] + engine.close() + reopened = Engine(engine.config) + try: + assert _search(reopened, "zebra") == ["Decision:cache"] + assert _search(reopened, "alpha") == ["Decision:cache"] + finally: + reopened.close() + + +def test_conflicts_are_reported_and_left_unchanged_on_every_run(engine): + define_schema(engine, engine.config, DefineSchemaRequest(node_tables=[ + NodeTableSpec(name="Decisions", properties=[PropertySpec(name="body")]), + NodeTableSpec(name="Task", properties=[PropertySpec(name="status", type="INT64")]), + NodeTableSpec(name="Question", primary_key="slug"), + ], rel_tables=[RelTableSpec(name="Insight", from_label="Task", to_label="Task")])) + before = {t: _columns(engine, t) for t in ("Decisions", "Task")} + added = ["Task.title", "Task.body", "Task.scope", "Question.title", "Question.body", "Question.status", "Question.scope"] + for expected in (added, []): + report = define_schema(engine, engine.config, PRESET).preset + assert report.created == [] and report.added == expected + conflicts = "\n".join(report.conflicts) + assert "Decision not created: similar table Decisions" in conflicts + assert "Insight exists as a relationship table" in conflicts + assert "Task.status is INT64" in conflicts + assert "Question key is slug:STRING" in conflicts + tables = {row[1] for row in engine.execute("CALL SHOW_TABLES() RETURN *").rows} + assert "Decision" not in tables + assert _columns(engine, "Decisions") == before["Decisions"] + assert _columns(engine, "Task")["status"] == "INT64" + assert _version(engine) == [["1"]] + # allow_similar creates the preset table beside the near-duplicate. + assert define_schema(engine, engine.config, DefineSchemaRequest(preset="memory", allow_similar=True)).preset.created == ["Decision"] + + +def test_newer_recorded_version_is_refused_without_changes(engine): + engine.execute_write("MERGE (m:_grag_meta {key: 'preset.memory.version'}) ON CREATE SET m.value = '2' ON MATCH SET m.value = '2'") + with pytest.raises(SchemaError, match="supports up to 1"): + define_schema(engine, engine.config, PRESET) + assert "Decision" not in {row[1] for row in engine.execute("CALL SHOW_TABLES() RETURN *").rows} + + +def test_migration_steps_run_once_when_crossing_their_version(engine, monkeypatch): + define_schema(engine, engine.config, PRESET) + calls = [] + monkeypatch.setitem(presets.PRESETS, "memory", (2, presets.PRESETS["memory"][1])) + monkeypatch.setitem(presets.MIGRATIONS, "memory", { + 1: lambda engine, report: calls.append(1), 2: lambda engine, report: calls.append(2)}) + report = define_schema(engine, engine.config, PRESET).preset + assert (report.previous_version, report.version, calls) == (1, 2, [2]) + define_schema(engine, engine.config, PRESET) + assert calls == [2] and _version(engine) == [["2"]] + + +def test_preset_is_its_own_call(): + with pytest.raises(ValidationError, match="separate call"): + DefineSchemaRequest(preset="memory", node_tables=[NodeTableSpec(name="Note")]) + + +def test_mcp_preset_needs_no_table_lists(tmp_path): + from grag.mcp_server import server as mcp + from grag.service import GragService + + svc = GragService(GragConfig(db_path=tmp_path / "mcp.lbdb", buffer_pool_size=128 * 1024**2)) + try: + text = mcp.define_schema(svc, preset="memory") + assert "Decision(" in text and '"preset":{"name":"memory","version":1' in text + finally: + svc.close() + From af792085d6414ecf8763c63150debaf7cbb8496f Mon Sep 17 00:00:00 2001 From: Krokz Date: Wed, 23 Sep 2026 17:22:11 +0300 Subject: [PATCH 2/3] feat(cli): add grag memory adopt and init --memory-preset --- src/grag/cli.py | 31 ++++++++++++++++-- src/grag/cli_graph.py | 21 +++++++++++++ tests/test_presets.py | 73 ++++++++++++++++++++++++++++++++++++++++++- tests/test_skill.py | 2 +- 4 files changed, 123 insertions(+), 4 deletions(-) diff --git a/src/grag/cli.py b/src/grag/cli.py index c3cf4ce..206179d 100644 --- a/src/grag/cli.py +++ b/src/grag/cli.py @@ -249,6 +249,15 @@ def main(argv: list[str] | None = None) -> int: retire.add_argument("--reason", help="reason for retiring this memory") retire.add_argument("--operation-id", help="durable retry ID for this exact retirement") retire.add_argument("--json", action="store_true") + memory = sub.add_parser("memory", help="manage the optional, versioned memory schema preset") + memory_sub = memory.add_subparsers(dest="memory_cmd", required=True) + adopt = memory_sub.add_parser( + "adopt", + help="add the memory preset (Decision/Insight/Task/Question with title, body, status, scope); " + "additive and repeatable: existing tables are never dropped or retyped, conflicts are reported", + ) + adopt.add_argument("--allow-similar", action="store_true", help="create preset tables even when a near-duplicate table exists") + adopt.add_argument("--json", action="store_true") search = sub.add_parser("search", help="retrieve knowledge from the selected graph") search.add_argument("query") search.add_argument("--label", action="append", dest="labels") @@ -429,6 +438,10 @@ def main(argv: list[str] | None = None) -> int: "--ingest-if-empty", action="store_true", help="index the checkout only when its graph is empty or contains only the init verification record", ) + init.add_argument( + "--memory-preset", action="store_true", + help="also adopt the optional memory schema preset (same as 'grag memory adopt')", + ) init.add_argument( "--global-skill", action="store_true", help="install only the user-level grag skill so it is available in new repos; no database or MCP registration", @@ -645,7 +658,7 @@ def main(argv: list[str] | None = None) -> int: except GragError as exc: print(str(exc), file=sys.stderr) return 1 - elif args.cmd in ("ingest", "ingest-code", "remember", "inspect", "retire", "search", "context"): + elif args.cmd in ("ingest", "ingest-code", "remember", "inspect", "retire", "memory", "search", "context"): from pydantic import ValidationError from grag.cli_graph import graph_command @@ -838,6 +851,8 @@ def _init_command(args: argparse.Namespace, cfg: GragConfig) -> int: project_root = find_root() if args.ingest_if_empty and (args.server_url or cfg.server_url or args.remove): raise ProjectConfigError("--ingest-if-empty needs a local checkout database and cannot be combined with --remove or a remote server.") + if args.memory_preset and (args.server_url or cfg.server_url or args.remove): + raise ProjectConfigError("--memory-preset needs a local checkout database and cannot be combined with --remove or a remote server; use 'grag --server-url memory adopt' for a remote graph.") if args.client == "codex" and not args.no_mcp and not args.remove: raise ProjectConfigError("Codex MCP registration is not managed by init. Use --client codex --no-mcp for local setup/CLI access, or --global-skill for the skill alone.") clients = ( @@ -944,6 +959,8 @@ def _init_command(args: argparse.Namespace, cfg: GragConfig) -> int: if args.dry_run: preview_ops(ops) + if args.memory_preset: + print(f"Would adopt the memory preset in {db_path}") return 0 if args.ingest_if_empty: @@ -968,6 +985,16 @@ def _init_command(args: argparse.Namespace, cfg: GragConfig) -> int: elif mcp_ops: print("Client connection unverified (--no-verify). Rerun init to verify it.") + if args.memory_preset: + from grag.cli_graph import preset_lines + from grag.client import GraphClient + from grag.core.types import DefineSchemaRequest + + cfg.db_path = db_path + with GraphClient(cfg) as client: + result = client.call("define_schema", DefineSchemaRequest(preset="memory")) + print("\n" + "\n".join(preset_lines(result["preset"], client.target))) + if args.ingest_if_empty: import json @@ -1015,7 +1042,7 @@ def _global_skill_command(args: argparse.Namespace) -> int: from grag.project import apply_ops, plan_global_skill_ops, preview_ops from grag.project_files import ProjectConfigError - if any((args.db, args.db_dir, args.port, args.ingest, args.ingest_if_empty, + if any((args.db, args.db_dir, args.port, args.ingest, args.ingest_if_empty, args.memory_preset, args.server_url, args.server_db, args.url, args.no_mcp, args.no_claude_md, args.no_skill, args.no_verify)): raise ProjectConfigError("--global-skill supports --client, --dry-run and --remove only; it does not configure a project or database.") diff --git a/src/grag/cli_graph.py b/src/grag/cli_graph.py index 9b9d31b..6e86f15 100644 --- a/src/grag/cli_graph.py +++ b/src/grag/cli_graph.py @@ -83,6 +83,21 @@ def _print_mutation(result: dict, client: GraphClient, args: argparse.Namespace, print(f"Warning: {warning}") +def preset_lines(report: dict, target: str) -> list[str]: + """Human summary of one preset adoption.""" + before = report.get("previous_version") + lines = [f"Memory preset v{report['version']} in {target}" + + (f" (was v{before})." if before not in (None, report["version"]) else ".")] + if report.get("created"): + lines.append(f"Created: {', '.join(report['created'])}") + if report.get("added"): + lines.append(f"Added properties: {', '.join(report['added'])}") + if not report.get("created") and not report.get("added"): + lines.append("No changes needed.") + lines.extend(f"Conflict (unchanged): {conflict}" for conflict in report.get("conflicts", [])) + return lines + + def graph_command(args: argparse.Namespace, cfg: GragConfig) -> int: if args.cmd == "ingest": from grag.ingest.loaders import ingest_paths @@ -114,6 +129,12 @@ def graph_command(args: argparse.Namespace, cfg: GragConfig) -> int: with GraphClient(cfg) as client: if args.cmd == "inspect": print(json.dumps(_inspect(client, args), ensure_ascii=False, indent=None if args.json else 2)) + elif args.cmd == "memory": + result = client.call("define_schema", DefineSchemaRequest(preset="memory", allow_similar=args.allow_similar)) + if args.json: + print(json.dumps(result["preset"], ensure_ascii=False)) + else: + print("\n".join(preset_lines(result["preset"], client.target))) elif args.cmd == "retire": label, key, _ = _node_schema(client, args.node_id) result = client.call("upsert_nodes", UpsertNodesRequest(nodes=[UpsertNode( diff --git a/tests/test_presets.py b/tests/test_presets.py index 11be207..af9c284 100644 --- a/tests/test_presets.py +++ b/tests/test_presets.py @@ -1,13 +1,20 @@ """M18: the opt-in memory preset is versioned, additive and reports conflicts.""" from __future__ import annotations +import json + import pytest +from fastapi.testclient import TestClient from pydantic import ValidationError +from grag import cli +from grag.admin import ServerInfo +from grag.api.main import create_app +from grag.client import GraphClient from grag.config import GragConfig from grag.core import presets from grag.core.engine import Engine -from grag.core.errors import SchemaError +from grag.core.errors import ConfigurationError, SchemaError from grag.core.mutate import define_schema, upsert_nodes from grag.core.schema import build_schema_document, schema_text from grag.core.types import ( @@ -137,3 +144,67 @@ def test_mcp_preset_needs_no_table_lists(tmp_path): finally: svc.close() + +@pytest.fixture() +def run(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("GRAG_EMBED_PROVIDER", "") + monkeypatch.setenv("GRAG_AUTO_REFRESH_CODE", "0") + monkeypatch.setattr("grag.admin.find_server", lambda db: None) + config = GragConfig(db_path=tmp_path / "memory.lbdb", buffer_pool_size=128 * 1024**2) + + def run(*args, success=True): + status = cli.main(["--db", str(config.db_path), *args]) + output = capsys.readouterr() + assert status == (0 if success else 1), output + return json.loads(output.out) if success and "--json" in args else output + + run.config = config + return run + + +def test_cli_adopt_json_and_human_output(run): + assert run("memory", "adopt", "--json")["created"] == ["Decision", "Insight", "Task", "Question"] + out = run("memory", "adopt").out + assert "Memory preset v1" in out and "No changes needed." in out + + +def test_client_refuses_an_owner_without_preset_support(run, monkeypatch): + with TestClient(create_app(run.config)) as http: + monkeypatch.setattr("grag.admin.find_server", lambda db: ServerInfo(8471, "0.12.0", True, None)) + + class OldOwner: + def request(self, method, path, **kwargs): + response = http.request(method, path, **kwargs) + if path == "/api/health": + body = response.json() + body["capabilities"].pop("memory_preset") + return type("Health", (), {"status_code": 200, "json": lambda self: body})() + if path == "/api/schema/define": + pytest.fail("sent a preset request to an owner without support") + return response + + def close(self): + pass + + monkeypatch.setattr("grag.client.httpx2.Client", lambda **kw: OldOwner()) + with GraphClient(run.config) as client, pytest.raises(ConfigurationError, match="schema presets"): + client.call("define_schema", PRESET) + + +def test_init_adopts_the_preset_and_dry_run_opens_no_database(tmp_path, monkeypatch, capsys): + root = tmp_path / "project" + (root / ".git").mkdir(parents=True) + for key in ("GRAG_DB_PATH", "GRAG_DB_DIR", "GRAG_SERVER_URL", "GRAG_EMBED_PROVIDER"): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("GRAG_AUTO_REFRESH_CODE", "0") + monkeypatch.setenv("GRAG_BUFFER_POOL_MB", "128") + monkeypatch.setattr("grag.admin.find_server", lambda db: None) + monkeypatch.chdir(root) + db = tmp_path / "project.lbdb" + args = ["--db", str(db), "init", "--client", "claude", "--no-verify", "--memory-preset"] + assert cli.main([*args, "--dry-run"]) == 0 + assert "Would adopt the memory preset" in capsys.readouterr().out and not db.exists() + assert cli.main(args) == 0 + assert "Created: Decision, Insight, Task, Question" in capsys.readouterr().out + assert cli.main([*args[:3], "--remove", "--memory-preset"]) == 1 diff --git a/tests/test_skill.py b/tests/test_skill.py index bef1c4a..49bf603 100644 --- a/tests/test_skill.py +++ b/tests/test_skill.py @@ -113,7 +113,7 @@ def test_global_collision_preserves_entire_existing_bundle(tmp_path, monkeypatch assert reference.read_text() == "My own memory procedures\n" -@pytest.mark.parametrize("extra", [["--ingest-if-empty"], ["--no-mcp"], ["--server-url", "http://127.0.0.1:1234"]]) +@pytest.mark.parametrize("extra", [["--ingest-if-empty"], ["--memory-preset"], ["--no-mcp"], ["--server-url", "http://127.0.0.1:1234"]]) def test_global_mode_refuses_project_flags(tmp_path, monkeypatch, extra): monkeypatch.setattr(Path, "home", lambda: tmp_path) assert main(["init", "--global-skill", *extra]) == 1 From d27a003c118175042acf27e943189e25592e5ab5 Mon Sep 17 00:00:00 2001 From: Krokz Date: Wed, 23 Sep 2026 17:22:11 +0300 Subject: [PATCH 3/3] docs: document the optional memory preset --- .agents/skills/grag/references/memory.md | 6 ++++-- .claude/skills/grag/references/memory.md | 6 ++++-- .cursor/skills/grag/references/memory.md | 6 ++++-- docs/architecture.md | 3 ++- docs/guides/memory.md | 3 ++- docs/reference/cli.md | 6 ++++-- docs/reference/mcp.md | 2 +- docs/reference/schema.md | 25 ++++++++++++++++++++++ src/grag/assets/skill/references/memory.md | 6 ++++-- src/grag/core/evidence.py | 2 +- src/grag/memory_view.py | 2 +- 11 files changed, 52 insertions(+), 15 deletions(-) diff --git a/.agents/skills/grag/references/memory.md b/.agents/skills/grag/references/memory.md index f110c65..f2eeb38 100644 --- a/.agents/skills/grag/references/memory.md +++ b/.agents/skills/grag/references/memory.md @@ -3,9 +3,11 @@ Use the graph's existing vocabulary. Common choices are Task (status and acceptance), Decision (choice, reason and rejected alternative), Insight (established finding), -Question (unresolved issue), and Concept/Integration. They are conventions, not built-in +Question (unresolved issue), and Concept/Integration. They are conventions, not required schema. Check unfamiliar schema before writes; define only missing types and -endpoint pairs. Topic search needs no schema preflight. +endpoint pairs. With no memory labels yet, `define_schema` with only `preset="memory"` +adds Decision/Insight/Task/Question (title, body, status, scope) without changing +existing tables. Topic search needs no schema preflight. ## Preserve the result of an investigation diff --git a/.claude/skills/grag/references/memory.md b/.claude/skills/grag/references/memory.md index f110c65..f2eeb38 100644 --- a/.claude/skills/grag/references/memory.md +++ b/.claude/skills/grag/references/memory.md @@ -3,9 +3,11 @@ Use the graph's existing vocabulary. Common choices are Task (status and acceptance), Decision (choice, reason and rejected alternative), Insight (established finding), -Question (unresolved issue), and Concept/Integration. They are conventions, not built-in +Question (unresolved issue), and Concept/Integration. They are conventions, not required schema. Check unfamiliar schema before writes; define only missing types and -endpoint pairs. Topic search needs no schema preflight. +endpoint pairs. With no memory labels yet, `define_schema` with only `preset="memory"` +adds Decision/Insight/Task/Question (title, body, status, scope) without changing +existing tables. Topic search needs no schema preflight. ## Preserve the result of an investigation diff --git a/.cursor/skills/grag/references/memory.md b/.cursor/skills/grag/references/memory.md index f110c65..f2eeb38 100644 --- a/.cursor/skills/grag/references/memory.md +++ b/.cursor/skills/grag/references/memory.md @@ -3,9 +3,11 @@ Use the graph's existing vocabulary. Common choices are Task (status and acceptance), Decision (choice, reason and rejected alternative), Insight (established finding), -Question (unresolved issue), and Concept/Integration. They are conventions, not built-in +Question (unresolved issue), and Concept/Integration. They are conventions, not required schema. Check unfamiliar schema before writes; define only missing types and -endpoint pairs. Topic search needs no schema preflight. +endpoint pairs. With no memory labels yet, `define_schema` with only `preset="memory"` +adds Decision/Insight/Task/Question (title, body, status, scope) without changing +existing tables. Topic search needs no schema preflight. ## Preserve the result of an investigation diff --git a/docs/architecture.md b/docs/architecture.md index 969b372..79ba1a0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,7 +21,8 @@ The core workflow is **ingest, retrieve and remember**: The agent harness owns planning, model execution and tool selection. Authored memory uses the project's vocabulary; grag does not require a task engine, -ontology or schema preset. Embeddings and the browser views are optional ways to +ontology or schema preset. An optional, versioned memory preset gives a new graph +a shared starting point. Embeddings and the browser views are optional ways to use the same local graph. Single-owner access, atomic writes, edit guards, backup and recovery support the diff --git a/docs/guides/memory.md b/docs/guides/memory.md index 3296b3e..95aca68 100644 --- a/docs/guides/memory.md +++ b/docs/guides/memory.md @@ -8,7 +8,8 @@ decision, correction or handoff. You can also explicitly ask it to remember. Reuse the project's labels and records; check unfamiliar schema before writes. Topic search needs no schema preflight. `Task`, `Decision`, `Insight` and `Question` -are conventions, not a required schema. +are conventions, not a required schema; the optional [memory +preset](../reference/schema.md#optional-memory-preset) creates them for you. | Intent | Existing MCP tools | |---|---| diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 692fe1b..56ef5fc 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -8,7 +8,7 @@ command syntax supported by your installed version. |---|---| | Connect | `init`, `relocate` | | Index | `ingest-code`, `ingest` | -| Save and retrieve | `remember`, `inspect`, `retire`, `search`, `context` | +| Save and retrieve | `remember`, `inspect`, `retire`, `search`, `context`, `memory adopt` | | Run a server | `serve`, `mcp`, `start`, `restart`, `stop` | | Diagnose | `status`, `doctor` | | Preserve data | `export`, `import`, `recover` | @@ -40,7 +40,8 @@ These commands use the same database resolution and owner routing as ingestion: | `search/context --tokens N` | `2000` | Estimated retrieval budget; this CLI default overrides `GRAG_TOKEN_BUDGET` for these commands. | | `search/context --hops N` | `1` | Graph-expansion depth. | | `search/context/inspect --freshness MODE` | `allow_stale` | `allow_stale`, `wait` or `require`; uses the request's default freshness timeout. | -| `remember/search/context/inspect/retire --json` | off | Emits compact machine-readable JSON instead of the human summary (`inspect` otherwise uses indented JSON). | +| `memory adopt` | — | Since 0.13.0: adopts the optional, versioned [memory preset](schema.md#optional-memory-preset); additive and repeatable. Conflicts are reported and left unchanged. `--allow-similar` creates preset tables beside near-duplicates. | +| `remember/search/context/inspect/retire/memory adopt --json` | off | Emits compact machine-readable JSON instead of the human summary (`inspect` otherwise uses indented JSON). | See the [everyday memory walkthrough](../guides/memory.md#cli-shortcuts) for guarded corrections, retry semantics and the difference between retraction and deletion. @@ -61,6 +62,7 @@ corrections, retry semantics and the difference between retraction and deletion. | `init --port PORT` | saved, otherwise derived per-project | Port written into generated MCP/shared-server configuration. New defaults derive from the database path (41000–49151); choose an explicit port if occupied. | | `init --ingest` | off | Also runs `ingest-code` on the resolved project root immediately. | | `init --ingest-if-empty` | off | Since 0.10.0: checks the selected local graph and sets up/indexes the checkout only when absent, empty or containing only the verified init marker. Other content skips setup and indexing. Mutually exclusive with `--ingest`; refuses remote servers and `--remove`. | +| `init --memory-preset` | off | Since 0.13.0: also runs `memory adopt` on the selected local database after setup. Refuses remote servers and `--remove`. | | `init --global-skill` | off | Since 0.10.0: installs only the user-level skill bundle so it can be invoked in new repos. Supports `--client`, `--dry-run` and `--remove`; does not open a database or configure MCP. Claude honors `CLAUDE_CONFIG_DIR` for its personal skill location. | | `init --remove` | off | Undoes init: removes the grag MCP entry and the CLAUDE.md block. | | `init --url` | off | Writes direct HTTP URL transport instead of stdio plus auto-serve; the shared server must already be running. | diff --git a/docs/reference/mcp.md b/docs/reference/mcp.md index 675da8b..ceab783 100644 --- a/docs/reference/mcp.md +++ b/docs/reference/mcp.md @@ -10,7 +10,7 @@ Any MCP client gets these 10 tools: | tool | purpose | |---|---| | `describe_schema` | compact schema: tables, property types, primary keys and directed endpoints. Optional full detail and revision-based reuse. Call before writing Cypher. | -| `define_schema` | create node/rel tables (LLM designs the graph for a domain); grag 0.10.0 validates reserved names and publishes the batch atomically | +| `define_schema` | create node/rel tables (LLM designs the graph for a domain); grag 0.10.0 validates reserved names and publishes the batch atomically; since 0.13.0, `preset="memory"` alone adopts the optional [memory preset](schema.md#optional-memory-preset) | | `upsert_nodes` / `upsert_edges` | atomic MERGE batches; optional retry IDs and revision checks; `_source` provenance automatic | | `cypher_query` | read-only Cypher; errors come back with correction hints | | `search_knowledge` | hybrid BM25 + vector seeds → RRF fusion → per-label diversity cap → k-hop expansion → cited, token-budgeted context | diff --git a/docs/reference/schema.md b/docs/reference/schema.md index 961c70e..6356bfb 100644 --- a/docs/reference/schema.md +++ b/docs/reference/schema.md @@ -4,6 +4,31 @@ Use the project's existing labels and properties. Task, Decision, Insight and Question are useful conventions, not a required schema. { .grag-lead } +## Optional memory preset + +**New in 0.13.0:** `grag memory adopt`, `grag init --memory-preset`, or +`define_schema` with only `preset="memory"` adopts version 1 of the memory +preset: `Decision`, `Insight`, `Task` and `Question` node tables, each keyed by a +STRING `id` with STRING `title`, `body`, `status` and `scope` properties. + +Adoption is additive and can be repeated: + +- Missing tables are created and registered as searchable. +- Existing tables gain missing properties. The table's text search index is + rebuilt to include new text properties. +- Nothing is dropped, renamed or retyped. A different primary key, a property + with another type, a relationship table with a preset name, or a near-duplicate + table (`Decisions`) is reported as a conflict and left unchanged. Pass + `allow_similar=true` (`--allow-similar`) to create a preset table beside a + near-duplicate. +- The adopted version is recorded in grag's internal metadata. A later grag + version adds its changes on the next adoption and refuses to run against a + database adopted by a newer version. + +The response lists created tables, added properties and conflicts. Export does not +carry the recorded version; re-adopting after an import is safe. A server that +predates presets is refused before any request is sent. + ## Reuse before defining `describe_schema` shows the current vocabulary. `define_schema` rejects a new diff --git a/src/grag/assets/skill/references/memory.md b/src/grag/assets/skill/references/memory.md index f110c65..f2eeb38 100644 --- a/src/grag/assets/skill/references/memory.md +++ b/src/grag/assets/skill/references/memory.md @@ -3,9 +3,11 @@ Use the graph's existing vocabulary. Common choices are Task (status and acceptance), Decision (choice, reason and rejected alternative), Insight (established finding), -Question (unresolved issue), and Concept/Integration. They are conventions, not built-in +Question (unresolved issue), and Concept/Integration. They are conventions, not required schema. Check unfamiliar schema before writes; define only missing types and -endpoint pairs. Topic search needs no schema preflight. +endpoint pairs. With no memory labels yet, `define_schema` with only `preset="memory"` +adds Decision/Insight/Task/Question (title, body, status, scope) without changing +existing tables. Topic search needs no schema preflight. ## Preserve the result of an investigation diff --git a/src/grag/core/evidence.py b/src/grag/core/evidence.py index b02de9f..538e2a5 100644 --- a/src/grag/core/evidence.py +++ b/src/grag/core/evidence.py @@ -1,6 +1,6 @@ """Explicit memory lifecycle and opt-in, transactional authored revision history. -No preset schema or background expiry worker. Untracked graphs keep working; +No required schema or background expiry worker. Untracked graphs keep working; expiry is evaluated at read time and never deletes evidence. History begins at adoption, with a baseline for an existing node, not invented earlier revisions. """ diff --git a/src/grag/memory_view.py b/src/grag/memory_view.py index 6e16417..f95e74a 100644 --- a/src/grag/memory_view.py +++ b/src/grag/memory_view.py @@ -1,4 +1,4 @@ -"""Bounded, schema-aware UI browsing. No preset schema or inferred memories.""" +"""Bounded, schema-aware UI browsing. No required schema or inferred memories.""" from __future__ import annotations