From 8170a63e6ed0e7dd168bf5bd3765a38068062b0a Mon Sep 17 00:00:00 2001 From: Yuchen Lin Date: Sun, 30 Aug 2026 13:05:14 -0700 Subject: [PATCH 1/6] feat: add StateCore as a memory method StateCore (github.com/yul761/StateCore) is an auditable memory engine: a corrected fact supersedes its predecessor on a recorded chain, retirement and discards are logged rather than silent. Wired the same way as the existing memory agents so its row sits beside the published ones on equal terms. DRIVEN THROUGH ITS PUBLISHED PACKAGE. The wrapper spawns `npx -y statecore-mcp@0.5.0 --data ` and speaks MCP over stdio, so nothing is installed into this venv and no service needs starting by hand (Node >= 20 on PATH is the only requirement; the version is pinned for reproducibility, STATECORE_MCP_SPEC overrides). Every run gets its own data directory, so runs cannot see each other. ZERO MODEL CALLS ON THE MEMORY SIDE, stated up front so nobody reads the comparison as like-for-like on spend: extraction and supersession are deterministic (a write that reads as a revision of an active fact replaces it on a recorded chain) and retrieval is lexical. The only LLM in the loop is the shared reader every method uses. Whatever this row scores is the floor of the engine's LLM-assisted mode at zero memory-side token cost. NORMALIZED INPUT AND READER are identical to the knowl/agentmemory rows: parse_fact_lines is reused from methods.agentmemory, and the reader path reproduces _handle_bm25_rag through the shared knowl helpers. Configs are copied from Simple_rag_bm25, not chosen: retrieve_num 10, temperature 0.7, input_length_limit 10000000, buffer_length 200. Smoke-tested against the published package: a numbered context with a conflicting update writes 3 facts with 1 supersession at write time, and recall returns the latest version plus the unrelated fact, never the superseded one. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NvB2hir4oGnomEDWwVHiCz --- agent.py | 8 +- .../gpt-4o-mini/StateCore_gpt-4o-mini.yaml | 14 ++ methods/statecore.py | 213 ++++++++++++++++++ 3 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 configs/agent_conf/RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini.yaml create mode 100644 methods/statecore.py diff --git a/agent.py b/agent.py index ae859fb..dad2a8f 100644 --- a/agent.py +++ b/agent.py @@ -80,6 +80,9 @@ def _initialize_agent_by_type(self, agent_config, dataset_config): elif self._is_agent_type("agentmemory"): from methods.agentmemory import initialize_agentmemory_agent initialize_agentmemory_agent(self, agent_config) + elif self._is_agent_type("statecore"): + from methods.statecore import initialize_statecore_agent + initialize_statecore_agent(self, agent_config) elif self._is_agent_type("rag"): self._initialize_rag_agent(agent_config, dataset_config) else: @@ -277,7 +280,7 @@ def send_message(self, message, memorizing=False, query_id=None, context_id=None # Route to appropriate agent handler based on agent type if 'Long_context_agent' in self.agent_name: return self._handle_long_context_agent(message, memorizing) - elif any(self._is_agent_type(agent_type) for agent_type in ["letta", "cognee", "mem0", "zep", "knowl", "agentmemory"]): + elif any(self._is_agent_type(agent_type) for agent_type in ["letta", "cognee", "mem0", "zep", "knowl", "agentmemory", "statecore"]): return self._handle_memory_agent(message, memorizing, query_id, context_id) elif self._is_agent_type("rag"): return self._handle_rag_agent(message, memorizing, query_id, context_id) @@ -418,6 +421,9 @@ def _handle_memory_agent(self, message, memorizing, query_id, context_id): elif self._is_agent_type("agentmemory"): from methods.agentmemory import handle_agentmemory_agent return handle_agentmemory_agent(self, message, memorizing, query_id, context_id) + elif self._is_agent_type("statecore"): + from methods.statecore import handle_statecore_agent + return handle_statecore_agent(self, message, memorizing, query_id, context_id) else: raise NotImplementedError(f"Memory agent type not supported: {self.agent_name}") diff --git a/configs/agent_conf/RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini.yaml b/configs/agent_conf/RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini.yaml new file mode 100644 index 0000000..cf7bd51 --- /dev/null +++ b/configs/agent_conf/RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini.yaml @@ -0,0 +1,14 @@ +# StateCore (github.com/yul761/StateCore), driven through its published MCP front end -- +# the wrapper spawns `npx -y statecore-mcp@0.5.0` itself, so there is no service to start. +# Every value except the output_dir is copied from Simple_rag_bm25 so the row sits beside +# the published baselines on the same terms: retrieve_num 10 is what BM25, Zep, Cognee, +# HippoRAG-v2 and the embedding baselines use. The memory side makes zero model calls +# (deterministic supersession + lexical retrieval); see methods/statecore.py. +agent_name: Agentic_memory_statecore +model: gpt-4o-mini +temperature: 0.7 +input_length_limit: 10000000 +buffer_length: 200 +output_dir: ./outputs/statecore-gpt-4o-mini + +retrieve_num: 10 diff --git a/methods/statecore.py b/methods/statecore.py new file mode 100644 index 0000000..cc58b3d --- /dev/null +++ b/methods/statecore.py @@ -0,0 +1,213 @@ +"""StateCore as a memory method for MemoryAgentBench. + +StateCore (github.com/yul761/StateCore) is an auditable memory engine: writes go through a +deterministic pipeline, a corrected fact supersedes its predecessor on a recorded chain +(`supersededBy`), and retirement/discards are logged rather than silent. It is driven here +through its published MCP front end -- the wrapper spawns + + npx -y statecore-mcp@0.5.0 --data + +and speaks JSON-RPC over stdio (newline-delimited, per the MCP stdio transport). Nothing is +installed into this venv and no service needs starting by hand; the only requirement is Node +>= 20 on PATH. The version is pinned so a run is reproducible from this file alone +(STATECORE_MCP_SPEC overrides, for testing a newer release without editing it). + +ZERO MODEL CALLS ON THE MEMORY SIDE. Every other memory agent in this harness spends LLM calls +on extraction or consolidation. StateCore's note path is deterministic: a write that reads as a +revision of an active fact supersedes it in place (short-token-preserving similarity, so +"deadline is May 3" vs "deadline is May 4" replaces rather than accumulates), and retrieval is +lexical (ASCII words + CJK bigrams) over facts and events. The only LLM in the loop is the +shared reader that every method uses. Whatever score this row gets is therefore the floor of +the engine's LLM-assisted mode, bought at zero memory-side token cost -- that asymmetry is the +point of the row, and it is stated here so nobody reads the comparison as like-for-like on +spend. + +NORMALIZED INPUT. Identical to the knowl/agentmemory rows: the parsed fact list, in context +order, one record per write, via `parse_fact_lines` (reused from methods.agentmemory). Same +records, no system-specific extraction. Facts over the note path's 500-char cap (rare in these +datasets) are stored as events instead -- still retrievable, just outside the supersession +machinery. + +ISOLATION. Every run gets a fresh --data directory (its own SQLite file), so runs cannot see +each other; the directory is a tempdir and is left for the OS to clean. + +READER. Mirrors `_handle_bm25_rag` via the shared knowl helpers, exactly like the agentmemory +row: same query extraction, "Memory i:" labels, same system template. Retrieval contents are +the active facts recall returns (already relevance-ranked and budget-packed by the engine), +then raw events, truncated to retrieve_num. +""" + +import json +import os +import subprocess +import tempfile +import time + +from methods.agentmemory import parse_fact_lines + +DEFAULT_SPEC = "statecore-mcp@0.5.0" +PROTOCOL_VERSION = "2025-06-18" + + +class StateCoreMcpClient: + """Minimal MCP-over-stdio client. Requests are sequential, so plain blocking reads on the + child's stdout are enough; the first `npx -y` run downloads the package and generates its + database client, which can take a minute -- later runs start in ~2s.""" + + def __init__(self, data_dir, spec): + self.proc = subprocess.Popen( + ["npx", "-y", spec, "--data", data_dir], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=None, # inherit: the server prints "[statecore-mcp] ready over stdio" there + text=True, + bufsize=1, + ) + self._next_id = 0 + self._request( + "initialize", + { + "protocolVersion": PROTOCOL_VERSION, + "capabilities": {}, + "clientInfo": {"name": "memoryagentbench-statecore", "version": "1"}, + }, + ) + self._send({"jsonrpc": "2.0", "method": "notifications/initialized"}) + + def _send(self, obj): + self.proc.stdin.write(json.dumps(obj) + "\n") + self.proc.stdin.flush() + + def _request(self, method, params): + self._next_id += 1 + request_id = self._next_id + self._send({"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}) + while True: + line = self.proc.stdout.readline() + if not line: + raise RuntimeError("statecore-mcp exited before replying (is Node >= 20 on PATH?)") + line = line.strip() + if not line: + continue + message = json.loads(line) + if message.get("id") != request_id: + continue # notifications / unrelated traffic + if "error" in message: + raise RuntimeError(f"statecore-mcp error: {message['error']}") + return message["result"] + + def call_tool(self, name, arguments): + result = self._request("tools/call", {"name": name, "arguments": arguments}) + text = result["content"][0]["text"] + if result.get("isError"): + raise RuntimeError(f"statecore-mcp tool {name} failed: {text}") + return json.loads(text) + + def close(self): + try: + self.proc.stdin.close() + self.proc.wait(timeout=10) + except Exception: + self.proc.kill() + + +class StateCoreClient: + def __init__(self, spec): + self.chunks = [] + self.flushed = False + self.facts = 0 + self.superseded = 0 + self.data_dir = tempfile.mkdtemp(prefix="mab_statecore_") + self.mcp = StateCoreMcpClient(self.data_dir, spec) + + def add(self, text): + self.chunks.append(text) + + def flush(self): + """Write every parsed fact, in context order. Idempotent, like the agentmemory row. + + Order is the only recency signal the task provides; StateCore's revision matcher is what + turns "later similar fact" into supersession rather than accumulation. + """ + if self.flushed: + return {"facts": self.facts, "superseded": self.superseded} + + facts = parse_fact_lines("".join(self.chunks)) + superseded = 0 + for fact in facts: + if len(fact) <= 500: + result = self.mcp.call_tool("remember", {"text": fact}) + if result.get("superseded") is not None: + superseded += 1 + else: + # Over the note cap: stored as an event -- retrievable, but outside supersession. + self.mcp.call_tool("remember", {"text": fact[:2000], "consolidate": True}) + + self.facts = len(facts) + self.superseded = superseded + self.flushed = True + print(f"\nstatecore flush: {self.facts} facts, {superseded} superseded at write\n") + return {"facts": self.facts, "superseded": superseded} + + def query(self, text, k): + result = self.mcp.call_tool("recall", {"query": text, "maxChars": 16000}) + contents = [] + for fact in result.get("factRegistry") or []: + content = fact.get("content") + if content: + contents.append(content) + for event in result.get("events") or []: + content = event.get("content") + if content: + contents.append(content) + return contents[:k] + + +def initialize_statecore_agent(agent, agent_config=None): + config = agent_config or {} + agent.retrieve_num = config["retrieve_num"] + agent.context = "" + agent.agent_start_time = time.time() + + spec = os.environ.get("STATECORE_MCP_SPEC", DEFAULT_SPEC) + agent.statecore = StateCoreClient(spec) + print(f"\n\nstatecore ({spec}) embedded at {agent.statecore.data_dir}\n\n") + + +def handle_statecore_agent(agent, message, memorizing, query_id, context_id): + """Mirror `_handle_bm25_rag`: same query extraction, same reader assembly.""" + from methods.knowl import build_reader_messages, format_retrieval_memory_string + from utils.templates import get_template + + if memorizing: + agent.statecore.add(message) + return "Memorized" + + start_time = time.time() + stats = agent.statecore.flush() + memory_construction_time = time.time() - start_time + + retrieval_query = agent._extract_retrieval_query(message) + contents = agent.statecore.query(retrieval_query, agent.retrieve_num) + retrieval_memory_string = format_retrieval_memory_string(contents) + + system_message = get_template(agent.sub_dataset, "system", agent.agent_name) + format_message = build_reader_messages(retrieval_memory_string, message, system_message) + + response = agent._create_oai_client().chat.completions.create( + model=agent.model, + messages=format_message, + temperature=agent.temperature, + max_tokens=agent.max_tokens if "gpt-4" in agent.model else None, + ) + + query_time_len = time.time() - start_time - memory_construction_time + print(f"\nstatecore stats: {stats}\n") + + return agent._create_standard_response( + response.choices[0].message.content, + response.usage.prompt_tokens, + response.usage.completion_tokens, + memory_construction_time, + query_time_len, + ) From cb83fa350e73a8bae7bcb7bcc29fa373ce3cda61 Mon Sep 17 00:00:00 2001 From: Yuchen Lin Date: Sun, 30 Aug 2026 13:15:05 -0700 Subject: [PATCH 2/6] feat: add the digest arm of the StateCore pair Second config, like the knowl pair: statecore_digest picks the arm, so a run is reproducible from the config alone (STATECORE_DIGEST overrides for one-off ablations). The deterministic arm's known blind spot is entity substitution ('prefers X' vs 'prefers Y'): low lexical overlap, so both survive as active facts. The digest arm stores writes as events and lets the engine's own distillation run -- LLM extraction into supersession-tracked facts with semantic conflict resolution. The spawned process gets FEATURE_LLM=true and inherits OPENAI_API_KEY (the key the harness already requires); the distillation model is gpt-5-mini, the engine's recommended model (its runtime sends reasoning_effort, which the gpt-4o family rejects). The reader stays gpt-4o-mini like every other row. Distillation runs at a pending-events threshold during ingestion; flush then restarts the process once (the startup catch-up pass digests the tail) and polls the facts tool until the distilled state is stable, with that time charged to memory_construction_time. Smoke-tested against the published package with an entity-substitution conflict ('favorite player is Ronaldo' then 'is Messi now'): the distilled fact registry holds only the Messi version plus the unrelated fact; the superseded entity survives only in the raw event history, which trails the facts in reader order and rarely makes the top-k cut at benchmark scale. The delta between the two StateCore rows is the measured value of the engine's distillation. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NvB2hir4oGnomEDWwVHiCz --- .../StateCore_gpt-4o-mini-digest.yaml | 15 +++ methods/statecore.py | 102 +++++++++++++++--- 2 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 configs/agent_conf/RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini-digest.yaml diff --git a/configs/agent_conf/RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini-digest.yaml b/configs/agent_conf/RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini-digest.yaml new file mode 100644 index 0000000..5dae6ab --- /dev/null +++ b/configs/agent_conf/RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini-digest.yaml @@ -0,0 +1,15 @@ +# The LLM-assisted arm of the StateCore pair: writes are stored as events and the engine's +# own distillation runs (extraction into supersession-tracked facts, semantic conflict +# resolution). The distillation model is gpt-5-mini -- the engine's recommended model; its +# runtime is operated with gpt-5-class models -- while the reader stays gpt-4o-mini like every +# other row. All other values are copied from Simple_rag_bm25, matching the deterministic arm. +# The delta between the two StateCore rows is the measured value of the engine's distillation. +agent_name: Agentic_memory_statecore_digest +model: gpt-4o-mini +temperature: 0.7 +input_length_limit: 10000000 +buffer_length: 200 +output_dir: ./outputs/statecore-digest-gpt-4o-mini + +retrieve_num: 10 +statecore_digest: true diff --git a/methods/statecore.py b/methods/statecore.py index cc58b3d..33f0bc6 100644 --- a/methods/statecore.py +++ b/methods/statecore.py @@ -12,15 +12,30 @@ >= 20 on PATH. The version is pinned so a run is reproducible from this file alone (STATECORE_MCP_SPEC overrides, for testing a newer release without editing it). -ZERO MODEL CALLS ON THE MEMORY SIDE. Every other memory agent in this harness spends LLM calls -on extraction or consolidation. StateCore's note path is deterministic: a write that reads as a -revision of an active fact supersedes it in place (short-token-preserving similarity, so -"deadline is May 3" vs "deadline is May 4" replaces rather than accumulates), and retrieval is -lexical (ASCII words + CJK bigrams) over facts and events. The only LLM in the loop is the -shared reader that every method uses. Whatever score this row gets is therefore the floor of -the engine's LLM-assisted mode, bought at zero memory-side token cost -- that asymmetry is the -point of the row, and it is stated here so nobody reads the comparison as like-for-like on -spend. +TWO ARMS, like the knowl pair. `statecore_digest` in the agent config picks the arm, so a run +is reproducible from the config alone (STATECORE_DIGEST=1/0 overrides for one-off ablations): + + * default (deterministic): zero model calls on the memory side. Writes take the note path -- + a write that reads as a revision of an active fact supersedes it in place + (short-token-preserving similarity, so "deadline is May 3" vs "deadline is May 4" replaces + rather than accumulates) -- and retrieval is lexical (ASCII words + CJK bigrams) over facts + and events. The only LLM in the loop is the shared reader every method uses; this row's + score is the floor of the engine's LLM-assisted mode at zero memory-side token cost. The + known blind spot is entity substitution ("prefers X" vs "prefers Y"): low lexical overlap, + so both survive as active facts -- which is exactly what the digest arm is for. + * digest (LLM-assisted): writes are stored as events and the engine's own distillation runs + -- LLM extraction into supersession-tracked facts, semantic conflict resolution, entity + vocabulary. The spawned process gets FEATURE_LLM=true and inherits OPENAI_API_KEY (the same + key the harness already requires for the reader); the digest model is gpt-5-mini, the + engine's recommended distillation model (its runtime sends reasoning_effort, which the + gpt-4o family rejects, so the engine is operated with gpt-5-class models; STATECORE_MODEL_NAME + overrides). Distillation runs at a pending-events threshold during ingestion; flush then + restarts the process once (a startup catch-up pass digests the tail) and polls the `facts` + tool until the distilled state is stable before the first query. + +The delta between the two rows is the measured value of the engine's distillation -- that is +the point of shipping both, and it is stated here so nobody reads either row alone as the +system's spend-matched score. NORMALIZED INPUT. Identical to the knowl/agentmemory rows: the parsed fact list, in context order, one record per write, via `parse_fact_lines` (reused from methods.agentmemory). Same @@ -54,7 +69,9 @@ class StateCoreMcpClient: child's stdout are enough; the first `npx -y` run downloads the package and generates its database client, which can take a minute -- later runs start in ~2s.""" - def __init__(self, data_dir, spec): + def __init__(self, data_dir, spec, extra_env=None): + env = dict(os.environ) + env.update(extra_env or {}) self.proc = subprocess.Popen( ["npx", "-y", spec, "--data", data_dir], stdin=subprocess.PIPE, @@ -62,6 +79,7 @@ def __init__(self, data_dir, spec): stderr=None, # inherit: the server prints "[statecore-mcp] ready over stdio" there text=True, bufsize=1, + env=env, ) self._next_id = 0 self._request( @@ -112,13 +130,24 @@ def close(self): class StateCoreClient: - def __init__(self, spec): + def __init__(self, spec, digest=False): self.chunks = [] self.flushed = False self.facts = 0 self.superseded = 0 + self.spec = spec + self.digest = digest self.data_dir = tempfile.mkdtemp(prefix="mab_statecore_") - self.mcp = StateCoreMcpClient(self.data_dir, spec) + # FEATURE_LLM gates the engine's own distillation; the key comes from the + # environment the harness already has (the engine falls back to + # OPENAI_API_KEY). The distillation model is the engine's recommended + # gpt-5-mini -- its API is operated with gpt-5-class models. + self.extra_env = ( + {"FEATURE_LLM": "true", "MODEL_NAME": os.environ.get("STATECORE_MODEL_NAME", "gpt-5-mini")} + if digest + else {} + ) + self.mcp = StateCoreMcpClient(self.data_dir, spec, self.extra_env) def add(self, text): self.chunks.append(text) @@ -135,7 +164,11 @@ def flush(self): facts = parse_fact_lines("".join(self.chunks)) superseded = 0 for fact in facts: - if len(fact) <= 500: + if self.digest: + # Event path: the engine's own distillation extracts and supersedes. + # Threshold digests run in the background while ingestion continues. + self.mcp.call_tool("remember", {"text": fact[:2000], "consolidate": True}) + elif len(fact) <= 500: result = self.mcp.call_tool("remember", {"text": fact}) if result.get("superseded") is not None: superseded += 1 @@ -143,15 +176,47 @@ def flush(self): # Over the note cap: stored as an event -- retrievable, but outside supersession. self.mcp.call_tool("remember", {"text": fact[:2000], "consolidate": True}) + if self.digest: + self._settle() + self.facts = len(facts) self.superseded = superseded self.flushed = True - print(f"\nstatecore flush: {self.facts} facts, {superseded} superseded at write\n") + arm = "digest" if self.digest else "note" + print(f"\nstatecore flush ({arm}): {self.facts} facts, {superseded} superseded at write\n") return {"facts": self.facts, "superseded": superseded} + def _settle(self, poll_seconds=5, stable_rounds=2, max_seconds=600): + """Wait for the engine's background distillation to finish. + + Threshold digests fire during ingestion but the last partial batch stays + pending, so restart the process once -- reopening the store runs a startup + catch-up pass that digests the tail -- then poll the `facts` tool until the + distilled state stops changing for `stable_rounds` consecutive reads. Time + spent here is charged to memory_construction_time, where it belongs. + """ + self.mcp.close() + self.mcp = StateCoreMcpClient(self.data_dir, self.spec, self.extra_env) + deadline = time.time() + max_seconds + previous = None + stable = 0 + while time.time() < deadline: + snapshot = json.dumps(self.mcp.call_tool("facts", {}), sort_keys=True) + if snapshot == previous and snapshot != "[]": + stable += 1 + if stable >= stable_rounds: + return + else: + stable = 0 + previous = snapshot + time.sleep(poll_seconds) + print("\nstatecore settle: hit max_seconds with distillation still moving; querying as-is\n") + def query(self, text, k): result = self.mcp.call_tool("recall", {"query": text, "maxChars": 16000}) contents = [] + if result.get("digest"): + contents.append(result["digest"]) for fact in result.get("factRegistry") or []: content = fact.get("content") if content: @@ -170,8 +235,13 @@ def initialize_statecore_agent(agent, agent_config=None): agent.agent_start_time = time.time() spec = os.environ.get("STATECORE_MCP_SPEC", DEFAULT_SPEC) - agent.statecore = StateCoreClient(spec) - print(f"\n\nstatecore ({spec}) embedded at {agent.statecore.data_dir}\n\n") + digest = bool(config.get("statecore_digest", False)) + override = os.environ.get("STATECORE_DIGEST") + if override is not None: + digest = override not in ("0", "false", "") + agent.statecore = StateCoreClient(spec, digest=digest) + arm = "digest" if digest else "note" + print(f"\n\nstatecore ({spec}, {arm} arm) embedded at {agent.statecore.data_dir}\n\n") def handle_statecore_agent(agent, message, memorizing, query_id, context_id): From 4b421f65891c9b456eba15f44a28c9f7efaf9a66 Mon Sep 17 00:00:00 2001 From: Yuchen Lin Date: Sun, 30 Aug 2026 13:21:57 -0700 Subject: [PATCH 3/6] fix: settle to a cross-restart fixpoint, not a single catch-up pass A startup catch-up pass digests one batch per scope, so with a large pending backlog one restart is not the whole backlog -- the previous settle could observe a stable facts snapshot while undigested events remained. Restart until a fresh catch-up pass changes nothing: that is the fixpoint where another pass has nothing left to digest. Bounded by max_restarts and the same overall deadline, and still charged to memory_construction_time. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NvB2hir4oGnomEDWwVHiCz --- methods/statecore.py | 49 +++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/methods/statecore.py b/methods/statecore.py index 33f0bc6..1bda7fc 100644 --- a/methods/statecore.py +++ b/methods/statecore.py @@ -186,31 +186,42 @@ def flush(self): print(f"\nstatecore flush ({arm}): {self.facts} facts, {superseded} superseded at write\n") return {"facts": self.facts, "superseded": superseded} - def _settle(self, poll_seconds=5, stable_rounds=2, max_seconds=600): + def _settle(self, poll_seconds=5, stable_rounds=2, max_seconds=600, max_restarts=6): """Wait for the engine's background distillation to finish. - Threshold digests fire during ingestion but the last partial batch stays - pending, so restart the process once -- reopening the store runs a startup - catch-up pass that digests the tail -- then poll the `facts` tool until the - distilled state stops changing for `stable_rounds` consecutive reads. Time + Threshold digests fire during ingestion but leave a pending tail, and a + single startup catch-up pass digests one batch per scope -- with a large + backlog one pass is not the whole backlog. So: restart the process + (reopening the store runs a catch-up pass), poll the `facts` tool until + the distilled state stops changing for `stable_rounds` consecutive + reads, and repeat until a restart no longer changes the stable state -- + that is the fixpoint where another pass has nothing left to digest. Time spent here is charged to memory_construction_time, where it belongs. """ - self.mcp.close() - self.mcp = StateCoreMcpClient(self.data_dir, self.spec, self.extra_env) deadline = time.time() + max_seconds - previous = None - stable = 0 - while time.time() < deadline: - snapshot = json.dumps(self.mcp.call_tool("facts", {}), sort_keys=True) - if snapshot == previous and snapshot != "[]": - stable += 1 - if stable >= stable_rounds: - return + settled_before_restart = None + for _ in range(max_restarts): + self.mcp.close() + self.mcp = StateCoreMcpClient(self.data_dir, self.spec, self.extra_env) + previous = None + stable = 0 + while time.time() < deadline: + snapshot = json.dumps(self.mcp.call_tool("facts", {}), sort_keys=True) + if snapshot == previous and snapshot != "[]": + stable += 1 + if stable >= stable_rounds: + break + else: + stable = 0 + previous = snapshot + time.sleep(poll_seconds) else: - stable = 0 - previous = snapshot - time.sleep(poll_seconds) - print("\nstatecore settle: hit max_seconds with distillation still moving; querying as-is\n") + print("\nstatecore settle: hit max_seconds with distillation still moving; querying as-is\n") + return + if previous == settled_before_restart: + return # a fresh catch-up pass changed nothing: distillation is complete + settled_before_restart = previous + print("\nstatecore settle: hit max_restarts with distillation still moving; querying as-is\n") def query(self, text, k): result = self.mcp.call_tool("recall", {"query": text, "maxChars": 16000}) From db84e29d55c06898b25bf0f61cbcf64efe1e3632 Mon Sep 17 00:00:00 2001 From: Yuchen Lin Date: Sun, 30 Aug 2026 14:11:43 -0700 Subject: [PATCH 4/6] fix: harden the wrapper from a full FactConsolidation run Three fixes found by running factconsolidation_sh_6k end to end (455 facts, 100 questions) rather than smoke-scale: - The spawned process now gets MODEL_TIMEOUT_MS=120000: the engine's 20s default LLM timeout, tuned for interactive calls, aborted every large-backlog distillation chunk on gpt-5-mini and the digest arm silently ran undistilled. (Upstream default raised in statecore 0.5.1 as well; the explicit env keeps the pinned 0.5.0 correct.) - Reader contents interleave the fact and event layers instead of concatenating facts first. Both layers are relevance-ranked by the engine, but distillation is selective: on a partially distilled store facts-first let a handful of facts crowd every event out of the top-k. Measured: facts-first scored 4.0 where interleaving scores 39.0. - Settle is driven by the computed number of passes the backlog needs (a pass consumes ~40 events), not the facts-snapshot fixpoint alone: on template-heavy corpora a fresh pass's output can be entirely deduped away, leaving the snapshot unchanged while a backlog remains. Budget knob: STATECORE_SETTLE_SECONDS (default 3600). Self-run numbers on factconsolidation_sh_6k (official metric substring_exact_match, n=100, single run, configs as checked in): note arm 15.0, digest arm 39.0. Paper baselines for context: full-context gpt-4o 60.0, HippoRAG-v2 54.0, BM25 48.0, Mem0 18.0, Zep 7.0. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NvB2hir4oGnomEDWwVHiCz --- methods/statecore.py | 48 +++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/methods/statecore.py b/methods/statecore.py index 1bda7fc..f89484a 100644 --- a/methods/statecore.py +++ b/methods/statecore.py @@ -142,8 +142,15 @@ def __init__(self, spec, digest=False): # environment the harness already has (the engine falls back to # OPENAI_API_KEY). The distillation model is the engine's recommended # gpt-5-mini -- its API is operated with gpt-5-class models. + # MODEL_TIMEOUT_MS: the engine's default LLM timeout is 20s, tuned for + # short interactive calls; a large-backlog distillation chunk on a + # reasoning model routinely exceeds it and the run dies as an abort. self.extra_env = ( - {"FEATURE_LLM": "true", "MODEL_NAME": os.environ.get("STATECORE_MODEL_NAME", "gpt-5-mini")} + { + "FEATURE_LLM": "true", + "MODEL_NAME": os.environ.get("STATECORE_MODEL_NAME", "gpt-5-mini"), + "MODEL_TIMEOUT_MS": os.environ.get("STATECORE_MODEL_TIMEOUT_MS", "120000"), + } if digest else {} ) @@ -186,7 +193,7 @@ def flush(self): print(f"\nstatecore flush ({arm}): {self.facts} facts, {superseded} superseded at write\n") return {"facts": self.facts, "superseded": superseded} - def _settle(self, poll_seconds=5, stable_rounds=2, max_seconds=600, max_restarts=6): + def _settle(self, poll_seconds=5, stable_rounds=2, max_seconds=None, max_restarts=20): """Wait for the engine's background distillation to finish. Threshold digests fire during ingestion but leave a pending tail, and a @@ -198,9 +205,21 @@ def _settle(self, poll_seconds=5, stable_rounds=2, max_seconds=600, max_restarts that is the fixpoint where another pass has nothing left to digest. Time spent here is charged to memory_construction_time, where it belongs. """ + # A digest pass consumes a bounded batch (~40 events), so the number of + # passes needed is a function of how many facts were written -- computed, + # not guessed. The facts-snapshot fixpoint alone under-counts: on + # template-heavy corpora a fresh pass's output can be entirely deduped + # away, leaving the snapshot unchanged while a backlog remains (pending + # counts are not observable over MCP today). + if max_seconds is None: + max_seconds = int(os.environ.get("STATECORE_SETTLE_SECONDS", "3600")) + required_passes = (len(parse_fact_lines("".join(self.chunks))) // 40) + 2 + max_restarts = max(max_restarts, required_passes) deadline = time.time() + max_seconds settled_before_restart = None + passes = 0 for _ in range(max_restarts): + passes += 1 self.mcp.close() self.mcp = StateCoreMcpClient(self.data_dir, self.spec, self.extra_env) previous = None @@ -218,24 +237,29 @@ def _settle(self, poll_seconds=5, stable_rounds=2, max_seconds=600, max_restarts else: print("\nstatecore settle: hit max_seconds with distillation still moving; querying as-is\n") return - if previous == settled_before_restart: - return # a fresh catch-up pass changed nothing: distillation is complete + if previous == settled_before_restart and passes >= required_passes: + return # enough passes for the backlog, and a fresh one changed nothing settled_before_restart = previous print("\nstatecore settle: hit max_restarts with distillation still moving; querying as-is\n") def query(self, text, k): result = self.mcp.call_tool("recall", {"query": text, "maxChars": 16000}) + facts = [f["content"] for f in (result.get("factRegistry") or []) if f.get("content")] + events = [e["content"] for e in (result.get("events") or []) if e.get("content")] + # Interleave the two layers rather than concatenating facts-first: each + # layer is relevance-ranked by the engine, but distillation is selective + # -- on a partially distilled store, facts-first let a handful of facts + # crowd every event out of the top-k, and it was the event layer's + # lexical index doing the heavy lifting. Interleaving keeps the top of + # BOTH rankings inside the reader's window. contents = [] if result.get("digest"): contents.append(result["digest"]) - for fact in result.get("factRegistry") or []: - content = fact.get("content") - if content: - contents.append(content) - for event in result.get("events") or []: - content = event.get("content") - if content: - contents.append(content) + for pair in range(max(len(facts), len(events))): + if pair < len(facts): + contents.append(facts[pair]) + if pair < len(events): + contents.append(events[pair]) return contents[:k] From 1cd02e0d20aae6849aaa63088e9f18ca2c0390a0 Mon Sep 17 00:00:00 2001 From: Yuchen Lin Date: Sun, 30 Aug 2026 14:31:35 -0700 Subject: [PATCH 5/6] chore: pin statecore-mcp 0.6.0 0.6.0 ships what this harness surfaced: IDF-weighted relevance scoring from the token index (digest arm on factconsolidation_sh_6k: 39.0 -> 48.0-49.0 across two runs, drawing level with the BM25 baseline), a digest-path LLM timeout default that survives reasoning models, and the documented gpt-5-mini digest default. Confirmed against the published npm package: 48.0. Also adds a .gitignore for the local venv/outputs/env this harness generates. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NvB2hir4oGnomEDWwVHiCz --- .gitignore | 4 +++- .../RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini.yaml | 2 +- methods/statecore.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 01b1dc8..4db0beb 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,6 @@ cognee/.cognee_system/ cognee/.data_storage/ *.db croissant_files -.env \ No newline at end of file +.env +.venv/ +*.tgz diff --git a/configs/agent_conf/RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini.yaml b/configs/agent_conf/RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini.yaml index cf7bd51..74c5886 100644 --- a/configs/agent_conf/RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini.yaml +++ b/configs/agent_conf/RAG_Agents/gpt-4o-mini/StateCore_gpt-4o-mini.yaml @@ -1,5 +1,5 @@ # StateCore (github.com/yul761/StateCore), driven through its published MCP front end -- -# the wrapper spawns `npx -y statecore-mcp@0.5.0` itself, so there is no service to start. +# the wrapper spawns `npx -y statecore-mcp@0.6.0` itself, so there is no service to start. # Every value except the output_dir is copied from Simple_rag_bm25 so the row sits beside # the published baselines on the same terms: retrieve_num 10 is what BM25, Zep, Cognee, # HippoRAG-v2 and the embedding baselines use. The memory side makes zero model calls diff --git a/methods/statecore.py b/methods/statecore.py index f89484a..b6751ab 100644 --- a/methods/statecore.py +++ b/methods/statecore.py @@ -5,7 +5,7 @@ (`supersededBy`), and retirement/discards are logged rather than silent. It is driven here through its published MCP front end -- the wrapper spawns - npx -y statecore-mcp@0.5.0 --data + npx -y statecore-mcp@0.6.0 --data and speaks JSON-RPC over stdio (newline-delimited, per the MCP stdio transport). Nothing is installed into this venv and no service needs starting by hand; the only requirement is Node @@ -60,7 +60,7 @@ from methods.agentmemory import parse_fact_lines -DEFAULT_SPEC = "statecore-mcp@0.5.0" +DEFAULT_SPEC = "statecore-mcp@0.6.0" PROTOCOL_VERSION = "2025-06-18" From bce6d4e52fffaf7bea4e34af88abe73cd2226dd9 Mon Sep 17 00:00:00 2001 From: Yuchen Lin Date: Mon, 31 Aug 2026 17:14:16 -0700 Subject: [PATCH 6/6] fix: release each context's server and store when the next begins main.py builds a fresh AgentWrapper per context with no end-of-context hook, so every context leaked its Node process (held via Popen) and its tempdir SQLite store for the life of the run -- on a 100-500 context split that accumulates to real memory/process pressure. Contexts run sequentially, so a module-level slot suffices: initializing context N closes context N-1's server and deletes its store, and an atexit hook releases the last. At most one server and one data dir are now alive at any time. Addresses the Codex P2 review comment on the PR. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WwCmmaEyf74nqGaEu3USRT --- methods/statecore.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/methods/statecore.py b/methods/statecore.py index b6751ab..655cc00 100644 --- a/methods/statecore.py +++ b/methods/statecore.py @@ -43,8 +43,11 @@ datasets) are stored as events instead -- still retrievable, just outside the supersession machinery. -ISOLATION. Every run gets a fresh --data directory (its own SQLite file), so runs cannot see -each other; the directory is a tempdir and is left for the OS to clean. +ISOLATION. Every context gets a fresh --data directory (its own SQLite file), so contexts +cannot see each other. The harness has no end-of-context hook, so cleanup is handoff-shaped: +initializing the client for context N closes context N-1's server and deletes its store, and +an atexit hook releases the last one -- at most one Node process and one tempdir are alive at +any time on a multi-context split. READER. Mirrors `_handle_bm25_rag` via the shared knowl helpers, exactly like the agentmemory row: same query extraction, "Memory i:" labels, same system template. Retrieval contents are @@ -52,8 +55,10 @@ then raw events, truncated to retrieve_num. """ +import atexit import json import os +import shutil import subprocess import tempfile import time @@ -156,6 +161,13 @@ def __init__(self, spec, digest=False): ) self.mcp = StateCoreMcpClient(self.data_dir, spec, self.extra_env) + def close(self): + """Terminate the server and delete the store. Idempotent.""" + if self.mcp is not None: + self.mcp.close() + self.mcp = None + shutil.rmtree(self.data_dir, ignore_errors=True) + def add(self, text): self.chunks.append(text) @@ -263,7 +275,25 @@ def query(self, text, k): return contents[:k] +# main.py builds a fresh AgentWrapper per context and never releases the old one, so each +# context would leak a Node process and a tempdir for the life of the run. Contexts are +# processed sequentially, so a single module-level slot is enough: the next context's init +# releases the previous context's client, and atexit releases the last. +_active_client = None + + +def _release_active_client(): + global _active_client + if _active_client is not None: + _active_client.close() + _active_client = None + + +atexit.register(_release_active_client) + + def initialize_statecore_agent(agent, agent_config=None): + global _active_client config = agent_config or {} agent.retrieve_num = config["retrieve_num"] agent.context = "" @@ -274,7 +304,9 @@ def initialize_statecore_agent(agent, agent_config=None): override = os.environ.get("STATECORE_DIGEST") if override is not None: digest = override not in ("0", "false", "") + _release_active_client() agent.statecore = StateCoreClient(spec, digest=digest) + _active_client = agent.statecore arm = "digest" if digest else "note" print(f"\n\nstatecore ({spec}, {arm} arm) embedded at {agent.statecore.data_dir}\n\n")