diff --git a/docs/design/agent/agent-improvement-log.md b/docs/design/agent/agent-improvement-log.md new file mode 100644 index 00000000..06229c6b --- /dev/null +++ b/docs/design/agent/agent-improvement-log.md @@ -0,0 +1,1278 @@ +# Agent improvement log + +This log records the agent improvement project step by step. Each entry +has a date, an action, and a result. The harness in `tools/agent-lab/` +appends measurement entries. The goal of the project: fix context +pollution, fix response latency, keep memory access. The full review is +in `review-2026-09.md`. Source material for a future blog post. + +## 2026-09-15 - Project start + +- Problem: simple questions take 3 to 5 minutes. Context bleeds between + threads and speakers. The todo flow is not reliable. +- We reviewed the code and the current research. Result: five latency + multipliers stack on one endpoint. See `review-2026-09.md` section 1. +- Decision: build a validation harness first. Measure before each + change. Log all results here. +- Rule for this log: Simplified Technical English (ASD-STE100). + +## 2026-09-15 - Harness built + +- New tool: `tools/agent-lab/lab.py` (stdlib only). +- `probe` finds the usage fields that oMLX reports. +- `cache` tests the central hypothesis: the `lean_state` history rewrite + invalidates the oMLX prefix cache and forces a full re-prefill. +- Method: four phases. Cold sends a fresh synthetic conversation. Warm + and warm2 only append turns. Mutate rewrites one early message, as + `lean_state.py:64-103` does with prior tool results. + +## 2026-09-15 12:00 UTC - lab probe + +- endpoint: `http://localhost:8888/v1`, model: `Qwen3.6-35B-A3B-UD-MLX-4bit` +- raw result: `tools/agent-lab/results/20260915T120024Z-probe.json` + +| phase | ttft_s | total_s | prompt_tokens | completion_tokens | cache fields | +|---|---|---|---|---|---| +| first | 1.092 | 1.093 | 34 | 2 | {"input_tokens": 34, "output_tokens": 2, "prompt_tokens_details": {"cached_tokens": 0}, "time_to_first_token": 1.09, "total_time": 1.09, "prompt_eval_duration": 1.09, "generation_duration": 0.0, "prompt_tokens_per_second": 31.33, "generation_tokens_per_second": 6961.56} | +| second (identical prompt) | 0.441 | 0.441 | 34 | 2 | {"input_tokens": 34, "output_tokens": 2, "prompt_tokens_details": {"cached_tokens": 0}, "time_to_first_token": 0.44, "total_time": 0.44, "prompt_eval_duration": 0.44, "generation_duration": 0.0, "prompt_tokens_per_second": 77.78, "generation_tokens_per_second": 7650.61} | + +- note: first probe: find cache fields in usage + +## 2026-09-15 12:01 UTC - lab cache + +- endpoint: `http://localhost:8888/v1`, model: `Qwen3.6-35B-A3B-UD-MLX-4bit` +- raw result: `tools/agent-lab/results/20260915T120103Z-cache.json` + +| phase | ttft_s | total_s | prompt_tokens | completion_tokens | cache fields | +|---|---|---|---|---|---| +| cold | 10.419 | 10.922 | 4413 | 31 | {"input_tokens": 4413, "output_tokens": 31, "prompt_tokens_details": {"cached_tokens": 0}, "time_to_first_token": 10.35, "total_time": 10.85, "prompt_eval_duration": 10.35, "generation_duration": 0.51, "prompt_tokens_per_second": 426.53, "generation_tokens_per_second": 61.19} | +| warm | 1.555 | 2.303 | 4461 | 43 | {"input_tokens": 4461, "output_tokens": 43, "prompt_tokens_details": {"cached_tokens": 4096}, "time_to_first_token": 1.51, "total_time": 2.26, "prompt_eval_duration": 1.51, "generation_duration": 0.75, "prompt_tokens_per_second": 2951.74, "generation_tokens_per_second": 57.15} | +| warm2 | 1.697 | 1.766 | 4520 | 8 | {"input_tokens": 4520, "output_tokens": 8, "prompt_tokens_details": {"cached_tokens": 4096}, "time_to_first_token": 1.66, "total_time": 1.73, "prompt_eval_duration": 1.66, "generation_duration": 0.07, "prompt_tokens_per_second": 2728.16, "generation_tokens_per_second": 110.34} | +| mutate | 9.986 | 10.044 | 4464 | 7 | {"input_tokens": 4464, "output_tokens": 7, "prompt_tokens_details": {"cached_tokens": 0}, "time_to_first_token": 9.95, "total_time": 10.01, "prompt_eval_duration": 9.95, "generation_duration": 0.06, "prompt_tokens_per_second": 448.7, "generation_tokens_per_second": 113.26} | + +- note: hypothesis test: append-only turns hit the prefix cache, the lean_state-style rewrite forces re-prefill + +## 2026-09-15 12:07 UTC - lab replay + +- endpoint: `http://localhost:8888/v1`, model: `Qwen3.6-35B-A3B-UD-MLX-4bit` +- raw result: `tools/agent-lab/results/20260915T120700Z-replay.json` + +**lean_state (production)** + +| phase | ttft_s | total_s | prompt_tokens | cached_tokens | completion_tokens | +|---|---|---|---|---|---| +| turn 1 | 7.926 | 8.758 | 3390 | 0 | 48 | +| turn 2 | 7.656 | 8.25 | 3537 | 0 | 35 | +| turn 3 | 8.026 | 8.554 | 3684 | 0 | 32 | +| turn 4 | 8.26 | 8.794 | 3831 | 0 | 32 | + +**append-only** + +| phase | ttft_s | total_s | prompt_tokens | cached_tokens | completion_tokens | +|---|---|---|---|---|---| +| turn 1 | 7.276 | 8.088 | 3392 | 0 | 48 | +| turn 2 | 8.025 | 8.653 | 3728 | 0 | 37 | +| turn 3 | 8.805 | 9.368 | 4066 | 0 | 32 | +| turn 4 | 9.725 | 10.298 | 4403 | 0 | 34 | + +- note: replay with the real lean_messages transform vs append-only; same scripted tool conversation + +## 2026-09-15 12:09 UTC - lab replay + +- endpoint: `http://localhost:8888/v1`, model: `Qwen3.6-35B-A3B-UD-MLX-4bit` +- raw result: `tools/agent-lab/results/20260915T120930Z-replay.json` + +**lean_state (production)** + +| phase | ttft_s | total_s | prompt_tokens | cached_tokens | completion_tokens | +|---|---|---|---|---|---| +| turn 1 | 15.955 | 16.764 | 6717 | 0 | 48 | +| turn 2 | 7.05 | 7.532 | 6864 | 4096 | 29 | +| turn 3 | 7.427 | 7.963 | 7010 | 4096 | 32 | +| turn 4 | 7.714 | 8.235 | 7157 | 4096 | 31 | + +**append-only** + +| phase | ttft_s | total_s | prompt_tokens | cached_tokens | completion_tokens | +|---|---|---|---|---|---| +| turn 1 | 14.86 | 15.694 | 6709 | 0 | 48 | +| turn 2 | 7.49 | 8.147 | 7044 | 4096 | 38 | +| turn 3 | 8.251 | 8.834 | 7379 | 4096 | 34 | +| turn 4 | 9.116 | 9.776 | 7715 | 4096 | 38 | + +- note: block-size hypothesis: with a ~8K-token prefix the shared prefix spans 4096-token blocks; append-only should now hit, lean_state should hit only the system-prompt blocks + +## 2026-09-15 - Findings from the endpoint experiments + +Interpretation of the three runs above. + +1. oMLX reports full cache telemetry in `usage`: `cached_tokens`, + `time_to_first_token`, `prompt_eval_duration`, and token rates. The + agent never reads this today. Step 1 of the plan stays valid. +2. The prefix cache works and it is fast. An append-only follow-up turn + on a ~4.4K-token prompt: TTFT 1.5 s instead of 10.4 s. +3. An in-place rewrite of one early message removes the full cache hit. + TTFT goes back to 10.0 s. This confirms the `lean_state` hypothesis. +4. New finding: the cache reuses the prefix in 4096-token blocks. A + shared prefix below 4096 tokens gives `cached_tokens: 0`. The first + replay run (prompts 3.4K to 4.4K tokens) hit zero cache in BOTH + arms. The second run (~6.7K-token prefix) hit exactly 4096 in both + arms. Consequence: each turn re-prefills up to 4096 tail tokens + (~9 s at the measured 430-460 tokens/s). The block size is a + serving-side lever. We must check if oMLX can use smaller blocks. +5. Measured prefill rate: 430-460 tokens/s cold on the 35B MoE. The + production ~10K-token prefix therefore costs ~23 s per cold call. + Twelve tool iterations with cache loss explain multi-minute turns + without any failed tool call. + +## 2026-09-15 - Next tier: isolated end-to-end rig + +- Rationale: the endpoint lab proves mechanisms, but it simulates the + agent. The user wants measurements through the real nanobot harness. +- Design: run the production agent image with the real runtime shims, + driven by `nanobot agent -m` (a direct turn, no Matrix channel). + A demo vault (fabricated data, git repo) replaces the family vault. + `lab-api.py` (port 42011) replaces the production famstack-api for + `stack memory` commands, with the same line protocol and exit codes, + and the same optional LLM keyword-rewrite hop. `proxy.py` (port + 42012) sits between the agent and oMLX and records the byte-exact + request and the usage object for every call. +- The proxy answers a standing requirement: see the exact context the + agent sends on each request. +- Product goal recorded: lists (todo, shopping, checklist) must work + seamlessly, similar to Apple's Lists. The rig scenario is therefore + a shopping list. The rig write path (vault writes through lab-api) + is the next rig increment. + +## 2026-09-15 12:19 UTC - rig turn (rig:lists) + +- message: `What is still open on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 59.77 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 11281 | 0 | 58 | 27.87 | 29.082 | +| 2 | 4 | 11423 | 8192 | 55 | 9.31 | 10.437 | +| 3 | 6 | 11631 | 8192 | 80 | 9.91 | 11.554 | + +- note: rig smoke test: first end-to-end turn through the real nanobot image, demo vault, lab-api, proxy + +## 2026-09-15 12:20 UTC - rig turn (rig:lists) + +- message: `Who usually adds items to the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 58.34 s, 4 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 8 | 11592 | 8192 | 88 | 10.42 | 12.256 | +| 2 | 10 | 11698 | 8192 | 67 | 10.09 | 11.492 | +| 3 | 12 | 11874 | 8192 | 98 | 10.62 | 12.636 | +| 4 | 14 | 12096 | 8192 | 46 | 11.6 | 12.556 | + +- note: turn 2 in the same session: cross-turn cache with lean_state active; also a grounded question that needs person pages + +## 2026-09-15 12:24 UTC - rig turn (dm:homer) + +- message: `What is still on my errands list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 54.22 s, 4 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 11281 | 8192 | 61 | 9.49 | 10.809 | +| 2 | 4 | 11457 | 8192 | 54 | 9.57 | 10.664 | +| 3 | 6 | 11552 | 8192 | 52 | 9.66 | 10.722 | +| 4 | 8 | 11757 | 8192 | 89 | 10.18 | 11.984 | + +- note: DM scenario: personal list, not topic bound; fresh state after vault extension + +## 2026-09-15 12:25 UTC - rig turn (topic:camping) + +- message: `Bart: what is still open on the packing list for the camping trip?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 41.58 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 11288 | 8192 | 72 | 9.54 | 11.006 | +| 2 | 4 | 11511 | 8192 | 55 | 9.5 | 10.59 | +| 3 | 6 | 11729 | 8192 | 98 | 10.1 | 12.099 | + +- note: multi-person topic scenario: speaker prefix convention, camping packing list (homer+bart) + +## 2026-09-15 - Findings from the first rig runs + +The rig is live: real agent image, real shims, demo vault, lab-api, +logging proxy. Four turns measured (entries above). Results: + +1. The real stable prefix is ~11,280 tokens, not the ~10K the runtime + README estimates. It spans two full cache blocks (8192 tokens). + These two blocks hit on every call, in every session, even a fresh + one. Cross-session prefix sharing works. +2. The third block never stabilizes. The prompt ends at ~11.3K-12.1K + tokens, and the tail past 8192 contains the end of the system + prompt plus the mutable history. Every LLM call therefore + re-prefills ~3.1K-3.9K tokens: ~9.5-11.6 s TTFT per call, on every + call, cold or warm. +3. A simple list question costs 3-4 LLM calls (initial + one per tool + iteration): 42-60 s wall time. This is the production latency + floor. The observed 3-5 min turns are this floor times more + iterations, plus the rewrite hop, plus contention. +4. Quality is good in the rig: correct list answers, correct person + attribution, and grounded citations ("Sources: ...") in all four + turns. The grounded-answers product goal seems reachable with the + current model; latency is the blocker, not capability. +5. Levers, ranked by measured effect: (a) shrink the stable prefix + below one block boundary distance from the prompt end, or shrink it + in total (11.2K tokens for four small files plus tool schemas is a + lot); (b) check if oMLX can use a smaller cache block size than + 4096; (c) fewer tool iterations per question (better retrieval + instructions); (d) remove the lean_state rewrite so blocks past the + prefix can stabilize between turns. +- Rig gap noted: lab-api's keyword-rewrite calls go to oMLX directly, + so the proxy does not count them. Point lab-api at the proxy in the + next runs. + +Next scenario work: multi-person sessions with the speaker-prefix +convention, the gift-ideas pollution trap, and the rig write path for +list mutations (tick off, add item). + +## 2026-09-15 12:35 UTC - rig turn (topic:birthday) + +- message: `What is still open on the party checklist?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 41.0 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 11281 | 8192 | 54 | 9.43 | 10.569 | +| 2 | 4 | 11489 | 8192 | 56 | 9.72 | 10.91 | +| 3 | 6 | 11701 | 8192 | 70 | 10.12 | 11.51 | + +- note: sanity check of the new AGENT_LEAN_STATE=0 switch: append-only history, birthday checklist scenario + +## 2026-09-15 - oMLX settings analysis, switch from measuring to fixing + +We inspected the oMLX v0.6.4 settings (UI screenshot and +`~/.omlx/settings.json`). Findings: + +1. `scheduler.prefill_priority` is `"context"` (Max Context). This + mode trades prefill speed for large-prompt headroom. Our measured + 430-460 tokens/s possibly runs in this slow mode. +2. `scheduler.chunked_prefill` is `false`. Enabled, it reduces TTFT + when requests overlap. The keyword-rewrite hop overlaps with agent + calls, so this applies. +3. The kernel GPU limit is 51.8 GB (`iogpu.wired_limit_mb`). The UI + recommends `sudo sysctl iogpu.wired_limit_mb=59392`. The value does + not survive a reboot; persistence needs a LaunchDaemon (note: this + Mac reboots monthly). +4. There is no cache block-size setting. The `gdn_*` cache entries + (Gated DeltaNet sidecars) indicate the model uses hybrid linear + attention. Linear-attention layers carry a recurrent state, so the + cache can only restore at state snapshots. This explains the strict + 4096-token cache granularity. The partial-block re-prefill per call + is a model-architecture property, not a config mistake. +5. Hot cache is 6 GB, SSD cache 372 GB, `preserve_mid_system_cache` + is on. + +Decision: stop broad measuring, apply fixes one at a time, and re-run +the three rig scenarios (groceries, dm:homer, topic:camping) after +each change as the A/B check. + +Fix order: +1. Serving side (operator action, needs an oMLX restart): + `prefill_priority: "speed"`, `chunked_prefill: true`, raise the + kernel limit. Expected effect: every call gets faster; this + multiplies with all later fixes. +2. Retrieval consolidation (code): one documented retrieval route, + align workspace AGENTS.md and SKILL.md, so a list question costs + 1-2 calls instead of 3-4. +3. lean_state A/B: the runtime now has an `AGENT_LEAN_STATE=0` switch + (append-only history). Sanity check passed (entry above). The + effect shows in longer sessions, once the prompt crosses the next + 4096-token boundary; short sessions are dominated by the partial + block anyway. +4. Prefix diet: the stable prefix is 11,281 tokens. Target: well under + 8192, so the whole prefix fits two stable blocks and the mutable + tail starts earlier and costs less. + +## 2026-09-15 12:42 UTC - lab cache + +- endpoint: `http://localhost:8888/v1`, model: `Qwen3.6-35B-A3B-UD-MLX-4bit` +- raw result: `tools/agent-lab/results/20260915T124224Z-cache.json` + +| phase | ttft_s | total_s | prompt_tokens | cached_tokens | completion_tokens | +|---|---|---|---|---|---| +| cold | 1.897 | 2.413 | 4413 | 4096 | 31 | +| warm | 1.47 | 2.213 | 4461 | 4096 | 43 | +| warm2 | 1.644 | 1.71 | 4520 | 4096 | 8 | +| mutate | 1.537 | 1.585 | 4464 | 4096 | 7 | + +- note: post-restart check: did the scheduler changes take effect? baseline cold prefill was 426-453 tok/s + +## 2026-09-15 12:43 UTC - lab cache + +- endpoint: `http://localhost:8888/v1`, model: `Qwen3.6-35B-A3B-UD-MLX-4bit` +- raw result: `tools/agent-lab/results/20260915T124303Z-cache.json` + +| phase | ttft_s | total_s | prompt_tokens | cached_tokens | completion_tokens | +|---|---|---|---|---|---| +| cold | 10.879 | 11.32 | 4416 | 0 | 28 | +| warm | 1.593 | 2.447 | 4461 | 4096 | 48 | +| warm2 | 1.678 | 1.725 | 4525 | 4096 | 7 | +| mutate | 10.206 | 10.243 | 4468 | 0 | 6 | + +- note: true cold measurement with fresh seed 99: prompts never seen, no SSD reuse possible; checks whether scheduler changes took effect + +## 2026-09-15 - Serving-side settings: what we change and why + +Change set for oMLX v0.6.4 (config: `~/.omlx/settings.json`) and the +kernel. Rationale per setting: + +| Setting | From | To | Why | +|---|---|---|---| +| `scheduler.prefill_priority` | `context` | `speed` | "Context" trades prefill speed for large-prompt headroom under memory pressure. Our prompts are ~12K tokens, far below any limit. Prefill speed multiplies every LLM call, so the agent pays this cost 3-4 times per question. | +| `scheduler.chunked_prefill` | `false` | `true` | Interleaves prefill chunks with decode steps. Reduces TTFT when requests overlap. The agent's keyword-rewrite hop overlaps with agent calls, and family members ask in parallel. | +| kernel `iogpu.wired_limit_mb` | 51.8 GB ceiling | 59392 | The oMLX UI recommends this value. It raises the GPU memory ceiling, so the memory guard throttles prefill later. Caution: the value resets on reboot, and this Mac reboots monthly. Persistence needs a LaunchDaemon. | + +Verification after the restart (entries above): + +1. `iogpu.wired_limit_mb` is 59392. Applied. +2. The settings file still shows `prefill_priority: "context"` and + `chunked_prefill: false`. The UI toggles did not persist. +3. Measurement agrees: true cold prefill (fresh seed 99) is 408 + tokens/s, the same as the 426-453 baseline. The scheduler changes + are NOT active. Re-apply and re-verify before any A/B comparison. +4. Positive side finding: the SSD cache tier survives the server + restart. A byte-identical prompt from an earlier run hit the cache + in every phase, including the mutate phase. Deterministic prompts + plus the cold tier give cross-restart reuse. For the agent this + means the stable prefix stays warm across oMLX restarts. + +## 2026-09-15 12:48 UTC - lab cache + +- endpoint: `http://localhost:8888/v1`, model: `Qwen3.6-35B-A3B-UD-MLX-4bit` +- raw result: `tools/agent-lab/results/20260915T124818Z-cache.json` + +| phase | ttft_s | total_s | prompt_tokens | cached_tokens | completion_tokens | +|---|---|---|---|---|---| +| cold | 10.144 | 10.601 | 4410 | 0 | 23 | +| warm | 1.551 | 2.096 | 4450 | 4096 | 32 | +| warm2 | 1.578 | 1.724 | 4498 | 4096 | 12 | +| mutate | 9.731 | 9.887 | 4447 | 0 | 7 | + +- note: scheduler settings verified active (speed + chunked prefill): true cold measurement with fresh seed 123 + +## 2026-09-15 - Scheduler settings active: no single-request gain + +The re-applied settings are saved (`prefill_priority: "speed"`, +`chunked_prefill: true`) and verified in the file. Fresh-seed cold +measurement: 437 tokens/s, identical to the 426-453 baseline. + +Conclusion: "context" mode only trades speed under memory pressure, +and this machine is not under pressure with ~12K-token prompts. The +prefill rate ~440 tokens/s is the compute-bound hardware rate for +this 35B model on the M1 Max. Chunked prefill remains useful for +overlapping requests (rewrite hop, parallel family questions), not +for single-request speed. + +Consequence: the serving side is done. All remaining latency levers +are in famstack code: +1. Fewer LLM calls per question (retrieval consolidation) - next. +2. Shorter stable prefix (11,281 tokens today). +3. Stable blocks across turns (lean_state A/B in long sessions). + +## 2026-09-15 12:52 UTC - rig turn (topic:groceries) + +- message: `What is still open on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 72.83 s, 4 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 11411 | 0 | 59 | 28.44 | 29.624 | +| 2 | 4 | 11490 | 8192 | 57 | 9.54 | 10.701 | +| 3 | 6 | 11618 | 8192 | 58 | 9.88 | 11.05 | +| 4 | 8 | 11829 | 8192 | 76 | 10.42 | 11.949 | + +- note: A/B after retrieval consolidation (brief pointers, one-search rule, no --nl hop); baseline: 59.8s / 3 calls + +## 2026-09-15 12:58 UTC - rig turn (topic:groceries) + +- message: `What is still open on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 39.01 s, 4 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 9184 | 4096 | 56 | 13.73 | 14.95 | +| 2 | 4 | 9260 | 8192 | 49 | 3.5 | 4.463 | +| 3 | 6 | 9380 | 8192 | 59 | 3.85 | 5.019 | +| 4 | 8 | 9592 | 8192 | 65 | 4.36 | 5.625 | + +- note: prefix diet A/B: disabled builtin skills memory+my, minimal USER.md, compressed AGENTS.md and family-memory skill; baseline 11281-token prefix, 59.8s / 3 calls + +## 2026-09-15 13:00 UTC - rig turn (topic:groceries) + +- message: `What is still open on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 33.36 s, 4 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 9184 | 8192 | 68 | 5.03 | 8.736 | +| 2 | 4 | 9367 | 8192 | 56 | 3.76 | 4.865 | +| 3 | 6 | 9494 | 8192 | 58 | 4.16 | 5.337 | +| 4 | 8 | 9705 | 8192 | 54 | 4.65 | 5.695 | + +- note: A/B rerun after lab-api regex fix; diet prefix; baseline 59.8s / 3 calls + +## 2026-09-15 13:01 UTC - rig turn (topic:groceries) + +- message: `What is still open on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 23.48 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 9184 | 8192 | 60 | 4.14 | 5.3 | +| 2 | 4 | 9376 | 8192 | 59 | 3.79 | 4.974 | +| 3 | 6 | 9588 | 8192 | 54 | 4.42 | 5.493 | + +- note: A/B rerun 2: lab-api date ranking fixed; diet prefix + +## 2026-09-15 13:04 UTC - rig turn (dm:homer) + +- message: `What is still on my errands list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 24.28 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 9184 | 8192 | 64 | 4.15 | 5.415 | +| 2 | 4 | 9387 | 8192 | 89 | 3.81 | 5.609 | +| 3 | 6 | 9629 | 8192 | 80 | 4.52 | 6.127 | + +- note: A/B: DM scenario after diet + consolidation; baseline 54.2s / 4 calls + +## 2026-09-15 13:04 UTC - rig turn (topic:camping) + +- message: `Bart: what is still open on the packing list for the camping trip?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 23.79 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 9191 | 8192 | 80 | 4.29 | 5.886 | +| 2 | 4 | 9348 | 8192 | 51 | 3.77 | 4.772 | +| 3 | 6 | 9562 | 8192 | 73 | 4.32 | 5.789 | + +- note: A/B: camping scenario after diet + consolidation; baseline 41.6s / 3 calls + +## 2026-09-15 - Fix 2+3 results: retrieval consolidation and prefix diet + +Changes measured together (entries above): + +1. Retrieval consolidation: brief-pointer-first and one-search rule in + workspace AGENTS.md and SKILL.md; `memory_search` builds an OR-regex + from model keywords in code, which removes the hidden second LLM + call (`--nl` rewrite) from every search. +2. Prefix diet: `disabled_skills: ["memory", "my"]` in config.json + (nanobot built-in skills we never used), minimal USER.md seed + instead of nanobot's ~1K-char profile template, token-lean rewrite + of AGENTS.md and the family-memory skill. + +Rig fidelity fixes on the way (lab-api, not agent code): treat the +query as a regex like the real CLI, and rank hits by frontmatter date +descending like the real CLI. Both bugs distorted one A/B run each. + +Result table, same three scenarios, same questions: + +| Scenario | Baseline | After fixes | +|---|---|---| +| topic:groceries | 59.8 s, 3 calls | 23.5 s, 3 calls | +| dm:homer | 54.2 s, 4 calls | 24.3 s, 3 calls | +| topic:camping | 41.6 s, 3 calls | 23.8 s, 3 calls | + +- Stable prefix: 11,281 -> 9,184 tokens. System prompt: 21,988 -> + 13,319 chars. +- Warm calls: 9.5-11.6 s TTFT -> 3.5-4.7 s TTFT. The mutable tail + past the 8192-token cache boundary shrank from ~3.4K to ~1.1K + tokens. +- All replies stayed correct (verified from the session store). +- Open quality item: only 1 of 3 replies kept the Sources line; the + baseline cited in 3 of 3. The skill compression likely cut citation + compliance. Watch it; strengthen the Sources rule with few tokens + if it stays weak. +- Remaining floor: 3 calls x ~5 s + container start. Next reductions: + a 2-call turn (brief pointer straight to read_file; needs the room + label, which the rig CLI mode does not provide), and a shorter tail. +- Queued (user request): research pragmatic retrieval/search + upgrades. Candidates: SQLite FTS5/BM25 index over the vault, and + hybrid search with a small embedding model. oMLX already serves + embedding models (Qwen3-Embedding-8B, bge-m3, LFM2.5-Embedding). + +## 2026-09-15 13:23 UTC - rig turn (topic:groceries) + +- message: `What is still open on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 47.07 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 7442 | 0 | 57 | 17.49 | 18.697 | +| 2 | 4 | 7631 | 4096 | 58 | 8.9 | 10.023 | +| 3 | 6 | 7842 | 4096 | 65 | 9.48 | 10.732 | + +- note: A/B: tool trim (6 schemas dropped) + Sources compliance fix + +## 2026-09-15 13:24 UTC - rig turn (dm:homer) + +- message: `What is still on my errands list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 45.8 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 7442 | 4096 | 66 | 9.34 | 10.616 | +| 2 | 4 | 7647 | 4096 | 81 | 8.88 | 13.323 | +| 3 | 6 | 7881 | 4096 | 95 | 9.56 | 14.437 | + +- note: A/B: tool trim (6 schemas dropped) + Sources compliance fix + +## 2026-09-15 13:24 UTC - rig turn (topic:camping) + +- message: `Bart: what is still open on the packing list for the camping trip?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 38.42 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 7449 | 4096 | 67 | 9.24 | 10.545 | +| 2 | 4 | 7655 | 4096 | 45 | 8.95 | 9.813 | +| 3 | 6 | 7863 | 4096 | 79 | 9.46 | 10.997 | + +- note: A/B: tool trim (6 schemas dropped) + Sources compliance fix + +## 2026-09-15 13:27 UTC - rig turn (topic:groceries) + +- message: `What is still open on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 17.5 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 8364 | 8192 | 69 | 2.05 | 3.426 | +| 2 | 4 | 8565 | 8192 | 59 | 1.74 | 2.889 | +| 3 | 6 | 8777 | 8192 | 77 | 2.34 | 3.862 | + +- note: A/B: tool trim + cache-pad alignment (prefix ~8330, two stable blocks) + +## 2026-09-15 13:27 UTC - rig turn (dm:homer) + +- message: `What is still on my errands list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 33.8 s, 6 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 8364 | 8192 | 90 | 2.09 | 3.86 | +| 2 | 4 | 8474 | 8192 | 52 | 1.53 | 2.547 | +| 3 | 6 | 8567 | 8192 | 52 | 1.74 | 2.756 | +| 4 | 8 | 8772 | 8192 | 83 | 2.33 | 3.974 | +| 5 | 10 | 8898 | 8192 | 56 | 2.65 | 3.754 | +| 6 | 12 | 8993 | 8192 | 186 | 2.79 | 6.511 | + +- note: A/B: tool trim + cache-pad alignment (prefix ~8330, two stable blocks) + +## 2026-09-15 13:27 UTC - rig turn (topic:camping) + +- message: `Bart: what is still open on the packing list for the camping trip?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 16.73 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 8371 | 8192 | 69 | 2.1 | 3.46 | +| 2 | 4 | 8517 | 8192 | 46 | 1.66 | 2.555 | +| 3 | 6 | 8726 | 8192 | 74 | 2.21 | 3.685 | + +- note: A/B: tool trim + cache-pad alignment (prefix ~8330, two stable blocks) + +## 2026-09-15 - Fix 4: tool trim, and the cache-block alignment finding + +Tool trim: a new runtime shim (tool_trim.py, AGENT_TOOL_TRIM=0 to +disable) drops six unused tool schemas the config cannot disable: +cron, long_task, complete_goal, spawn, write_stdin, +list_exec_sessions. Prefix: 9,184 -> 7,442 tokens. + +Result: wall times got WORSE (47/46/38 s). The rows show why: with a +7,442-token prefix only one 4096-token cache block lies fully inside +the stable prefix, so every call re-prefilled ~3.4K tokens (~9 s). +The 9,184-token prefix sat just past the 8,192 boundary and paid only +a ~1K tail. + +Rule for this serving stack (4096-token snapshot blocks): +per-call prefill cost ~= (prefix mod 4096) + new history tokens. +A smaller prefix is only faster if it lands just ABOVE a block +boundary. Blind token cutting can cross a boundary and double the +per-call cost. + +Fix: a declarative alignment pad, workspace skill `zz-cache-pad` +(inert text, always-on, sorts last among skills). Tuned in the rig to +land the prefix at ~8,330 tokens, just past 8,192. + +Result table (same scenarios; baseline from this morning): + +| Scenario | Baseline | Diet (9,184) | Trim (7,442) | Trim+pad (~8,330) | +|---|---|---|---|---| +| topic:groceries | 59.8 s | 23.5 s | 47.1 s | 17.5 s | +| dm:homer | 54.2 s | 24.3 s | 45.8 s | 33.8 s* | +| topic:camping | 41.6 s | 23.8 s | 38.4 s | 16.7 s | + +Warm TTFT is now 1.5-2.8 s per call with both blocks cached. + +*The DM turn took 6 calls: a rig artifact. Without a Matrix sender +the brief cannot name the speaker, so the model searched for who +"my" refers to. Production supplies the speaker line per turn. Rig +convention from now on: prefix the speaker in every message, DMs +included. + +## 2026-09-15 - Retrieval research result (queued work) + +Research verdict for the search upgrade (full brief in the session, +key sources: sqlite.org/fts5, arXiv 2605.15184 "Is Grep All You +Need?", Liquid AI LFM2.5-Embedding): + +1. First iteration: SQLite FTS5 + BM25, one DB file next to the vault + clone, incremental upsert on git change, `unicode61 + remove_diacritics 2` tokenizer (German umlauts), quoted prefix + queries from the model's 2-4 keywords, snippet() output. Stdlib + only, <10 ms per query. Optional trigram companion table for + German compound words. +2. Stretch: hybrid via sqlite-vec + LFM2.5-Embedding-350M (already + served by oMLX), RRF fusion, whole-page embeddings. ~50-250 ms. + Evidence says hybrid mainly cuts search iterations on paraphrase + and cross-language queries; lexical-first is the 2026 consensus + for iterating agents. +3. Skip: Meilisearch/Typesense (extra service), DuckDB FTS (no + incremental update), tantivy (unneeded below ~100K docs). + +Next work package (user priority): map the context build for threaded +communication (nanobot session keys, max_messages replay, idle +compaction), fold the thread root into the session key, and cap long +threads. + +## 2026-09-15 - Fix 5: thread-scoped sessions and the context-build map + +How nanobot 0.2.2 builds context (mapped from the pinned source): + +1. Session key: `channel:chat_id` (the room id), unless the inbound + message carries `session_key_override` - a seam upstream added + exactly for thread-scoped sessions, unused by the Matrix channel. +2. History replay: last `max_messages` (default 120) messages, then a + token-budget slice from the tail. +3. Idle compaction: after `session_ttl_minutes` (default 15) idle, + the session is archived and an LLM summary replaces it on resume. + This is compaction at a natural boundary; it also resets the + cache at a moment nobody is waiting. + +Changes: + +1. New runtime shim thread_session.py (AGENT_THREAD_SESSIONS=0 to + disable): threaded messages get + `matrix:#thread:` as session key. Threads stop + sharing the room transcript; replies still route by room id. + Known v1 gap: a new thread session does not carry the root + message's text (it lives in the room session). +2. `max_messages: 60` in config.json: a hard cap for long threads on + top of the idle compaction. When the cap engages, the history + window slides and costs a cache re-prefill; the 15-minute + compaction keeps that case rare. +3. Verification: all four patch points confirmed in the built image + (handle_with_thread_session, register_trimmed, + _build_messages_lean, _runtime_lines). + +Validation gap: the rig cannot produce real Matrix thread events. +Thread isolation needs a manual Matrix test (two parallel threads in +one room, check that answers do not cross) or the e2e rig lane. + +## 2026-09-15 14:02 UTC - rig turn (topic:birthday) + +- message: `Marge: what is still open on the party checklist, and is the science kit gift wrapped yet?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 59.45 s, 5 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 8469 | 0 | 132 | 20.5 | 23.141 | +| 2 | 5 | 8702 | 8192 | 108 | 2.09 | 4.27 | +| 3 | 8 | 8988 | 8192 | 134 | 2.85 | 5.534 | +| 4 | 11 | 9185 | 8192 | 128 | 3.33 | 5.912 | +| 5 | 14 | 10016 | 8192 | 187 | 5.46 | 9.224 | + +- note: batch-search test: two independent lookups in one turn via queries[] + +## 2026-09-15 - Fix 6: batched search, cap lowered to 40 + +Question 1: is max_messages 60 too long? Grounded answer: the token +budget in nanobot's history replay only activates when a context +window is configured, and ours is not. So max_messages is the only +cap. 60 messages of chat plus tool results is ~5-9K history tokens on +top of the ~8.3K prefix; with lean_state on, that tail re-prefills +every call (~11-20 s/call at the top). Thread-scoped sessions and the +15-minute idle compaction keep typical sessions far below the cap, so +the cap is a backstop. Lowered to 40: rare loss, bounded worst case. + +Question 2: multiple searches per tool call? nanobot executes ALL +tool calls of one assistant message in one iteration, so parallel +calls already collapse round trips. Added both levers: +1. memory_search gains an optional `queries` array (up to three + keyword sets, run concurrently, labeled result blocks). +2. One skill line instructs batching independent lookups. + +Rig test (two-part question, topic:birthday): the model emitted TWO +parallel memory_search calls in iteration 1. Both question parts +answered correctly with correct Sources. 5 iterations total, because +part two lives in marge's personal bucket and the family/ scope +guidance made the first probes miss. + +Two observations for the backlog: +1. Scope-first guidance costs iterations when the answer is in a + personal bucket. The FTS5 upgrade with ranked global search will + remove most of this class. +2. Privacy gap found by the gift-ideas trap: Marge asked in the + shared birthday room, and the reply named her private gift note + content and linked the page. No page-audience concept exists yet. + Needs a design (audience frontmatter or bucket rules) before + private notes and shared rooms mix in production. + +## 2026-09-15 14:07 UTC - rig turn (topic:groceries) + +- message: `Homer: please add butter to the shopping list` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 30.26 s, 5 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 8458 | 8192 | 72 | 2.18 | 3.606 | +| 2 | 4 | 8642 | 8192 | 55 | 1.98 | 3.043 | +| 3 | 6 | 8850 | 8192 | 101 | 2.5 | 4.526 | +| 4 | 8 | 8975 | 8192 | 88 | 2.81 | 4.568 | +| 5 | 10 | 9084 | 8192 | 37 | 3.07 | 3.788 | + +- note: write-path scenario 1: add an item + +## 2026-09-15 14:08 UTC - rig turn (topic:groceries) + +- message: `Homer: how many items are still open on the shopping list, and which ones?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 18.3 s, 2 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 12 | 9258 | 8192 | 76 | 4.22 | 5.74 | +| 2 | 14 | 9512 | 8192 | 105 | 4.19 | 6.259 | + +- note: staleness test, lean_state ON: list changed out of band after the agent's last read; correct answer is 6 open incl. eggs+flour, rye bread ticked + +## 2026-09-15 14:10 UTC - rig turn (stale:off) + +- message: `Homer: how many items are still open on the shopping list, and which ones?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 10.89 s, 1 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 8 | 9013 | 8192 | 77 | 3.77 | 5.313 | + +- note: staleness test, lean_state OFF: prior read still verbatim in context; correct answer excludes oat milk and includes jam + +## 2026-09-15 - Rig write path, and the staleness A/B + +Write path shipped in the rig: lab-api now implements +`memory write --by [--patch] [--dry-run]` with the +production buffer-file contract, checkbox-aware diff sentences +("added 1: butter"), and git commits authored as the actor. +Scenario "Homer: please add butter to the shopping list": correct +apply_patch, correct file state, correct commit, reply relays the +store's sentence. 5 calls, 30.3 s. + +Staleness A/B (the question: what happens when the agent must rely +on current data but stale data sits in its context). Setup: agent +reads the list, another person edits the list out of band, agent is +asked again. + +| Arm | Behavior | Answer | +|---|---|---| +| lean_state ON (production) | prior read decayed to a re-run pointer; model re-read the page (2 calls, 18.3 s) | CURRENT list: eggs+flour present, rye bread gone. Cosmetic slip: wrote "7" above a correct 6-item list. | +| lean_state OFF (append-only) | one LLM call, no re-fetch | STALE list: oat milk still open, jam missing, and a Sources line citing the page it never re-read | + +Decision: lean_state STAYS. The review's removal recommendation is +reversed. With the aligned prefix its rewrites land in the tail that +re-prefills anyway, so it costs nothing until history crosses the +next 4096 boundary, which the 40-message cap and idle compaction +bound. The AGENT_LEAN_STATE switch remains for experiments. + +Open staleness risk noted: the idle-compaction summary is written by +the consolidator LLM and could bake point-in-time facts ("the list +has 5 items") into the resumed context with no pointer to decay. +Check the consolidator prompt; if needed, shim a "state facts as +of-then, not as current" instruction into it. + +## 2026-09-15 14:15 UTC - rig turn (topic:groceries) + +- message: `Homer: what is on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 17.58 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 8458 | 8192 | 68 | 2.14 | 3.472 | +| 2 | 4 | 8638 | 8192 | 45 | 1.93 | 2.811 | +| 3 | 6 | 8836 | 8192 | 61 | 2.5 | 3.702 | + +- note: list-lifecycle test 1/4: read + +## 2026-09-15 14:15 UTC - rig turn (topic:groceries) + +- message: `Homer: please add butter, eggs and flour to the list` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 21.25 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 8 | 8776 | 8192 | 63 | 3.19 | 4.418 | +| 2 | 10 | 8992 | 8192 | 127 | 2.83 | 5.359 | +| 3 | 12 | 9144 | 8192 | 44 | 3.27 | 4.128 | + +- note: list-lifecycle test 2/4: add three items + +## 2026-09-15 14:16 UTC - rig turn (topic:groceries) + +- message: `Homer: please restructure the list by grocery category (dairy, bakery, produce, household)` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 33.98 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 14 | 9273 | 8192 | 55 | 4.35 | 5.44 | +| 2 | 16 | 9505 | 8192 | 500 | 4.09 | 14.273 | +| 3 | 18 | 10028 | 8192 | 70 | 5.51 | 6.917 | + +- note: list-lifecycle test 3/4: restructure by category via write_file + +## 2026-09-15 14:17 UTC - rig turn (topic:groceries) + +- message: `Homer: show me the new list, and cross off the apples` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 45.54 s, 4 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 20 | 10193 | 8192 | 56 | 6.71 | 7.82 | +| 2 | 22 | 10478 | 8192 | 127 | 6.73 | 9.285 | +| 3 | 24 | 10661 | 8192 | 174 | 7.15 | 10.688 | +| 4 | 26 | 10858 | 8192 | 119 | 7.78 | 10.213 | + +- note: list-lifecycle test 4/4: show restructured list and tick one item + +## 2026-09-15 14:21 UTC - rig turn (topic:groceries) + +- message: `Homer: what is on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 28.73 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 8502 | 4096 | 70 | 11.97 | 13.468 | +| 2 | 4 | 8684 | 8192 | 45 | 2.06 | 2.941 | +| 3 | 6 | 8882 | 8192 | 64 | 2.59 | 3.856 | + +- note: list-lifecycle rerun after frontmatter guard + restore rule + +## 2026-09-15 14:21 UTC - rig turn (topic:groceries) + +- message: `Homer: please add butter, eggs and flour to the list` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 22.07 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 8 | 8822 | 8192 | 63 | 3.3 | 4.565 | +| 2 | 10 | 9038 | 8192 | 121 | 3.0 | 5.4 | +| 3 | 12 | 9184 | 8192 | 44 | 3.37 | 4.237 | + +- note: list-lifecycle rerun after frontmatter guard + restore rule + +## 2026-09-15 14:22 UTC - rig turn (topic:groceries) + +- message: `Homer: please restructure the list by grocery category (dairy, bakery, produce, household)` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 77.29 s, 7 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 14 | 9313 | 8192 | 55 | 4.53 | 5.636 | +| 2 | 16 | 9545 | 8192 | 614 | 4.29 | 16.957 | +| 3 | 18 | 10182 | 8192 | 108 | 5.97 | 8.168 | +| 4 | 20 | 10332 | 8192 | 88 | 6.32 | 8.075 | +| 5 | 22 | 10476 | 8192 | 62 | 6.74 | 7.967 | +| 6 | 24 | 10750 | 8192 | 169 | 7.44 | 10.871 | +| 7 | 26 | 10943 | 8192 | 145 | 7.92 | 10.884 | + +- note: list-lifecycle rerun after frontmatter guard + restore rule + +## 2026-09-15 14:23 UTC - rig turn (topic:groceries) + +- message: `Homer: show me the new list, and cross off the apples` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 57.04 s, 4 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 28 | 11250 | 8192 | 56 | 9.7 | 10.837 | +| 2 | 30 | 11519 | 8192 | 81 | 9.49 | 11.112 | +| 3 | 32 | 11656 | 8192 | 174 | 9.97 | 13.546 | +| 4 | 34 | 11853 | 8192 | 106 | 10.47 | 12.651 | + +- note: list-lifecycle rerun after frontmatter guard + restore rule + +## 2026-09-15 14:35 UTC - rig turn (topic:groceries) + +- message: `Homer: what is on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 25.78 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 8252 | 4096 | 56 | 11.28 | 12.528 | +| 2 | 4 | 8440 | 8192 | 55 | 1.37 | 2.459 | +| 3 | 6 | 8648 | 8192 | 65 | 2.0 | 3.301 | + +- note: list_edit experiment, prose instructions + +## 2026-09-15 14:35 UTC - rig turn (topic:groceries) + +- message: `Homer: please add butter, eggs and flour to the list` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 34.14 s, 4 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 8 | 8559 | 4096 | 87 | 12.34 | 14.183 | +| 2 | 10 | 8667 | 8192 | 69 | 2.0 | 3.382 | +| 3 | 12 | 8757 | 8192 | 68 | 2.27 | 3.658 | +| 4 | 14 | 8846 | 8192 | 26 | 2.47 | 2.967 | + +- note: list_edit experiment, prose instructions + +## 2026-09-15 14:36 UTC - rig turn (topic:groceries) + +- message: `Homer: please restructure the list by grocery category (dairy, bakery, produce, household)` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 56.44 s, 6 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 16 | 9136 | 8192 | 64 | 4.15 | 5.418 | +| 2 | 18 | 9377 | 8192 | 279 | 3.76 | 9.41 | +| 3 | 20 | 9679 | 8192 | 97 | 4.66 | 6.609 | +| 4 | 22 | 9988 | 8192 | 129 | 5.48 | 8.145 | +| 5 | 24 | 10173 | 8192 | 174 | 6.01 | 9.629 | +| 6 | 26 | 10371 | 8192 | 82 | 6.5 | 8.15 | + +- note: list_edit experiment, prose instructions + +## 2026-09-15 14:37 UTC - rig turn (topic:groceries) + +- message: `Homer: show me the new list, and cross off the apples` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 35.05 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 28 | 10483 | 8192 | 61 | 7.58 | 8.81 | +| 2 | 30 | 10757 | 8192 | 78 | 7.45 | 9.02 | +| 3 | 32 | 10858 | 8192 | 117 | 7.78 | 10.178 | + +- note: list_edit experiment, prose instructions + +## 2026-09-15 14:40 UTC - rig turn (topic:groceries) + +- message: `Homer: what is on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 25.41 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 8222 | 4096 | 59 | 11.14 | 12.407 | +| 2 | 4 | 8413 | 8192 | 55 | 1.29 | 2.368 | +| 3 | 6 | 8621 | 8192 | 66 | 1.9 | 3.205 | + +- note: list_edit + pseudocode skill variant + +## 2026-09-15 14:40 UTC - rig turn (topic:groceries) + +- message: `Homer: please add butter, eggs and flour to the list` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 34.21 s, 4 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 8 | 8532 | 4096 | 91 | 12.3 | 14.225 | +| 2 | 10 | 8644 | 8192 | 70 | 1.99 | 3.374 | +| 3 | 12 | 8735 | 8192 | 69 | 2.18 | 3.538 | +| 4 | 14 | 8825 | 8192 | 26 | 2.4 | 2.892 | + +- note: list_edit + pseudocode skill variant + +## 2026-09-15 14:41 UTC - rig turn (topic:groceries) + +- message: `Homer: please restructure the list by grocery category (dairy, bakery, produce, household)` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 36.75 s, 4 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 16 | 9115 | 8192 | 61 | 4.06 | 5.273 | +| 2 | 18 | 9353 | 8192 | 329 | 3.75 | 10.399 | +| 3 | 20 | 9705 | 8192 | 88 | 4.68 | 6.459 | +| 4 | 22 | 10021 | 8192 | 99 | 5.46 | 7.438 | + +- note: list_edit + pseudocode skill variant + +## 2026-09-15 14:42 UTC - rig turn (topic:groceries) + +- message: `Homer: show me the new list, and cross off the apples` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 31.08 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 24 | 10020 | 8192 | 77 | 6.41 | 7.962 | +| 2 | 26 | 10120 | 8192 | 55 | 5.79 | 6.885 | +| 3 | 28 | 10404 | 8192 | 112 | 6.52 | 8.781 | + +- note: list_edit + pseudocode skill variant + +## 2026-09-15 14:43 UTC - rig turn (topic:groceries) + +- message: `Homer: what is on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 26.47 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 8222 | 4096 | 66 | 11.1 | 12.541 | +| 2 | 4 | 8400 | 8192 | 55 | 1.3 | 2.389 | +| 3 | 6 | 8608 | 8192 | 67 | 1.89 | 3.222 | + +- note: final validation: list_edit + pseudocode skill + untick guard + +## 2026-09-15 14:44 UTC - rig turn (topic:groceries) + +- message: `Homer: please add butter, eggs and flour to the list` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 30.75 s, 2 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 8 | 8552 | 4096 | 212 | 12.32 | 16.697 | +| 2 | 12 | 8805 | 8192 | 33 | 3.24 | 3.877 | + +- note: final validation: list_edit + pseudocode skill + untick guard + +## 2026-09-15 14:45 UTC - rig turn (topic:groceries) + +- message: `Homer: please restructure the list by grocery category (dairy, bakery, produce, household)` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 48.85 s, 5 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 14 | 9102 | 8192 | 70 | 4.07 | 5.463 | +| 2 | 16 | 9349 | 8192 | 237 | 3.73 | 8.499 | +| 3 | 18 | 9644 | 8192 | 170 | 4.56 | 8.0 | +| 4 | 20 | 9873 | 8192 | 213 | 5.09 | 9.449 | +| 5 | 22 | 10110 | 8192 | 78 | 5.75 | 7.313 | + +- note: final validation: list_edit + pseudocode skill + untick guard + +## 2026-09-15 14:45 UTC - rig turn (topic:groceries) + +- message: `Homer: show me the new list, and cross off the apples` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 26.64 s, 2 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 24 | 10445 | 8192 | 109 | 7.47 | 9.693 | +| 2 | 27 | 10809 | 8192 | 105 | 7.54 | 9.696 | + +- note: final validation: list_edit + pseudocode skill + untick guard + +## 2026-09-15 - List-lifecycle experiments: verdict + +Test: one conversation - read the list, add three items, restructure +by category, show and tick one item. Oracle: the file's end state. + +| Variant | Calls | Wall | End state | +|---|---|---|---| +| Free-form editing (baseline) | 17 | 185 s | WRONG first run (lost [x], broken frontmatter); correct after guards + repair rule, at 7-call restructures | +| list_edit tool, prose skill | 16 | 151 s | correct (one untick + model repair) | +| list_edit tool, pseudocode skill | 14 | 127 s | WRONG: restructure unticked an item, model did not repair | +| list_edit + pseudocode + untick guard | 12 | 133 s | CORRECT, no repair needed | + +Shipped from this: +1. list_edit tool (runtime/list_tool.py): add/tick/untick/remove, one + item per call; the store matches the item, refuses ambiguity with + candidates, preserves all other state by construction. +2. Pseudocode skill: the family-memory skill rewritten as terse rule + constructs. ~300 tokens saved, fewer calls, Sources kept. +3. Untick guard at the write seam (memory/cli/write.py, mirrored in + lab-api): a whole-page todos write that reopens or removes items + is refused with the items named. Lesson: routing rules compress + into pseudocode fine; safety rules move OUT of the prompt into + deterministic code. The pseudocode variant's one failure was a + safety rule; the guard closed it. +4. Cache pad retuned twice (53 -> 80 lines); prefix at ~8,213. + +Production gap: `stack memory list-edit` exists only in the rig's +lab-api. The memory stacklet needs the same verb (same contract, +matching, sentences) plus a famstack-api allowlist entry before the +agent image ships. + +On a dedicated list service (user question): recommendation is no. +The fights were about model discipline on whole-page rewrites and +missing item verbs, not about storage. Both are now solved at the +write seam, and the end state is correct with service-grade +semantics. A separate service would fork the source of truth away +from the vault (files, git audit, wiki, human editing) and re-import +a sync problem. The pragmatic endpoint of the current path delivers +the same reliability: item ops via list-edit (done), and later a +server-side list-restructure op (model sends {section: [items]} as +data, the store rebuilds the page deterministically) so write_file +disappears from list handling entirely. + +## 2026-09-15 - Prior-art research for the pseudocode post (material only) + +Archived for the separate blog session. Key points from the research +brief (full sources in the session transcript): + +- Pseudocode prompting is established: Mishra et al., EMNLP 2023 + (+7-16 F1 on non-instruction-tuned models); Puerto et al., EMNLP + 2024 (code format helps conditional reasoning, +7 to +22 points); + arXiv:2411.10541 (format alone swings up to 40%, small models are + the format-sensitive ones). SudoLang (2023) is the practitioner + canon. Do not claim novelty for pseudocode prompts. +- Guards-over-prompts is also established: 12-Factor Agents, NeMo + Guardrails, Claude Code hooks. Do not claim novelty there either. +- The defensible novel angle is the interaction, with our measured + failure as the worked example: compressing prose to pseudocode + preferentially drops rare-firing conditionals, which is exactly the + rule class the guardrails literature says should not live in + prompts at all. Nearest published analogue: KV-cache compression + degrading a defense instruction first (arXiv:2510.00231). Nobody + has published this for prompt-style compression on small local + models. n=1, one model: state that up front. +- pi's <1K-token prompt argues minimal prompts because frontier + models are strong; our result is the inversion for weak local + models: not less prompt, denser encoding plus code guards. +- Candidate framing ranked first by the research: "Pseudocode is + lossy compression, and it drops exactly the rules you can least + afford to lose." + +## 2026-09-15 15:53 UTC - rig turn (topic:groceries) + +- message: `Homer: was steht auf der Einkaufsliste?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 21.55 s, 2 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 8306 | 4096 | 61 | 11.4 | 12.721 | +| 2 | 4 | 8520 | 8192 | 68 | 1.64 | 2.951 | + +- note: German test run incl. clear-done and link-on-touch + +## 2026-09-15 15:53 UTC - rig turn (topic:groceries) + +- message: `Homer: bitte setz noch Butter und Mehl auf die Liste` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 17.07 s, 2 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 6 | 8525 | 8192 | 143 | 2.52 | 5.395 | +| 2 | 9 | 8728 | 8192 | 24 | 2.67 | 3.121 | + +- note: German test run incl. clear-done and link-on-touch + +## 2026-09-15 15:53 UTC - rig turn (topic:groceries) + +- message: `Homer: hak die Äpfel ab` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 22.97 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 11 | 8911 | 8192 | 80 | 3.48 | 5.081 | +| 2 | 13 | 9043 | 8192 | 84 | 2.96 | 4.668 | +| 3 | 15 | 9164 | 8192 | 59 | 3.33 | 4.509 | + +- note: German test run incl. clear-done and link-on-touch + +## 2026-09-15 15:54 UTC - rig turn (topic:groceries) + +- message: `Homer: wir haben alles gekauft, bitte räum die Liste auf` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 18.59 s, 2 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 17 | 9339 | 8192 | 80 | 4.68 | 6.325 | +| 2 | 19 | 9458 | 8192 | 53 | 4.13 | 5.212 | + +- note: German test run incl. clear-done and link-on-touch + +## 2026-09-15 - Clear-list, link-on-touch, German run + +Two quality-of-life additions. + +1. Bulk list ops on the list-edit verb: `clear-done` (remove all + ticked items, named) and `reset` (reopen all). "Clear the list" + and "we bought everything" now cost one call, and the write guard + no longer blocks the flow. +2. Link on touch: every edit verb (list-edit and write) prints the + touched page's wiki link. New helper stack.links.go_page maps a + vault path to a /topic or /person route, "" when no stable route + exists. The skill routes that link into the reply's Sources, so a + change is as citable as a read. + +German test run (4 turns, topic:groceries): +- read, add two items, tick, clear-done: 2-3 calls each, 17-23 s. +- Umlauts preserved ("Äpfel", "Mehl"). Replies fully in German. +- clear-done tool result: "REMOVED 2: apples; coffee beans" plus the + link line, relayed correctly. +- A follow-up add cited Sources with the wiki link. +Note: the add kept the model's capitalization ("Butter", "Mehl") +rather than lowercasing; harmless, matches the family's own words. + +Tests: 11 list-edit module tests, links doctests, test_links all green. + +## 2026-09-15 16:18 UTC - rig turn (topic:groceries) + +- message: `Homer: what is on the shopping list?` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 35.54 s, 3 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 2 | 8333 | 0 | 66 | 20.3 | 21.599 | +| 2 | 4 | 8511 | 8192 | 55 | 1.58 | 2.676 | +| 3 | 6 | 8719 | 8192 | 66 | 2.19 | 3.504 | + +- note: complex lifecycle revalidation after single-tool items[] batch + +## 2026-09-15 16:18 UTC - rig turn (topic:groceries) + +- message: `Homer: please add butter, eggs and flour to the list` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 14.66 s, 2 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 8 | 8663 | 8192 | 98 | 2.84 | 4.793 | +| 2 | 10 | 8800 | 8192 | 27 | 2.34 | 2.845 | + +- note: complex lifecycle revalidation after single-tool items[] batch + +## 2026-09-15 16:19 UTC - rig turn (topic:groceries) + +- message: `Homer: please restructure the list by grocery category (dairy, bakery, produce, household)` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 36.74 s, 4 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 12 | 8957 | 8192 | 63 | 3.61 | 4.862 | +| 2 | 14 | 9197 | 8192 | 228 | 3.39 | 8.002 | +| 3 | 16 | 9484 | 8192 | 215 | 4.13 | 8.505 | +| 4 | 18 | 9737 | 8192 | 81 | 4.73 | 6.339 | + +- note: complex lifecycle revalidation after single-tool items[] batch + +## 2026-09-15 16:19 UTC - rig turn (topic:groceries) + +- message: `Homer: show me the new list, and cross off the apples` +- model: `Qwen3.6-35B-A3B-UD-MLX-4bit`, wall time 24.2 s, 2 LLM call(s), exit 0 + +| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s | +|---|---|---|---|---|---|---| +| 1 | 20 | 9999 | 8192 | 113 | 6.3 | 8.596 | +| 2 | 23 | 10367 | 8192 | 94 | 6.36 | 8.268 | + +- note: complex lifecycle revalidation after single-tool items[] batch + +## 2026-09-15 - Batch list-edit validated end-to-end + +Single-tool items[] confirmed on the wire: the add turn was one call, +list_edit(op=add, items=["butter","eggs","flour"]) -> "added 3" in one +commit (was 3 calls, 3 commits). Full lifecycle correct: every item +kept, [x] preserved, clean commit trail. Rig now runs the production +transform, so this exercised the real store code. diff --git a/docs/design/agent/review-2026-09.md b/docs/design/agent/review-2026-09.md new file mode 100644 index 00000000..5e90bf0a --- /dev/null +++ b/docs/design/agent/review-2026-09.md @@ -0,0 +1,224 @@ +# Agent review, September 2026 + +Status: assessment. No code changed. + +A review of the agent stacklet against its own targets (plan.md: median < 8 s, +p95 < 20 s), triggered by observed 3-5 minute responses to simple questions, +unreliable todo handling, and context pollution across threads and speakers. +Covers: root causes in the current code, the 2025/26 state of the art it should +follow, and a staged first iteration. Two experiments are scoped at the end: +a containerised pi agent and an established OSS memory layer (Hindsight). + +## 1. Why responses take minutes + +No single bug. Five multipliers stack, all contending for one 35B MoE +(Qwen3.6-35B-A3B-UD-MLX-4bit) on one Mac: + +| # | Multiplier | Where | Effect | +|---|---|---|---| +| 1 | Nested LLM call inside retrieval | `stacklets/memory/cli/search.py:182-215` -> `rewrite.py` | Every multi-word `memory_search` runs a second chat completion (keyword rewrite) on the same GPU before the search even walks the vault. `grep` under `vault/` is silently rerouted into the same path (`runtime/grep_tool.py:16-85`), and `AGENTS.md:4` tells the model grep is the primary lookup. | +| 2 | KV cache self-invalidation | `runtime/lean_state.py:64-103` | Prior tool results are retroactively rewritten to `[prior result of ...]` placeholders. The message list at turn N+1 diverges from turn N at the first prior tool message, so oMLX re-prefills from that point on every turn that follows a tool-using turn. The late-brief injection work (`sitecustomize.py:112-124`, measured 4096/4746 tokens cached) is largely given back here. | +| 3 | Iteration count | `config.json:17` | `max_tool_iterations: 12`, each iteration re-sends the ~10K-token prefix plus history. No total-turn budget exists. | +| 4 | Tool transport | container -> TCP socket -> famstack-api -> fresh `./stack` interpreter -> `docker exec` into bot-runner | Five hops, two cold Python starts, per tool call. Timeout ladder: 60 s (rewrite) < 120 s (host CLI) < 125 s (socket) < 130 s (tool). | +| 5 | Event-loop blocking | `runtime/vault_write.py:87-95` (sync `subprocess.run`, up to 120 s), `runtime/brief.py:64-74` (git log per turn, 5 s) | Blocks Matrix sync and the agent loop while running. | + +Prefill economics make #2 and #3 fatal on their own: prefill on Apple Silicon +is compute-bound, roughly 7.6 s per 1K tokens at the small end of our own +measurements (`lib/stack/ai/client.py:169-213` comment), and oMLX's paged +prefix cache only helps when the prompt is byte-stable and append-only. +"Failed tool calls" are a real but secondary contributor: the honest-exit-code +work already fixed the worst retry loops; the transcripts now show flailing +(15 `read_file` probes for one path) caused by four competing retrieval +routes, not by protocol errors. + +## 2. Why context pollutes + +1. **Session key is the room id.** All threads and all speakers in a family + room share one transcript (`addressing.md:68-70` flags it; the designed fix, + folding the thread root into the session key, never shipped). DMs are clean + only because a DM is its own room. +2. **Model-writable prefix.** nanobot's `USER.md` and `memory/MEMORY.md` sit + inside the cached prefix and the model can rewrite them. Any self-edit both + pollutes every future conversation and permanently invalidates the cache. +3. **Partial decay.** `lean_state` placeholders only cover tool results. An + answer recited from the injected brief has no tool call to anchor decay, so + stale data is indistinguishable from conversation (post-mortem in + `pointer-memory.md:63-70`). + +The Chroma context-rot data (all 18 tested models degrade monotonically with +input length, distractors actively mislead) makes isolation a quality fix, +not just hygiene. + +## 3. Why we cannot see any of this + +- The only prompt record is `llm-state.log`: one line per message, clipped at + 320 chars, arguments dropped. Shape and ordering only, never the bytes sent. +- `resp.usage` is read nowhere in production code. We have no token counts, no + cached-token counts, no per-call latency, no iteration index. The KV cache + hit rate, the single most important production-agent metric (Manus, 2025), + is invisible. + +## 4. What the field says (research summary, Sept 2026) + +- **Context engineering consensus** (Manus 07/2025, Anthropic 09/2025, + unchanged into 2026): byte-stable prefix, fixed tool list, append-only + history, no timestamps or dynamic content early in the prompt, compaction + only at natural boundaries. baibot's trick applies to us: inject + conversation *start* time once, never "now" per request. +- **oMLX already has the machinery**: vLLM-style paged prefix cache with SSD + tier. Reported TTFT 1-3 s on long contexts when prompts are append-only. + Nothing on our side configures or measures it. +- **Memory**: markdown files + grep + a nightly consolidation pass is the + proven pragmatic tier (Claude Code pattern; Letta's sleep-time-compute + validates background consolidation, +18% accuracy in their study). Heavy + graph systems (Zep CE is discontinued; Graphiti needs a graph DB) are + overkill for a family. The one established system worth a trial here is + **Hindsight** (see section 7). +- **Two-model router**: evidence leans against it as a first move. Docker's + 2025 eval (21 models, 3570 cases) found generalist Qwen3 8B/14B beat + dedicated tool-calling models (Qwen3-14B F1 0.971 vs Watt-Tool-8B 0.484). + A second resident model costs RAM and splits the KV cache on one Mac. Our + latency is a cache/prefill problem, not a model-size problem. Revisit only + if triage latency still hurts after the cache fixes (candidates then: + xLAM-2, Hammer, Qwen3.5-4B). +- **Minimal harnesses**: the field moved to tiny loops with owned context + (pi, smolagents, nanobot itself). Our nine monkeypatches over pinned + nanobot 0.2.2 internals are the opposite of owning the loop; ADR-012 + records the failure mode (shims fail soft, tests stay green while + production breaks). + +## 5. First iteration + +Ordered so each step is measurable before the next. Steps 1-3 are small +changes to the existing stacklet; no architecture swap yet. + +### Step 1: see the context (observability) + +- Dump the exact request: full `messages` JSON per LLM call to a per-room + JSONL under `{data_dir}/agent/prompts/`, with turn id, iteration index, and + wall time. Replaces guessing with bytes. +- Read `resp.usage` on every call and log prompt tokens, completion tokens, + and cached tokens if oMLX reports them. Log TTFT. +- Success check: for two consecutive turns in one room, the second request's + prompt is a byte-prefix of the first plus appended messages, and cached + tokens ~= previous prompt tokens. + +### Step 2: stop breaking the cache, stop stacking calls + +- Delete the `lean_state` retroactive rewrite. History becomes append-only. + Accept larger transcripts; compact only at a natural boundary (size + threshold), which costs one deliberate cache reset instead of one per turn. +- Drop the NL-rewrite hop from the agent's search path: `memory_search` runs + the literal/regex search directly; the model itself already chooses + keywords. Keep `--nl` for human CLI use. +- One retrieval route. Un-instruct grep-then-read in `workspace/AGENTS.md`, + align `SKILL.md`, keep `memory_search`/`memory_person`/`memory_history` as + the only documented path. Fixes the flailing transcripts and the doc drift + in one move. +- `max_tool_iterations` 12 -> 6, plus a total turn budget (fail loud into the + room rather than grind for five minutes). +- Async the blocking calls (`vault_write` subprocess, brief git log; cache the + git log line for a few minutes). +- Freeze the prefix: seed `USER.md`/`MEMORY.md` read-only or route nanobot's + self-memory writes outside the prompt prefix. + +### Step 3: isolate conversations + +- Session key: `(room_id, thread_root)` for threads, `(room_id, sender)` in + multi-user rooms outside threads, `room_id` for DMs. This is the already + designed but unshipped step from addressing.md layer 2. +- Cross-conversation knowledge flows only through the vault (write in one + thread, `memory_search` in another), never through a shared transcript. + +### Step 4: measure + +Re-run the plan.md targets with the step-1 logging in place: median < 8 s, +p95 < 20 s for no-tool turns; one-tool turns < 30 s. Only if this fails does +the router idea (section 4) come back on the table. + +## 6. Experiment A: containerised pi agent + +Assessment: worth running. pi (earendil-works, ex badlogic/pi-mono, MIT, +v0.85.1 as of 2026-09-05) is a maintained minimal harness that gives us, +out of the box, most of what the shim stack fights for: + +- **Headless operation**: `pi -p` one-shot, `--mode json` (JSONL events), and + `--mode rpc` (long-lived process, JSONL over stdin/stdout, `prompt` / + `steer` / `abort` / `switch_session`). No TTY needed. RPC fits a + per-conversation resident process better than spawn-per-message. +- **Sessions are files**: JSONL trees under a mountable dir + (`PI_CODING_AGENT_SESSION_DIR`). One session per (room, thread) key gives + the step-3 isolation for free, inspectable with `jq`. +- **oMLX is a config entry**: `models.json` provider with + `baseUrl: http://host.docker.internal:8888/v1`, `api: openai-completions`, + explicit `contextWindow`. No code. +- **Cache-friendly by design**: system prompt < 1K tokens, append-only + sessions, compaction is the only prefix-invalidating event. +- **Extensible without monkeypatching**: `pi.registerTool()` for + `memory_search` etc. (TypeScript extensions), AGENTS.md and the Skills + standard supported natively, `.pi/SYSTEM.md` replaces the system prompt. +- **Containerisation**: documented pattern, `node:24-bookworm-slim` base, + named volume for `/root/.pi/agent`, `PI_OFFLINE=1`. No official image, so + we build and pin our own line per house rules. + +Experiment shape (test rig, not production): a thin Python bridge keeps one +`pi --mode rpc` process per conversation key, forwards Matrix messages as +`prompt` commands, posts `message_end` events back. Vault mounted read-only; +the write path stays `stack memory write` via exec. Compare against the +nanobot stacklet on the same prompts: TTFT, cached tokens, turn latency, +retrieval sanity. + +Risks: Node runtime in the stack, tool extensions are TypeScript (new +language surface), Matrix bridging is ours to write (pi has no channel +concept), project is young even if very active. + +Decision point: if steps 1-3 get nanobot to target latency, pi remains a +bench experiment. If the shim stack keeps fighting us (ADR-012), pi becomes +the migration target instead of a nanobot fork. + +## 7. Experiment B: established OSS memory layer (Hindsight) + +Candidates reviewed: Mem0 OSS, Letta, Zep CE/Graphiti, LangMem, Hindsight, +cognee, Memvid. Verdicts: + +| System | Verdict | +|---|---| +| Hindsight (vectorize-io, MIT, v0.10) | Trial first. One container + pgvector. The only candidate with published open-model benchmarks (83.2% LoCoMo with a 20B open model). In-process local embeddings, so oMLX only serves chat. Banks map to per-person/per-room isolation. Markdown projection matches the vault philosophy. | +| Mem0 OSS (Apache-2.0) | Fallback. Cleanest `add/search(user_id)` API, but independently measured OSS quality (32.4% LongMemEval) trails its marketing, and small-model JSON extraction is its documented failure mode. Default constructor silently phones OpenAI; must be configured away. | +| Graphiti (Zep CE is discontinued) | Only if temporal graph semantics prove necessary. Needs a graph DB; small-model JSON extraction is fragile. | +| Letta | Skip. It is an agent runtime, not a memory sidecar; wrong shape next to nanobot/pi. | +| LangMem | Skip. Pre-1.0 library that drags the LangChain tree in; closer to hand-rolled than established. | +| cognee, Memvid | Wrong shape (document ETL / storage format). | + +Trial shape (shadow mode, test rig): run Hindsight (pinned 0.10 line) + +pgvector alongside the vault. Feed the same room transcripts into banks +(`person:`, `room:`, `family:shared`) via `retain()` in the +background; at question time call `recall()` in parallel with the existing +`memory_search` and log both result sets. Judge after two weeks: recall +quality on German family content (the default bge-small-en-v1.5 embedder is +English-only; configure `intfloat/multilingual-e5-small`), retain throughput +on the oMLX model, and whether the markdown projection is usable next to the +vault. The vault stays the source of truth either way; Hindsight would be +the retrieval/consolidation layer, not the store. + +## 8. Deferred and cleanup + +Deferred: two-model router (decision gate in step 4); nanobot 0.3 migration +(the 0.2.2 pin is the current safety mechanism; the handover doc predates the +pin and was never executed). + +Doc drift found during review, worth a cleanup pass: + +- `docs/design/handover/nanobot-0.3-shim-migration.md`: untracked, stale, + superseded by the pin eight minutes after it was written. Reads as + current-and-broken. +- `docs/design/agent/plan.md`: specifies MicroBot bridge, tools-server + endpoint, agent-notes repo, and the invariant "the agent never mutates + family/memory", all overtaken by what shipped. +- `workspace/AGENTS.md` and `SKILL.md` disagree on the primary retrieval + route and neither names the `memory_search` tool (fixed by step 2). +- `runtime/README.md` says "two independent shims"; sitecustomize carries + nine. +- `todo-strike-flaky.md` reads as an open proposal but its alternative was + implemented. +- `runtime/__pycache__/` and `hooks/__pycache__/` are present in the tree. diff --git a/lib/stack/links.py b/lib/stack/links.py index d94dcaf9..170e9f3d 100644 --- a/lib/stack/links.py +++ b/lib/stack/links.py @@ -108,6 +108,38 @@ def go_person(slug: str, leaf: str | None = None) -> str: return _entity_path("person", slug, leaf) +def go_page(repo_path: str) -> str: + """Logical link for a vault page path, or "" when no stable route exists. + + Edit verbs answer with the page they touched. The reply carries this + link so the family can open the page in the wiki. Only pages with a + resolver route get a link; anything else returns "" rather than a + path-keyed link that breaks on the next rename. + + >>> go_page("family/camping/todos.md") + '/topic/family/camping/todo' + >>> go_page("family/camping/about.md") + '/topic/family/camping' + >>> go_page("homer/about.md") + '/person/homer' + >>> go_page("homer/notes/2026/09/errands.md") + '' + """ + parts = [p for p in repo_path.strip("/").split("/") if p] + if len(parts) < 2: + return "" + if parts[-1] == "todos.md": + leaf = "todo" + elif parts[-1] == "about.md": + leaf = None + else: + return "" + scope = "/".join(parts[:-1]) + if len(parts) == 2 and parts[0] != "family": + return go_person(parts[0], leaf) + return go_topic(scope, leaf) + + def _entity_path(kind: str, scope: str, leaf: str | None) -> str: """Join `kind`, the scope segments, and an optional leaf into a path. diff --git a/stacklets/agent/config.json b/stacklets/agent/config.json index e2969fda..e6af6e6f 100644 --- a/stacklets/agent/config.json +++ b/stacklets/agent/config.json @@ -14,7 +14,9 @@ "agents": { "defaults": { "model_preset": "primary", - "max_tool_iterations": 12 + "max_tool_iterations": 12, + "max_messages": 40, + "disabled_skills": ["memory", "my"] } }, "tools": { diff --git a/stacklets/agent/entrypoint.sh b/stacklets/agent/entrypoint.sh index e55f3bed..52e8c607 100644 --- a/stacklets/agent/entrypoint.sh +++ b/stacklets/agent/entrypoint.sh @@ -19,6 +19,11 @@ for f in SOUL.md AGENTS.md; do [ -f "$NB/workspace/$f" ] || sed "s/__AGENT_NAME__/$NAME/g" "/app/workspace-seed/$f" > "$NB/workspace/$f" done +# Seed a minimal USER.md so nanobot does not install its default profile +# template. The template costs ~1K prompt characters per request and holds +# no data: the runtime brief carries the per-turn speaker instead. +[ -f "$NB/workspace/USER.md" ] || printf '# User Profile\n\nThe runtime brief names the current speaker each turn. No data here.\n' > "$NB/workspace/USER.md" + # Product-defined skills always refresh from the image seed (not user-editable). if [ -d /app/workspace-seed/skills ]; then mkdir -p "$NB/workspace/skills" diff --git a/stacklets/agent/runtime/list_tool.py b/stacklets/agent/runtime/list_tool.py new file mode 100644 index 00000000..34583cf4 --- /dev/null +++ b/stacklets/agent/runtime/list_tool.py @@ -0,0 +1,123 @@ +"""Agent runtime tool for item-level list edits. + +Free-form page editing loses state on restructures (measured +2026-09-15: a category rewrite unticked a done item). For item +operations the model only names the item; the store matches it and +preserves everything else by construction. One route per intent: +`list_edit` for items, `write_file` for restructures. The old +structured CLI verbs failed because two routes competed for one +intent (docs/design/agent/todo-strike-flaky.md); this tool is the +only documented item route. +""" + +from __future__ import annotations + +import asyncio + +from nanobot.agent.tools.base import Tool, tool_parameters +from nanobot.agent.tools.schema import ( + ArraySchema, + StringSchema, + tool_parameters_schema, +) + + +@tool_parameters( + tool_parameters_schema( + page=StringSchema( + "The list page, such as vault/family/groceries/todos.md.", + min_length=1, + ), + op=StringSchema( + "Item operation, or bulk: clear-done removes all ticked " + "items, reset reopens them.", + enum=("add", "tick", "untick", "remove", "clear-done", "reset"), + ), + items=ArraySchema( + StringSchema("One item's text, as it appears on the list."), + description="The item(s) for this op, one or more, in the " + "family's words. Omit for bulk ops (clear-done, " + "reset).", + nullable=True, + ), + section=StringSchema( + "Optional section heading for add, such as Dairy.", + nullable=True, + ), + ) +) +class ListEditTool(Tool): + """Change one item on a family list.""" + + _scopes = {"core"} + + @property + def name(self) -> str: + return "list_edit" + + @property + def description(self) -> str: + return ( + "Add, tick, untick, or remove ONE item on a family list page. " + "The store matches the item and answers with what actually " + "changed. Use this for every item change; use write_file only " + "to restructure a whole list." + ) + + @property + def read_only(self) -> bool: + return False + + async def execute( + self, + page: str, + op: str, + items: list[str] | None = None, + section: str | None = None, + ) -> str: + target = page.strip() + for marker in ("workspace/vault/", "vault/"): + if marker in target: + target = target.split(marker, 1)[1] + break + try: + import brief + actor = getattr(brief, "speaking_with", "") or "someone" + except Exception: + actor = "someone" + + names = [i for i in (items or []) if i and i.strip()] + args = ["stack", "memory", "list-edit", target, "--op", op, "--by", actor] + for name in names or [""]: + args.extend(["--item", name]) + if section: + args.extend(["--section", section]) + + proc = await asyncio.create_subprocess_exec( + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=60) + out = stdout.decode(errors="replace").strip() + err = stderr.decode(errors="replace").strip() + # Exit 1 carries an instructive answer (ambiguous item, unknown + # item with candidates). The model reads it and retries once. + if proc.returncode not in (0, 1): + return f"Error: list edit failed with exit {proc.returncode}: {err or out}" + return out or "(no answer from the list store)" + + +def install() -> None: + """Append ListEditTool to nanobot discovery, same pattern as memory_tool.""" + from nanobot.agent.tools.loader import ToolLoader + + original = ToolLoader.discover + + def discover_with_list_edit(self: ToolLoader) -> list[type[Tool]]: + tools = list(original(self)) + if ListEditTool not in tools: + tools.append(ListEditTool) + return tools + + ToolLoader.discover = discover_with_list_edit diff --git a/stacklets/agent/runtime/memory_tool.py b/stacklets/agent/runtime/memory_tool.py index 7006529d..8f5d5d2b 100644 --- a/stacklets/agent/runtime/memory_tool.py +++ b/stacklets/agent/runtime/memory_tool.py @@ -3,17 +3,31 @@ from __future__ import annotations import asyncio +import re from nanobot.agent.tools.base import Tool, tool_parameters -from nanobot.agent.tools.schema import IntegerSchema, StringSchema, tool_parameters_schema +from nanobot.agent.tools.schema import ( + ArraySchema, + IntegerSchema, + StringSchema, + tool_parameters_schema, +) @tool_parameters( tool_parameters_schema( query=StringSchema( - "Natural-language question or keywords to search across the family vault.", + "Two to four literal keywords, words that appear on the page. " + "Not a full question.", min_length=1, ), + queries=ArraySchema( + StringSchema("Keyword set for one search."), + description="Up to three independent keyword sets, searched in " + "one call. Use instead of repeated search calls.", + max_items=3, + nullable=True, + ), limit=IntegerSchema( 5, description="Maximum number of results to return.", @@ -59,22 +73,48 @@ def read_only(self) -> bool: async def execute( self, query: str, + queries: list[str] | None = None, limit: int | None = None, scope: str | None = None, person: str | None = None, tag: str | None = None, ) -> str: - # `--nl` is what makes the parameter description above true. The - # CLI's default query language is a regex, so a question sent - # without it asks for those exact words, adjacent, and matches - # nothing. The CLI skips the model itself on a single word, so - # passing this always costs nothing on keyword lookups. + # `queries` batches independent lookups into one tool call, so + # one LLM iteration answers a question that needs two or three + # searches. Each iteration costs a prompt prefill; the searches + # themselves are cheap and run concurrently. + batch = [q for q in (queries or []) if q and q.strip()] or [query] + batch = batch[:3] + results = await asyncio.gather( + *(self._search_one(q, limit, scope, person, tag) for q in batch) + ) + if len(batch) == 1: + return results[0] + return "\n\n".join( + f"## {q}\n{r}" for q, r in zip(batch, results) + ) + + async def _search_one( + self, + query: str, + limit: int | None, + scope: str | None, + person: str | None, + tag: str | None, + ) -> str: + # The CLI's query language is a regex, and adjacent words match + # nothing. Join the model's keywords with `|` so each keyword + # matches on its own. This replaces the CLI's `--nl` rewrite, + # which made a second LLM call inside every multi-word search + # (measured 2026-09-15: one full model call per search, on the + # same GPU as the turn). The model now supplies the keywords. + words = [re.escape(w) for w in query.split()] + pattern = "|".join(words) if len(words) > 1 else query args = [ "stack", "memory", "search", - query, - "--nl", + pattern, "--limit", str(limit or 5), ] diff --git a/stacklets/agent/runtime/sitecustomize.py b/stacklets/agent/runtime/sitecustomize.py index 9172ef57..1dd234d1 100644 --- a/stacklets/agent/runtime/sitecustomize.py +++ b/stacklets/agent/runtime/sitecustomize.py @@ -146,11 +146,19 @@ def _runtime_lines(state, msg, workspace, *, skip=False): # readable on the host for analysis, one appended block per turn. _STATE_LOG = _os.path.expanduser("~/.nanobot/llm-state.log") + # Experiment switch: AGENT_LEAN_STATE=0 disables the history rewrite and + # keeps the message list append-only. The rewrite invalidates the oMLX + # prefix cache from the first rewritten message on (measured 2026-09-15, + # see docs/design/agent/agent-improvement-log.md). Default stays on. + _LEAN_ENABLED = _os.environ.get("AGENT_LEAN_STATE", "1") != "0" + def _build_messages_lean(self, *args, **kwargs): # Post-process the assembled message list: stale prior-turn derived data # (tool results and tool-synthesized answers) become pointers; the # current turn stays intact. - messages = _lean_messages(_orig_build_messages(self, *args, **kwargs)) + messages = _orig_build_messages(self, *args, **kwargs) + if _LEAN_ENABLED: + messages = _lean_messages(messages) try: # a debug view; never worth breaking a turn over stamp = _dt.datetime.now().isoformat(timespec="seconds") with open(_STATE_LOG, "a", encoding="utf-8") as fh: @@ -178,6 +186,9 @@ def _build_messages_lean(self, *args, **kwargs): ("history_tool", "memory_history tool"), ("grep_tool", "vault grep -> memory_search routing"), ("vault_write", "write_file on a vault page -> stack memory write"), + ("list_tool", "list_edit item tool -> stack memory list-edit"), + ("tool_trim", "unused-tool trim (AGENT_TOOL_TRIM=0 to disable)"), + ("thread_session", "thread-scoped sessions (AGENT_THREAD_SESSIONS=0 to disable)"), ): try: importlib.import_module(_module_name).install() diff --git a/stacklets/agent/runtime/thread_session.py b/stacklets/agent/runtime/thread_session.py new file mode 100644 index 00000000..807c6672 --- /dev/null +++ b/stacklets/agent/runtime/thread_session.py @@ -0,0 +1,45 @@ +"""Thread-scoped sessions: one context per Matrix thread. + +nanobot keys a session as `channel:chat_id`, so every thread and every +speaker in a room shares one transcript. That is the measured context +pollution (docs/design/agent/review-2026-09.md section 2). nanobot +0.2.2 already has the seam: `InboundMessage.session_key_override`. +This shim sets it for threaded messages, so a thread gets its own +session. Replies still route by `chat_id`; the room id stays pure. + +Known v1 gap: the thread's root message was handled in the room +session, so a new thread session starts without the root's text. The +model sees the first threaded reply only. + +Set AGENT_THREAD_SESSIONS=0 to fall back to room-scoped sessions. + +PIN: patches MatrixChannel._handle_message (inherited from +channels/base.py). Re-verify on a nanobot bump. +""" + +from __future__ import annotations + +import os + + +def install() -> None: + if os.environ.get("AGENT_THREAD_SESSIONS", "1") == "0": + return + import nanobot.channels.matrix as matrix + + original = matrix.MatrixChannel._handle_message + + async def handle_with_thread_session( + self, sender_id, chat_id, content, + media=None, metadata=None, session_key=None, is_dm=False, + ): + root = (metadata or {}).get("thread_root_event_id") + if root and not session_key: + session_key = f"{self.name}:{chat_id}#thread:{root}" + return await original( + self, sender_id, chat_id, content, + media=media, metadata=metadata, + session_key=session_key, is_dm=is_dm, + ) + + matrix.MatrixChannel._handle_message = handle_with_thread_session diff --git a/stacklets/agent/runtime/tool_trim.py b/stacklets/agent/runtime/tool_trim.py new file mode 100644 index 00000000..f9f02d17 --- /dev/null +++ b/stacklets/agent/runtime/tool_trim.py @@ -0,0 +1,41 @@ +"""Drop unused nanobot tools from the model's tool list. + +nanobot 0.2.2 has no config gate for these tools. They enable with the +gateway services and put their JSON schemas into every request. The +family agent does not use them. Fewer schemas make a shorter and more +stable prompt prefix. + +Set AGENT_TOOL_TRIM=0 to keep the full tool list. +""" + +from __future__ import annotations + +import os + +# Tool names, as the model sees them. `message` stays: the gateway can +# send mid-turn channel messages through it. +TRIMMED = { + "cron", # agent-set reminders, not used yet + "long_task", # background goal loop, not used + "complete_goal", # pairs with long_task + "spawn", # subagents, not used + "write_stdin", # interactive exec sessions, agent runs one-shot CLI + "list_exec_sessions", # same group +} + + +def install() -> None: + """Filter tool registration. The registry is the one seam every + tool passes through, so class names stay out of it.""" + if os.environ.get("AGENT_TOOL_TRIM", "1") == "0": + return + from nanobot.agent.tools.registry import ToolRegistry + + original = ToolRegistry.register + + def register_trimmed(self: ToolRegistry, tool) -> None: + if getattr(tool, "name", None) in TRIMMED: + return + original(self, tool) + + ToolRegistry.register = register_trimmed diff --git a/stacklets/agent/workspace/AGENTS.md b/stacklets/agent/workspace/AGENTS.md index f0634e89..6f935bcc 100644 --- a/stacklets/agent/workspace/AGENTS.md +++ b/stacklets/agent/workspace/AGENTS.md @@ -1,16 +1,16 @@ # How I answer -- Before I answer anything about a person, a topic, a plan, or the family, I look - it up in my workspace under `vault/` first. To find something, `grep` inside - `vault/`; to read a page, `read_file` it. I never claim I don't know about the - family, or that a profile is blank, without searching `vault/` first. -- When someone asks about themselves ("me", "my", "I"), that is the person whose - handle sent the message. I look up their own page at `vault//about.md`. -- I answer the question that was asked, at the length it asks for. "One line" - means one line. Otherwise a sentence or two, and more only when asked. -- I don't describe myself or my tone. I just answer. -- No sign-offs like "what can I do for you". I stop once I've answered. -- No em dashes, ever. I use commas or short sentences instead. +- Family questions: look it up first. If my briefing names the page, `read_file` + it directly. Otherwise one `memory_search` (2-4 literal keywords), then + `read_file`. If the search misses, change keywords once; after that I say what + I looked for and ask. I never probe the same question three ways, and never + claim a profile is blank without looking. Every answer that used a vault + page ends with its Sources line. +- "me", "my", "I" is the sender. Their page: `vault//about.md`. +- I answer at the asked length. "One line" means one line; otherwise a sentence + or two. +- I don't describe myself or my tone. No sign-offs. I stop once I've answered. +- No em dashes, ever. Commas or short sentences. - When I do something (tick a todo, look something up), I say what I did in one line. - I never send anything outside the house. Drafts stay drafts until a person sends them. - A thumbs-down on something I did means undo it. diff --git a/stacklets/agent/workspace/skills/family-memory/SKILL.md b/stacklets/agent/workspace/skills/family-memory/SKILL.md index e5bb34b5..7da2d6e3 100644 --- a/stacklets/agent/workspace/skills/family-memory/SKILL.md +++ b/stacklets/agent/workspace/skills/family-memory/SKILL.md @@ -1,119 +1,46 @@ --- name: family-memory -description: How I look things up about the family. I use this before answering anything about a person, topic, plan, or the family. +description: How I look things up about the family and edit their lists. metadata: {"nanobot": {"always": true}} --- -# Looking things up - -Everything I know about the family is in the vault. I look it up before I answer -anything about a person, a topic, a plan, or the family. I never say "I don't -know" or "your profile is blank" without searching first. - -## Find, then read -- To find something across the family's notes, bookmarks and documents, run: - `stack memory search ""` - It prints dated, attributed results with a snippet of the matching line. Add - `--paths` for just the file paths, `--limit N` to cap results, or - `--scope family/` to look **within one topic first**. When the question - is about the topic I am in, I scope to it and widen only if that finds nothing. -- To read a full page, `read_file` on `vault/`. Search prints paths - relative to the vault, so `homer/about.md` is read as `vault/homer/about.md`. -- For the **full source document** behind a briefing (a scanned letter, a PDF), - the vault page's frontmatter carries a `paperless_id`. I fetch the original - body with `stack docs show --content` — only when the briefing itself is - not enough, since the source can be long. - -## Saying where I got it -Every answer that came out of the vault ends with the pages it came from, so -the family can open them and check me. Search prints each hit's link on the -last line of its block, and my answer ends with one line in this shape: - - Sources: []() - -with the sources separated by commas when there is more than one. - -The links are long, and every one of them is different. Copying one from -memory, shortening one with `…`, or reusing one id for two different pages -produces a link that goes nowhere, which is worse than citing nothing. So -each source gets the full line that search printed for that page, character -for character, pasted from the output I am looking at. - -- I link **only pages I actually read** for this answer. Not everything the - search returned, and never a page I decided against. -- If a page has no link line, it has no durable link. I name it by its title - and leave it unlinked rather than inventing one from its file path. -- Two or three sources is a list. More than that means I should have read less. -- Nothing from the vault, nothing to cite: a greeting or a question about - myself gets no Sources line. - -## Where things live -- A person: `vault//about.md` (a full profile). -- A shared topic or plan: `vault/family//about.md`, with its open items in - `vault/family//todos.md`. - -## What changed, and when -The vault keeps every version of everything, so `memory_history` answers the -questions a search cannot: "lately", "since when", "who did that", "what's -new". Search ranks pages by what they say *now*, so it always returns -something plausible for a question about change, and that answer is wrong -without looking wrong. - -"What has Homer been up to lately" is `memory_history` scoped to homer, not a -re-read of his profile. The profile says what is true; the history says what -is new. Some questions want both. - -I never guess a date, and I do not turn "he saved three articles about -hiking" into "he has taken up hiking". I say what the history actually shows. - -## Changing a list (add, tick off, split, tidy) -A topic's list is a page, and I change it by editing the page. I do it right -away without asking permission, because every version is kept and nothing is -lost. - -I always `read_file` on `vault/family//todos.md` first. Then I pick by -how much of the page is changing: - -- **Most of the time: `apply_patch`.** Ticking something off, adding an item, - fixing a word. I name the exact line in `old_text` and give the new one. - This is the safer tool and I reach for it by default, because it only - touches the lines I name and cannot disturb the rest. -- **For a real restructure: `write_file`** with the complete new contents. - Splitting one list into two, reordering the whole thing, a proper tidy-up. - Here the shape *is* the change and there is no smaller way to say it. - -There is no add or strike command. Adding is a new `- [ ] ` line, ticking off -is changing `- [ ]` to `- [x]`, and splitting one list into two is adding -`## ` headings. Ordinary markdown, which is why I should get it right. - -Rules I hold myself to: - -- **A patch that does not fit means the page moved, not that I should force - it.** If I am told `old_text` was not found, someone edited the list while - I was reading it. I read it again and patch what is actually there now. I - never fall back to `write_file` to get around it, because that would wipe - out whatever they just did. -- **I write the page back in full** when I use `write_file`. Whatever I leave - out is gone, so I carry over every line I was not asked to change, exactly - as it was. -- **I keep the order they put things in.** A list is not mine to sort. Unless - someone asks me to reorder it, every line stays where it was, and anything - new goes at the end of the section it belongs to. -- **I keep the family's words.** If the line says "Kühlbox", it stays - "Kühlbox". I do not improve it into "Kühlbox mitbringen". Their wording is - how they recognise their own list. -- **I tick off rather than delete.** "We did that one" means `- [x]`, not - removing the line. I only delete when someone asks me to. -- **I read what the edit tells me.** It answers with what actually changed - ("ticked off 2: ...", or "REMOVED 1: ..."). That answer is the truth about - what I did, and it is what I relay, in one short line. If it says something - was removed that I did not mean to remove, I say so and put it back. -- **I never claim a change I did not make.** If I did not call `apply_patch` - or `write_file` and read its answer, then nothing happened, however sure I - feel. - -The change commits to the family's store as the person I am replying to, so it -is theirs and shows up everywhere. `stack memory topic todo` lists the -items if I only need to read them; my briefing says a list exists, not what is -on it. - -I answer only from what I actually read, and I keep it short. +# Family memory (rules as pseudocode) + +Vault = all family knowledge. Look before answering. + +``` +LOOKUP: + brief names the page -> read_file(page) # no search + else -> memory_search(2-4 literal keywords) + scope family/ first; miss -> widen + independent lookups -> ONE call, queries=[..] # max 3 + miss -> retry once, new keywords; then say tried + ask + profile -> memory_person(name) + "lately|since when|who did" -> memory_history # search ranks NOW, not change + full source document -> paperless_id frontmatter + `stack docs show --content` + +PATHS: + person -> vault//about.md + topic -> vault/family//about.md ; items in .../todos.md + search prints vault-relative paths -> read as vault/ + +SOURCES (answer used a vault page, searched or read): + end answer with: Sources: []() + only pages I read; never shorten|reuse|invent a link; no link line -> title only + max 3 sources; nothing from vault -> no Sources line + +LISTS: + item op add|tick|untick|remove -> list_edit, items=[one or more] + "we did that" -> tick, never remove + "clear the list|bought all" -> list_edit op=clear-done # removes ticked + "start the week|fresh list" -> list_edit op=reset # reopens all + answer "ambiguous|no item" -> pick from named candidates, retry once + restructure ONLY -> read_file then write_file(complete page) + keep: frontmatter verbatim at column one, every [x], their order, their words + write answer reports unintended unticked|REMOVED -> restore now, then say so + edit answer = the truth -> relay it in one line + edit answer prints a link line -> that page goes in Sources + no tool call + answer read -> nothing happened, whatever I believe + edits commit as the person I reply to +``` + +I answer only from what I read, and I keep it short. diff --git a/stacklets/agent/workspace/skills/zz-cache-pad/SKILL.md b/stacklets/agent/workspace/skills/zz-cache-pad/SKILL.md new file mode 100644 index 00000000..d5bd0e3f --- /dev/null +++ b/stacklets/agent/workspace/skills/zz-cache-pad/SKILL.md @@ -0,0 +1,90 @@ +--- +name: zz-cache-pad +description: KV-cache block alignment padding. No instructions. +metadata: {"nanobot": {"always": true}} +--- +# Cache alignment padding + +This section aligns the prompt prefix to a KV-cache block boundary of +the serving stack. It contains no instructions. Ignore it. + +sensor socket buffer report channel record sensor thread. +buffer signal filter module state filter state index. +sensor system metric index report sensor state metric. +system module metric window metric thread sensor packet. +sensor thread metric index metric state table handle. +thread value state metric system handle window window. +metric state table buffer timer channel report index. +module filter state system record state filter thread. +buffer buffer module module index thread table thread. +window channel state timer module metric handle signal. +sensor metric buffer channel value metric handle system. +report channel signal packet window filter value module. +timer record record sensor window index record metric. +sensor state sensor record timer signal record metric. +state socket signal sensor report module window report. +metric socket timer packet record state filter table. +thread value state filter table index filter handle. +record module report signal packet report channel metric. +report timer socket metric system metric channel record. +channel packet filter packet signal system index signal. +value filter state sensor value report thread value. +report metric report index metric window sensor metric. +record index socket sensor channel socket filter report. +socket filter system socket window module table module. +window thread sensor buffer module window signal window. +channel sensor signal state thread filter value record. +value index sensor timer index packet table system. +value state index handle window system metric thread. +buffer metric handle socket report system index table. +metric state thread handle thread sensor timer filter. +sensor filter buffer socket channel buffer state thread. +timer record state handle filter window packet value. +signal buffer sensor handle table thread module state. +packet sensor timer report signal sensor record system. +value metric packet value buffer record system signal. +filter packet report record system packet timer module. +sensor metric packet module window signal state system. +module window channel report packet sensor module signal. +signal signal state socket thread value state index. +index handle thread window record filter packet module. +metric timer module packet index state signal record. +socket handle sensor handle buffer metric filter system. +packet state value signal module value packet value. +channel table value system index sensor thread value. +handle sensor thread state module system state filter. +table buffer filter sensor metric system timer handle. +socket channel timer buffer channel index module state. +state packet index socket state metric channel metric. +filter buffer socket value system packet report handle. +sensor value window thread packet handle timer signal. +packet socket thread packet packet system window metric. +socket filter state module socket packet system metric. +signal channel packet buffer table buffer timer signal. +module state thread value state signal timer table. +channel thread channel packet packet sensor socket channel. +record socket thread report metric table filter channel. +channel table metric handle report metric window system. +module filter buffer sensor value system handle table. +system metric state report buffer table timer index. +buffer handle index report report signal signal filter. +sensor report table socket buffer value filter sensor. +signal window sensor sensor window value table state. +index module window packet buffer thread window thread. +packet filter buffer window buffer state report signal. +filter system handle report metric metric thread thread. +filter handle packet index system sensor index thread. +filter filter timer system table table module module. +handle timer thread timer record state metric signal. +channel record filter timer window metric handle value. +index buffer record socket socket state packet record. +buffer filter handle socket sensor packet sensor module. +buffer socket timer index handle value system thread. +value module index metric sensor packet record value. +channel packet handle buffer buffer report module metric. +state signal sensor module state value timer table. +filter value system buffer signal table system channel. +system window system thread signal handle thread table. +filter module table timer record socket timer index. +packet metric report filter index metric timer sensor. +system packet channel report window record report packet. diff --git a/stacklets/core/famstack-api.py b/stacklets/core/famstack-api.py index b841a3ba..fe231130 100644 --- a/stacklets/core/famstack-api.py +++ b/stacklets/core/famstack-api.py @@ -52,6 +52,9 @@ # ... and the way it changes one. A page is handed over whole and the # reply names what the edit did, so a rewrite that drops items says so. ["memory", "write"], + # One item on a list, by name. The store matches the item and keeps + # the rest of the page intact, so a tick cannot lose other state. + ["memory", "list-edit"], # What changed, when, and who did it. The vault is a git repo and has # always known this; nothing read it back until now. ["memory", "history"], diff --git a/stacklets/memory/cli/list-edit.py b/stacklets/memory/cli/list-edit.py new file mode 100644 index 00000000..583dd497 --- /dev/null +++ b/stacklets/memory/cli/list-edit.py @@ -0,0 +1,229 @@ +"""stack memory list-edit — change one item on a list page. + +The item verb for lists. `stack memory write` hands a page over whole. +That is right for a restructure and wrong for one tick: a whole-page +write can lose state the caller never meant to touch (measured +2026-09-15 in the agent lab: rewrites drop `[x]` marks, and prompt +rules do not reliably repair them). Here the caller only names the +item. Matching, state preservation, and the commit happen at the +store, so an item change cannot damage the rest of the page. + +Matching is exact first, then substring, both case-insensitive. An +ambiguous name is an answer, not a guess: the reply names the +candidates and the caller retries with more of the item's own words. + +Exit codes: 0 = changed, or an honest no-op ("already ticked"); +1 = instructive refusal (unknown page, no match, ambiguous match); +2 = usage error. +""" + +HELP = "Change one item on a family list page" + +import argparse +import re +import sys +from pathlib import Path + +# Sibling-import pattern used by the other memory CLI plugins: plugins +# run on the host's stdlib-only python3, so we manipulate sys.path +# instead of relying on a package install. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from lib import update_memory # noqa: E402 + +from stack.links import go_page, public # noqa: E402 +from write import _commit_message # noqa: E402 + +_BOX = re.compile(r"^(\s*)- \[([ xX])\]\s*(.+?)\s*$") + +_OPS = ("add", "tick", "untick", "remove", "clear-done", "reset") + + +def apply_list_edit(text: str, op: str, item: str, + section: str | None = None) -> tuple[str, str, str]: + """Apply one item operation, or one bulk operation, to a page's text. + + Pure transform, so the behavior is testable without the store. + Returns (new_text, sentence, kind). kind is one of: + changed the page text changed; sentence says what happened + noop nothing to do; sentence says why ("already ticked") + refuse the operation cannot proceed; sentence instructs the caller + """ + lines = text.splitlines() + boxes = [(i, m.group(2).lower() == "x", m.group(3)) + for i, ln in enumerate(lines) if (m := _BOX.match(ln))] + + # Bulk operations: the everyday list flows. "We bought everything" + # is clear-done; a recurring list starts the week with reset. Both + # name every affected item, so the reply stays checkable. + if op == "clear-done": + done = [(i, t) for i, d, t in boxes if d] + if not done: + return text, "nothing is ticked; the list is already clear", "noop" + for i, _ in reversed(done): + del lines[i] + names = "; ".join(t for _, t in done) + return ("\n".join(lines) + "\n", + f"REMOVED {len(done)}: {names}", "changed") + if op == "reset": + done = [(i, t) for i, d, t in boxes if d] + if not done: + return text, "nothing is ticked; the list is already open", "noop" + for i, _ in done: + lines[i] = re.sub(r"- \[[xX]\]", "- [ ]", lines[i], count=1) + names = "; ".join(t for _, t in done) + return ("\n".join(lines) + "\n", + f"reopened {len(done)}: {names}", "changed") + + if op == "add": + if any(item.casefold() == t.casefold() for _, _, t in boxes): + return text, f"'{item}' is already on the list", "noop" + insert_at = len(lines) + if section: + heads = [i for i, ln in enumerate(lines) + if ln.lstrip().startswith("#") + and ln.lstrip("# ").strip().casefold() == section.casefold()] + if heads: + insert_at = heads[0] + 1 + while (insert_at < len(lines) + and not lines[insert_at].lstrip().startswith("#")): + insert_at += 1 + # Keep the blank line that separates sections below the item. + while insert_at > heads[0] + 1 and not lines[insert_at - 1].strip(): + insert_at -= 1 + lines.insert(insert_at, f"- [ ] {item}") + return "\n".join(lines) + "\n", f"added 1: {item}", "changed" + + matches = [b for b in boxes if item.casefold() == b[2].casefold()] + if not matches: + matches = [b for b in boxes if item.casefold() in b[2].casefold()] + if not matches: + open_items = "; ".join(t for _, done, t in boxes if not done) + return text, (f"no item matching '{item}'. " + f"Open items: {open_items or '(none)'}"), "refuse" + if len(matches) > 1: + names = "; ".join(t for _, _, t in matches) + return text, (f"'{item}' is ambiguous, it matches: {names}. " + "Name the item more exactly."), "refuse" + + idx, done, name = matches[0] + if op == "tick": + if done: + return text, f"'{name}' is already ticked", "noop" + lines[idx] = lines[idx].replace("- [ ]", "- [x]", 1) + sentence = f"ticked off 1: {name}" + elif op == "untick": + if not done: + return text, f"'{name}' is already open", "noop" + lines[idx] = re.sub(r"- \[[xX]\]", "- [ ]", lines[idx], count=1) + sentence = f"reopened 1: {name}" + else: # remove + del lines[idx] + sentence = f"REMOVED 1: {name}" + return "\n".join(lines) + "\n", sentence, "changed" + + +_VERB = {"add": "added", "tick": "ticked off", "untick": "reopened", + "remove": "REMOVED"} + + +def apply_list_edits(text: str, op: str, items: list[str], + section: str | None = None) -> tuple[str, str, str]: + """Apply one op to several items, in one page write. + + Each item runs against the page as the previous item left it, so a + batch is exactly the same result as the calls run in order, at one + commit. The sentence groups the items that changed and appends the + ones that did not (already ticked, no match), so nothing is hidden. + kind is "changed" if any item changed, else "noop" if all were + no-ops, else "refuse". + """ + if len(items) == 1: + return apply_list_edit(text, op, items[0], section) + changed: list[str] = [] + notes: list[str] = [] + kinds: set[str] = set() + cur = text + for it in items: + cur, sentence, kind = apply_list_edit(cur, op, it, section) + kinds.add(kind) + if kind == "changed": + changed.append(sentence.split(": ", 1)[1]) + else: + notes.append(sentence) + parts: list[str] = [] + if changed: + parts.append(f"{_VERB[op]} {len(changed)}: {'; '.join(changed)}") + parts.extend(notes) + sentence = "; ".join(parts) + kind = "changed" if changed else ("refuse" if "refuse" in kinds else "noop") + return cur, sentence, kind + + +def run(args, stacklet, config): + parser = argparse.ArgumentParser(prog="stack memory list-edit", + add_help=False) + parser.add_argument("page") + parser.add_argument("--op", required=True, choices=list(_OPS)) + parser.add_argument("--item", action="append", default=[]) + parser.add_argument("--section", default=None) + parser.add_argument("--by", default="someone") + try: + ns = parser.parse_args(list(args or [])) + except SystemExit: + print(parser.format_usage().strip()) + sys.exit(2) + + repo_path = ns.page.strip().removeprefix("vault/").lstrip("/") + if not repo_path.endswith(".md"): + print(f"{repo_path!r} is not a page (expected a .md path)") + sys.exit(2) + actor = ns.by.strip().split(":")[0].lstrip("@") or "someone" + items = [i.strip() for i in ns.item if i.strip()] + if not items and ns.op not in ("clear-done", "reset"): + print("--item must name the item") + sys.exit(2) + + # The transform runs against the canonical page Forgejo hands back, + # not a local clone, so the match cannot lose a race with another + # writer. Refusals and no-ops return the page unchanged; an + # unchanged page does not commit. + outcome: dict = {} + + def _apply(prior: str) -> str: + if not (prior or "").strip(): + outcome["sentence"], outcome["kind"] = f"no such page: {repo_path}", "refuse" + return prior or "" + if ns.op in ("clear-done", "reset"): + new, sentence, kind = apply_list_edit(prior, ns.op, "", ns.section) + else: + new, sentence, kind = apply_list_edits(prior, ns.op, items, ns.section) + outcome["sentence"], outcome["kind"] = sentence, kind + return new + + result = update_memory( + config, repo_path, _apply, actor=actor, + message=lambda before, after: _commit_message( + actor, repo_path, outcome.get("sentence", "list edit")), + ) + if "error" in result: + return {"error": f"could not edit {repo_path}: {result['error']}"} + + sentence, kind = outcome.get("sentence", ""), outcome.get("kind", "") + if kind == "refuse": + print(sentence) + sys.exit(1) + if kind == "noop" or not result.get("committed"): + print(sentence or "no change") + return {"ok": True, "committed": False, "path": repo_path} + + # Same phrasing as `memory write`: the mirror lag is a delay, never + # a doubt, or the model re-runs the edit and duplicates it. The link + # line names the touched page in the wiki, for the reply's Sources. + lag = "" if result.get("mirrored") else "\n The wiki and the vault mount catch up shortly." + home_url = (config or {}).get("home_url", "") + url = public(go_page(repo_path), f"{home_url}/go" if home_url else "") + link = f"\n {url}" if url else "" + print(f"{sentence} (by {actor}){link}{lag}") + return {"ok": True, "committed": True, "path": repo_path, + "by": actor, "summary": sentence} diff --git a/stacklets/memory/cli/write.py b/stacklets/memory/cli/write.py index dfe6016e..f8da7bf8 100644 --- a/stacklets/memory/cli/write.py +++ b/stacklets/memory/cli/write.py @@ -60,6 +60,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from lib import update_memory # noqa: E402 +from stack.frontmatter import FrontmatterError, parse as parse_frontmatter # noqa: E402 +from stack.links import go_page, public # noqa: E402 from stack.list_doc import diff # noqa: E402 from stack.page_patch import apply_edits # noqa: E402 @@ -73,6 +75,25 @@ " e.g. stack memory write family/camping/todos.md --by marge") +def _check_frontmatter(before: str, after: str, page: str) -> None: + """Reject a write that breaks or drops the page's frontmatter.""" + if after.lstrip().startswith("---") and not after.startswith("---\n"): + raise ValueError( + f"{page} must start with '---' at column one; the write begins " + "with whitespace before the frontmatter. Re-send the page with " + "the frontmatter block exactly as you read it.") + if before.startswith("---\n") and not after.startswith("---\n"): + raise ValueError( + f"{page} has frontmatter and this write drops it. Keep the " + "frontmatter block exactly as you read it, then the body.") + try: + parse_frontmatter(after) + except FrontmatterError as e: + raise ValueError( + f"{page}: frontmatter does not parse ({e}). Keep the " + "frontmatter block exactly as you read it.") from e + + def run(args, stacklet, config): argv = list(args or []) actor, argv = _opt(argv, "--by") @@ -123,6 +144,26 @@ def _replace(prior: str) -> str: seen["before"] = prior or "" after = apply_edits(prior or "", edits) if as_patch else content seen["after"] = after if after.endswith("\n") else after + "\n" + # The wiki refuses a page whose frontmatter does not parse, so a + # bad write silently drops the page from the family's view. This + # seam sees every mutation, so the check lives here. ValueError + # becomes the error message the caller relays to the model. + _check_frontmatter(seen["before"], seen["after"], repo_path) + # A whole-page rewrite of a list is a restructure. A restructure + # never legitimately reopens or removes an item (measured + # 2026-09-15: rewrites drop [x] marks, and prompt rules do not + # reliably repair them). Targeted edits go through --patch or + # the list-edit verb, which stay allowed. + if not as_patch and repo_path.endswith("todos.md"): + change = diff(seen["before"], seen["after"]) + lost = list(change.reopened) + list(change.removed) + if lost: + raise ValueError( + f"this rewrite reopens or removes items: " + f"{'; '.join(lost)}. A restructure keeps every item " + "and every [x]. Resend the full page with them " + "unchanged. To reopen or remove an item on purpose, " + "use list_edit.") # A preview still wants the *current* page to compare against, so it # takes the same trip and then hands back what was already there: # an unchanged file is a no-op, and a no-op does not commit. @@ -165,7 +206,11 @@ def _replace(prior: str) -> str: # again, and the second write is a duplicate nobody asked for. mirrored = bool(result.get("mirrored")) lag = "" if mirrored else "\n The wiki and the vault mount catch up shortly." - print(f"Wrote {repo_path} (by {actor})\n {told}{lag}") + # The touched page's wiki link, so the reply can cite it. + home_url = (config or {}).get("home_url", "") + url = public(go_page(repo_path), f"{home_url}/go" if home_url else "") + link = f"\n {url}" if url else "" + print(f"Wrote {repo_path} (by {actor})\n {told}{link}{lag}") return { "ok": True, "committed": True, "path": repo_path, "by": actor, "summary": told, "mirrored": mirrored, diff --git a/tests/stacklets/conftest.py b/tests/stacklets/conftest.py index 0e053ad4..f69061fd 100644 --- a/tests/stacklets/conftest.py +++ b/tests/stacklets/conftest.py @@ -73,6 +73,12 @@ class ToolLoader: def discover(self): return [] + class ToolRegistry: + # The seam tool_trim patches: register() is where a tool + # enters the model's list, so the trim filters there. + def register(self, tool): + pass + class GrepTool: async def execute(self, *args, **kwargs): return "stock grep" @@ -126,8 +132,16 @@ async def _on_room_invite(self, room, event): # Stock nanobot: join, say nothing. self.joined.append(room.room_id) - async def _handle_message(self, **kwargs): - self.handled.append(kwargs) + async def _handle_message(self, sender_id=None, chat_id=None, + content=None, media=None, metadata=None, + session_key=None, is_dm=False): + # Real nanobot's positional signature (channels/base.py), so + # the thread_session shim can wrap it and forward the + # session_key it derives from the thread root. + self.handled.append(dict( + sender_id=sender_id, chat_id=chat_id, content=content, + media=media, metadata=metadata, session_key=session_key, + is_dm=is_dm)) mods: dict[str, types.ModuleType] = {} @@ -145,9 +159,10 @@ def mod(name, **attrs): mod("nanobot.agent.tools") mod("nanobot.agent.tools.base", Tool=Tool, tool_parameters=tool_parameters) mod("nanobot.agent.tools.schema", - StringSchema=_Schema, IntegerSchema=_Schema, + StringSchema=_Schema, IntegerSchema=_Schema, ArraySchema=_Schema, tool_parameters_schema=tool_parameters_schema) mod("nanobot.agent.tools.loader", ToolLoader=ToolLoader) + mod("nanobot.agent.tools.registry", ToolRegistry=ToolRegistry) mod("nanobot.agent.tools.search", GrepTool=GrepTool) mod("nanobot.agent.tools.filesystem", WriteFileTool=WriteFileTool, EditFileTool=EditFileTool) diff --git a/tests/stacklets/test_agent_runtime_shims.py b/tests/stacklets/test_agent_runtime_shims.py index 33f4f7b8..f9a31d47 100644 --- a/tests/stacklets/test_agent_runtime_shims.py +++ b/tests/stacklets/test_agent_runtime_shims.py @@ -25,7 +25,8 @@ SHIMMED_MODULES = ("sitecustomize", "brief", "lean_state", "memory_tool", "person_tool", "history_tool", "grep_tool", - "name_trigger", "thread_trigger", "join_greeting", "vault_write") + "name_trigger", "thread_trigger", "join_greeting", "vault_write", + "list_tool", "tool_trim", "thread_session") # The stub nanobot itself lives in conftest as `nanobot_stub`, shared with @@ -72,7 +73,7 @@ def test_vault_tools_are_registered(nanobot): """ mods = nanobot() assert _discovered(mods) == {"MemorySearchTool", "MemoryPersonTool", - "MemoryHistoryTool"} + "MemoryHistoryTool", "ListEditTool"} def test_asking_the_vault_when_something_happened_is_a_tool(nanobot): @@ -272,7 +273,7 @@ def test_a_moved_symbol_does_not_take_the_others_down(nanobot): mods = nanobot(drop="nanobot.agent.tools.search.GrepTool") assert _discovered(mods) == {"MemorySearchTool", "MemoryPersonTool", - "MemoryHistoryTool"} + "MemoryHistoryTool", "ListEditTool"} def test_the_stub_can_actually_express_a_detached_shim(nanobot): diff --git a/tests/stacklets/test_memory_list_edit.py b/tests/stacklets/test_memory_list_edit.py new file mode 100644 index 00000000..eea9cf67 --- /dev/null +++ b/tests/stacklets/test_memory_list_edit.py @@ -0,0 +1,173 @@ +"""`stack memory list-edit` — one item changes, nothing else moves. + +The verb exists because whole-page rewrites lose state the caller never +meant to touch: in the agent lab, a category restructure dropped an item's +`[x]` mark, and prompt rules did not reliably repair it. The contract under +test: the caller only names an item; the store finds it, changes exactly +that line, and answers with what happened. Ambiguity and misses are +instructive refusals, never guesses, because a wrong guess strikes a +family member's item silently. + +These tests drive the pure transform through its public shape +(text in, text out, sentence, kind) — the same calls the CLI makes. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(_REPO_ROOT / "lib")) +sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "memory")) +sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "memory" / "cli")) + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +list_edit = _load( + "memory_cli_list_edit", + _REPO_ROOT / "stacklets" / "memory" / "cli" / "list-edit.py", +) + +PAGE = """--- +type: note +date: 2026-09-14 +--- + +# Shopping list + +## Dairy + +- [ ] oat milk +- [x] coffee beans + +## Household + +- [ ] dish soap +""" + + +def test_tick_by_partial_name_changes_only_that_line(): + new, sentence, kind = list_edit.apply_list_edit(PAGE, "tick", "oat") + assert kind == "changed" + assert sentence == "ticked off 1: oat milk" + assert "- [x] oat milk" in new + # Everything else is untouched, byte for byte. + assert new.replace("- [x] oat milk", "- [ ] oat milk") == PAGE + + +def test_ambiguous_name_is_refused_and_names_the_candidates(): + page = PAGE + "- [ ] oat cookies\n" + new, sentence, kind = list_edit.apply_list_edit(page, "tick", "oat") + assert kind == "refuse" + assert new == page + assert "oat milk" in sentence and "oat cookies" in sentence + + +def test_exact_match_wins_over_substring(): + page = PAGE + "- [ ] milk\n" + new, sentence, kind = list_edit.apply_list_edit(page, "tick", "milk") + assert kind == "changed" + assert sentence == "ticked off 1: milk" + assert "- [ ] oat milk" in new + + +def test_unknown_item_is_refused_and_lists_what_is_open(): + new, sentence, kind = list_edit.apply_list_edit(PAGE, "tick", "bananas") + assert kind == "refuse" + assert new == PAGE + assert "oat milk" in sentence and "dish soap" in sentence + # Done items are not offered as candidates for a tick. + assert "coffee beans" not in sentence + + +def test_add_lands_in_the_named_section(): + new, sentence, kind = list_edit.apply_list_edit( + PAGE, "add", "butter", section="Dairy") + assert kind == "changed" + assert sentence == "added 1: butter" + dairy = new.split("## Dairy")[1].split("## Household")[0] + assert "- [ ] butter" in dairy + + +def test_add_duplicate_is_an_honest_noop(): + new, sentence, kind = list_edit.apply_list_edit(PAGE, "add", "Oat Milk") + assert kind == "noop" + assert new == PAGE + assert "already on the list" in sentence + + +def test_remove_names_what_it_destroyed(): + new, sentence, kind = list_edit.apply_list_edit(PAGE, "remove", "dish soap") + assert kind == "changed" + assert sentence == "REMOVED 1: dish soap" + assert "dish soap" not in new + + +def test_clear_done_removes_only_ticked_items_and_names_them(): + page = PAGE.replace("- [ ] dish soap", "- [x] dish soap") + new, sentence, kind = list_edit.apply_list_edit(page, "clear-done", "") + assert kind == "changed" + assert sentence == "REMOVED 2: coffee beans; dish soap" + assert "coffee beans" not in new and "dish soap" not in new + assert "- [ ] oat milk" in new + + +def test_clear_done_on_a_clean_list_is_a_noop(): + page = PAGE.replace("- [x] coffee beans", "- [ ] coffee beans") + new, sentence, kind = list_edit.apply_list_edit(page, "clear-done", "") + assert kind == "noop" + assert new == page + + +def test_reset_reopens_every_ticked_item(): + new, sentence, kind = list_edit.apply_list_edit(PAGE, "reset", "") + assert kind == "changed" + assert sentence == "reopened 1: coffee beans" + assert "- [x]" not in new + + +def test_untick_reports_a_reopening(): + new, sentence, kind = list_edit.apply_list_edit(PAGE, "untick", "coffee") + assert kind == "changed" + assert sentence == "reopened 1: coffee beans" + assert "- [ ] coffee beans" in new + + +def test_batch_add_is_one_change_naming_every_item(): + new, sentence, kind = list_edit.apply_list_edits( + PAGE, "add", ["butter", "flour"]) + assert kind == "changed" + assert sentence == "added 2: butter; flour" + assert "- [ ] butter" in new and "- [ ] flour" in new + + +def test_batch_tick_across_the_page(): + new, sentence, kind = list_edit.apply_list_edits( + PAGE, "tick", ["oat milk", "dish soap"]) + assert kind == "changed" + assert sentence == "ticked off 2: oat milk; dish soap" + assert "- [x] oat milk" in new and "- [x] dish soap" in new + + +def test_batch_reports_changed_and_misses_together(): + new, sentence, kind = list_edit.apply_list_edits( + PAGE, "tick", ["oat milk", "bananas"]) + assert kind == "changed" + assert "ticked off 1: oat milk" in sentence + assert "no item matching 'bananas'" in sentence + assert "- [x] oat milk" in new + + +def test_batch_of_one_matches_the_single_form(): + a = list_edit.apply_list_edits(PAGE, "tick", ["oat milk"]) + b = list_edit.apply_list_edit(PAGE, "tick", "oat milk") + assert a == b diff --git a/tests/stacklets/test_memory_write_cli.py b/tests/stacklets/test_memory_write_cli.py index 8022df6c..f808c257 100644 --- a/tests/stacklets/test_memory_write_cli.py +++ b/tests/stacklets/test_memory_write_cli.py @@ -120,19 +120,19 @@ def test_a_rewrite_replaces_the_page_and_says_what_it_did(store, tmp_path): assert store.page.count("- [x] Wetter checken") == 1 -def test_a_rewrite_that_drops_an_item_names_it_in_full(store, tmp_path): - """The failure this whole path exists to catch. +def test_a_rewrite_that_drops_an_item_is_refused(store, tmp_path): + """A whole-page rewrite must never lose a list item. - An edit that loses items must never render as an ordinary success, - because the caller relays this sentence to the family verbatim. + Removals go through list_edit, so a write that would drop an item is + refused with the item named, rather than committed and reported. This + is the guard that makes a restructure safe to hand a model. """ result = _run(store, "# Camping\n\n- [ ] Wetter checken\n", tmp=tmp_path) - assert result["destructive"] is True - assert result["removed"] == ["Kühlbox mitbringen"] - assert result["summary"].startswith("REMOVED"), ( - "a loss has to lead the sentence, not trail it" - ) + assert "error" in result + assert "Kühlbox mitbringen" in result["error"] + assert "list_edit" in result["error"] + assert store.commits == [], "a refused write commits nothing" # ── patching a page ────────────────────────────────────────────────────── @@ -267,23 +267,20 @@ def test_a_page_that_is_not_a_list_still_says_something_true(store, tmp_path): ) -def test_a_long_description_moves_below_the_subject_intact(store, tmp_path): +def test_a_long_description_moves_below_the_subject_intact(): """A removal names every item, so it is the case that overflows. Truncating would drop exactly the detail worth keeping, so the long form moves into the commit body, which git and Forgejo both show. """ - store.page = ("# Camping\n\n" - + "".join(f"- [ ] Ausruestungsgegenstand Nummer {n}\n" - for n in range(1, 6))) - result = _run(store, "# Camping\n\n- [ ] Ausruestungsgegenstand Nummer 1\n", - tmp=tmp_path) + told = ("REMOVED 5: " + "; ".join( + f"Ausruestungsgegenstand Nummer {n}" for n in range(1, 6))) + message = write_cli._commit_message("marge", "family/camping/todos.md", told) - subject, _, body = store.last_subject.partition("\n\n") + subject, _, body = message.partition("\n\n") assert len(subject) <= 72, "the subject line stays readable" assert "family/camping/todos.md" in subject - for gone in result["removed"]: - assert gone in body, "every lost item survives in the body" + assert told in body, "every lost item survives in the body" def test_a_path_that_is_not_a_page_is_refused(store, tmp_path): diff --git a/tools/agent-lab/README.md b/tools/agent-lab/README.md new file mode 100644 index 00000000..04fcab2c --- /dev/null +++ b/tools/agent-lab/README.md @@ -0,0 +1,37 @@ +# agent-lab + +A validation harness for agent experiments. It measures the LLM serving +path in isolation. It sends only synthetic data to the oMLX endpoint. +It does not touch stack services, Matrix, or family data. + +The harness uses only the Python standard library. It reads endpoint +defaults from `stack.toml [ai]`. Flags override the defaults. + +## Commands + +``` +python3 tools/agent-lab/lab.py probe +python3 tools/agent-lab/lab.py cache --prefix-words 3000 --history-turns 6 +``` + +- `probe`: sends one small completion two times. It shows the raw usage + object. Use it to find the cache fields that oMLX reports. +- `cache`: measures prefix-cache behavior in four phases: cold, warm, + warm2, mutate. The mutate phase rewrites one early message in place. + This simulates the `lean_state` rewrite in the agent runtime. + Expected result: warm turns have a low TTFT. The mutate turn has a + TTFT near the cold value. + +## Output + +Each run writes two artifacts: + +- A raw JSON record in `tools/agent-lab/results/`. +- A summary entry in `docs/design/agent/agent-improvement-log.md`. + Use `--no-log` to skip the log entry. Use `--note` to add context. + +## Rules + +- Synthetic data only. Do not point the harness at family content. +- The endpoint runs on the production Mac. Keep runs short. Do not run + the harness while the family uses the agent. diff --git a/tools/agent-lab/lab.py b/tools/agent-lab/lab.py new file mode 100644 index 00000000..76268936 --- /dev/null +++ b/tools/agent-lab/lab.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +"""Validation harness for agent experiments. + +This tool measures the LLM serving path in isolation. It sends only +synthetic data to the oMLX endpoint. It does not touch stack services, +Matrix, or family data. + +Subcommands: + probe Send one small completion. Show the raw usage object. + cache Measure KV-cache behavior: cold, append-only, and mutated prompts. + +Each run writes a raw JSON result to tools/agent-lab/results/ and +appends a summary to docs/design/agent/agent-improvement-log.md. +The tool uses only the Python standard library. +""" + +import argparse +import datetime as dt +import json +import random +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +RESULTS_DIR = Path(__file__).resolve().parent / "results" +LOG_PATH = REPO_ROOT / "docs" / "design" / "agent" / "agent-improvement-log.md" + +# oMLX serves Qwen models. The stack disables thinking for all calls. +# The harness does the same so measurements match production behavior. +NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}} + +# Fixed vocabulary for deterministic filler text. The seed is fixed, so +# every run builds byte-identical prompts. Byte-identical prompts are +# necessary to compare cache behavior across runs. +_WORDS = ( + "system module sensor value record window channel signal filter state " + "buffer packet metric report table index handle timer socket thread " + "queue cache token prompt model layer weight block page memory disk" +).split() + + +def _config_defaults() -> dict: + """Read endpoint defaults from stack.toml. Flags override these.""" + import tomllib + + path = REPO_ROOT / "stack.toml" + if not path.exists(): + return {} + ai = tomllib.loads(path.read_text()).get("ai", {}) + return { + "url": ai.get("openai_url"), + "key": ai.get("openai_key"), + "model": ai.get("default"), + } + + +def filler_text(words: int, seed: int) -> str: + """Build deterministic text with approximately the given word count.""" + rng = random.Random(seed) + out = [] + while len(out) < words: + sentence = [rng.choice(_WORDS) for _ in range(rng.randint(6, 12))] + sentence[0] = sentence[0].capitalize() + out.extend(sentence) + out[-1] += "." + return " ".join(out[:words]) + "." + + +class Endpoint: + """One OpenAI-compatible chat endpoint.""" + + def __init__(self, url: str, key: str, model: str, timeout: float): + self.url = url.rstrip("/") + self.key = key + self.model = model + self.timeout = timeout + + def _request(self, path: str, body: dict | None = None) -> urllib.request.Request: + req = urllib.request.Request( + self.url + path, + data=json.dumps(body).encode() if body is not None else None, + method="POST" if body is not None else "GET", + ) + req.add_header("Authorization", f"Bearer {self.key}") + req.add_header("Content-Type", "application/json") + return req + + def models(self) -> dict: + with urllib.request.urlopen(self._request("/models"), timeout=10) as resp: + return json.load(resp) + + def chat(self, messages: list[dict], max_tokens: int, stream: bool) -> dict: + """Send one chat completion. Return timing and usage data. + + Timing fields: + t_first_chunk time to the first SSE chunk (stream only) + t_first_token time to the first content delta (stream only) + t_total time to the full response + """ + body: dict = { + "model": self.model, + "messages": messages, + "max_tokens": max_tokens, + "temperature": 0, + **NO_THINKING, + } + if stream: + body["stream"] = True + body["stream_options"] = {"include_usage": True} + + start = time.monotonic() + req = self._request("/chat/completions", body) + result: dict = { + "t_first_chunk": None, + "t_first_token": None, + "t_total": None, + "usage": None, + "text": "", + "error": None, + } + try: + with urllib.request.urlopen(req, timeout=self.timeout) as resp: + if not stream: + data = json.load(resp) + result["t_total"] = time.monotonic() - start + result["usage"] = data.get("usage") + result["text"] = data["choices"][0]["message"].get("content") or "" + return result + for raw in resp: + line = raw.decode("utf-8", "replace").strip() + if not line.startswith("data:"): + continue + payload = line[5:].strip() + if payload == "[DONE]": + break + chunk = json.loads(payload) + now = time.monotonic() - start + if result["t_first_chunk"] is None: + result["t_first_chunk"] = now + if chunk.get("usage"): + result["usage"] = chunk["usage"] + for choice in chunk.get("choices", []): + delta = choice.get("delta", {}) + content = delta.get("content") + if content: + if result["t_first_token"] is None: + result["t_first_token"] = now + result["text"] += content + result["t_total"] = time.monotonic() - start + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + result["t_total"] = time.monotonic() - start + return result + + +def build_conversation(prefix_words: int, history_turns: int, seed: int) -> list[dict]: + """Build a synthetic conversation: one large system prompt plus history. + + The shape follows the production agent: a stable ~10K-token prefix and + a multi-turn transcript. All content is synthetic. + """ + messages = [{"role": "system", "content": filler_text(prefix_words, seed)}] + for i in range(history_turns): + messages.append( + {"role": "user", "content": f"Question {i}: " + filler_text(60, seed + 10 + i)} + ) + messages.append( + {"role": "assistant", "content": f"Answer {i}: " + filler_text(80, seed + 100 + i)} + ) + return messages + + +def _phase_row(name: str, r: dict) -> dict: + usage = r.get("usage") or {} + return { + "phase": name, + "ttft_s": round(r["t_first_token"], 3) if r["t_first_token"] else None, + "total_s": round(r["t_total"], 3) if r["t_total"] else None, + "usage": usage, + "error": r.get("error"), + } + + +def cmd_probe(ep: Endpoint, args) -> dict: + """Check the endpoint. Report the raw usage object and its cache fields.""" + models = ep.models() + served = [m.get("id") for m in models.get("data", [])] + print(f"endpoint: {ep.url}") + print(f"served models: {served}") + msgs = [ + {"role": "system", "content": "You are a test endpoint. Answer in one short sentence."}, + {"role": "user", "content": "Say the word ready."}, + ] + first = ep.chat(msgs, max_tokens=16, stream=args.stream) + second = ep.chat(msgs, max_tokens=16, stream=args.stream) + print("first call usage:", json.dumps(first.get("usage"), indent=2)) + print("second call usage:", json.dumps(second.get("usage"), indent=2)) + return { + "served_models": served, + "first": _phase_row("first", first), + "second": _phase_row("second (identical prompt)", second), + } + + +def cmd_cache(ep: Endpoint, args) -> dict: + """Measure prefix-cache behavior in four phases. + + cold: full synthetic conversation, first contact. + warm: the same conversation plus the real reply and one new turn. + warm2: one more append-only turn. + mutate: the warm2 conversation with one early message rewritten, + plus one new turn. This simulates the lean_state rewrite. + Expected result: warm and warm2 have a low TTFT. mutate has a TTFT + near the cold value, because the prefix diverges early. + """ + msgs = build_conversation(args.prefix_words, args.history_turns, args.seed) + rows = [] + + def turn(question: str) -> dict: + msgs.append({"role": "user", "content": question}) + r = ep.chat(msgs, max_tokens=args.max_tokens, stream=args.stream) + msgs.append({"role": "assistant", "content": r["text"] or "(empty)"}) + return r + + print("phase cold ...") + rows.append(_phase_row("cold", turn("Summarize the topic of this text in one sentence."))) + print("phase warm ...") + rows.append(_phase_row("warm", turn("Give one more sentence about it."))) + print("phase warm2 ...") + rows.append(_phase_row("warm2", turn("Give a final short sentence."))) + + # Rewrite one early assistant message in place. This is the same + # operation lean_state applies to prior tool results. + msgs[2]["content"] = "[prior result of tool(args); re-run for the current value]" + print("phase mutate ...") + rows.append(_phase_row("mutate", turn("Give one more short sentence."))) + + for row in rows: + print(json.dumps(row)) + return {"phases": rows, "prefix_words": args.prefix_words, "history_turns": args.history_turns} + + +def build_scripted_history(seed: int, prefix_words: int, turns: int) -> list[list[dict]]: + """Build a scripted tool-using conversation, one block list per turn. + + Each turn has the shape nanobot produces: user question, assistant + tool call, tool result, assistant answer. All content is synthetic. + The script is deterministic, so both replay arms see identical bytes. + """ + blocks = [[{"role": "system", "content": filler_text(prefix_words, seed)}]] + for t in range(turns): + call_id = f"call_{t}" + args = json.dumps({"query": f"topic {t} " + " ".join(filler_text(4, seed + t).split())}) + blocks.append([ + {"role": "user", "content": f"Question {t}: " + filler_text(30, seed + 10 + t)}, + {"role": "assistant", "content": "", "tool_calls": [{ + "id": call_id, "type": "function", + "function": {"name": "memory_search", "arguments": args}, + }]}, + {"role": "tool", "tool_call_id": call_id, + "content": f"Result {t}: " + filler_text(150, seed + 100 + t)}, + {"role": "assistant", "content": f"Answer {t}: " + filler_text(60, seed + 200 + t)}, + ]) + return blocks + + +def cmd_replay(ep: Endpoint, args) -> dict: + """Replay the real lean_state transform against an append-only arm. + + Arm A builds each turn's payload with lean_messages() from the agent + runtime, as production does. Arm B sends the same history verbatim. + The two arms use different seeds, so they do not share cache blocks. + Expected result: arm B hits the prefix cache from turn 2 on. Arm A + misses it on every turn after the first tool result. + """ + sys.path.insert(0, str(REPO_ROOT / "stacklets" / "agent" / "runtime")) + from lean_state import lean_messages + + arms = {} + for arm, (transform, seed) in { + "lean_state (production)": (lean_messages, args.seed), + "append-only": (lambda m: m, args.seed + 1000), + }.items(): + blocks = build_scripted_history(seed, args.prefix_words, args.turns) + rows = [] + print(f"arm: {arm}") + for t in range(1, args.turns + 1): + # The payload at turn t: system + t-1 full turns + this turn's + # user message. This is the state at the start of a turn. + history = [m for block in blocks[:t] for m in block] + payload = transform(history + [blocks[t][0]]) + r = ep.chat(payload, max_tokens=args.max_tokens, stream=args.stream) + rows.append(_phase_row(f"turn {t}", r)) + print(json.dumps(rows[-1])) + arms[arm] = rows + return {"arms": arms, "prefix_words": args.prefix_words, "turns": args.turns} + + +def append_log(entry_title: str, ep: Endpoint, payload: dict, note: str, result_file: Path): + """Append one STE summary entry to the improvement log.""" + stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + lines = [f"\n## {stamp} - {entry_title}\n"] + lines.append(f"- endpoint: `{ep.url}`, model: `{ep.model}`") + lines.append(f"- raw result: `tools/agent-lab/results/{result_file.name}`") + groups = payload.get("arms") or {"": payload.get("phases") or + [v for v in payload.values() if isinstance(v, dict) and "phase" in v]} + for group, rows in groups.items(): + if not rows: + continue + if group: + lines.append(f"\n**{group}**") + lines.append("") + lines.append("| phase | ttft_s | total_s | prompt_tokens | cached_tokens | completion_tokens |") + lines.append("|---|---|---|---|---|---|") + for r in rows: + u = r.get("usage") or {} + cached = (u.get("prompt_tokens_details") or {}).get("cached_tokens") + lines.append( + f"| {r['phase']} | {r['ttft_s']} | {r['total_s']} | " + f"{u.get('prompt_tokens')} | {cached} | {u.get('completion_tokens')} |" + ) + if note: + lines.append(f"\n- note: {note}") + with LOG_PATH.open("a") as fh: + fh.write("\n".join(lines) + "\n") + print(f"logged to {LOG_PATH.relative_to(REPO_ROOT)}") + + +def main() -> int: + defaults = _config_defaults() + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", default=defaults.get("url"), help="OpenAI-compatible base URL") + parser.add_argument("--key", default=defaults.get("key") or "none") + parser.add_argument("--model", default=defaults.get("model")) + parser.add_argument("--timeout", type=float, default=600.0) + parser.add_argument("--no-stream", dest="stream", action="store_false") + parser.add_argument("--no-log", dest="log", action="store_false", + help="do not append to the improvement log") + parser.add_argument("--note", default="", help="note for the log entry") + sub = parser.add_subparsers(dest="cmd", required=True) + + sub.add_parser("probe", help="endpoint sanity check, show raw usage") + + p_cache = sub.add_parser("cache", help="KV-cache experiment: cold/warm/mutate") + p_cache.add_argument("--prefix-words", type=int, default=3000, + help="system prompt size in words (~1.3 tokens per word)") + p_cache.add_argument("--history-turns", type=int, default=6) + p_cache.add_argument("--max-tokens", type=int, default=48) + p_cache.add_argument("--seed", type=int, default=42) + + p_replay = sub.add_parser("replay", help="replay real lean_state vs append-only") + p_replay.add_argument("--prefix-words", type=int, default=3000) + p_replay.add_argument("--turns", type=int, default=4) + p_replay.add_argument("--max-tokens", type=int, default=48) + p_replay.add_argument("--seed", type=int, default=7) + + args = parser.parse_args() + if not args.url or not args.model: + print("error: no endpoint or model. Set --url/--model or stack.toml [ai].", file=sys.stderr) + return 2 + + ep = Endpoint(args.url, args.key, args.model, args.timeout) + payload = {"probe": cmd_probe, "cache": cmd_cache, "replay": cmd_replay}[args.cmd](ep, args) + + RESULTS_DIR.mkdir(exist_ok=True) + stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + result_file = RESULTS_DIR / f"{stamp}-{args.cmd}.json" + record = { + "at": stamp, + "cmd": args.cmd, + "endpoint": ep.url, + "model": ep.model, + "args": {k: v for k, v in vars(args).items() if k not in ("key",)}, + "payload": payload, + } + result_file.write_text(json.dumps(record, indent=2) + "\n") + print(f"raw result: {result_file.relative_to(REPO_ROOT)}") + + if args.log: + append_log(f"lab {args.cmd}", ep, payload, args.note, result_file) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/agent-lab/results/20260915T120024Z-probe.json b/tools/agent-lab/results/20260915T120024Z-probe.json new file mode 100644 index 00000000..e8457060 --- /dev/null +++ b/tools/agent-lab/results/20260915T120024Z-probe.json @@ -0,0 +1,91 @@ +{ + "at": "20260915T120024Z", + "cmd": "probe", + "endpoint": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "args": { + "url": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "timeout": 600.0, + "stream": true, + "log": true, + "note": "first probe: find cache fields in usage", + "cmd": "probe" + }, + "payload": { + "served_models": [ + "Gemma-4-31B-JANG_4M-CRACK", + "LFM2-24B-A2B-MLX-4bit", + "LFM2.5-Embedding-350M-4bit", + "Meta-Llama-3.1-8B-Instruct-4bit", + "Qwen3-30B-A3B-Instruct-2507-MLX-4bit", + "Qwen3-30B-A3B-Instruct-2507-MLX-fp16", + "Qwen3-30B-A3B-MLX-4bit", + "Qwen3-Embedding-8B-4bit-DWQ", + "Qwen3-VL-8B-Instruct-MLX-4bit", + "Qwen3.5-27B-4bit", + "Qwen3.5-27B-Claude-4.6-Opus-Reasoning-Distilled-qx64-hi-mlx", + "Qwen3.5-27B-heretic-8bit", + "Qwen3.5-2B-bf16", + "Qwen3.5-35B-A3B-4bit", + "Qwen3.5-35B-A3B-4bit-fp16", + "Qwen3.5-9B-MLX-4bit", + "Qwen3.5-9B-MLX-8bit", + "Qwen3.6-35B-A3B-4bit", + "Qwen3.6-35B-A3B-4bit-fp16", + "Qwen3.6-35B-A3B-UD-MLX-4bit", + "Qwen3.6-35B-A3B-UD-MLX-4bit-fp16", + "bge-m3-mlx-8bit", + "gemma-3-12b-it-qat-4bit", + "gemma-3-12b-it-qat-fp16", + "gemma-4-26b-a4b-it-4bit", + "gemma-4-31b-it-MLX-8bit", + "mlx-community--Qwen3.5-35B-A3B-4bit", + "mlx-community--gemma-3-12b-it-qat-4bit" + ], + "first": { + "phase": "first", + "ttft_s": 1.092, + "total_s": 1.093, + "usage": { + "prompt_tokens": 34, + "completion_tokens": 2, + "total_tokens": 36, + "input_tokens": 34, + "output_tokens": 2, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 1.09, + "total_time": 1.09, + "prompt_eval_duration": 1.09, + "generation_duration": 0.0, + "prompt_tokens_per_second": 31.33, + "generation_tokens_per_second": 6961.56 + }, + "error": null + }, + "second": { + "phase": "second (identical prompt)", + "ttft_s": 0.441, + "total_s": 0.441, + "usage": { + "prompt_tokens": 34, + "completion_tokens": 2, + "total_tokens": 36, + "input_tokens": 34, + "output_tokens": 2, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 0.44, + "total_time": 0.44, + "prompt_eval_duration": 0.44, + "generation_duration": 0.0, + "prompt_tokens_per_second": 77.78, + "generation_tokens_per_second": 7650.61 + }, + "error": null + } + } +} diff --git a/tools/agent-lab/results/20260915T120103Z-cache.json b/tools/agent-lab/results/20260915T120103Z-cache.json new file mode 100644 index 00000000..f500112a --- /dev/null +++ b/tools/agent-lab/results/20260915T120103Z-cache.json @@ -0,0 +1,113 @@ +{ + "at": "20260915T120103Z", + "cmd": "cache", + "endpoint": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "args": { + "url": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "timeout": 600.0, + "stream": true, + "log": true, + "note": "hypothesis test: append-only turns hit the prefix cache, the lean_state-style rewrite forces re-prefill", + "cmd": "cache", + "prefix_words": 3000, + "history_turns": 6, + "max_tokens": 48, + "seed": 42 + }, + "payload": { + "phases": [ + { + "phase": "cold", + "ttft_s": 10.419, + "total_s": 10.922, + "usage": { + "prompt_tokens": 4413, + "completion_tokens": 31, + "total_tokens": 4444, + "input_tokens": 4413, + "output_tokens": 31, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 10.35, + "total_time": 10.85, + "prompt_eval_duration": 10.35, + "generation_duration": 0.51, + "prompt_tokens_per_second": 426.53, + "generation_tokens_per_second": 61.19 + }, + "error": null + }, + { + "phase": "warm", + "ttft_s": 1.555, + "total_s": 2.303, + "usage": { + "prompt_tokens": 4461, + "completion_tokens": 43, + "total_tokens": 4504, + "input_tokens": 4461, + "output_tokens": 43, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 1.51, + "total_time": 2.26, + "prompt_eval_duration": 1.51, + "generation_duration": 0.75, + "prompt_tokens_per_second": 2951.74, + "generation_tokens_per_second": 57.15 + }, + "error": null + }, + { + "phase": "warm2", + "ttft_s": 1.697, + "total_s": 1.766, + "usage": { + "prompt_tokens": 4520, + "completion_tokens": 8, + "total_tokens": 4528, + "input_tokens": 4520, + "output_tokens": 8, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 1.66, + "total_time": 1.73, + "prompt_eval_duration": 1.66, + "generation_duration": 0.07, + "prompt_tokens_per_second": 2728.16, + "generation_tokens_per_second": 110.34 + }, + "error": null + }, + { + "phase": "mutate", + "ttft_s": 9.986, + "total_s": 10.044, + "usage": { + "prompt_tokens": 4464, + "completion_tokens": 7, + "total_tokens": 4471, + "input_tokens": 4464, + "output_tokens": 7, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 9.95, + "total_time": 10.01, + "prompt_eval_duration": 9.95, + "generation_duration": 0.06, + "prompt_tokens_per_second": 448.7, + "generation_tokens_per_second": 113.26 + }, + "error": null + } + ], + "prefix_words": 3000, + "history_turns": 6 + } +} diff --git a/tools/agent-lab/results/20260915T120700Z-replay.json b/tools/agent-lab/results/20260915T120700Z-replay.json new file mode 100644 index 00000000..848bfdfa --- /dev/null +++ b/tools/agent-lab/results/20260915T120700Z-replay.json @@ -0,0 +1,205 @@ +{ + "at": "20260915T120700Z", + "cmd": "replay", + "endpoint": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "args": { + "url": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "timeout": 600.0, + "stream": true, + "log": true, + "note": "replay with the real lean_messages transform vs append-only; same scripted tool conversation", + "cmd": "replay", + "prefix_words": 3000, + "turns": 4, + "max_tokens": 48, + "seed": 7 + }, + "payload": { + "arms": { + "lean_state (production)": [ + { + "phase": "turn 1", + "ttft_s": 7.926, + "total_s": 8.758, + "usage": { + "prompt_tokens": 3390, + "completion_tokens": 48, + "total_tokens": 3438, + "input_tokens": 3390, + "output_tokens": 48, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 7.86, + "total_time": 8.7, + "prompt_eval_duration": 7.86, + "generation_duration": 0.83, + "prompt_tokens_per_second": 431.11, + "generation_tokens_per_second": 57.5 + }, + "error": null + }, + { + "phase": "turn 2", + "ttft_s": 7.656, + "total_s": 8.25, + "usage": { + "prompt_tokens": 3537, + "completion_tokens": 35, + "total_tokens": 3572, + "input_tokens": 3537, + "output_tokens": 35, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 7.62, + "total_time": 8.22, + "prompt_eval_duration": 7.62, + "generation_duration": 0.6, + "prompt_tokens_per_second": 464.02, + "generation_tokens_per_second": 58.48 + }, + "error": null + }, + { + "phase": "turn 3", + "ttft_s": 8.026, + "total_s": 8.554, + "usage": { + "prompt_tokens": 3684, + "completion_tokens": 32, + "total_tokens": 3716, + "input_tokens": 3684, + "output_tokens": 32, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 7.99, + "total_time": 8.53, + "prompt_eval_duration": 7.99, + "generation_duration": 0.53, + "prompt_tokens_per_second": 460.86, + "generation_tokens_per_second": 60.13 + }, + "error": null + }, + { + "phase": "turn 4", + "ttft_s": 8.26, + "total_s": 8.794, + "usage": { + "prompt_tokens": 3831, + "completion_tokens": 32, + "total_tokens": 3863, + "input_tokens": 3831, + "output_tokens": 32, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 8.23, + "total_time": 8.77, + "prompt_eval_duration": 8.23, + "generation_duration": 0.54, + "prompt_tokens_per_second": 465.6, + "generation_tokens_per_second": 59.51 + }, + "error": null + } + ], + "append-only": [ + { + "phase": "turn 1", + "ttft_s": 7.276, + "total_s": 8.088, + "usage": { + "prompt_tokens": 3392, + "completion_tokens": 48, + "total_tokens": 3440, + "input_tokens": 3392, + "output_tokens": 48, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 7.25, + "total_time": 8.06, + "prompt_eval_duration": 7.25, + "generation_duration": 0.82, + "prompt_tokens_per_second": 467.99, + "generation_tokens_per_second": 58.83 + }, + "error": null + }, + { + "phase": "turn 2", + "ttft_s": 8.025, + "total_s": 8.653, + "usage": { + "prompt_tokens": 3728, + "completion_tokens": 37, + "total_tokens": 3765, + "input_tokens": 3728, + "output_tokens": 37, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 7.99, + "total_time": 8.63, + "prompt_eval_duration": 7.99, + "generation_duration": 0.63, + "prompt_tokens_per_second": 466.36, + "generation_tokens_per_second": 58.62 + }, + "error": null + }, + { + "phase": "turn 3", + "ttft_s": 8.805, + "total_s": 9.368, + "usage": { + "prompt_tokens": 4066, + "completion_tokens": 32, + "total_tokens": 4098, + "input_tokens": 4066, + "output_tokens": 32, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 8.77, + "total_time": 9.34, + "prompt_eval_duration": 8.77, + "generation_duration": 0.57, + "prompt_tokens_per_second": 463.58, + "generation_tokens_per_second": 56.41 + }, + "error": null + }, + { + "phase": "turn 4", + "ttft_s": 9.725, + "total_s": 10.298, + "usage": { + "prompt_tokens": 4403, + "completion_tokens": 34, + "total_tokens": 4437, + "input_tokens": 4403, + "output_tokens": 34, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 9.68, + "total_time": 10.26, + "prompt_eval_duration": 9.68, + "generation_duration": 0.58, + "prompt_tokens_per_second": 454.83, + "generation_tokens_per_second": 58.94 + }, + "error": null + } + ] + }, + "prefix_words": 3000, + "turns": 4 + } +} diff --git a/tools/agent-lab/results/20260915T120930Z-replay.json b/tools/agent-lab/results/20260915T120930Z-replay.json new file mode 100644 index 00000000..984ef7bf --- /dev/null +++ b/tools/agent-lab/results/20260915T120930Z-replay.json @@ -0,0 +1,205 @@ +{ + "at": "20260915T120930Z", + "cmd": "replay", + "endpoint": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "args": { + "url": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "timeout": 600.0, + "stream": true, + "log": true, + "note": "block-size hypothesis: with a ~8K-token prefix the shared prefix spans 4096-token blocks; append-only should now hit, lean_state should hit only the system-prompt blocks", + "cmd": "replay", + "prefix_words": 6000, + "turns": 4, + "max_tokens": 48, + "seed": 21 + }, + "payload": { + "arms": { + "lean_state (production)": [ + { + "phase": "turn 1", + "ttft_s": 15.955, + "total_s": 16.764, + "usage": { + "prompt_tokens": 6717, + "completion_tokens": 48, + "total_tokens": 6765, + "input_tokens": 6717, + "output_tokens": 48, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 15.86, + "total_time": 16.68, + "prompt_eval_duration": 15.86, + "generation_duration": 0.81, + "prompt_tokens_per_second": 423.4, + "generation_tokens_per_second": 59.03 + }, + "error": null + }, + { + "phase": "turn 2", + "ttft_s": 7.05, + "total_s": 7.532, + "usage": { + "prompt_tokens": 6864, + "completion_tokens": 29, + "total_tokens": 6893, + "input_tokens": 6864, + "output_tokens": 29, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 6.99, + "total_time": 7.47, + "prompt_eval_duration": 6.99, + "generation_duration": 0.49, + "prompt_tokens_per_second": 982.11, + "generation_tokens_per_second": 59.7 + }, + "error": null + }, + { + "phase": "turn 3", + "ttft_s": 7.427, + "total_s": 7.963, + "usage": { + "prompt_tokens": 7010, + "completion_tokens": 32, + "total_tokens": 7042, + "input_tokens": 7010, + "output_tokens": 32, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 7.37, + "total_time": 7.91, + "prompt_eval_duration": 7.37, + "generation_duration": 0.54, + "prompt_tokens_per_second": 950.82, + "generation_tokens_per_second": 59.23 + }, + "error": null + }, + { + "phase": "turn 4", + "ttft_s": 7.714, + "total_s": 8.235, + "usage": { + "prompt_tokens": 7157, + "completion_tokens": 31, + "total_tokens": 7188, + "input_tokens": 7157, + "output_tokens": 31, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 7.66, + "total_time": 8.19, + "prompt_eval_duration": 7.66, + "generation_duration": 0.53, + "prompt_tokens_per_second": 934.11, + "generation_tokens_per_second": 58.98 + }, + "error": null + } + ], + "append-only": [ + { + "phase": "turn 1", + "ttft_s": 14.86, + "total_s": 15.694, + "usage": { + "prompt_tokens": 6709, + "completion_tokens": 48, + "total_tokens": 6757, + "input_tokens": 6709, + "output_tokens": 48, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 14.81, + "total_time": 15.65, + "prompt_eval_duration": 14.81, + "generation_duration": 0.84, + "prompt_tokens_per_second": 453.09, + "generation_tokens_per_second": 57.14 + }, + "error": null + }, + { + "phase": "turn 2", + "ttft_s": 7.49, + "total_s": 8.147, + "usage": { + "prompt_tokens": 7044, + "completion_tokens": 38, + "total_tokens": 7082, + "input_tokens": 7044, + "output_tokens": 38, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 7.43, + "total_time": 8.09, + "prompt_eval_duration": 7.43, + "generation_duration": 0.66, + "prompt_tokens_per_second": 948.39, + "generation_tokens_per_second": 57.4 + }, + "error": null + }, + { + "phase": "turn 3", + "ttft_s": 8.251, + "total_s": 8.834, + "usage": { + "prompt_tokens": 7379, + "completion_tokens": 34, + "total_tokens": 7413, + "input_tokens": 7379, + "output_tokens": 34, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 8.2, + "total_time": 8.78, + "prompt_eval_duration": 8.2, + "generation_duration": 0.59, + "prompt_tokens_per_second": 900.25, + "generation_tokens_per_second": 57.96 + }, + "error": null + }, + { + "phase": "turn 4", + "ttft_s": 9.116, + "total_s": 9.776, + "usage": { + "prompt_tokens": 7715, + "completion_tokens": 38, + "total_tokens": 7753, + "input_tokens": 7715, + "output_tokens": 38, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 9.05, + "total_time": 9.72, + "prompt_eval_duration": 9.05, + "generation_duration": 0.66, + "prompt_tokens_per_second": 852.03, + "generation_tokens_per_second": 57.21 + }, + "error": null + } + ] + }, + "prefix_words": 6000, + "turns": 4 + } +} diff --git a/tools/agent-lab/results/20260915T124224Z-cache.json b/tools/agent-lab/results/20260915T124224Z-cache.json new file mode 100644 index 00000000..078a23d7 --- /dev/null +++ b/tools/agent-lab/results/20260915T124224Z-cache.json @@ -0,0 +1,113 @@ +{ + "at": "20260915T124224Z", + "cmd": "cache", + "endpoint": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "args": { + "url": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "timeout": 600.0, + "stream": true, + "log": true, + "note": "post-restart check: did the scheduler changes take effect? baseline cold prefill was 426-453 tok/s", + "cmd": "cache", + "prefix_words": 3000, + "history_turns": 6, + "max_tokens": 48, + "seed": 42 + }, + "payload": { + "phases": [ + { + "phase": "cold", + "ttft_s": 1.897, + "total_s": 2.413, + "usage": { + "prompt_tokens": 4413, + "completion_tokens": 31, + "total_tokens": 4444, + "input_tokens": 4413, + "output_tokens": 31, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 1.84, + "total_time": 2.36, + "prompt_eval_duration": 1.84, + "generation_duration": 0.52, + "prompt_tokens_per_second": 2398.45, + "generation_tokens_per_second": 59.51 + }, + "error": null + }, + { + "phase": "warm", + "ttft_s": 1.47, + "total_s": 2.213, + "usage": { + "prompt_tokens": 4461, + "completion_tokens": 43, + "total_tokens": 4504, + "input_tokens": 4461, + "output_tokens": 43, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 1.45, + "total_time": 2.2, + "prompt_eval_duration": 1.45, + "generation_duration": 0.75, + "prompt_tokens_per_second": 3079.95, + "generation_tokens_per_second": 57.55 + }, + "error": null + }, + { + "phase": "warm2", + "ttft_s": 1.644, + "total_s": 1.71, + "usage": { + "prompt_tokens": 4520, + "completion_tokens": 8, + "total_tokens": 4528, + "input_tokens": 4520, + "output_tokens": 8, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 1.62, + "total_time": 1.69, + "prompt_eval_duration": 1.62, + "generation_duration": 0.07, + "prompt_tokens_per_second": 2783.57, + "generation_tokens_per_second": 115.56 + }, + "error": null + }, + { + "phase": "mutate", + "ttft_s": 1.537, + "total_s": 1.585, + "usage": { + "prompt_tokens": 4464, + "completion_tokens": 7, + "total_tokens": 4471, + "input_tokens": 4464, + "output_tokens": 7, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 1.52, + "total_time": 1.57, + "prompt_eval_duration": 1.52, + "generation_duration": 0.05, + "prompt_tokens_per_second": 2943.78, + "generation_tokens_per_second": 136.72 + }, + "error": null + } + ], + "prefix_words": 3000, + "history_turns": 6 + } +} diff --git a/tools/agent-lab/results/20260915T124303Z-cache.json b/tools/agent-lab/results/20260915T124303Z-cache.json new file mode 100644 index 00000000..c66bc5cf --- /dev/null +++ b/tools/agent-lab/results/20260915T124303Z-cache.json @@ -0,0 +1,113 @@ +{ + "at": "20260915T124303Z", + "cmd": "cache", + "endpoint": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "args": { + "url": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "timeout": 600.0, + "stream": true, + "log": true, + "note": "true cold measurement with fresh seed 99: prompts never seen, no SSD reuse possible; checks whether scheduler changes took effect", + "cmd": "cache", + "prefix_words": 3000, + "history_turns": 6, + "max_tokens": 48, + "seed": 99 + }, + "payload": { + "phases": [ + { + "phase": "cold", + "ttft_s": 10.879, + "total_s": 11.32, + "usage": { + "prompt_tokens": 4416, + "completion_tokens": 28, + "total_tokens": 4444, + "input_tokens": 4416, + "output_tokens": 28, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 10.82, + "total_time": 11.27, + "prompt_eval_duration": 10.82, + "generation_duration": 0.45, + "prompt_tokens_per_second": 408.08, + "generation_tokens_per_second": 62.69 + }, + "error": null + }, + { + "phase": "warm", + "ttft_s": 1.593, + "total_s": 2.447, + "usage": { + "prompt_tokens": 4461, + "completion_tokens": 48, + "total_tokens": 4509, + "input_tokens": 4461, + "output_tokens": 48, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 1.57, + "total_time": 2.43, + "prompt_eval_duration": 1.57, + "generation_duration": 0.86, + "prompt_tokens_per_second": 2839.48, + "generation_tokens_per_second": 55.89 + }, + "error": null + }, + { + "phase": "warm2", + "ttft_s": 1.678, + "total_s": 1.725, + "usage": { + "prompt_tokens": 4525, + "completion_tokens": 7, + "total_tokens": 4532, + "input_tokens": 4525, + "output_tokens": 7, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 1.66, + "total_time": 1.71, + "prompt_eval_duration": 1.66, + "generation_duration": 0.05, + "prompt_tokens_per_second": 2730.33, + "generation_tokens_per_second": 138.72 + }, + "error": null + }, + { + "phase": "mutate", + "ttft_s": 10.206, + "total_s": 10.243, + "usage": { + "prompt_tokens": 4468, + "completion_tokens": 6, + "total_tokens": 4474, + "input_tokens": 4468, + "output_tokens": 6, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 10.19, + "total_time": 10.23, + "prompt_eval_duration": 10.19, + "generation_duration": 0.04, + "prompt_tokens_per_second": 438.66, + "generation_tokens_per_second": 149.1 + }, + "error": null + } + ], + "prefix_words": 3000, + "history_turns": 6 + } +} diff --git a/tools/agent-lab/results/20260915T124818Z-cache.json b/tools/agent-lab/results/20260915T124818Z-cache.json new file mode 100644 index 00000000..c3564931 --- /dev/null +++ b/tools/agent-lab/results/20260915T124818Z-cache.json @@ -0,0 +1,113 @@ +{ + "at": "20260915T124818Z", + "cmd": "cache", + "endpoint": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "args": { + "url": "http://localhost:8888/v1", + "model": "Qwen3.6-35B-A3B-UD-MLX-4bit", + "timeout": 600.0, + "stream": true, + "log": true, + "note": "scheduler settings verified active (speed + chunked prefill): true cold measurement with fresh seed 123", + "cmd": "cache", + "prefix_words": 3000, + "history_turns": 6, + "max_tokens": 48, + "seed": 123 + }, + "payload": { + "phases": [ + { + "phase": "cold", + "ttft_s": 10.144, + "total_s": 10.601, + "usage": { + "prompt_tokens": 4410, + "completion_tokens": 23, + "total_tokens": 4433, + "input_tokens": 4410, + "output_tokens": 23, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 10.08, + "total_time": 10.55, + "prompt_eval_duration": 10.08, + "generation_duration": 0.46, + "prompt_tokens_per_second": 437.29, + "generation_tokens_per_second": 49.56 + }, + "error": null + }, + { + "phase": "warm", + "ttft_s": 1.551, + "total_s": 2.096, + "usage": { + "prompt_tokens": 4450, + "completion_tokens": 32, + "total_tokens": 4482, + "input_tokens": 4450, + "output_tokens": 32, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 1.53, + "total_time": 2.07, + "prompt_eval_duration": 1.53, + "generation_duration": 0.55, + "prompt_tokens_per_second": 2916.12, + "generation_tokens_per_second": 58.34 + }, + "error": null + }, + { + "phase": "warm2", + "ttft_s": 1.578, + "total_s": 1.724, + "usage": { + "prompt_tokens": 4498, + "completion_tokens": 12, + "total_tokens": 4510, + "input_tokens": 4498, + "output_tokens": 12, + "prompt_tokens_details": { + "cached_tokens": 4096 + }, + "time_to_first_token": 1.56, + "total_time": 1.71, + "prompt_eval_duration": 1.56, + "generation_duration": 0.15, + "prompt_tokens_per_second": 2888.67, + "generation_tokens_per_second": 80.6 + }, + "error": null + }, + { + "phase": "mutate", + "ttft_s": 9.731, + "total_s": 9.887, + "usage": { + "prompt_tokens": 4447, + "completion_tokens": 7, + "total_tokens": 4454, + "input_tokens": 4447, + "output_tokens": 7, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "time_to_first_token": 9.71, + "total_time": 9.87, + "prompt_eval_duration": 9.71, + "generation_duration": 0.16, + "prompt_tokens_per_second": 457.98, + "generation_tokens_per_second": 43.55 + }, + "error": null + } + ], + "prefix_words": 3000, + "history_turns": 6 + } +} diff --git a/tools/agent-lab/rig/.gitignore b/tools/agent-lab/rig/.gitignore new file mode 100644 index 00000000..dc3b76b2 --- /dev/null +++ b/tools/agent-lab/rig/.gitignore @@ -0,0 +1 @@ +state/ diff --git a/tools/agent-lab/rig/README.md b/tools/agent-lab/rig/README.md new file mode 100644 index 00000000..06d0028c --- /dev/null +++ b/tools/agent-lab/rig/README.md @@ -0,0 +1,69 @@ +# agent-lab rig + +An isolated end-to-end rig for the famstack agent. It runs the real +agent image, the real runtime shims, and the real tool loop. It +replaces every production dependency: + +| Production | Rig | +|---|---| +| Matrix channel | `nanobot agent -m` (direct turn, no channel) | +| family vault | `demo-vault/` (fabricated data, git repo) | +| famstack-api (port 42001) | `lab-api.py` (port 42011) | +| oMLX direct | `proxy.py` (port 42012, logs every request) | + +The proxy records the byte-exact request and the usage object for +every LLM call in `state/proxy-log.jsonl`. This shows the full context +the agent sends, per request. + +## Usage + +``` +python3 tools/agent-lab/rig/rig.py build +python3 tools/agent-lab/rig/rig.py reset +python3 tools/agent-lab/rig/lab-api.py --model & # port 42011 +python3 tools/agent-lab/rig/proxy.py & # port 42012 +python3 tools/agent-lab/rig/rig.py turn "What is open on the shopping list?" --session rig:lists +``` + +`turn` prints the agent reply, then a metrics summary: wall time, LLM +calls, prompt tokens, cached tokens, TTFT per call. Each turn appends +an entry to `docs/design/agent/agent-improvement-log.md` (`--no-log` +to skip, `--note` to add context). + +## Scenarios + +The demo vault holds multiple lists across topics and persons: + +| Scenario | Session | Data | +|---|---|---| +| Shared topic list | `topic:groceries` | `family/groceries/todos.md` (homer, marge) | +| Multi-person topic | `topic:camping` | `family/camping/todos.md` (homer, bart) | +| Multi-person topic | `topic:birthday` | `family/birthday/todos.md` (marge, lisa) | +| DM, not topic bound | `dm:homer` | `homer/notes/2026/09/errands.md` | +| DM, private data | `dm:marge` | `marge/notes/2026/09/gift-ideas.md` | + +Conventions: + +- One session per simulated room or DM: `--session topic:camping`, + `--session dm:homer`. +- In multi-person sessions, prefix the message with the speaker: + `"Bart: what is open on the packing list?"`. +- The gift-ideas page is a pollution trap. A correct agent does not + surface it in the birthday topic session. + +## Limitations (rig v1) + +- Read path only. Vault writes (`stack memory write`) return exit 126 + from lab-api. The write path is the next rig increment; the + seamless-lists goal needs it. +- No real Matrix sender. In production, the brief derives the speaker + from the Matrix mxid. The CLI turn has no sender, so the speaker + line in the brief can differ from production. +- lab-api reimplements `memory search/person/history` over the demo + vault. Same protocol and exit codes, simpler ranking. + +## Rules + +- Fabricated data only in `demo-vault/`. Never copy family content. +- The rig binds lab services to 127.0.0.1 and uses its own ports. + It must never point at port 42001 (production famstack-api). diff --git a/tools/agent-lab/rig/config.json b/tools/agent-lab/rig/config.json new file mode 100644 index 00000000..f940c152 --- /dev/null +++ b/tools/agent-lab/rig/config.json @@ -0,0 +1,35 @@ +{ + "providers": { + "custom": { + "api_base": "${AGENT_OPENAI_URL}", + "api_key": "${AGENT_OPENAI_KEY}" + } + }, + "model_presets": { + "primary": { + "provider": "custom", + "model": "${AGENT_MODEL}" + } + }, + "agents": { + "defaults": { + "model_preset": "primary", + "max_tool_iterations": 12, + "max_messages": 40, + "disabled_skills": ["memory", "my"] + } + }, + "tools": { + "exec": {"enable": true}, + "web": {"enable": false}, + "file": {"enable": true}, + "cli_apps": {"enable": false}, + "my": {"enable": false}, + "image_generation": {"enable": false} + }, + "channels": { + "matrix": { + "enabled": false + } + } +} diff --git a/tools/agent-lab/rig/demo-vault/bart/about.md b/tools/agent-lab/rig/demo-vault/bart/about.md new file mode 100644 index 00000000..f65a5e8f --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/bart/about.md @@ -0,0 +1,14 @@ +--- +type: person +generated: true +date: 2026-09-01 +persons: + - bart +--- + +Demo kid. Plans the camping trip with Homer. + +## About + +Bart is a fabricated demo person for the agent lab. +He owns the packing list for the camping trip. diff --git a/tools/agent-lab/rig/demo-vault/family/about.md b/tools/agent-lab/rig/demo-vault/family/about.md new file mode 100644 index 00000000..bff44d98 --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/family/about.md @@ -0,0 +1,11 @@ +--- +type: topic +date: 2026-09-01 +tags: + - family +--- + +# The demo family + +This is a fabricated demo family for the agent lab. No real data. +Homer and Marge live here with their demo household. diff --git a/tools/agent-lab/rig/demo-vault/family/birthday/about.md b/tools/agent-lab/rig/demo-vault/family/birthday/about.md new file mode 100644 index 00000000..563706a6 --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/family/birthday/about.md @@ -0,0 +1,14 @@ +--- +type: topic +date: 2026-09-08 +persons: + - marge + - lisa +tags: + - birthday +--- + +# Birthday party + +Marge and Lisa organize the party on 2026-10-03. +The checklist lives in [todos.md](todos.md). diff --git a/tools/agent-lab/rig/demo-vault/family/birthday/todos.md b/tools/agent-lab/rig/demo-vault/family/birthday/todos.md new file mode 100644 index 00000000..8bdf908f --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/family/birthday/todos.md @@ -0,0 +1,18 @@ +--- +type: note +date: 2026-09-13 +persons: + - marge + - lisa +tags: + - birthday + - todos +--- + +# Party checklist + +- [x] book the community hall +- [ ] send invitations +- [ ] order the cake +- [ ] buy decorations +- [ ] playlist diff --git a/tools/agent-lab/rig/demo-vault/family/camping/about.md b/tools/agent-lab/rig/demo-vault/family/camping/about.md new file mode 100644 index 00000000..910d7920 --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/family/camping/about.md @@ -0,0 +1,14 @@ +--- +type: topic +date: 2026-09-05 +persons: + - homer + - bart +tags: + - camping +--- + +# Camping trip + +Homer and Bart plan a camping weekend in October. +The packing list lives in [todos.md](todos.md). diff --git a/tools/agent-lab/rig/demo-vault/family/camping/todos.md b/tools/agent-lab/rig/demo-vault/family/camping/todos.md new file mode 100644 index 00000000..4cbc1a23 --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/family/camping/todos.md @@ -0,0 +1,19 @@ +--- +type: note +date: 2026-09-12 +persons: + - homer + - bart +tags: + - camping + - todos +--- + +# Packing list + +- [x] tent +- [ ] sleeping bags +- [ ] camping stove +- [ ] flashlights +- [ ] first aid kit +- [x] cooler box diff --git a/tools/agent-lab/rig/demo-vault/family/groceries/about.md b/tools/agent-lab/rig/demo-vault/family/groceries/about.md new file mode 100644 index 00000000..8ffefbb9 --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/family/groceries/about.md @@ -0,0 +1,11 @@ +--- +type: topic +date: 2026-09-01 +tags: + - groceries +--- + +# Groceries + +Shopping list and grocery notes for the demo household. +The list lives in [todos.md](todos.md). diff --git a/tools/agent-lab/rig/demo-vault/family/groceries/todos.md b/tools/agent-lab/rig/demo-vault/family/groceries/todos.md new file mode 100644 index 00000000..4234887a --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/family/groceries/todos.md @@ -0,0 +1,18 @@ +--- +type: note +date: 2026-09-14 +persons: + - homer + - marge +tags: + - groceries + - todos +--- + +# Shopping list + +- [ ] oat milk +- [ ] rye bread +- [ ] apples +- [x] coffee beans +- [ ] dish soap diff --git a/tools/agent-lab/rig/demo-vault/homer/about.md b/tools/agent-lab/rig/demo-vault/homer/about.md new file mode 100644 index 00000000..5282344d --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/homer/about.md @@ -0,0 +1,14 @@ +--- +type: person +generated: true +date: 2026-09-01 +persons: + - homer +--- + +Demo parent. Works at the plant. Likes simple dinners. + +## About + +Homer is a fabricated demo person for the agent lab. +He handles the groceries on Saturdays. diff --git a/tools/agent-lab/rig/demo-vault/homer/notes/2026/09/errands.md b/tools/agent-lab/rig/demo-vault/homer/notes/2026/09/errands.md new file mode 100644 index 00000000..eb918597 --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/homer/notes/2026/09/errands.md @@ -0,0 +1,17 @@ +--- +type: note +date: 2026-09-10 +persons: + - homer +tags: + - todos +--- + +# Homer's errands + +Personal list, not bound to a topic. + +- [ ] renew car inspection +- [ ] return library books +- [x] dentist appointment +- [ ] fix the garden gate diff --git a/tools/agent-lab/rig/demo-vault/lisa/about.md b/tools/agent-lab/rig/demo-vault/lisa/about.md new file mode 100644 index 00000000..e9f43607 --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/lisa/about.md @@ -0,0 +1,14 @@ +--- +type: person +generated: true +date: 2026-09-01 +persons: + - lisa +--- + +Demo kid. Organizes the birthday party with Marge. + +## About + +Lisa is a fabricated demo person for the agent lab. +She keeps the party checklist current. diff --git a/tools/agent-lab/rig/demo-vault/marge/about.md b/tools/agent-lab/rig/demo-vault/marge/about.md new file mode 100644 index 00000000..1f70b286 --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/marge/about.md @@ -0,0 +1,14 @@ +--- +type: person +generated: true +date: 2026-09-01 +persons: + - marge +--- + +Demo parent. Plans the week. Keeps the family calendar. + +## About + +Marge is a fabricated demo person for the agent lab. +She adds most items to the shopping list. diff --git a/tools/agent-lab/rig/demo-vault/marge/notes/2026/09/gift-ideas.md b/tools/agent-lab/rig/demo-vault/marge/notes/2026/09/gift-ideas.md new file mode 100644 index 00000000..d42721e9 --- /dev/null +++ b/tools/agent-lab/rig/demo-vault/marge/notes/2026/09/gift-ideas.md @@ -0,0 +1,17 @@ +--- +type: note +date: 2026-09-11 +persons: + - marge +tags: + - todos + - birthday +--- + +# Gift ideas + +Private list. Do not surface this in the birthday topic room. + +- [ ] saxophone reed set for Lisa +- [ ] book voucher +- [x] wrap the science kit diff --git a/tools/agent-lab/rig/lab-api.py b/tools/agent-lab/rig/lab-api.py new file mode 100644 index 00000000..63f43e7c --- /dev/null +++ b/tools/agent-lab/rig/lab-api.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Lab replacement for the famstack API, for the isolated rig only. + +The rig agent's `stack` shim sends one shlex line over TCP. This server +answers `stack memory ...` commands from the demo vault. It never talks +to the production API, Forgejo, or the family vault. + +Protocol: read one line until EOF. Send plain text back. On a non-zero +exit, append a trailing `stack-exit: N` line. This copies the contract +of stacklets/core/famstack-api.py. + +The `--nl` flag makes one keyword-rewrite call to the LLM endpoint, +like the production search path does. This keeps the latency shape. + +Usage: lab-api.py [--listen 42011] [--vault DIR] [--llm URL] [--model M] +""" + +import argparse +import json +import re +import shlex +import socketserver +import subprocess +import importlib.util +import sys +import urllib.request +from pathlib import Path + +ARGS = None + +# Run the real store transform, so the rig cannot drift from production. +# The transforms are pure; the module also imports store deps at load +# time, so put their paths on sys.path before executing it. +_REPO = Path(__file__).resolve().parents[3] +for _p in ("lib", "stacklets/memory", "stacklets/memory/cli"): + sys.path.insert(0, str(_REPO / _p)) +_spec = importlib.util.spec_from_file_location( + "prod_list_edit", _REPO / "stacklets" / "memory" / "cli" / "list-edit.py") +_prod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_prod) +_list_edit_transform = _prod.apply_list_edit +_list_edit_batch = _prod.apply_list_edits + + +def _body_only(text: str) -> str: + """Strip YAML frontmatter, so field names do not match every page.""" + if text.startswith("---\n"): + end = text.find("\n---\n", 4) + if end != -1: + return text[end + 5:] + return text + + +def _rewrite_keywords(query: str) -> list[str]: + """One LLM call that turns a question into 2-4 literal keywords.""" + body = { + "model": ARGS.model, + "messages": [{"role": "user", "content": + "Give 2 to 4 literal search keywords for this query. " + "Reply with the keywords only, separated by spaces.\n\n" + f"Query: {query}"}], + "max_tokens": 32, + "temperature": 0, + "chat_template_kwargs": {"enable_thinking": False}, + } + req = urllib.request.Request( + ARGS.llm.rstrip("/") + "/chat/completions", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": f"Bearer {ARGS.key}"}, + ) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + text = json.load(resp)["choices"][0]["message"]["content"] or "" + words = [w.strip(".,;:!?\"'") for w in text.split()] + return [w for w in words if w][:4] or query.split() + except Exception: + return query.split() + + +def memory_search(argv: list[str]) -> tuple[str, int]: + parser = argparse.ArgumentParser(prog="stack memory search", add_help=False) + parser.add_argument("query") + parser.add_argument("--nl", action="store_true") + parser.add_argument("--limit", type=int, default=5) + parser.add_argument("--scope", default=None) + parser.add_argument("--person", default=None) + parser.add_argument("--tag", default=None) + try: + ns = parser.parse_args(argv) + except SystemExit: + return parser.format_usage(), 2 + + if ns.nl and len(ns.query.split()) > 1: + terms = _rewrite_keywords(ns.query) + pattern = re.compile("|".join(re.escape(t) for t in terms), re.IGNORECASE) + else: + # The real CLI treats the query as a regex (lib.py search engine). + # Keep that contract; fall back to a literal match on a bad regex. + try: + pattern = re.compile(ns.query, re.IGNORECASE) + except re.error: + pattern = re.compile(re.escape(ns.query), re.IGNORECASE) + + vault = Path(ARGS.vault) + hits = [] + for path in sorted(vault.rglob("*.md")): + rel = path.relative_to(vault) + if ns.scope and not str(rel).startswith(ns.scope): + continue + text = path.read_text() + body = _body_only(text) + lines = [ln for ln in body.splitlines() if pattern.search(ln)] + if lines: + m = re.search(r"^date:\s*(\S+)", text, re.MULTILINE) + date = m.group(1) if m else "" + hits.append((date, f"— vault/{rel}\n" + + "\n".join(f" {ln}" for ln in lines[:3]))) + if not hits: + return "no results\n", 1 + # The real CLI sorts by frontmatter date, newest first, then limits. + hits.sort(key=lambda h: h[0], reverse=True) + return "\n".join(h[1] for h in hits[: ns.limit]) + "\n", 0 + + +def memory_person(argv: list[str]) -> tuple[str, int]: + if not argv: + return "usage: stack memory person \n", 2 + page = Path(ARGS.vault) / argv[0].lower() / "about.md" + if not page.exists(): + return f"no page for {argv[0]}\n", 1 + return page.read_text(), 0 + + +def memory_history(argv: list[str]) -> tuple[str, int]: + cmd = ["git", "-C", ARGS.vault, "log", "--oneline", "-n", "10"] + if argv: + cmd += ["--", argv[0]] + r = subprocess.run(cmd, capture_output=True, text=True, timeout=10) + return (r.stdout or "no history\n"), (0 if r.returncode == 0 else 1) + + +_BOX = re.compile(r"^\s*- \[([ xX])\]\s*(.+?)\s*$") + + +def _boxes(text: str) -> dict[str, bool]: + """Map checkbox item text -> done state.""" + out = {} + for ln in text.splitlines(): + if m := _BOX.match(ln): + out[m.group(2)] = m.group(1).lower() == "x" + return out + + +def _diff_sentence(old: str, new: str, page: str) -> str: + """Describe what a write actually changed, like the real store does.""" + ob, nb = _boxes(old), _boxes(new) + ticked = [t for t in ob if t in nb and not ob[t] and nb[t]] + unticked = [t for t in ob if t in nb and ob[t] and not nb[t]] + added = [t for t in nb if t not in ob] + removed = [t for t in ob if t not in nb] + parts = [] + if ticked: + parts.append(f"ticked off {len(ticked)}: " + "; ".join(ticked)) + if unticked: + parts.append(f"unticked {len(unticked)}: " + "; ".join(unticked)) + if added: + parts.append(f"added {len(added)}: " + "; ".join(added)) + if removed: + parts.append(f"REMOVED {len(removed)}: " + "; ".join(removed)) + return "; ".join(parts) if parts else f"updated {page}" + + +def memory_write(argv: list[str]) -> tuple[str, int]: + parser = argparse.ArgumentParser(prog="stack memory write", add_help=False) + parser.add_argument("page") + parser.add_argument("--by", default="someone") + parser.add_argument("--patch", action="store_true") + parser.add_argument("--dry-run", action="store_true") + try: + ns = parser.parse_args(argv) + except SystemExit: + return parser.format_usage(), 2 + + buffer = Path(ARGS.buffer) + if not buffer.exists(): + return "no write buffer\n", 3 + payload = buffer.read_text() + path = Path(ARGS.vault) / ns.page + old = path.read_text() if path.exists() else "" + + if ns.patch: + try: + edits = json.loads(payload) + except json.JSONDecodeError as e: + return f"patch is not valid JSON: {e}\n", 2 + new = old + for edit in edits: + old_text = edit.get("old_text") or "" + if old_text and old_text not in new: + return (f"old_text was not found in {ns.page}; the page has " + f"changed. Read it again and patch what is there " + f"now.\n"), 1 + if old_text: + new = new.replace(old_text, edit.get("new_text") or "", 1) + else: + new = new + (edit.get("new_text") or "") + else: + new = payload + + # Same frontmatter guard as the production write seam. + if new.lstrip().startswith("---") and not new.startswith("---\n"): + return (f"{ns.page} must start with '---' at column one; the write " + "begins with whitespace before the frontmatter. Re-send the " + "page with the frontmatter block exactly as you read it.\n"), 1 + if old.startswith("---\n") and not new.startswith("---\n"): + return (f"{ns.page} has frontmatter and this write drops it. Keep " + "the frontmatter block exactly as you read it.\n"), 1 + + # Same restructure guard as the production write seam: a whole-page + # rewrite of a list must not reopen or remove items. + if not ns.patch and ns.page.endswith("todos.md"): + ob, nb = _boxes(old), _boxes(new) + lost = [t for t in ob if (t in nb and ob[t] and not nb[t]) or t not in nb] + if lost: + return (f"this rewrite reopens or removes items: " + f"{'; '.join(lost)}. A restructure keeps every item and " + "every [x]. Resend the full page with them unchanged. To " + "reopen or remove an item on purpose, use list_edit.\n"), 1 + + sentence = _diff_sentence(old, new, ns.page) + if ns.dry_run: + return f"dry run: {sentence}\n", 0 + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(new) + env = {"GIT_AUTHOR_NAME": ns.by, "GIT_AUTHOR_EMAIL": f"{ns.by}@demo.invalid", + "GIT_COMMITTER_NAME": ns.by, "GIT_COMMITTER_EMAIL": f"{ns.by}@demo.invalid"} + subprocess.run(["git", "add", ns.page], cwd=ARGS.vault, env=env, timeout=10) + subprocess.run(["git", "commit", "-q", "-m", f"docs(memory): {sentence[:60]}"], + cwd=ARGS.vault, env=env, timeout=10) + return sentence + _link_line(ns.page) + "\n", 0 + + +def _link_line(page: str) -> str: + """Rig stand-in for the production wiki link line on touched pages.""" + parts = page.strip("/").split("/") + if parts[-1] == "todos.md": + return f"\n https://rig.invalid/go/topic/{'/'.join(parts[:-1])}/todo" + if parts[-1] == "about.md": + return f"\n https://rig.invalid/go/topic/{'/'.join(parts[:-1])}" + return "" + + +def _commit(page: str, actor: str, sentence: str) -> None: + env = {"GIT_AUTHOR_NAME": actor, "GIT_AUTHOR_EMAIL": f"{actor}@demo.invalid", + "GIT_COMMITTER_NAME": actor, "GIT_COMMITTER_EMAIL": f"{actor}@demo.invalid"} + subprocess.run(["git", "add", page], cwd=ARGS.vault, env=env, timeout=10) + subprocess.run(["git", "commit", "-q", "-m", f"docs(memory): {sentence[:60]}"], + cwd=ARGS.vault, env=env, timeout=10) + + +def memory_list_edit(argv: list[str]) -> tuple[str, int]: + """Item-level and bulk list operations. + + Runs the production transform (stacklets/memory/cli/list-edit.py) + so the rig cannot drift from the real store contract. + """ + parser = argparse.ArgumentParser(prog="stack memory list-edit", add_help=False) + parser.add_argument("page") + parser.add_argument("--op", required=True, + choices=["add", "tick", "untick", "remove", + "clear-done", "reset"]) + parser.add_argument("--item", action="append", default=[]) + parser.add_argument("--section", default=None) + parser.add_argument("--by", default="someone") + try: + ns = parser.parse_args(argv) + except SystemExit: + return parser.format_usage(), 2 + + path = Path(ARGS.vault) / ns.page + if not path.exists(): + return f"no such page: {ns.page}\n", 1 + items = [i.strip() for i in ns.item if i.strip()] + text = path.read_text() + if ns.op in ("clear-done", "reset"): + new, sentence, kind = _list_edit_transform(text, ns.op, "", ns.section) + else: + new, sentence, kind = _list_edit_batch(text, ns.op, items, ns.section) + + if kind == "refuse": + return sentence + "\n", 1 + if kind == "noop": + return sentence + "\n", 0 + path.write_text(new) + _commit(ns.page, ns.by, sentence) + return sentence + _link_line(ns.page) + "\n", 0 + + +def dispatch(line: str) -> tuple[str, int]: + try: + argv = shlex.split(line) + except ValueError as e: + return f"parse error: {e}\n", 2 + if len(argv) >= 2 and argv[0] == "memory": + handlers = {"search": memory_search, "person": memory_person, + "history": memory_history, "write": memory_write, + "list-edit": memory_list_edit} + if argv[1] in handlers: + return handlers[argv[1]](argv[2:]) + return f"lab-api: command not allowed in the rig: {' '.join(argv[:2])}\n", 126 + + +class Handler(socketserver.StreamRequestHandler): + def handle(self): + line = self.rfile.read().decode("utf-8", "replace").strip() + print(f"lab-api: {line}") + text, code = dispatch(line) + if code != 0: + text = text + f"stack-exit: {code}\n" + self.wfile.write(text.encode()) + + +def main(): + global ARGS + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--listen", type=int, default=42011) + parser.add_argument("--vault", default=str(Path(__file__).parent / "state" / "vault")) + parser.add_argument("--buffer", default=str( + Path(__file__).parent / "state" / "nanobot" / ".write-buffer")) + parser.add_argument("--llm", default="http://localhost:8888/v1") + parser.add_argument("--key", default="none") + parser.add_argument("--model", default=None) + ARGS = parser.parse_args() + socketserver.ThreadingTCPServer.allow_reuse_address = True + server = socketserver.ThreadingTCPServer(("127.0.0.1", ARGS.listen), Handler) + print(f"lab-api: 127.0.0.1:{ARGS.listen}, vault {ARGS.vault}") + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/tools/agent-lab/rig/proxy.py b/tools/agent-lab/rig/proxy.py new file mode 100644 index 00000000..da60a8ca --- /dev/null +++ b/tools/agent-lab/rig/proxy.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Logging proxy between the rig agent and oMLX. + +The proxy forwards every request to the real endpoint. It records the +byte-exact request body, the usage object, and the duration in a JSONL +file. This shows the full context the agent sends on each request. + +Usage: proxy.py [--listen 42012] [--target http://localhost:8888] [--log FILE] +""" + +import argparse +import json +import threading +import time +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +_LOCK = threading.Lock() + + +def _extract_usage(body: bytes, content_type: str) -> dict | None: + """Read the usage object from a JSON or SSE response body.""" + text = body.decode("utf-8", "replace") + if "text/event-stream" in content_type: + usage = None + for line in text.splitlines(): + if line.startswith("data:") and '"usage"' in line: + try: + chunk = json.loads(line[5:].strip()) + usage = chunk.get("usage") or usage + except json.JSONDecodeError: + pass + return usage + try: + return json.loads(text).get("usage") + except json.JSONDecodeError: + return None + + +class Handler(BaseHTTPRequestHandler): + target = "http://localhost:8888" + log_path = Path("proxy-log.jsonl") + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def _forward(self, body: bytes | None): + url = self.target + self.path + req = urllib.request.Request(url, data=body, method=self.command) + for h in ("Authorization", "Content-Type", "Accept"): + if self.headers.get(h): + req.add_header(h, self.headers[h]) + start = time.monotonic() + try: + with urllib.request.urlopen(req, timeout=900) as resp: + data = resp.read() + status = resp.status + ctype = resp.headers.get("Content-Type", "") + except urllib.error.HTTPError as e: + data, status, ctype = e.read(), e.code, "" + except Exception as e: + data, status, ctype = json.dumps({"error": str(e)}).encode(), 502, "application/json" + duration = time.monotonic() - start + + self.send_response(status) + self.send_header("Content-Type", ctype or "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + record = { + "at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "path": self.path, + "status": status, + "duration_s": round(duration, 3), + } + if body and self.path.endswith("/chat/completions"): + try: + payload = json.loads(body) + record["model"] = payload.get("model") + record["n_messages"] = len(payload.get("messages", [])) + record["messages"] = payload.get("messages") + record["tools"] = [t.get("function", {}).get("name") + for t in payload.get("tools") or []] + except json.JSONDecodeError: + record["raw_request"] = body.decode("utf-8", "replace")[:2000] + record["usage"] = _extract_usage(data, ctype) + with _LOCK, self.log_path.open("a") as fh: + fh.write(json.dumps(record) + "\n") + + def do_POST(self): + length = int(self.headers.get("Content-Length") or 0) + self._forward(self.rfile.read(length)) + + def do_GET(self): + self._forward(None) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--listen", type=int, default=42012) + parser.add_argument("--target", default="http://localhost:8888") + parser.add_argument("--log", default=str(Path(__file__).parent / "state" / "proxy-log.jsonl")) + args = parser.parse_args() + Handler.target = args.target.rstrip("/") + Handler.log_path = Path(args.log) + Handler.log_path.parent.mkdir(parents=True, exist_ok=True) + server = ThreadingHTTPServer(("127.0.0.1", args.listen), Handler) + print(f"proxy: 127.0.0.1:{args.listen} -> {Handler.target}, log {Handler.log_path}") + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/tools/agent-lab/rig/rig.py b/tools/agent-lab/rig/rig.py new file mode 100644 index 00000000..aeae9476 --- /dev/null +++ b/tools/agent-lab/rig/rig.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Isolated end-to-end rig for the famstack agent. + +The rig runs the real agent image, the real runtime shims, and the real +tool loop. It replaces every production dependency: + + Matrix -> `nanobot agent -m` (direct turn, no channel) + family vault -> demo vault with fabricated data (git repo) + famstack-api -> lab-api.py on port 42011 + oMLX direct -> proxy.py on port 42012 (logs every request) + +Subcommands: + build Build the agent image from stacklets/agent/. + reset Wipe and seed the rig state (workspace, config, demo vault). + turn Run one agent turn. Print the reply and the per-call metrics. + +Start lab-api.py and proxy.py before the first turn. +""" + +import argparse +import datetime as dt +import json +import shutil +import subprocess +import sys +import time +import tomllib +from pathlib import Path + +RIG = Path(__file__).resolve().parent +REPO = RIG.parents[2] +STATE = RIG / "state" +IMAGE = "famstack-agent-lab" +LOG_PATH = REPO / "docs" / "design" / "agent" / "agent-improvement-log.md" +PROXY_LOG = STATE / "proxy-log.jsonl" + + +def _ai_config() -> dict: + ai = tomllib.loads((REPO / "stack.toml").read_text()).get("ai", {}) + return {"key": ai.get("openai_key", "none"), "model": ai.get("default", "")} + + +def cmd_build(_args): + subprocess.run(["docker", "build", "-t", IMAGE, + str(REPO / "stacklets" / "agent")], check=True) + + +def cmd_reset(_args): + """Seed a fresh rig state. Keeps nothing from earlier runs.""" + shutil.rmtree(STATE, ignore_errors=True) + ws = STATE / "nanobot" / "workspace" + ws.mkdir(parents=True) + shutil.copy(RIG / "config.json", STATE / "nanobot" / "config.json") + + seed = REPO / "stacklets" / "agent" / "workspace" + for name in ("SOUL.md", "AGENTS.md"): + text = (seed / name).read_text().replace("__AGENT_NAME__", "Stacky") + (ws / name).write_text(text) + shutil.copytree(seed / "skills", ws / "skills") + # Same minimal USER.md the production entrypoint seeds. + (ws / "USER.md").write_text( + "# User Profile\n\n" + "The runtime brief names the current speaker each turn. No data here.\n") + + # The brief reads git history from the vault. Make the demo vault a + # real git repo, with the same shape as the production clone. + vault = STATE / "vault" + shutil.copytree(RIG / "demo-vault", vault) + env = {"GIT_AUTHOR_NAME": "homer", "GIT_AUTHOR_EMAIL": "homer@demo.invalid", + "GIT_COMMITTER_NAME": "homer", "GIT_COMMITTER_EMAIL": "homer@demo.invalid"} + for cmd in (["git", "init", "-q", "-b", "main"], + ["git", "add", "-A"], + ["git", "commit", "-q", "-m", "docs(memory): seed demo vault"]): + subprocess.run(cmd, cwd=vault, check=True, env=env) + print(f"rig state ready: {STATE}") + + +def _proxy_offset() -> int: + return PROXY_LOG.stat().st_size if PROXY_LOG.exists() else 0 + + +def _proxy_calls(offset: int) -> list[dict]: + if not PROXY_LOG.exists(): + return [] + with PROXY_LOG.open() as fh: + fh.seek(offset) + return [json.loads(ln) for ln in fh if ln.strip()] + + +def cmd_turn(args): + cfg = _ai_config() + offset = _proxy_offset() + docker_cmd = [ + "docker", "run", "--rm", + "--add-host", "host.docker.internal:host-gateway", + "-v", f"{STATE / 'nanobot'}:/home/nanobot/.nanobot", + "-v", f"{STATE / 'vault'}:/home/nanobot/.nanobot/workspace/vault:ro", + "-e", "AGENT_OPENAI_URL=http://host.docker.internal:42012/v1", + "-e", f"AGENT_OPENAI_KEY={cfg['key']}", + "-e", f"AGENT_MODEL={cfg['model']}", + "-e", "STACK_API_ADDR=host.docker.internal:42011", + *[x for pair in (args.env or []) for x in ("-e", pair)], + "--entrypoint", "nanobot", + IMAGE, "agent", "-m", args.message, "-s", args.session, + "--no-markdown", + ] + if args.logs: + docker_cmd.append("--logs") + start = time.monotonic() + r = subprocess.run(docker_cmd, capture_output=True, text=True) + wall = time.monotonic() - start + print(r.stdout) + if r.returncode != 0: + print(r.stderr, file=sys.stderr) + + calls = [c for c in _proxy_calls(offset) if c.get("path", "").endswith("/chat/completions")] + rows = [] + for i, c in enumerate(calls, 1): + u = c.get("usage") or {} + rows.append({ + "call": i, + "n_messages": c.get("n_messages"), + "prompt_tokens": u.get("prompt_tokens"), + "cached_tokens": (u.get("prompt_tokens_details") or {}).get("cached_tokens"), + "completion_tokens": u.get("completion_tokens"), + "ttft_s": u.get("time_to_first_token"), + "duration_s": c.get("duration_s"), + }) + summary = {"wall_s": round(wall, 2), "llm_calls": len(calls), "rows": rows, + "exit": r.returncode} + print(json.dumps(summary, indent=2)) + + if args.log: + _log_turn(args, cfg, summary) + return 0 if r.returncode == 0 else 1 + + +def _log_turn(args, cfg, summary): + stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + lines = [f"\n## {stamp} - rig turn ({args.session})\n"] + lines.append(f"- message: `{args.message}`") + lines.append(f"- model: `{cfg['model']}`, wall time {summary['wall_s']} s, " + f"{summary['llm_calls']} LLM call(s), exit {summary['exit']}") + if summary["rows"]: + lines.append("") + lines.append("| call | messages | prompt_tokens | cached_tokens | completion_tokens | ttft_s | duration_s |") + lines.append("|---|---|---|---|---|---|---|") + for r in summary["rows"]: + lines.append(f"| {r['call']} | {r['n_messages']} | {r['prompt_tokens']} | " + f"{r['cached_tokens']} | {r['completion_tokens']} | " + f"{r['ttft_s']} | {r['duration_s']} |") + if args.note: + lines.append(f"\n- note: {args.note}") + with LOG_PATH.open("a") as fh: + fh.write("\n".join(lines) + "\n") + print(f"logged to {LOG_PATH.relative_to(REPO)}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="cmd", required=True) + sub.add_parser("build") + sub.add_parser("reset") + p_turn = sub.add_parser("turn") + p_turn.add_argument("message") + p_turn.add_argument("--session", default="rig:main") + p_turn.add_argument("--logs", action="store_true", help="show nanobot runtime logs") + p_turn.add_argument("--no-log", dest="log", action="store_false") + p_turn.add_argument("--note", default="") + p_turn.add_argument("--env", action="append", metavar="KEY=VAL", + help="extra env for the agent container, repeatable") + args = parser.parse_args() + return {"build": cmd_build, "reset": cmd_reset, "turn": cmd_turn}[args.cmd](args) or 0 + + +if __name__ == "__main__": + sys.exit(main())