fix(dream): refine runs the rewrite guard; cluster joins at 0.75; abstractions get real names (1.11.0)

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 ## [Unreleased]
 
+## [1.11.0] — 2026-10-25
+
+### Fixed
+
+- **Refine runs the rewrite guard (#85).** Only hindsight called `checkRewrite`, so refine could drop what a definition committed to or append an observation verbatim, and the signature gate still called the run clean. In one live run, a paper summary was appended to an unrelated memory and its opening "Virgil's memoir-agent idea:" was dropped; another memory lost its closing "(v0.8.1)". Refine now declines both and keeps the old definition. Declined rewrites are counted in `phases.refine.declined`.
+- **The guard catches more:**
+  - **Version numbers:** the number check now sees them ("v0.8.1" → "0.8.1"; before, `\b` found no boundary after the `v`).
+  - **Dropped named entities:** a capitalised word from mid-sentence in the old definition that is gone from the new one.
+  - **Lost opening subject:** the subject the old definition opened with ("Virgil's …") is gone.
+  - **Growth:** with `limitGrowth` (refine sets it), a definition that grows past max(1.4×, +300 chars) is declined.
+  - **Case:** entity checks ignore case and match whole words, so "Cortex" opening a sentence is not new next to "the cortex API".
+- **Cluster joins an existing memory only at similarity ≥ 0.75**, up from 0.70. A wrong join hands refine unrelated text; a missed join only mints a separate memory. In the run above, the bad join scored 0.712 and the legitimate one 0.797. Set it per namespace with `cluster_threshold`, which `dream` and auto-consolidation now pass through.
+- **Unlabelled abstractions get a real name.** With no `Pattern:` line in the model output, the name used to be the definition's first clause cut short ("The deeper pattern is that systems—whether…"). Abstract now asks `deriveName` for a concept label, the way create names memories. `parseAbstraction` returns `labelled`.
+- `phases.hindsight.declined` is now reported by `dreamConsolidate`. It was counted but dropped from the result.
+
 ## [1.10.0] — 2026-10-25
 
 ### Fixed

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fozikio/cortex-engine",
-  "version": "1.10.0",
+  "version": "1.11.0",
   "description": "Portable cognitive engine for AI agents — storage, embeddings, memory, FSRS, and MCP server",
   "type": "module",
   "main": "dist/index.js",

src/core/config.ts

@@ -27,6 +27,14 @@ export interface NamespaceConfig {
    * the paraphrase pairs and the genuinely distinct ones.
    */
   abstraction_dedupe_threshold?: number;
+  /**
+   * Similarity an observation needs to join an existing memory in dream's
+   * cluster phase and become evidence for refining it (default: 0.75). A
+   * wrong join lets refine fold unrelated text into a memory; a missed join
+   * only mints a separate memory. Measure the nearest-memory scores of a
+   * run's observations and put the line above the joins that do not belong.
+   */
+  cluster_threshold?: number;
   /** Whether other namespaces can read this namespace's memories (default: false). */
   queryable?: boolean;
 }

src/engines/auto-consolidate.ts

@@ -161,6 +161,7 @@ export class SessionConsolidator {
       observation_limit: 50,
       similarity_merge: nsConfig.similarity_merge,
       similarity_link: nsConfig.similarity_link,
+      cluster_threshold: nsConfig.cluster_threshold,
     }).then(() => {}).catch((err: unknown) => {
       // Auto-consolidation is best-effort — never crash the serving process.
       if (process.env['CORTEX_DEBUG']) {

src/engines/cognition-abstract.test.ts

@@ -25,6 +25,7 @@ describe('parseAbstraction', () => {
     expect(parsed).toEqual({
       name: 'Silent Success as a Structural Failure Mode',
       definition: 'This abstraction connects the silent wander failure and the markdown folder. It matters because invisible failure becomes systemic.',
+      labelled: true,
     });
   });
 
@@ -49,6 +50,8 @@ describe('parseAbstraction', () => {
     const parsed = parseAbstraction(text);
     expect(parsed?.name).toBe('Silent failure is structural, not incidental');
     expect(parsed?.definition).toBe(text);
+    // Unlabelled: abstractCrossDomain asks deriveName for a real label (#85).
+    expect(parsed?.labelled).toBe(false);
   });
 
   it('returns null for a title with no body', () => {

src/engines/cognition-refine-guard.test.ts

@@ -0,0 +1,172 @@
+/**
+ * #85: refine runs the rewrite guard, cluster joins only above 0.75, and an
+ * unlabelled abstraction gets a real name.
+ *
+ * The refine fixture is the shape one live run produced: an observation that
+ * shared vocabulary with a memory (a paper summary vs. a memoir idea) joined
+ * it, and refine dropped the memory's subject and appended the observation.
+ * The signature gate called that clean; the guard has to refuse it.
+ */
+
+import { describe, it, expect, vi } from 'vitest';
+import { dreamPhaseA, abstractCrossDomain } from './cognition.js';
+import { newFSRSState } from './fsrs.js';
+import { SqliteCortexStore } from '../stores/sqlite.js';
+import type { EmbedProvider } from '../core/embed.js';
+import type { LLMProvider } from '../core/llm.js';
+import type { Memory, MemoryCategory } from '../core/types.js';
+
+const MEMO = [1, 0, 0];
+const OTHER_A = [0, 0, 1];
+const OTHER_B = [0, 0.6, 0.8];
+
+const MEMOIR_DEF =
+  "Virgil's memoir-agent idea: feed someone's draft memoir into a cortex agent and write from the lived experience. A product direction nobody else could do.";
+
+function memory(overrides: Partial<Omit<Memory, 'id'>>): Omit<Memory, 'id'> {
+  const now = new Date();
+  return {
+    name: "Virgil's memoir-agent idea",
+    definition: MEMOIR_DEF,
+    category: 'belief',
+    salience: 0.7,
+    confidence: 0.7,
+    access_count: 0,
+    created_at: now,
+    updated_at: now,
+    last_accessed: now,
+    source_files: [],
+    embedding: MEMO,
+    tags: [],
+    fsrs: newFSRSState(),
+    ...overrides,
+  };
+}
+
+/** A memory with two outgoing edges, so cluster treats its schema as established. */
+async function seedDenseMemoir(store: SqliteCortexStore): Promise<string> {
+  const id = await store.putMemory(memory({}));
+  const a = await store.putMemory(memory({ name: 'A', definition: 'Alpha is first.', embedding: OTHER_A }));
+  const b = await store.putMemory(memory({ name: 'B', definition: 'Beta is second.', embedding: OTHER_B }));
+  for (const target of [a, b]) {
+    await store.putEdge({ source_id: id, target_id: target, relation: 'supports', weight: 0.5, evidence: 'x', created_at: new Date() });
+  }
+  return id;
+}
+
+async function observe(store: SqliteCortexStore, content: string, embedding: number[]): Promise<void> {
+  await store.putObservation({
+    content, source_file: '', source_section: '', salience: 0.5, processed: false,
+    prediction_error: null, created_at: new Date(), updated_at: new Date(), embedding, keywords: [],
+  });
+}
+
+function llmReturning(text: string): LLMProvider & { generate: ReturnType<typeof vi.fn> } {
+  return {
+    name: 'fake', modelId: 'fake',
+    generate: vi.fn(async () => text),
+    generateJSON: vi.fn(async () => ({ category: 'belief', valid: true })),
+  } as unknown as LLMProvider & { generate: ReturnType<typeof vi.fn> };
+}
+
+const embed: EmbedProvider = { embed: vi.fn(async () => MEMO) };
+
+/** Unit vector at cosine `c` from MEMO. */
+const at = (c: number): number[] => [c, Math.sqrt(1 - c * c), 0];
+
+describe('refine runs the rewrite guard (#85)', () => {
+  it('declines a refinement that drops the subject and appends the observation', async () => {
+    const store = new SqliteCortexStore(':memory:');
+    const id = await seedDenseMemoir(store);
+    const paper = 'Generative Agents (Park et al., UIST 2023) keep a memory stream scored by recency, importance and relevance, and reflect when summed importance exceeds 150.';
+    await observe(store, paper, at(0.9));
+    const bled = `Feed someone's draft memoir into a cortex agent and write from the lived experience. A product direction nobody else could do. ${paper} Cortex differs in using FSRS retrievability, novelty gating and dream consolidation over a graph, but lacks evidence pointers in reflections and any importance-based reflection schedule.`;
+
+    const result = await dreamPhaseA(store, embed, llmReturning(bled));
+
+    expect(result.cluster.clustered).toBe(1);
+    expect(result.refine.refined).toBe(0);
+    expect(result.refine.declined).toBe(1);
+    expect((await store.getMemory(id))?.definition).toBe(MEMOIR_DEF);
+    expect(await store.getBeliefHistory(id)).toHaveLength(0);
+  });
+
+  it('declines a refinement that drops a version number', async () => {
+    const store = new SqliteCortexStore(':memory:');
+    const def = 'The agency check became an ops decision loaded at every session start (v0.8.1). It runs before anything else.';
+    const id = await seedDenseMemoir(store);
+    await store.updateMemory(id, { definition: def, name: 'Agency check wiring' });
+    await observe(store, 'The agency check is logged with ops_append at session end.', at(0.9));
+
+    const result = await dreamPhaseA(store, embed, llmReturning(
+      'The agency check became an ops decision loaded at every session start, and it is logged with ops_append at session end.',
+    ));
+
+    expect(result.refine.declined).toBe(1);
+    expect((await store.getMemory(id))?.definition).toBe(def);
+  });
+
+  it('still applies a refinement that integrates the evidence', async () => {
+    const store = new SqliteCortexStore(':memory:');
+    const id = await seedDenseMemoir(store);
+    await observe(store, 'Virgil wants the memoir agent to keep the writer’s own voice.', at(0.9));
+    const next = `${MEMOIR_DEF} It keeps the writer's own voice.`;
+
+    const result = await dreamPhaseA(store, embed, llmReturning(next));
+
+    expect(result.refine.refined).toBe(1);
+    expect(result.refine.declined).toBe(0);
+    expect((await store.getMemory(id))?.definition).toBe(next);
+  });
+});
+
+describe('cluster join floor (#85)', () => {
+  it('does not join an observation at 0.72 by default; it stays novel', async () => {
+    const store = new SqliteCortexStore(':memory:');
+    await seedDenseMemoir(store);
+    await observe(store, 'A paper about agent memory streams.', at(0.72));
+
+    const result = await dreamPhaseA(store, embed, llmReturning('unused'));
+
+    expect(result.cluster.clustered).toBe(0);
+    expect(result.cluster.unclustered).toBe(1);
+  });
+
+  it('joins at 0.72 when the namespace sets cluster_threshold 0.70', async () => {
+    const store = new SqliteCortexStore(':memory:');
+    await seedDenseMemoir(store);
+    await observe(store, 'A paper about agent memory streams.', at(0.72));
+
+    const result = await dreamPhaseA(store, embed, llmReturning(MEMOIR_DEF), { cluster_threshold: 0.70 });
+
+    expect(result.cluster.clustered).toBe(1);
+  });
+});
+
+describe('abstract names an unlabelled synthesis (#85)', () => {
+  it('asks for a concept label instead of clipping the first clause', async () => {
+    const store = new SqliteCortexStore(':memory:');
+    const seeds: Array<[string, string, MemoryCategory, number[]]> = [
+      ['The sky is blue', 'The sky is blue on clear days.', 'belief', [1, 0, 0]],
+      ['The sea is deep', 'The sea is deep and cold.', 'topic', [0, 1, 0]],
+      ['Clouds are white', 'Clouds are white and drift.', 'value', [0, 0, 1]],
+    ];
+    for (const [name, definition, category, embedding] of seeds) {
+      await store.putMemory(memory({ name, definition, category, embedding }));
+    }
+    const synthesis =
+      'The deeper pattern is that the sky, the sea and the clouds are one weather system. The blue sky, the deep sea and the white clouds each show a part of it, and that matters because no part explains the weather alone.';
+    const llm = {
+      name: 'fake', modelId: 'fake',
+      // First call is the synthesis, the second is deriveName's label.
+      generate: vi.fn().mockResolvedValueOnce(synthesis).mockResolvedValue('One Weather System'),
+      generateJSON: vi.fn(async () => ({})),
+    } as unknown as LLMProvider;
+
+    await abstractCrossDomain(store, { embed: vi.fn(async () => [0.5, 0.5, 0.5]) }, llm, { abstraction_attempts: 1 });
+
+    const minted = (await store.getAllMemories()).find((m) => m.memory_origin === 'abstract');
+    expect(minted?.name).toBe('One Weather System');
+    expect(minted?.definition).toBe(synthesis);
+  });
+});