fix(dream): abstractions from related members with a concrete shared mechanism, at salience 0.5 (1.12.0)

CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## [Unreleased]
 
+## [1.12.0] — 2026-10-25
+
+### Changed
+
+- **Abstract samples members that already relate (#147).** Before, each member came from a different random category, so the model was handed unrelated memories and asked what connects them. It always found something by climbing to a theme: one live run linked a failed MCP integration to a person's family history. Each attempt now seeds on a recent memory, and its members are the seed's graph neighbours (any relation but `exemplifies`) plus near neighbours at cosine ≥ `abstraction_relate_floor` (default 0.50). Members from new categories are preferred. With fewer than two related members, the model is not asked at all.
+  - Why 0.50: on one store, random pairs scored p90 0.43 and edge-linked pairs a median of 0.37–0.42, so the graph carries most of the signal.
+  - Existing abstractions are never members, so syntheses don't compound on syntheses.
+- **An abstraction must name a concrete shared mechanism.** `abstract-synthesis` v3 asks for JSON (`mechanism`, `members`, `name`, `abstraction`) and makes "none" the expected answer. The new `checkMechanism` keeps a synthesis only when its mechanism contains a term that at least two cited members' own text uses. The term must not be an abstract noun such as "complexity", "friction" or "identity", and in pools of 20 or more it must appear in no more than 25% of memories. Members that restate one fact (term-set Jaccard ≥ 0.25) count as one member: the first measured run wrote a third retelling of two near-identical memories. Refused syntheses are counted in `phases.abstract.rejected`.
+- **The phase now writes few abstractions, on purpose.** I measured 20 attempts on one store with qwen3:14b. With this prompt it wrote none: the model answered "none" for 18 related groups, and its one synthesis was a restatement, which was refused. A variant without the "most groups have none" line wrote 8, of which about half were the same theme-glue under concrete-sounding names. A phase that runs unattended should be quiet rather than wrong. Retrieval feedback is the better signal for loosening it later.
+- **Abstractions start at salience 0.5** (`ABSTRACTION_SALIENCE`), down from 0.8, so an unverified synthesis no longer outranks most organic memories. They link `exemplifies` only to the members the mechanism spans, and each edge's evidence names the shared terms.
+
 ## [1.11.0] — 2026-10-25
 
 ### Fixed

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fozikio/cortex-engine",
-  "version": "1.11.0",
+  "version": "1.12.0",
   "description": "Portable cognitive engine for AI agents — storage, embeddings, memory, FSRS, and MCP server",
   "type": "module",
   "main": "dist/index.js",

src/engines/abstraction-check.test.ts

@@ -0,0 +1,101 @@
+/**
+ * Shared-mechanism check (#147). The rejected fixtures are the shapes of two
+ * live abstractions: a theme ("friction and identity") gluing a technical
+ * memory to a personal one, and a term every memory in the store uses.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { checkMechanism, documentFrequency } from './abstraction-check.js';
+
+const PUSH = { name: 'Push gate fixed', text: 'The push gate hook denied every Bash command containing git push; fixed by matching the command, not the text.' };
+const GUARD = { name: 'Guardrail hook blocks deletes', text: 'The guardrail hook blocks rm with a tilde path before the command runs.' };
+const FAMILY = { name: 'Family background', text: 'Virgil grew up moving between cities and learned to rebuild routines from scratch.' };
+const members = [PUSH, GUARD, FAMILY];
+
+const smallPool = { df: documentFrequency(members.map((m) => `${m.name} ${m.text}`)), poolSize: 3 };
+
+describe('checkMechanism', () => {
+  it('accepts a mechanism that names a term two cited members share', () => {
+    const r = checkMechanism({
+      mechanism: 'a hook that blocks a command before it runs',
+      cited: ['Push gate fixed', 'Guardrail hook blocks deletes'],
+      members, ...smallPool,
+    });
+    expect(r.ok).toBe(true);
+    expect(r.sharedTerms).toEqual(expect.arrayContaining(['hook', 'command']));
+    expect(r.citedMembers.map((m) => m.name)).toEqual(['Push gate fixed', 'Guardrail hook blocks deletes']);
+  });
+
+  it('rejects a theme that no cited member uses', () => {
+    const r = checkMechanism({
+      mechanism: 'systems develop through friction and identity',
+      cited: ['Push gate fixed', 'Family background'],
+      members, ...smallPool,
+    });
+    expect(r.ok).toBe(false);
+    expect(r.reasons).toContain('mechanism names no concrete term the cited members share');
+  });
+
+  it('rejects a term too common in a real-sized pool to count as shared', () => {
+    const filler = Array.from({ length: 40 }, (_, i) => `Memory ${i} about a hook in some command.`);
+    const df = documentFrequency([...members.map((m) => `${m.name} ${m.text}`), ...filler]);
+    const r = checkMechanism({
+      mechanism: 'a hook on a command',
+      cited: ['Push gate fixed', 'Guardrail hook blocks deletes'],
+      members, df, poolSize: 43,
+    });
+    expect(r.ok).toBe(false);
+  });
+
+  it('needs at least two cited members that were actually shown', () => {
+    const r = checkMechanism({
+      mechanism: 'a hook that blocks a command',
+      cited: ['Push gate fixed', 'Something never shown'],
+      members, ...smallPool,
+    });
+    expect(r.ok).toBe(false);
+    expect(r.reasons[0]).toMatch(/spans 1 of the shown members/);
+  });
+
+  it('matches cited names loosely (case, punctuation, shortened)', () => {
+    const r = checkMechanism({
+      mechanism: 'a hook that blocks a command',
+      cited: ['push gate', 'GUARDRAIL HOOK BLOCKS DELETES.'],
+      members, ...smallPool,
+    });
+    expect(r.ok).toBe(true);
+  });
+
+  it('rejects a missing mechanism or a non-array members field', () => {
+    expect(checkMechanism({ mechanism: '', cited: ['Push gate fixed', 'Guardrail hook blocks deletes'], members, ...smallPool }).reasons)
+      .toContain('no mechanism stated');
+    expect(checkMechanism({ mechanism: 'a hook', cited: 'Push gate fixed', members, ...smallPool }).ok).toBe(false);
+  });
+});
+
+describe('checkMechanism — restatements (#147)', () => {
+  // Shapes of the two Muzak memories one copy run "abstracted" into a third copy.
+  const A = { name: 'Muzak was designed to sedate anxious elevator riders', text: "Muzak was designed to sedate anxious elevator riders through functional acoustic engineering that used patented 'Stimulus Progression', a system of 15-minute blocks engineered to create a subconscious sense of forward movement." };
+  const B = { name: 'Muzak Stimulus Progression was a patented system', text: "Muzak's 'Stimulus Progression' was a patented system for behavioral engineering through ambient sound, using 15-minute blocks of music engineered to create a subconscious sense of forward movement." };
+
+  it('counts two retellings of one fact as one member', () => {
+    const r = checkMechanism({
+      mechanism: 'Stimulus Progression, 15-minute blocks of engineered music',
+      cited: [A.name, B.name],
+      members: [A, B], df: documentFrequency([A.text, B.text]), poolSize: 2,
+    });
+    expect(r.ok).toBe(false);
+    expect(r.reasons.join(' ')).toMatch(/restate one another/);
+  });
+
+  it('keeps "stimulus" whole', () => {
+    const r = checkMechanism({
+      mechanism: 'a hook that blocks a command',
+      cited: ['Push gate fixed', 'Guardrail hook blocks deletes'],
+      members, ...smallPool,
+    });
+    expect(r.ok).toBe(true);
+    expect(checkMechanism({ mechanism: 'stimulus', cited: [A.name, PUSH.name], members: [A, PUSH], df: new Map(), poolSize: 2 }).sharedTerms)
+      .not.toContain('stimulu');
+  });
+});

src/engines/abstraction-check.ts

@@ -0,0 +1,139 @@
+/**
+ * Shared-mechanism check for dream abstractions (#147).
+ *
+ * Asked for "the pattern that connects these concepts", a model nearly always
+ * finds one by climbing to a level where anything connects: "systems develop
+ * through friction and identity", "subtraction over addition", "agency and
+ * constraint". One live run linked a failed MCP integration to a person's
+ * family history that way. The words that carry such a synthesis are abstract
+ * nouns that no member actually uses, or words every memory in the store uses.
+ *
+ * So the model has to name the mechanism and the members it spans, and the
+ * mechanism has to hold a term that at least two of those members' own text
+ * contains — not an abstract noun, and not a word common to a large share of
+ * the store. A synthesis that can't point to a term the members share is
+ * glue, however fluent.
+ */
+
+import { extractKeywords } from './keywords.js';
+
+/** Nouns a synthesis reaches for when it has no concrete link. */
+const ABSTRACT_NOUNS = new Set([
+  'system', 'pattern', 'complexity', 'integrity', 'authenticity', 'clarity', 'efficiency', 'friction',
+  'identity', 'constraint', 'agency', 'autonomy', 'alignment', 'balance', 'tension', 'interplay',
+  'evolution', 'growth', 'adaptation', 'adaptability', 'coherence', 'consistency', 'reliability',
+  'structure', 'structured', 'process', 'approach', 'dynamic', 'mechanism', 'framework', 'principle',
+  'concept', 'context', 'nature', 'value', 'purpose', 'meaning', 'trust', 'boundary', 'boundarie',
+  'change', 'flow', 'emergence', 'emergent', 'resilience', 'transformation', 'optimization',
+  'optimisation', 'performance', 'simplicity', 'subtraction', 'addition', 'design', 'development',
+  'relationship', 'connection', 'interaction', 'feedback', 'loop', 'learning', 'knowledge',
+  'understanding', 'insight', 'deeper', 'underlying', 'fundamental', 'effective', 'meaningful',
+  'complex', 'iterative', 'intentional', 'organizational', 'organisational', 'technical', 'personal',
+  'cognitive', 'creative', 'shift', 'role', 'need', 'importance', 'impact', 'focus', 'state', 'level',
+]);
+
+/** Share of the pool above which a term is too common to count as shared. */
+export const MAX_TERM_SHARE = 0.25;
+/** Pool size below which the share rule is not applied. */
+export const MIN_POOL_FOR_SHARE = 20;
+/**
+ * Term-set Jaccard at or above which two members restate one fact and count
+ * as one. On one store, edge-linked pairs sat at p95 0.094 / p99 0.294; two
+ * memories retelling Muzak's "Stimulus Progression" scored 0.308, and their
+ * "abstraction" was a third copy of the same fact.
+ */
+export const RESTATE_JACCARD = 0.25;
+
+function jaccard(a: Set<string>, b: Set<string>): number {
+  let shared = 0;
+  for (const x of a) if (b.has(x)) shared++;
+  const union = a.size + b.size - shared;
+  return union === 0 ? 0 : shared / union;
+}
+
+/** Light normalisation so "hooks"/"hook" and "edges"/"edge" meet; "stimulus", "analysis", "class" stay whole. */
+export function normaliseTerm(word: string): string {
+  return word.length > 4 && word.endsWith('s') && !/(?:ss|us|is)$/.test(word) ? word.slice(0, -1) : word;
+}
+
+export function termSet(text: string): Set<string> {
+  return new Set(extractKeywords(text, 500).map(normaliseTerm));
+}
+
+/** Document frequency of each term over a pool of texts. */
+export function documentFrequency(texts: string[]): Map<string, number> {
+  const df = new Map<string, number>();
+  for (const t of texts) for (const term of termSet(t)) df.set(term, (df.get(term) ?? 0) + 1);
+  return df;
+}
+
+export interface MechanismMember {
+  name: string;
+  text: string;
+}
+
+export interface MechanismCheckInput {
+  /** The mechanism as the model stated it. */
+  mechanism: unknown;
+  /** Names of the members the model says the mechanism spans. */
+  cited: unknown;
+  /** The members the model was shown. */
+  members: MechanismMember[];
+  /** Document frequency over the pool, and the pool's size. */
+  df: Map<string, number>;
+  poolSize: number;
+}
+
+export interface MechanismCheckResult {
+  ok: boolean;
+  reasons: string[];
+  /** Members the model cited that match a shown member, in shown order. */
+  citedMembers: MechanismMember[];
+  /** Mechanism terms found in at least two cited members. */
+  sharedTerms: string[];
+}
+
+const norm = (s: string): string => s.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
+
+export function checkMechanism(input: MechanismCheckInput): MechanismCheckResult {
+  const reasons: string[] = [];
+  const mechanism = typeof input.mechanism === 'string' ? input.mechanism.trim() : '';
+  const cited = Array.isArray(input.cited) ? input.cited.filter((c): c is string => typeof c === 'string') : [];
+
+  // A cited name matches a shown member when either contains the other
+  // (models shorten long names, or add the category tag).
+  const citedMembers = input.members.filter((m) => {
+    const n = norm(m.name);
+    return cited.some((c) => {
+      const cn = norm(c);
+      return cn.length > 0 && (cn.includes(n) || n.includes(cn));
+    });
+  });
+
+  if (!mechanism) reasons.push('no mechanism stated');
+  if (citedMembers.length < 2) reasons.push(`mechanism spans ${citedMembers.length} of the shown members (need 2)`);
+
+  // Members that restate one fact count once: a "synthesis" of two retellings
+  // is a third retelling, not an abstraction.
+  const memberTerms: Array<Set<string>> = [];
+  for (const m of citedMembers) {
+    const terms = termSet(`${m.name} ${m.text}`);
+    if (!memberTerms.some((t) => jaccard(t, terms) >= RESTATE_JACCARD)) memberTerms.push(terms);
+  }
+  if (citedMembers.length >= 2 && memberTerms.length < 2) {
+    reasons.push(`cited members restate one another (term overlap >= ${RESTATE_JACCARD})`);
+  }
+  // "Too common" means something only in a pool big enough to have a
+  // background; in a handful of memories any shared term is a large share.
+  const limit = input.poolSize < MIN_POOL_FOR_SHARE ? Infinity : Math.floor(input.poolSize * MAX_TERM_SHARE);
+  const sharedTerms = [...termSet(mechanism)].filter((term) =>
+    !ABSTRACT_NOUNS.has(term)
+    && memberTerms.filter((s) => s.has(term)).length >= 2
+    && (input.df.get(term) ?? 0) <= limit,
+  );
+  if (mechanism && memberTerms.length >= 2 && sharedTerms.length === 0) {
+    reasons.push('mechanism names no concrete term the cited members share');
+  }
+
+  return { ok: reasons.length === 0, reasons, citedMembers, sharedTerms };
+}