host: the memory review's two-ended cut spends its budget in characters

changelog.d/2553-memory-review-line-budget.fixed.md

@@ -0,0 +1 @@
+The memory review's two-ended cut of the user's own line now spends its 3,000-character budget in characters at both ends: a line of 3,500 emoji keeps 3,000 characters (2,000 head, 1,000 tail) instead of 1,503, so the review no longer drops half of a long message written with characters outside the BMP. ([#2553](https://github.com/ThinkFlowLab/FlyBot/issues/2553))

core/src/host/memory.js

@@ -28,7 +28,7 @@ import { BlockAssembler } from '@deepseek-ai/dsh-llm'
 import { currentParticipants } from './room-membership.js'
 import { defineTool } from './tool-schema.js'
 import { clip, codePoints, splitRef, unquote } from './text.js'
-import { codePointStart, cut, fits } from '../shared/text-boundary.js'
+import { cut, fits } from '../shared/text-boundary.js'
 import { output } from './tools.js'
 import { retiredBot } from './store.js'
 import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js'
@@ -1267,13 +1267,26 @@ export function install(rt) {
     return cursors
   }
   const saveCursors = () => (writeFolder(root) ? writeText(cursorsPath, JSON.stringify(cursors)) : Promise.resolve())
+  // The UTF-16 index of the n-th character: a budget spent in characters needs an index, and
+  // `text.length` counts units, so a character outside the BMP would spend two of it.
+  // Stepping whole code points makes the index a boundary by construction, so no cut lands
+  // between the halves of a surrogate pair.
+  const charIndex = (text, chars) => {
+    let index = 0
+    for (let n = 0; n < chars && index < text.length; n += 1) index += text.codePointAt(index) > 0xffff ? 2 : 1
+    return index
+  }
+
   // A long message often ends with what the user wants of it, so both ends stay.
   const ends = (text, max) => {
     if (fits(text, max)) return text
-    const tail = text.slice(codePointStart(text, text.length - Math.floor(max / 3)))
-    // Stepping back onto a boundary can keep one more unit at the tail; the head gives it back.
-    const head = cut(text, Math.min(Math.ceil(max * 2 / 3), max - tail.length))
-    return `${head.text} …(middle cut)… ${tail}`
+    // Flattened first, as `cut` did: text that arrives with an unpaired surrogate is stored
+    // as U+FFFD rather than half a character.
+    const flat = text.toWellFormed()
+    const tailChars = Math.floor(max / 3)
+    const tail = flat.slice(charIndex(flat, codePoints(flat) - tailChars))
+    const head = flat.slice(0, charIndex(flat, max - tailChars))
+    return `${head} …(middle cut)… ${tail}`
   }
 
   // Lines say who spoke. The user's lines are kept longest: what they say is what

core/test/memory.test.js

@@ -606,6 +606,30 @@ test('a long user line cut for the review keeps whole characters at both ends (#
     assert.deepEqual(Buffer.from(keptHead, 'utf8'), Buffer.from(text.slice(0, keptHead.length), 'utf8'))
     assert.deepEqual(Buffer.from(keptTail, 'utf8'), Buffer.from(text.slice(-keptTail.length), 'utf8'))
   }
+  // The two ends spend the budget in characters too, not in UTF-16 units: 3,500 emoji are 7,000
+  // units, so unit arithmetic kept 1,516 characters of the 3,000-character budget, a
+  // 1,000-character head and a 500-character tail. The whole line is `MAX` characters plus the
+  // marker, the head is the rendered line's first 2,000 characters (the `User: ` label included)
+  // and the tail its last 1,000. ASCII of the same length is the control: it spent the budget
+  // correctly before the change too.
+  for (const [text, sample] of [['🚀'.repeat(3500), 'User: 🚀'], ['a'.repeat(3500), 'User: a']]) {
+    const prompt = await render(text)
+    const line = prompt.split('\n').find(one => one.startsWith(sample))
+    assert.ok(line !== undefined, `the user line is in the review context:\n${prompt}`)
+    assert.equal(line.isWellFormed(), true)
+    assert.equal(line.includes('\ufffd'), false)
+    const mark = ' …(middle cut)… '
+    const at = line.indexOf(mark)
+    assert.ok(at > 0, line.slice(0, 40))
+    const head = [...line.slice(0, at)]
+    const tail = [...line.slice(at + mark.length)]
+    assert.equal(head.length + tail.length, MAX, 'the view keeps the whole 3,000-character budget')
+    assert.equal(head.length, Math.ceil((MAX * 2) / 3), 'the head keeps its 2,000 characters')
+    assert.equal(tail.length, Math.floor(MAX / 3), 'the tail keeps its 1,000 characters')
+    const message = [...text]
+    assert.equal(head.slice('User: '.length).join(''), message.slice(0, head.length - 'User: '.length).join(''))
+    assert.equal(tail.join(''), message.slice(-tail.length).join(''))
+  }
 })
 
 test('when a turn ends, the memory review keeps what the user said, through the same rules', async (t) => {