diff --git a/changelog.d/2556-memory-review-flattened-lines.fixed.md b/changelog.d/2556-memory-review-flattened-lines.fixed.md new file mode 100644 index 00000000..ca2e1149 --- /dev/null +++ b/changelog.d/2556-memory-review-flattened-lines.fixed.md @@ -0,0 +1 @@ +A memory review line that carried an unpaired surrogate now reaches the prompt as one whole character instead of half a pair: both of the review's render paths flatten the text first ([#2556](https://github.com/ThinkFlowLab/FlyBot/issues/2556)). diff --git a/core/src/host/memory.js b/core/src/host/memory.js index cf40d1f6..22d18cae 100644 --- a/core/src/host/memory.js +++ b/core/src/host/memory.js @@ -1279,10 +1279,11 @@ export function install(rt) { // A long message often ends with what the user wants of it, so both ends stay. const ends = (text, max) => { - if (fits(text, max)) return text - // Flattened first, as `cut` did: text that arrives with an unpaired surrogate is stored - // as U+FFFD rather than half a character. + // Flattened on both paths, as `cut` did: text that arrives with an unpaired surrogate is + // stored as U+FFFD rather than half a character. A line that fits is the likelier carrier โ€” + // it is the one this branch used to answer untouched. const flat = text.toWellFormed() + if (fits(flat, max)) return flat const tailChars = Math.floor(max / 3) const tail = flat.slice(charIndex(flat, codePoints(flat) - tailChars)) const head = flat.slice(0, charIndex(flat, max - tailChars)) @@ -1293,7 +1294,7 @@ export function install(rt) { // the review keeps; the rest is there to make it understood. const reviewLine = (who, text, bot) => (who === 'user' ? ends(text, REVIEW_USER_LINE_MAX) - : clip(who === 'you' ? `${bot.name}: ${text.replace(/^You: /, '')}` : text, REVIEW_OTHER_LINE_MAX)) + : clip((who === 'you' ? `${bot.name}: ${text.replace(/^You: /, '')}` : text).toWellFormed(), REVIEW_OTHER_LINE_MAX)) // What is new in a Bot's own chat, read newest first from the Session. async function chatBatch(sessionId, bot, since) { diff --git a/core/test/memory.test.js b/core/test/memory.test.js index c6d14ad1..135e1583 100644 --- a/core/test/memory.test.js +++ b/core/test/memory.test.js @@ -607,11 +607,12 @@ test('a long user line cut for the review keeps whole characters at both ends (# assert.deepEqual(Buffer.from(keptTail, 'utf8'), Buffer.from(text.slice(-keptTail.length), 'utf8')) } // The two ends spend the budget in characters too, not in UTF-16 units: 3,500 emoji are 7,000 - // units, so unit arithmetic kept 1,516 characters of the 3,000-character budget, a - // 1,000-character head and a 500-character tail. The whole line is `MAX` characters plus the - // marker, the head is the rendered line's first 2,000 characters (the `User: ` label included) - // and the tail its last 1,000. ASCII of the same length is the control: it spent the budget - // correctly before the change too. + // units, so unit arithmetic kept 1,503 characters of the 3,000-character budget โ€” a + // 1,003-character head (`User: ` included, the rendered line's first 2,000 units) and a + // 500-character tail. After the fix the whole line is `MAX` characters plus the marker, the + // head is the rendered line's first 2,000 characters (the `User: ` label included) and the + // tail its last 1,000. ASCII of the same length is the control: it spent the budget correctly + // before the change too. for (const [text, sample] of [['๐Ÿš€'.repeat(3500), 'User: ๐Ÿš€'], ['a'.repeat(3500), 'User: a']]) { const prompt = await render(text) const line = prompt.split('\n').find(one => one.startsWith(sample)) @@ -632,6 +633,52 @@ test('a long user line cut for the review keeps whole characters at both ends (# } }) +// A line can arrive carrying an unpaired surrogate: a stream that ended mid-character, a fixture +// written by hand. The review's render path handed it straight through โ€” the `fits()` branch of +// `ends` answered the line itself, and `clip` only ever flattened what it cut. Both ends flatten +// first now, so the prompt holds U+FFFD where the half pair was, never the half pair itself. +test('an unpaired surrogate in a reviewed line reaches the prompt as one whole character', async (t) => { + const userText = `${'x'.repeat(100)}\ud800${'y'.repeat(100)}` + const botText = `${'z'.repeat(80)}\ud800${'w'.repeat(80)}` + const history = { + b2: [ + { event: { seq: 20, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Start here.' }] } } }, + { event: { seq: 21, type: 'assistant/message', data: { message: { content: [{ type: 'text', text: botText }] } } } }, + { event: { seq: 22, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: userText }] } } }, + ], + } + const replies = [JSON.stringify({ changes: [] })] + const { ctx, call, requests, stop } = start({ replies, history, config: { memoryReviewDelayMs: 0 } }) + t.after(() => stop()) + await call('state', {}) + ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' }) + await reviewed(call) + const prompt = requests[0].messages[0].content[0].text + const replacements = (text) => [...text].filter(ch => ch === '\ufffd').length + + // The user's line is 201 characters โ€” 207 rendered โ€” so it takes the fits branch of `ends` + // instead of its cut, and that branch used to answer the line untouched. + const userLine = prompt.split('\n').find(one => one.startsWith('User: x')) + assert.ok(userLine !== undefined, `the user line is in the review context:\n${prompt}`) + assert.equal(userLine.isWellFormed(), true) + assert.equal(replacements(userLine), 1) + assert.equal(userLine.indexOf('\ufffd'), 'User: '.length + 100) + assert.equal(userLine.slice('User: '.length, 'User: '.length + 100), 'x'.repeat(100)) + assert.equal(userLine.slice('User: '.length + 101, 'User: '.length + 201), 'y'.repeat(100)) + + // The Bot's line is 160 characters under the 800 limit and carries its label, so it takes the + // other branch of `reviewLine`, which handed it to `clip` without flattening it first. + const botLine = prompt.split('\n').find(one => one.startsWith('Writer: z')) + assert.ok(botLine !== undefined, `the Bot line is in the review context:\n${prompt}`) + const label = botLine.slice(0, botLine.indexOf('z')) + assert.equal(botLine.isWellFormed(), true) + assert.equal(replacements(botLine), 1) + assert.equal(botLine.indexOf('\ufffd'), label.length + 80) + assert.equal(botLine.slice(label.length, label.length + 80), 'z'.repeat(80)) + assert.equal(botLine.slice(label.length + 81, label.length + 161), 'w'.repeat(80)) + assert.equal(prompt.isWellFormed(), true, 'the prompt holds no half pair') +}) + test('when a turn ends, the memory review keeps what the user said, through the same rules', async (t) => { const history = { b2: [ diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index 92f06a7e..2199d960 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -38,7 +38,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | --- | --- | | `core/index.js` | Server entry. | | `core/src/host/` | Host-side helpers (non-hooks). | -| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit โ€” one character costs one whatever its UTF-16 width โ€” and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it โ€” the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` โ€” so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it โ€” the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) โ€” each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-character context `clip` cuts through `cut`, and its 3,000-character two-ended `ends` spends the budget in characters at both ends โ€” `charIndex` steps whole code points to find the head's end and the tail's start, so the head is the rendered line's first 2,000 characters and the tail its last 1,000, and the text is flattened with `toWellFormed()` first as `cut` did โ€” so the prompt and the log hold whole characters within the same character budget: a line of 3,500 emoji keeps 3,000 characters where unit arithmetic kept 1,503 ([#2553](https://github.com/ThinkFlowLab/FlyBot/issues/2553)); the exported `clip`'s reason is optional there too, so a two-argument call reads `โ€ฆ (cut)` instead of `โ€ฆ (cut; undefined)` while a call with a reason is unchanged. Since #1923 the four bounds whose text crosses an owner boundary go through the same `cut` โ€” `cleanPublishedName` in `hooks.js` (60, into `state.flybot.remote`), the group member line's `displayName` 60 and `ownerName` 40 in `tools.js`, `label` in `reach.js` (60, the `othersInRoom` labels), and the local `clip` in `shared/room-share.js` (its per-field caps included) โ€” so a published, listed or relayed name keeps whole characters and never reaches a UTF-8 encoder as U+FFFD ([#1923](https://github.com/ThinkFlowLab/FlyBot/issues/1923)). The same rule now covers the host's own line bounds: the busy ack's 600-unit task and 1,500-unit message text in `busy-ack.js`, the group line's notice (100), a Bot's job line (120), the group answer echo (300) and the rework line's note (200) in `tools.js`, and `cleanName`'s 60-unit peer name in `delivery.js`, which lands in `fenceUntrusted`'s `from="โ€ฆ"` attribute โ€” so none of them can end in half a surrogate pair. The five bounds `turns.js` keeps go through `cut` as well: a failure's 500-unit message and the 200-unit message of its line (`turnFailure`, `failureLine`), the 79-unit step a running call shows (the `โ€ฆ` is appended only when `cut` dropped something, never by a length guess), an approval card's 2,000-unit description and 4,000-unit command (the card stores the command "Allow once" runs rather than a preview of it, and one that still had to be cut ends with `โ€ฆ` inside that bound, so the card says what it abridged and never drops the fact, [#1992](https://github.com/ThinkFlowLab/FlyBot/issues/1992)), and the 160-unit reason a failed turn leaves for the inbox. Since #1963 `cut` also flattens its input before it bounds โ€” `String(text).toWellFormed()` first, `codePointEnd` on the flat string โ€” so a title, a note or a name that arrives already carrying an unpaired surrogate is stored as U+FFFD instead of the lone surrogate, and U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag ([#1963](https://github.com/ThinkFlowLab/FlyBot/issues/1963)). Since #1979 the six remaining bounds on text another home or another person supplied go through it too โ€” a connector item name (80, `connectors.js`), a role label another home set on a face-room seat (80, `face-lead.js`), a cross-home group name (200, `project-team.js`), a capability description from a peer server (2,000, `capabilities.js`), the remote home's own error text (157, `remote-chat.js`) and an abbreviated remote reply (80, same file) โ€” so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, while `remote-chat.js`'s `โ€ฆ` and trailing-period strip still apply to the cut value ([#1979](https://github.com/ThinkFlowLab/FlyBot/issues/1979)). The two bounds on a friend's Bot's own name in that file โ€” the name its chat is created with (40) and the name a refresh writes (40) โ€” go through `cut` as well, so a refresh cannot store a lone high surrogate and a create no longer hands `roster.createBot` half a pair to flatten into U+FFFD; the budget stays 40 and a shorter name is stored word for word. Since #1993 the last five bounds in `core/src/host/` go through it too โ€” the quoted first line of a checkpoint's kept tail (80, `steward.js`), an approval record's `reason` (500, `approvals.js`), a mention receipt's reason line (240, `hooks.js`), the title given to a DSH Agent's Session (80, `AGENT_SESSION_TITLE_MAX` in `agents.js`) and a schedule run's failure `reason` (200, `schedules.js`) โ€” so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, and none of the five maxima changes ([#1993](https://github.com/ThinkFlowLab/FlyBot/issues/1993)). Since #2002 the connector's own error text goes through it too: the `reason` in `core/src/host/connectors.js` โ€” the text an MCP server's failure leaves on the owner's connector card โ€” keeps the longest prefix that stops before a split pair, so it is well formed and a code-point prefix of what the server sent; since #2016 that prefix is 2,000 units rather than 300, and one that had to be cut ends with `โ€ฆ` inside the bound, so the card says what it abridged ([#2002](https://github.com/ThinkFlowLab/FlyBot/issues/2002), [#2016](https://github.com/ThinkFlowLab/FlyBot/issues/2016)). Since #2005 the five bounds the Project panels read go through it as well โ€” the assistant card's 77-unit title in `project-assistant.js`, the board's 200-unit one-line value and a decision title's 200 in `project-board.js`, and `capped`'s name, role, persona, model and color caps plus the dossier's 4,000-unit memory head in `project-team.js`, whose head is bounded where `memoryDigest` reads it (`memory.js`), because a cut at the dossier alone could only turn half a pair into U+FFFD and never drop the character whole โ€” so a dossier row or a board card keeps whole characters ([#2005](https://github.com/ThinkFlowLab/FlyBot/issues/2005)). Since [#2321](https://github.com/ThinkFlowLab/FlyBot/issues/2321) a bound also judges in code points, not only counts in them: `fits(text, max)` in `core/src/shared/text-boundary.js` is the one predicate โ€” a character outside the BMP spends one of the budget and not two โ€” and `clip` and `excerpt` judge with it while their cuts keep going through `codePointEnd`/`cut`, so a 300-character text is no longer cut by a 400-character limit on its way to `read_own_chat` and the search rows. Since [#2339](https://github.com/ThinkFlowLab/FlyBot/issues/2339) the two bounds in `core/src/host/` that still judged with `text.length` use it as well โ€” the memory review's 3,000-character two-ended `ends` in `memory.js` and `room-context.js`'s `oneLine` (40/80/24/60) โ€” so a user line or a board line of characters outside the BMP stays whole inside its own limit instead of being cut at half of it; both maxima are unchanged and a board line's cut stays on `cut`, so an ASCII text judges exactly as `length <= max` did. The memory review's own third `clip` is the exported one now, so its cut reads `โ€ฆ (cut)`, and a bound added later that judges with `text.length` fails `core/test/text-bounds.test.js` at its own `file:line`. | +| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit โ€” one character costs one whatever its UTF-16 width โ€” and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it โ€” the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` โ€” so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it โ€” the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) โ€” each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-character context `clip` cuts through `cut`, and its 3,000-character two-ended `ends` spends the budget in characters at both ends โ€” `charIndex` steps whole code points to find the head's end and the tail's start, so the head is the rendered line's first 2,000 characters and the tail its last 1,000, and the text is flattened with `toWellFormed()` first as `cut` did โ€” so the prompt and the log hold whole characters within the same character budget: a line of 3,500 emoji keeps 3,000 characters where unit arithmetic kept 1,503 ([#2553](https://github.com/ThinkFlowLab/FlyBot/issues/2553)); the exported `clip`'s reason is optional there too, so a two-argument call reads `โ€ฆ (cut)` instead of `โ€ฆ (cut; undefined)` while a call with a reason is unchanged. Since #1923 the four bounds whose text crosses an owner boundary go through the same `cut` โ€” `cleanPublishedName` in `hooks.js` (60, into `state.flybot.remote`), the group member line's `displayName` 60 and `ownerName` 40 in `tools.js`, `label` in `reach.js` (60, the `othersInRoom` labels), and the local `clip` in `shared/room-share.js` (its per-field caps included) โ€” so a published, listed or relayed name keeps whole characters and never reaches a UTF-8 encoder as U+FFFD ([#1923](https://github.com/ThinkFlowLab/FlyBot/issues/1923)). The same rule now covers the host's own line bounds: the busy ack's 600-unit task and 1,500-unit message text in `busy-ack.js`, the group line's notice (100), a Bot's job line (120), the group answer echo (300) and the rework line's note (200) in `tools.js`, and `cleanName`'s 60-unit peer name in `delivery.js`, which lands in `fenceUntrusted`'s `from="โ€ฆ"` attribute โ€” so none of them can end in half a surrogate pair. The five bounds `turns.js` keeps go through `cut` as well: a failure's 500-unit message and the 200-unit message of its line (`turnFailure`, `failureLine`), the 79-unit step a running call shows (the `โ€ฆ` is appended only when `cut` dropped something, never by a length guess), an approval card's 2,000-unit description and 4,000-unit command (the card stores the command "Allow once" runs rather than a preview of it, and one that still had to be cut ends with `โ€ฆ` inside that bound, so the card says what it abridged and never drops the fact, [#1992](https://github.com/ThinkFlowLab/FlyBot/issues/1992)), and the 160-unit reason a failed turn leaves for the inbox. Since #1963 `cut` also flattens its input before it bounds โ€” `String(text).toWellFormed()` first, `codePointEnd` on the flat string โ€” so a title, a note or a name that arrives already carrying an unpaired surrogate is stored as U+FFFD instead of the lone surrogate, and U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag ([#1963](https://github.com/ThinkFlowLab/FlyBot/issues/1963)). Since #1979 the six remaining bounds on text another home or another person supplied go through it too โ€” a connector item name (80, `connectors.js`), a role label another home set on a face-room seat (80, `face-lead.js`), a cross-home group name (200, `project-team.js`), a capability description from a peer server (2,000, `capabilities.js`), the remote home's own error text (157, `remote-chat.js`) and an abbreviated remote reply (80, same file) โ€” so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, while `remote-chat.js`'s `โ€ฆ` and trailing-period strip still apply to the cut value ([#1979](https://github.com/ThinkFlowLab/FlyBot/issues/1979)). The two bounds on a friend's Bot's own name in that file โ€” the name its chat is created with (40) and the name a refresh writes (40) โ€” go through `cut` as well, so a refresh cannot store a lone high surrogate and a create no longer hands `roster.createBot` half a pair to flatten into U+FFFD; the budget stays 40 and a shorter name is stored word for word. Since #1993 the last five bounds in `core/src/host/` go through it too โ€” the quoted first line of a checkpoint's kept tail (80, `steward.js`), an approval record's `reason` (500, `approvals.js`), a mention receipt's reason line (240, `hooks.js`), the title given to a DSH Agent's Session (80, `AGENT_SESSION_TITLE_MAX` in `agents.js`) and a schedule run's failure `reason` (200, `schedules.js`) โ€” so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, and none of the five maxima changes ([#1993](https://github.com/ThinkFlowLab/FlyBot/issues/1993)). Since #2002 the connector's own error text goes through it too: the `reason` in `core/src/host/connectors.js` โ€” the text an MCP server's failure leaves on the owner's connector card โ€” keeps the longest prefix that stops before a split pair, so it is well formed and a code-point prefix of what the server sent; since #2016 that prefix is 2,000 units rather than 300, and one that had to be cut ends with `โ€ฆ` inside the bound, so the card says what it abridged ([#2002](https://github.com/ThinkFlowLab/FlyBot/issues/2002), [#2016](https://github.com/ThinkFlowLab/FlyBot/issues/2016)). Since #2005 the five bounds the Project panels read go through it as well โ€” the assistant card's 77-unit title in `project-assistant.js`, the board's 200-unit one-line value and a decision title's 200 in `project-board.js`, and `capped`'s name, role, persona, model and color caps plus the dossier's 4,000-unit memory head in `project-team.js`, whose head is bounded where `memoryDigest` reads it (`memory.js`), because a cut at the dossier alone could only turn half a pair into U+FFFD and never drop the character whole โ€” so a dossier row or a board card keeps whole characters ([#2005](https://github.com/ThinkFlowLab/FlyBot/issues/2005)). Since [#2321](https://github.com/ThinkFlowLab/FlyBot/issues/2321) a bound also judges in code points, not only counts in them: `fits(text, max)` in `core/src/shared/text-boundary.js` is the one predicate โ€” a character outside the BMP spends one of the budget and not two โ€” and `clip` and `excerpt` judge with it while their cuts keep going through `codePointEnd`/`cut`, so a 300-character text is no longer cut by a 400-character limit on its way to `read_own_chat` and the search rows. Since [#2339](https://github.com/ThinkFlowLab/FlyBot/issues/2339) the two bounds in `core/src/host/` that still judged with `text.length` use it as well โ€” the memory review's 3,000-character two-ended `ends` in `memory.js` and `room-context.js`'s `oneLine` (40/80/24/60) โ€” so a user line or a board line of characters outside the BMP stays whole inside its own limit instead of being cut at half of it; both maxima are unchanged and a board line's cut stays on `cut`, so an ASCII text judges exactly as `length <= max` did. The memory review's own third `clip` is the exported one now, so its cut reads `โ€ฆ (cut)`, and a bound added later that judges with `text.length` fails `core/test/text-bounds.test.js` at its own `file:line`. Both render paths of the memory review flatten before they bound as well: the `fits()` answer of `ends` and the labelled Bot line on its way to `clip` pass through `toWellFormed()` first, so an unpaired surrogate that arrives in a message reaches the review prompt as U+FFFD, whole and in place, instead of as half a pair ([#2556](https://github.com/ThinkFlowLab/FlyBot/issues/2556)). | | `core/src/host/prompt.js` | The Bot system prompt; a Main Bot's section is headed `You are the Main Bot` and uses that one term, the one the UI shows (#276). The team rules state the idle default (#699): a Bot does nothing until the user addresses it, it is @-mentioned in a group, its parent gives it a task, or unfinished tasks it was already given are still on its board. It does not invent work. A coordinator acts on a user task rather than on its own initiative โ€” an addressed Bot answers as before. | | `core/src/host/room-membership.js` | Typed room participants (human / local bot / granted remote), authorship, membership-window history (#50); rooms with friends' Bots or other owners take granted local Bots only and evict a Bot that stops qualifying (#90). Face-to-face rooms (#83) reuse this model (`origin: 'face-to-face'`, room-only remotes without a person-grant, `upsertFaceRoom` / `syncFaceParticipants` / `applyFaceRoomLine`); a face-room local Bot is `viaCode` (brought by number), `evictUnshared` skips face rooms (membership is the server's), and a Bot that returns to the member list drops the empty local `left` line (#236). Refusing an ungranted local Bot next to a friend's Bot throws `ungrantedRefusal`: a message that names the Bot, the friend and the two ways out, plus `error.details` `{ reason: 'ungranted-local', bots, remotes }`; `channel.js` `fail` forwards a plain-object `details` (empty otherwise) and `client/api.js` puts it on the thrown error, where `client/new-chat.js` draws a card (**Copy invite link for {Bot}** via `__flybot__.social.createInvite`, **Leave {Bot} out**) read through `ungrantedLocalFix` in `shared/remote-label.js`; the same file's `copiedToClipboard` reports whether the invite link really reached the clipboard (none, or a refused write, is `false`), and the card then shows the link to copy by hand instead of saying it was copied. | | `core/src/host/face-mentions.js` | Who "@" calls in a face-to-face room (#287): every alias of a participant (`faceAliases`, kept in `shared/remote-label.js` so the bubble chips the same names, #305; bare name, the picker label `'s Chief` / ` ยท 's Bot`, ` ยท `, handle), local and friend's Bots told apart. `faceMentions().handles` is the handle of every Bot a line names (the explicit mention list a sender attaches, #293) and `mentionedByHandle` resolves a received list to my Bots (`known` counts handles the room has). | diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index 6c41e838..28208767 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -6155,7 +6155,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js if (text === '') throw new Error('Type a question or a change first') const numbered = [ ...allEntries(loadScope(ownScope(bot.id))).map(entry => ({ ...entry, scope: 'own' })), -@@ -1074,18 +1074,40 @@ export function install(rt) { +@@ -1074,24 +1074,47 @@ export function install(rt) { let cursors const cursorsNow = () => { if (cursors === undefined) { @@ -6191,10 +6191,11 @@ diff --git a/src/host/memory.js b/src/host/memory.js // A long message often ends with what the user wants of it, so both ends stay. - const ends = (text, max) => (text.length > max ? `${text.slice(0, Math.ceil(max * 2 / 3))} โ€ฆ(middle cut)โ€ฆ ${text.slice(-Math.floor(max / 3))}` : text) + const ends = (text, max) => { -+ if (fits(text, max)) return text -+ // Flattened first, as `cut` did: text that arrives with an unpaired surrogate is stored -+ // as U+FFFD rather than half a character. ++ // Flattened on both paths, as `cut` did: text that arrives with an unpaired surrogate is ++ // stored as U+FFFD rather than half a character. A line that fits is the likelier carrier โ€” ++ // it is the one this branch used to answer untouched. + const flat = text.toWellFormed() ++ if (fits(flat, max)) return flat + const tailChars = Math.floor(max / 3) + const tail = flat.slice(charIndex(flat, codePoints(flat) - tailChars)) + const head = flat.slice(0, charIndex(flat, max - tailChars)) @@ -6203,6 +6204,13 @@ diff --git a/src/host/memory.js b/src/host/memory.js // Lines say who spoke. The user's lines are kept longest: what they say is what // the review keeps; the rest is there to make it understood. + const reviewLine = (who, text, bot) => (who === 'user' + ? ends(text, REVIEW_USER_LINE_MAX) +- : clip(who === 'you' ? `${bot.name}: ${text.replace(/^You: /, '')}` : text, REVIEW_OTHER_LINE_MAX)) ++ : clip((who === 'you' ? `${bot.name}: ${text.replace(/^You: /, '')}` : text).toWellFormed(), REVIEW_OTHER_LINE_MAX)) + + // What is new in a Bot's own chat, read newest first from the Session. + async function chatBatch(sessionId, bot, since) { @@ -1123,17 +1123,28 @@ export function install(rt) { // What is new in a group, from its log: the member's Session holds the same @@ -12187,7 +12195,7 @@ diff --git a/test/memory.test.js b/test/memory.test.js const done = (await call('memory-view', { botId: 'b2' })).value assert.deepEqual([done.summary?.text, done.error], ['## Overview\nYou like tea.', null]) assert.deepEqual(routes(requests).slice(2), ['deepseek/flash@2048']) -@@ -519,12 +519,110 @@ test('a host that only reads the team shows memory but changes none of it', asyn +@@ -519,12 +519,157 @@ test('a host that only reads the team shows memory but changes none of it', asyn // Until no review waits or runs. async function reviewed(call) { for (let tries = 0; tries < 200; tries += 1) { @@ -12271,11 +12279,12 @@ diff --git a/test/memory.test.js b/test/memory.test.js + assert.deepEqual(Buffer.from(keptTail, 'utf8'), Buffer.from(text.slice(-keptTail.length), 'utf8')) + } + // The two ends spend the budget in characters too, not in UTF-16 units: 3,500 emoji are 7,000 -+ // units, so unit arithmetic kept 1,516 characters of the 3,000-character budget, a -+ // 1,000-character head and a 500-character tail. The whole line is `MAX` characters plus the -+ // marker, the head is the rendered line's first 2,000 characters (the `User: ` label included) -+ // and the tail its last 1,000. ASCII of the same length is the control: it spent the budget -+ // correctly before the change too. ++ // units, so unit arithmetic kept 1,503 characters of the 3,000-character budget โ€” a ++ // 1,003-character head (`User: ` included, the rendered line's first 2,000 units) and a ++ // 500-character tail. After the fix the whole line is `MAX` characters plus the marker, the ++ // head is the rendered line's first 2,000 characters (the `User: ` label included) and the ++ // tail its last 1,000. ASCII of the same length is the control: it spent the budget correctly ++ // before the change too. + for (const [text, sample] of [['๐Ÿš€'.repeat(3500), 'User: ๐Ÿš€'], ['a'.repeat(3500), 'User: a']]) { + const prompt = await render(text) + const line = prompt.split('\n').find(one => one.startsWith(sample)) @@ -12295,6 +12304,52 @@ diff --git a/test/memory.test.js b/test/memory.test.js + assert.equal(tail.join(''), message.slice(-tail.length).join('')) + } +}) ++ ++// A line can arrive carrying an unpaired surrogate: a stream that ended mid-character, a fixture ++// written by hand. The review's render path handed it straight through โ€” the `fits()` branch of ++// `ends` answered the line itself, and `clip` only ever flattened what it cut. Both ends flatten ++// first now, so the prompt holds U+FFFD where the half pair was, never the half pair itself. ++test('an unpaired surrogate in a reviewed line reaches the prompt as one whole character', async (t) => { ++ const userText = `${'x'.repeat(100)}\ud800${'y'.repeat(100)}` ++ const botText = `${'z'.repeat(80)}\ud800${'w'.repeat(80)}` ++ const history = { ++ b2: [ ++ { event: { seq: 20, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: 'Start here.' }] } } }, ++ { event: { seq: 21, type: 'assistant/message', data: { message: { content: [{ type: 'text', text: botText }] } } } }, ++ { event: { seq: 22, type: 'user/message', data: { source: { kind: 'user' }, content: [{ type: 'text', text: userText }] } } }, ++ ], ++ } ++ const replies = [JSON.stringify({ changes: [] })] ++ const { ctx, call, requests, stop } = start({ replies, history, config: { memoryReviewDelayMs: 0 } }) ++ t.after(() => stop()) ++ await call('state', {}) ++ ctx.emit('agent/status', { agent: { id: 'b2' }, status: 'idle' }) ++ await reviewed(call) ++ const prompt = requests[0].messages[0].content[0].text ++ const replacements = (text) => [...text].filter(ch => ch === '\ufffd').length ++ ++ // The user's line is 201 characters โ€” 207 rendered โ€” so it takes the fits branch of `ends` ++ // instead of its cut, and that branch used to answer the line untouched. ++ const userLine = prompt.split('\n').find(one => one.startsWith('User: x')) ++ assert.ok(userLine !== undefined, `the user line is in the review context:\n${prompt}`) ++ assert.equal(userLine.isWellFormed(), true) ++ assert.equal(replacements(userLine), 1) ++ assert.equal(userLine.indexOf('\ufffd'), 'User: '.length + 100) ++ assert.equal(userLine.slice('User: '.length, 'User: '.length + 100), 'x'.repeat(100)) ++ assert.equal(userLine.slice('User: '.length + 101, 'User: '.length + 201), 'y'.repeat(100)) ++ ++ // The Bot's line is 160 characters under the 800 limit and carries its label, so it takes the ++ // other branch of `reviewLine`, which handed it to `clip` without flattening it first. ++ const botLine = prompt.split('\n').find(one => one.startsWith('Writer: z')) ++ assert.ok(botLine !== undefined, `the Bot line is in the review context:\n${prompt}`) ++ const label = botLine.slice(0, botLine.indexOf('z')) ++ assert.equal(botLine.isWellFormed(), true) ++ assert.equal(replacements(botLine), 1) ++ assert.equal(botLine.indexOf('\ufffd'), label.length + 80) ++ assert.equal(botLine.slice(label.length, label.length + 80), 'z'.repeat(80)) ++ assert.equal(botLine.slice(label.length + 81, label.length + 161), 'w'.repeat(80)) ++ assert.equal(prompt.isWellFormed(), true, 'the prompt holds no half pair') ++}) + test('when a turn ends, the memory review keeps what the user said, through the same rules', async (t) => { const history = {