diff --git a/changelog.d/2553-memory-review-line-budget.fixed.md b/changelog.d/2553-memory-review-line-budget.fixed.md new file mode 100644 index 000000000..b60ac1fef --- /dev/null +++ b/changelog.d/2553-memory-review-line-budget.fixed.md @@ -0,0 +1 @@ +The memory review's two-ended cut of the user's own line now spends its 3,000-character budget in characters at both ends: a line of 3,500 emoji keeps 3,000 characters (2,000 head, 1,000 tail) instead of 1,503, so the review no longer drops half of a long message written with characters outside the BMP. ([#2553](https://github.com/ThinkFlowLab/FlyBot/issues/2553)) diff --git a/core/src/host/memory.js b/core/src/host/memory.js index 76c13c976..cf40d1f63 100644 --- a/core/src/host/memory.js +++ b/core/src/host/memory.js @@ -28,7 +28,7 @@ import { BlockAssembler } from '@deepseek-ai/dsh-llm' import { currentParticipants } from './room-membership.js' import { defineTool } from './tool-schema.js' import { clip, codePoints, splitRef, unquote } from './text.js' -import { codePointStart, cut, fits } from '../shared/text-boundary.js' +import { cut, fits } from '../shared/text-boundary.js' import { output } from './tools.js' import { retiredBot } from './store.js' import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js' @@ -1267,13 +1267,26 @@ export function install(rt) { return cursors } const saveCursors = () => (writeFolder(root) ? writeText(cursorsPath, JSON.stringify(cursors)) : Promise.resolve()) + // The UTF-16 index of the n-th character: a budget spent in characters needs an index, and + // `text.length` counts units, so a character outside the BMP would spend two of it. + // Stepping whole code points makes the index a boundary by construction, so no cut lands + // between the halves of a surrogate pair. + const charIndex = (text, chars) => { + let index = 0 + for (let n = 0; n < chars && index < text.length; n += 1) index += text.codePointAt(index) > 0xffff ? 2 : 1 + return index + } + // A long message often ends with what the user wants of it, so both ends stay. const ends = (text, max) => { if (fits(text, max)) return text - const tail = text.slice(codePointStart(text, text.length - Math.floor(max / 3))) - // Stepping back onto a boundary can keep one more unit at the tail; the head gives it back. - const head = cut(text, Math.min(Math.ceil(max * 2 / 3), max - tail.length)) - return `${head.text} โ€ฆ(middle cut)โ€ฆ ${tail}` + // Flattened first, as `cut` did: text that arrives with an unpaired surrogate is stored + // as U+FFFD rather than half a character. + const flat = text.toWellFormed() + const tailChars = Math.floor(max / 3) + const tail = flat.slice(charIndex(flat, codePoints(flat) - tailChars)) + const head = flat.slice(0, charIndex(flat, max - tailChars)) + return `${head} โ€ฆ(middle cut)โ€ฆ ${tail}` } // Lines say who spoke. The user's lines are kept longest: what they say is what diff --git a/core/test/memory.test.js b/core/test/memory.test.js index 03a8ba8b2..c6d14ad1d 100644 --- a/core/test/memory.test.js +++ b/core/test/memory.test.js @@ -606,6 +606,30 @@ test('a long user line cut for the review keeps whole characters at both ends (# assert.deepEqual(Buffer.from(keptHead, 'utf8'), Buffer.from(text.slice(0, keptHead.length), 'utf8')) assert.deepEqual(Buffer.from(keptTail, 'utf8'), Buffer.from(text.slice(-keptTail.length), 'utf8')) } + // The two ends spend the budget in characters too, not in UTF-16 units: 3,500 emoji are 7,000 + // units, so unit arithmetic kept 1,516 characters of the 3,000-character budget, a + // 1,000-character head and a 500-character tail. The whole line is `MAX` characters plus the + // marker, the head is the rendered line's first 2,000 characters (the `User: ` label included) + // and the tail its last 1,000. ASCII of the same length is the control: it spent the budget + // correctly before the change too. + for (const [text, sample] of [['๐Ÿš€'.repeat(3500), 'User: ๐Ÿš€'], ['a'.repeat(3500), 'User: a']]) { + const prompt = await render(text) + const line = prompt.split('\n').find(one => one.startsWith(sample)) + assert.ok(line !== undefined, `the user line is in the review context:\n${prompt}`) + assert.equal(line.isWellFormed(), true) + assert.equal(line.includes('\ufffd'), false) + const mark = ' โ€ฆ(middle cut)โ€ฆ ' + const at = line.indexOf(mark) + assert.ok(at > 0, line.slice(0, 40)) + const head = [...line.slice(0, at)] + const tail = [...line.slice(at + mark.length)] + assert.equal(head.length + tail.length, MAX, 'the view keeps the whole 3,000-character budget') + assert.equal(head.length, Math.ceil((MAX * 2) / 3), 'the head keeps its 2,000 characters') + assert.equal(tail.length, Math.floor(MAX / 3), 'the tail keeps its 1,000 characters') + const message = [...text] + assert.equal(head.slice('User: '.length).join(''), message.slice(0, head.length - 'User: '.length).join('')) + assert.equal(tail.join(''), message.slice(-tail.length).join('')) + } }) test('when a turn ends, the memory review keeps what the user said, through the same rules', async (t) => { diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index a4d394f29..92f06a7eb 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -38,7 +38,7 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | --- | --- | | `core/index.js` | Server entry. | | `core/src/host/` | Host-side helpers (non-hooks). | -| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit โ€” one character costs one whatever its UTF-16 width โ€” and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it โ€” the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` โ€” so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it โ€” the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) โ€” each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-unit context `clip` cuts through `cut`, and its 3,000-unit two-ended `ends` takes the head through `cut` and moves the tail's start back with `codePointStart`, giving the head back the unit a stepped-back tail keeps, so the prompt and the log hold whole characters within the same budget; the exported `clip`'s reason is optional there too, so a two-argument call reads `โ€ฆ (cut)` instead of `โ€ฆ (cut; undefined)` while a call with a reason is unchanged. Since #1923 the four bounds whose text crosses an owner boundary go through the same `cut` โ€” `cleanPublishedName` in `hooks.js` (60, into `state.flybot.remote`), the group member line's `displayName` 60 and `ownerName` 40 in `tools.js`, `label` in `reach.js` (60, the `othersInRoom` labels), and the local `clip` in `shared/room-share.js` (its per-field caps included) โ€” so a published, listed or relayed name keeps whole characters and never reaches a UTF-8 encoder as U+FFFD ([#1923](https://github.com/ThinkFlowLab/FlyBot/issues/1923)). The same rule now covers the host's own line bounds: the busy ack's 600-unit task and 1,500-unit message text in `busy-ack.js`, the group line's notice (100), a Bot's job line (120), the group answer echo (300) and the rework line's note (200) in `tools.js`, and `cleanName`'s 60-unit peer name in `delivery.js`, which lands in `fenceUntrusted`'s `from="โ€ฆ"` attribute โ€” so none of them can end in half a surrogate pair. The five bounds `turns.js` keeps go through `cut` as well: a failure's 500-unit message and the 200-unit message of its line (`turnFailure`, `failureLine`), the 79-unit step a running call shows (the `โ€ฆ` is appended only when `cut` dropped something, never by a length guess), an approval card's 2,000-unit description and 4,000-unit command (the card stores the command "Allow once" runs rather than a preview of it, and one that still had to be cut ends with `โ€ฆ` inside that bound, so the card says what it abridged and never drops the fact, [#1992](https://github.com/ThinkFlowLab/FlyBot/issues/1992)), and the 160-unit reason a failed turn leaves for the inbox. Since #1963 `cut` also flattens its input before it bounds โ€” `String(text).toWellFormed()` first, `codePointEnd` on the flat string โ€” so a title, a note or a name that arrives already carrying an unpaired surrogate is stored as U+FFFD instead of the lone surrogate, and U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag ([#1963](https://github.com/ThinkFlowLab/FlyBot/issues/1963)). Since #1979 the six remaining bounds on text another home or another person supplied go through it too โ€” a connector item name (80, `connectors.js`), a role label another home set on a face-room seat (80, `face-lead.js`), a cross-home group name (200, `project-team.js`), a capability description from a peer server (2,000, `capabilities.js`), the remote home's own error text (157, `remote-chat.js`) and an abbreviated remote reply (80, same file) โ€” so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, while `remote-chat.js`'s `โ€ฆ` and trailing-period strip still apply to the cut value ([#1979](https://github.com/ThinkFlowLab/FlyBot/issues/1979)). The two bounds on a friend's Bot's own name in that file โ€” the name its chat is created with (40) and the name a refresh writes (40) โ€” go through `cut` as well, so a refresh cannot store a lone high surrogate and a create no longer hands `roster.createBot` half a pair to flatten into U+FFFD; the budget stays 40 and a shorter name is stored word for word. Since #1993 the last five bounds in `core/src/host/` go through it too โ€” the quoted first line of a checkpoint's kept tail (80, `steward.js`), an approval record's `reason` (500, `approvals.js`), a mention receipt's reason line (240, `hooks.js`), the title given to a DSH Agent's Session (80, `AGENT_SESSION_TITLE_MAX` in `agents.js`) and a schedule run's failure `reason` (200, `schedules.js`) โ€” so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, and none of the five maxima changes ([#1993](https://github.com/ThinkFlowLab/FlyBot/issues/1993)). Since #2002 the connector's own error text goes through it too: the `reason` in `core/src/host/connectors.js` โ€” the text an MCP server's failure leaves on the owner's connector card โ€” keeps the longest prefix that stops before a split pair, so it is well formed and a code-point prefix of what the server sent; since #2016 that prefix is 2,000 units rather than 300, and one that had to be cut ends with `โ€ฆ` inside the bound, so the card says what it abridged ([#2002](https://github.com/ThinkFlowLab/FlyBot/issues/2002), [#2016](https://github.com/ThinkFlowLab/FlyBot/issues/2016)). Since #2005 the five bounds the Project panels read go through it as well โ€” the assistant card's 77-unit title in `project-assistant.js`, the board's 200-unit one-line value and a decision title's 200 in `project-board.js`, and `capped`'s name, role, persona, model and color caps plus the dossier's 4,000-unit memory head in `project-team.js`, whose head is bounded where `memoryDigest` reads it (`memory.js`), because a cut at the dossier alone could only turn half a pair into U+FFFD and never drop the character whole โ€” so a dossier row or a board card keeps whole characters ([#2005](https://github.com/ThinkFlowLab/FlyBot/issues/2005)). Since [#2321](https://github.com/ThinkFlowLab/FlyBot/issues/2321) a bound also judges in code points, not only counts in them: `fits(text, max)` in `core/src/shared/text-boundary.js` is the one predicate โ€” a character outside the BMP spends one of the budget and not two โ€” and `clip` and `excerpt` judge with it while their cuts keep going through `codePointEnd`/`cut`, so a 300-character text is no longer cut by a 400-character limit on its way to `read_own_chat` and the search rows. Since [#2339](https://github.com/ThinkFlowLab/FlyBot/issues/2339) the two bounds in `core/src/host/` that still judged with `text.length` use it as well โ€” the memory review's 3,000-unit two-ended `ends` in `memory.js` and `room-context.js`'s `oneLine` (40/80/24/60) โ€” so a user line or a board line of characters outside the BMP stays whole inside its own limit instead of being cut at half of it; both maxima and every `cut` call are unchanged, so an ASCII text judges exactly as `length <= max` did. The memory review's own third `clip` is the exported one now, so its cut reads `โ€ฆ (cut)`, and a bound added later that judges with `text.length` fails `core/test/text-bounds.test.js` at its own `file:line`. | +| `core/src/host/text.js` | The pure text helpers the host modules share: message text, names, model refs, cutting and estimating text, and the clock format of chat anchors. Since #1895 `codePoints(text)` (a step per code point, no array built since #1898) is how the host counts a limit โ€” one character costs one whatever its UTF-16 width โ€” and the bounds this change puts through it are: a memory entry of 300 and a `MEMORY.md` of 4,000 or a topic of 16,000 (the refusal and the size note print the code-point number), a scheduled task's name of 120, a message of 20,000, and the Project memory entry of 600 in both writers, so an emoji is never charged twice. Both cuts land on a code point boundary: `clip` and `excerpt` move back rather than keep half of a surrogate pair, so a clipped emoji reaches a UTF-8 encoder as the character and not as U+FFFD ([#1907](https://github.com/ThinkFlowLab/FlyBot/issues/1907)). Since #1912 that rule is one exported helper rather than a private detail: `cut(text, max)` answers `{ text, cut }` with the longest prefix that stops before a split pair, and the archive writers that used to `slice(0, max)` go through it โ€” the `oneLine` of `decisions.js` and `meetings.js` (into `decisions.jsonl` and `decisions/-.md`), `claimLine` in `groups.js`, the report cap in `diagnostics.js`, `cleanLabel` in `flybot-feed.js` and `clean` in `browser.js` โ€” so a decision or a meeting record holds the whole character instead of U+FFFD. Since #1916 the boundary rule itself lives in `core/src/shared/text-boundary.js` (`codePointStart`, `codePointEnd`, `cut`), and `host/text.js` builds its `cut` on it so a shared module can bound text too; the eight remaining `slice(0, max)` writers follow it โ€” the claim cue's fallback reference (60), the task lead's name in a member prompt (80), a failed group Session's error text (300), `read_own_chat`'s `query` (80) and `around` (40), the open-page address and title (2000/300), the diagnostics browser-log line and its code (500/40), and `cleanPeerName` in `shared/remote-label.js` (60) โ€” each keeping the longest prefix that stops before a split pair, `length <= max` and a byte prefix of the input. Since #1918 the memory review's own two bounds go through the same rule: its 800-character context `clip` cuts through `cut`, and its 3,000-character two-ended `ends` spends the budget in characters at both ends โ€” `charIndex` steps whole code points to find the head's end and the tail's start, so the head is the rendered line's first 2,000 characters and the tail its last 1,000, and the text is flattened with `toWellFormed()` first as `cut` did โ€” so the prompt and the log hold whole characters within the same character budget: a line of 3,500 emoji keeps 3,000 characters where unit arithmetic kept 1,503 ([#2553](https://github.com/ThinkFlowLab/FlyBot/issues/2553)); the exported `clip`'s reason is optional there too, so a two-argument call reads `โ€ฆ (cut)` instead of `โ€ฆ (cut; undefined)` while a call with a reason is unchanged. Since #1923 the four bounds whose text crosses an owner boundary go through the same `cut` โ€” `cleanPublishedName` in `hooks.js` (60, into `state.flybot.remote`), the group member line's `displayName` 60 and `ownerName` 40 in `tools.js`, `label` in `reach.js` (60, the `othersInRoom` labels), and the local `clip` in `shared/room-share.js` (its per-field caps included) โ€” so a published, listed or relayed name keeps whole characters and never reaches a UTF-8 encoder as U+FFFD ([#1923](https://github.com/ThinkFlowLab/FlyBot/issues/1923)). The same rule now covers the host's own line bounds: the busy ack's 600-unit task and 1,500-unit message text in `busy-ack.js`, the group line's notice (100), a Bot's job line (120), the group answer echo (300) and the rework line's note (200) in `tools.js`, and `cleanName`'s 60-unit peer name in `delivery.js`, which lands in `fenceUntrusted`'s `from="โ€ฆ"` attribute โ€” so none of them can end in half a surrogate pair. The five bounds `turns.js` keeps go through `cut` as well: a failure's 500-unit message and the 200-unit message of its line (`turnFailure`, `failureLine`), the 79-unit step a running call shows (the `โ€ฆ` is appended only when `cut` dropped something, never by a length guess), an approval card's 2,000-unit description and 4,000-unit command (the card stores the command "Allow once" runs rather than a preview of it, and one that still had to be cut ends with `โ€ฆ` inside that bound, so the card says what it abridged and never drops the fact, [#1992](https://github.com/ThinkFlowLab/FlyBot/issues/1992)), and the 160-unit reason a failed turn leaves for the inbox. Since #1963 `cut` also flattens its input before it bounds โ€” `String(text).toWellFormed()` first, `codePointEnd` on the flat string โ€” so a title, a note or a name that arrives already carrying an unpaired surrogate is stored as U+FFFD instead of the lone surrogate, and U+FFFD is one UTF-16 unit for the one it replaces, so a well-formed input keeps its exact prefix and its `cut` flag ([#1963](https://github.com/ThinkFlowLab/FlyBot/issues/1963)). Since #1979 the six remaining bounds on text another home or another person supplied go through it too โ€” a connector item name (80, `connectors.js`), a role label another home set on a face-room seat (80, `face-lead.js`), a cross-home group name (200, `project-team.js`), a capability description from a peer server (2,000, `capabilities.js`), the remote home's own error text (157, `remote-chat.js`) and an abbreviated remote reply (80, same file) โ€” so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, while `remote-chat.js`'s `โ€ฆ` and trailing-period strip still apply to the cut value ([#1979](https://github.com/ThinkFlowLab/FlyBot/issues/1979)). The two bounds on a friend's Bot's own name in that file โ€” the name its chat is created with (40) and the name a refresh writes (40) โ€” go through `cut` as well, so a refresh cannot store a lone high surrogate and a create no longer hands `roster.createBot` half a pair to flatten into U+FFFD; the budget stays 40 and a shorter name is stored word for word. Since #1993 the last five bounds in `core/src/host/` go through it too โ€” the quoted first line of a checkpoint's kept tail (80, `steward.js`), an approval record's `reason` (500, `approvals.js`), a mention receipt's reason line (240, `hooks.js`), the title given to a DSH Agent's Session (80, `AGENT_SESSION_TITLE_MAX` in `agents.js`) and a schedule run's failure `reason` (200, `schedules.js`) โ€” so each keeps the longest prefix that stops before a split pair, is well formed and is a code-point prefix of the input, and none of the five maxima changes ([#1993](https://github.com/ThinkFlowLab/FlyBot/issues/1993)). Since #2002 the connector's own error text goes through it too: the `reason` in `core/src/host/connectors.js` โ€” the text an MCP server's failure leaves on the owner's connector card โ€” keeps the longest prefix that stops before a split pair, so it is well formed and a code-point prefix of what the server sent; since #2016 that prefix is 2,000 units rather than 300, and one that had to be cut ends with `โ€ฆ` inside the bound, so the card says what it abridged ([#2002](https://github.com/ThinkFlowLab/FlyBot/issues/2002), [#2016](https://github.com/ThinkFlowLab/FlyBot/issues/2016)). Since #2005 the five bounds the Project panels read go through it as well โ€” the assistant card's 77-unit title in `project-assistant.js`, the board's 200-unit one-line value and a decision title's 200 in `project-board.js`, and `capped`'s name, role, persona, model and color caps plus the dossier's 4,000-unit memory head in `project-team.js`, whose head is bounded where `memoryDigest` reads it (`memory.js`), because a cut at the dossier alone could only turn half a pair into U+FFFD and never drop the character whole โ€” so a dossier row or a board card keeps whole characters ([#2005](https://github.com/ThinkFlowLab/FlyBot/issues/2005)). Since [#2321](https://github.com/ThinkFlowLab/FlyBot/issues/2321) a bound also judges in code points, not only counts in them: `fits(text, max)` in `core/src/shared/text-boundary.js` is the one predicate โ€” a character outside the BMP spends one of the budget and not two โ€” and `clip` and `excerpt` judge with it while their cuts keep going through `codePointEnd`/`cut`, so a 300-character text is no longer cut by a 400-character limit on its way to `read_own_chat` and the search rows. Since [#2339](https://github.com/ThinkFlowLab/FlyBot/issues/2339) the two bounds in `core/src/host/` that still judged with `text.length` use it as well โ€” the memory review's 3,000-character two-ended `ends` in `memory.js` and `room-context.js`'s `oneLine` (40/80/24/60) โ€” so a user line or a board line of characters outside the BMP stays whole inside its own limit instead of being cut at half of it; both maxima are unchanged and a board line's cut stays on `cut`, so an ASCII text judges exactly as `length <= max` did. The memory review's own third `clip` is the exported one now, so its cut reads `โ€ฆ (cut)`, and a bound added later that judges with `text.length` fails `core/test/text-bounds.test.js` at its own `file:line`. | | `core/src/host/prompt.js` | The Bot system prompt; a Main Bot's section is headed `You are the Main Bot` and uses that one term, the one the UI shows (#276). The team rules state the idle default (#699): a Bot does nothing until the user addresses it, it is @-mentioned in a group, its parent gives it a task, or unfinished tasks it was already given are still on its board. It does not invent work. A coordinator acts on a user task rather than on its own initiative โ€” an addressed Bot answers as before. | | `core/src/host/room-membership.js` | Typed room participants (human / local bot / granted remote), authorship, membership-window history (#50); rooms with friends' Bots or other owners take granted local Bots only and evict a Bot that stops qualifying (#90). Face-to-face rooms (#83) reuse this model (`origin: 'face-to-face'`, room-only remotes without a person-grant, `upsertFaceRoom` / `syncFaceParticipants` / `applyFaceRoomLine`); a face-room local Bot is `viaCode` (brought by number), `evictUnshared` skips face rooms (membership is the server's), and a Bot that returns to the member list drops the empty local `left` line (#236). Refusing an ungranted local Bot next to a friend's Bot throws `ungrantedRefusal`: a message that names the Bot, the friend and the two ways out, plus `error.details` `{ reason: 'ungranted-local', bots, remotes }`; `channel.js` `fail` forwards a plain-object `details` (empty otherwise) and `client/api.js` puts it on the thrown error, where `client/new-chat.js` draws a card (**Copy invite link for {Bot}** via `__flybot__.social.createInvite`, **Leave {Bot} out**) read through `ungrantedLocalFix` in `shared/remote-label.js`; the same file's `copiedToClipboard` reports whether the invite link really reached the clipboard (none, or a refused write, is `false`), and the card then shows the link to copy by hand instead of saying it was copied. | | `core/src/host/face-mentions.js` | Who "@" calls in a face-to-face room (#287): every alias of a participant (`faceAliases`, kept in `shared/remote-label.js` so the bubble chips the same names, #305; bare name, the picker label `'s Chief` / ` ยท 's Bot`, ` ยท `, handle), local and friend's Bots told apart. `faceMentions().handles` is the handle of every Bot a line names (the explicit mention list a sender attaches, #293) and `mentionedByHandle` resolves a received list to my Bots (`known` counts handles the room has). | diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index ddb3c5de8..6c41e8383 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -5655,7 +5655,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js +import { currentParticipants } from './room-membership.js' +import { defineTool } from './tool-schema.js' +import { clip, codePoints, splitRef, unquote } from './text.js' -+import { codePointStart, cut, fits } from '../shared/text-boundary.js' ++import { cut, fits } from '../shared/text-boundary.js' import { output } from './tools.js' +import { retiredBot } from './store.js' +import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js' @@ -6155,7 +6155,7 @@ diff --git a/src/host/memory.js b/src/host/memory.js if (text === '') throw new Error('Type a question or a change first') const numbered = [ ...allEntries(loadScope(ownScope(bot.id))).map(entry => ({ ...entry, scope: 'own' })), -@@ -1074,18 +1074,27 @@ export function install(rt) { +@@ -1074,18 +1074,40 @@ export function install(rt) { let cursors const cursorsNow = () => { if (cursors === undefined) { @@ -6178,14 +6178,27 @@ diff --git a/src/host/memory.js b/src/host/memory.js } - const clip = (text, max) => (text.length > max ? `${text.slice(0, max)} โ€ฆ(cut)` : text) + const saveCursors = () => (writeFolder(root) ? writeText(cursorsPath, JSON.stringify(cursors)) : Promise.resolve()) ++ // The UTF-16 index of the n-th character: a budget spent in characters needs an index, and ++ // `text.length` counts units, so a character outside the BMP would spend two of it. ++ // Stepping whole code points makes the index a boundary by construction, so no cut lands ++ // between the halves of a surrogate pair. ++ const charIndex = (text, chars) => { ++ let index = 0 ++ for (let n = 0; n < chars && index < text.length; n += 1) index += text.codePointAt(index) > 0xffff ? 2 : 1 ++ return index ++ } ++ // A long message often ends with what the user wants of it, so both ends stay. - const ends = (text, max) => (text.length > max ? `${text.slice(0, Math.ceil(max * 2 / 3))} โ€ฆ(middle cut)โ€ฆ ${text.slice(-Math.floor(max / 3))}` : text) + const ends = (text, max) => { + if (fits(text, max)) return text -+ const tail = text.slice(codePointStart(text, text.length - Math.floor(max / 3))) -+ // Stepping back onto a boundary can keep one more unit at the tail; the head gives it back. -+ const head = cut(text, Math.min(Math.ceil(max * 2 / 3), max - tail.length)) -+ return `${head.text} โ€ฆ(middle cut)โ€ฆ ${tail}` ++ // Flattened first, as `cut` did: text that arrives with an unpaired surrogate is stored ++ // as U+FFFD rather than half a character. ++ const flat = text.toWellFormed() ++ const tailChars = Math.floor(max / 3) ++ const tail = flat.slice(charIndex(flat, codePoints(flat) - tailChars)) ++ const head = flat.slice(0, charIndex(flat, max - tailChars)) ++ return `${head} โ€ฆ(middle cut)โ€ฆ ${tail}` + } // Lines say who spoke. The user's lines are kept longest: what they say is what @@ -12174,7 +12187,7 @@ diff --git a/test/memory.test.js b/test/memory.test.js const done = (await call('memory-view', { botId: 'b2' })).value assert.deepEqual([done.summary?.text, done.error], ['## Overview\nYou like tea.', null]) assert.deepEqual(routes(requests).slice(2), ['deepseek/flash@2048']) -@@ -519,12 +519,86 @@ test('a host that only reads the team shows memory but changes none of it', asyn +@@ -519,12 +519,110 @@ test('a host that only reads the team shows memory but changes none of it', asyn // Until no review waits or runs. async function reviewed(call) { for (let tries = 0; tries < 200; tries += 1) { @@ -12257,6 +12270,30 @@ diff --git a/test/memory.test.js b/test/memory.test.js + assert.deepEqual(Buffer.from(keptHead, 'utf8'), Buffer.from(text.slice(0, keptHead.length), 'utf8')) + assert.deepEqual(Buffer.from(keptTail, 'utf8'), Buffer.from(text.slice(-keptTail.length), 'utf8')) + } ++ // The two ends spend the budget in characters too, not in UTF-16 units: 3,500 emoji are 7,000 ++ // units, so unit arithmetic kept 1,516 characters of the 3,000-character budget, a ++ // 1,000-character head and a 500-character tail. The whole line is `MAX` characters plus the ++ // marker, the head is the rendered line's first 2,000 characters (the `User: ` label included) ++ // and the tail its last 1,000. ASCII of the same length is the control: it spent the budget ++ // correctly before the change too. ++ for (const [text, sample] of [['๐Ÿš€'.repeat(3500), 'User: ๐Ÿš€'], ['a'.repeat(3500), 'User: a']]) { ++ const prompt = await render(text) ++ const line = prompt.split('\n').find(one => one.startsWith(sample)) ++ assert.ok(line !== undefined, `the user line is in the review context:\n${prompt}`) ++ assert.equal(line.isWellFormed(), true) ++ assert.equal(line.includes('\ufffd'), false) ++ const mark = ' โ€ฆ(middle cut)โ€ฆ ' ++ const at = line.indexOf(mark) ++ assert.ok(at > 0, line.slice(0, 40)) ++ const head = [...line.slice(0, at)] ++ const tail = [...line.slice(at + mark.length)] ++ assert.equal(head.length + tail.length, MAX, 'the view keeps the whole 3,000-character budget') ++ assert.equal(head.length, Math.ceil((MAX * 2) / 3), 'the head keeps its 2,000 characters') ++ assert.equal(tail.length, Math.floor(MAX / 3), 'the tail keeps its 1,000 characters') ++ const message = [...text] ++ assert.equal(head.slice('User: '.length).join(''), message.slice(0, head.length - 'User: '.length).join('')) ++ assert.equal(tail.join(''), message.slice(-tail.length).join('')) ++ } +}) + test('when a turn ends, the memory review keeps what the user said, through the same rules', async (t) => {