host: the steward's long-message budget is characters at both ends

changelog.d/2552-steward-shorten-character-budget.fixed.md

@@ -0,0 +1 @@
+A long message the context steward reads now spends its 1,500-character budget in characters at both ends: the start keeps 1,000 characters and the end keeps 500, so a message dense in emoji shows the whole budget instead of only its first 1,000 characters, and the marker reports the 100 characters really left out instead of counting the surplus UTF-16 units a second time. ([#2552](https://github.com/ThinkFlowLab/FlyBot/issues/2552))

core/src/host/steward.js

@@ -7,7 +7,7 @@
 // Every line follows the model. Its declared window, less the room its reply
 // needs, is the usable context; the check line is a share of that.
 import { codePoints, cut, estimateTokens } from './text.js'
-import { codePointEnd, codePointStart, fits } from '../shared/text-boundary.js'
+import { fits } from '../shared/text-boundary.js'
 
 // For a model that declares no window.
 export const DEFAULT_WINDOW = 500_000
@@ -43,16 +43,26 @@ export const STEWARD_SYSTEM = [
   'Answer with JSON only, on one line: {"action":"switch"|"compact","reason":"one short sentence"}',
 ].join('\n')
 
+// The UTF-16 index the nth character starts at. The budget is counted in characters while
+// `slice` takes units, so a cut needs the index of a character, not a character count spent
+// as one. Stepping whole code points makes every index a boundary by construction (#2321,
+// #1912/#1916): a slice there never splits a surrogate pair.
+const charIndex = (text, chars) => {
+  let index = 0
+  for (let n = 0; n < chars && index < text.length; n += 1) index += text.codePointAt(index) > 0xffff ? 2 : 1
+  return index
+}
+
 // A long message keeps its start (what was asked) and its end (how the reply ended and
 // what it promised). The marker says the cut is this view's, so that a whole reply does
-// not read as one that stopped halfway. Counted in characters and cut on a character
-// boundary (#2321, #1912/#1916): `slice` counts UTF-16 units, so a unit offset splits a
-// surrogate pair and the steward reads half of it as U+FFFD.
+// not read as one that stopped halfway. Both bounds and the marker are characters.
 function shorten(text) {
   if (fits(text, STEWARD_LINE_MAX)) return text
-  const head = codePointEnd(text, Math.ceil((STEWARD_LINE_MAX * 2) / 3))
-  const tail = codePointStart(text, text.length - (STEWARD_LINE_MAX - head))
-  const left = codePoints(text) - codePoints(text.slice(0, head)) - codePoints(text.slice(tail))
+  const chars = codePoints(text)
+  const headChars = Math.ceil((STEWARD_LINE_MAX * 2) / 3)
+  const head = charIndex(text, headChars)
+  const tail = charIndex(text, chars - (STEWARD_LINE_MAX - headChars))
+  const left = chars - codePoints(text.slice(0, head)) - codePoints(text.slice(tail))
   return `${text.slice(0, head)}\n[… ${left} characters left out of this view …]\n${text.slice(tail)}`
 }
 

core/test/steward.test.js

@@ -128,8 +128,22 @@ test('a long message shortened for the steward cuts on characters, never inside
     assert.equal(request.isWellFormed(), true, 'a character at the cut stays whole')
   }
   const shown = stewardRequest({ lines: [{ who: 'user', text: head }], midTurn: false, checkpoints: 0, group: false })
-  assert.match(shown, /User: a+\n\[… 300 characters left out of this view …\]\nb+$/)
+  // The emoji is the 1,000th character, so the start keeps it whole: the bounds are characters,
+  // and the 300 characters this view leaves out are the middle of the b's, not the emoji.
+  assert.match(shown, /User: a+🚀\n\[… 300 characters left out of this view …\]\nb+$/)
   // The message that fits in 1,500 characters comes through whole, marker and all.
   const whole = stewardRequest({ lines: [{ who: 'user', text: tail }], midTurn: false, checkpoints: 0, group: false })
   assert.match(whole, /User: a+🚀b+$/)
+
+  // 1,600 characters in 2,400 UTF-16 units: two thirds of the 1,500-character budget is a
+  // character count, so the start keeps 1,000 characters, the end keeps 500, and the marker
+  // reports the other 100. Spending a unit offset as characters took the end of the view away
+  // with the surplus units and made the marker count those units a second time (600).
+  const dense = '🚀'.repeat(800) + 'a'.repeat(800)
+  const denseRequest = stewardRequest({ lines: [{ who: 'user', text: dense }], midTurn: false, checkpoints: 0, group: false })
+  const [, denseHead, denseLeft, denseTail] = /User: ([\s\S]*?)\n\[… (\d+) characters left out of this view …\]\n([\s\S]*)$/.exec(denseRequest)
+  assert.equal([...denseHead].length + [...denseTail].length, 1500, 'the view keeps the whole 1,500-character budget')
+  assert.equal([...denseHead].length, 1000, 'the start is two thirds of the budget in characters')
+  assert.equal([...denseTail].length, 500, 'the end is the rest of the budget in characters')
+  assert.equal(Number(denseLeft), 100, 'the marker counts the characters that are really left out')
 })

patches/ds-bot/0001-core-fixes.patch

@@ -7607,7 +7607,7 @@ diff --git a/src/host/steward.js b/src/host/steward.js
  // needs, is the usable context; the check line is a share of that.
 -import { estimateTokens } from './text.js'
 +import { codePoints, cut, estimateTokens } from './text.js'
-+import { codePointEnd, codePointStart, fits } from '../shared/text-boundary.js'
++import { fits } from '../shared/text-boundary.js'
 
  // For a model that declares no window.
  export const DEFAULT_WINDOW = 500_000
@@ -7618,22 +7618,34 @@ diff --git a/src/host/steward.js b/src/host/steward.js
  // Context sizes between requests are estimates, so the hard line stays a little
  // below what the model can actually take.
  const HARD_SHARE = 0.95
-@@ -42,11 +42,15 @@ export const STEWARD_SYSTEM = [
+@@ -40,13 +40,27 @@ export const STEWARD_SYSTEM = [
+   'Answer with JSON only, on one line: {"action":"switch"|"compact","reason":"one short sentence"}',
+ ].join('\n')
 
++// The UTF-16 index the nth character starts at. The budget is counted in characters while
++// `slice` takes units, so a cut needs the index of a character, not a character count spent
++// as one. Stepping whole code points makes every index a boundary by construction (#2321,
++// #1912/#1916): a slice there never splits a surrogate pair.
++const charIndex = (text, chars) => {
++  let index = 0
++  for (let n = 0; n < chars && index < text.length; n += 1) index += text.codePointAt(index) > 0xffff ? 2 : 1
++  return index
++}
++
  // A long message keeps its start (what was asked) and its end (how the reply ended and
  // what it promised). The marker says the cut is this view's, so that a whole reply does
 -// not read as one that stopped halfway.
-+// not read as one that stopped halfway. Counted in characters and cut on a character
-+// boundary (#2321, #1912/#1916): `slice` counts UTF-16 units, so a unit offset splits a
-+// surrogate pair and the steward reads half of it as U+FFFD.
++// not read as one that stopped halfway. Both bounds and the marker are characters.
  function shorten(text) {
 -  if (text.length <= STEWARD_LINE_MAX) return text
 -  const head = Math.ceil((STEWARD_LINE_MAX * 2) / 3)
 -  return `${text.slice(0, head)}\n[… ${text.length - STEWARD_LINE_MAX} characters left out of this view …]\n${text.slice(head - STEWARD_LINE_MAX)}`
 +  if (fits(text, STEWARD_LINE_MAX)) return text
-+  const head = codePointEnd(text, Math.ceil((STEWARD_LINE_MAX * 2) / 3))
-+  const tail = codePointStart(text, text.length - (STEWARD_LINE_MAX - head))
-+  const left = codePoints(text) - codePoints(text.slice(0, head)) - codePoints(text.slice(tail))
++  const chars = codePoints(text)
++  const headChars = Math.ceil((STEWARD_LINE_MAX * 2) / 3)
++  const head = charIndex(text, headChars)
++  const tail = charIndex(text, chars - (STEWARD_LINE_MAX - headChars))
++  const left = chars - codePoints(text.slice(0, head)) - codePoints(text.slice(tail))
 +  return `${text.slice(0, head)}\n[… ${left} characters left out of this view …]\n${text.slice(tail)}`
  }
 
@@ -14658,7 +14670,7 @@ diff --git a/test/steward.test.js b/test/steward.test.js
    }
    // The check line never passes the hard line.
    assert.equal(linesFor({ window: 100_000, reserve: 0, checkRatio: 0.99 }).check, 95_000)
-@@ -101,7 +101,35 @@ test('the checkpoint also reads the messages the backend keeps after it', () =>
+@@ -101,7 +101,49 @@ test('the checkpoint also reads the messages the backend keeps after it', () =>
    assert.ok(checkpointTailNote([say('user', '长'.repeat(300))]).includes(`"${'长'.repeat(80)}"`))
  })
 
@@ -14689,10 +14701,24 @@ diff --git a/test/steward.test.js b/test/steward.test.js
 +    assert.equal(request.isWellFormed(), true, 'a character at the cut stays whole')
 +  }
 +  const shown = stewardRequest({ lines: [{ who: 'user', text: head }], midTurn: false, checkpoints: 0, group: false })
-+  assert.match(shown, /User: a+\n\[… 300 characters left out of this view …\]\nb+$/)
++  // The emoji is the 1,000th character, so the start keeps it whole: the bounds are characters,
++  // and the 300 characters this view leaves out are the middle of the b's, not the emoji.
++  assert.match(shown, /User: a+🚀\n\[… 300 characters left out of this view …\]\nb+$/)
 +  // The message that fits in 1,500 characters comes through whole, marker and all.
 +  const whole = stewardRequest({ lines: [{ who: 'user', text: tail }], midTurn: false, checkpoints: 0, group: false })
 +  assert.match(whole, /User: a+🚀b+$/)
++
++  // 1,600 characters in 2,400 UTF-16 units: two thirds of the 1,500-character budget is a
++  // character count, so the start keeps 1,000 characters, the end keeps 500, and the marker
++  // reports the other 100. Spending a unit offset as characters took the end of the view away
++  // with the surplus units and made the marker count those units a second time (600).
++  const dense = '🚀'.repeat(800) + 'a'.repeat(800)
++  const denseRequest = stewardRequest({ lines: [{ who: 'user', text: dense }], midTurn: false, checkpoints: 0, group: false })
++  const [, denseHead, denseLeft, denseTail] = /User: ([\s\S]*?)\n\[… (\d+) characters left out of this view …\]\n([\s\S]*)$/.exec(denseRequest)
++  assert.equal([...denseHead].length + [...denseTail].length, 1500, 'the view keeps the whole 1,500-character budget')
++  assert.equal([...denseHead].length, 1000, 'the start is two thirds of the budget in characters')
++  assert.equal([...denseTail].length, 500, 'the end is the rest of the budget in characters')
++  assert.equal(Number(denseLeft), 100, 'the marker counts the characters that are really left out')
 +})
 diff --git a/test/text.test.js b/test/text.test.js
 --- a/test/text.test.js