core-host: count the three remaining host limits in code points (#1895)

changelog.d/1895-host-limits-code-points.fixed.md

@@ -0,0 +1 @@
+Host character limits count code points, so a character outside the BMP costs one instead of two ([#1895](https://github.com/ThinkFlowLab/FlyBot/issues/1895)): a memory entry at 300, a `MEMORY.md` at 4,000 and its size note, a scheduled task's name at 120, a Bot-to-Bot message at 20,000, and the Project memory entry at 600 in both writers now measure what a reader counts, and a refusal prints that code-point number.

core/src/host/hooks.js

@@ -23,6 +23,7 @@ import { FEED_EVENTS, installFeed } from './flybot-feed.js'
 import { createPluginModel } from './plugin-model.js'
 import { defineTool } from './tool-schema.js'
 import { retiredBot } from './store.js'
+import { codePoints } from './text.js'
 import { EPHEMERAL_DENIED_TOOLS } from './tools.js'
 
 export const FLYBOT_API_VERSION = 1
@@ -728,7 +729,8 @@ export function install(rt) {
   async function send({ from, to, text, role = 'request', hop = 0, via = SHARED, meta, attachments } = {}) {
     await rt.load()
     if (typeof text !== 'string' || text.trim() === '') throw new Error('flybot: send needs text')
-    if (text.length > MAX_SEND_TEXT) throw new Error(`flybot: a message is at most ${MAX_SEND_TEXT} characters`)
+    // #1895: the bound counts code points, so a message of emoji is measured as a reader counts it.
+    if (codePoints(text) > MAX_SEND_TEXT) throw new Error(`flybot: a message is at most ${MAX_SEND_TEXT} characters`)
     if (role !== 'request' && role !== 'reply') throw new Error(`flybot: unsupported role ${role}`)
     if (!isRemote(from) && rt.botOf(from) === undefined) throw new Error(`flybot: unknown sender ${from}`)
     if (!isRemote(to) && rt.botOf(to) === undefined) throw new Error(`flybot: unknown Bot ${to}`)

core/src/host/memory.js

@@ -27,7 +27,7 @@ import { dirname, join } from 'node:path'
 import { BlockAssembler } from '@deepseek-ai/dsh-llm'
 import { currentParticipants } from './room-membership.js'
 import { defineTool } from './tool-schema.js'
-import { splitRef, unquote } from './text.js'
+import { codePoints, splitRef, unquote } from './text.js'
 import { output } from './tools.js'
 import { retiredBot } from './store.js'
 import { botNode, buildGraph, buildIndex, relatedOf, searchIndex, topicNode } from './memory-search.js'
@@ -530,7 +530,10 @@ export function install(rt) {
   const secretEntry = text => (rt.redactSecrets?.(text) ?? text) !== text || looksSecret(text)
   function entryProblem(text) {
     if (text === '') return 'Write the entry to save.'
-    if (text.length > MEMORY_LIMITS.entry) return `An entry holds at most ${MEMORY_LIMITS.entry} characters; this one has ${text.length}. Keep one fact per entry, or save the details as several entries in a topic.`
+    // #1895: one character costs one whatever its UTF-16 width, and the number the refusal
+    // prints is the one a reader would count.
+    const size = codePoints(text)
+    if (size > MEMORY_LIMITS.entry) return `An entry holds at most ${MEMORY_LIMITS.entry} characters; this one has ${size}. Keep one fact per entry, or save the details as several entries in a topic.`
     if (secretEntry(text)) return 'This looks like a key, token, password, or ID number, which memory never keeps. Keys go to request_secret.'
     return undefined
   }
@@ -540,14 +543,17 @@ export function install(rt) {
 
   function checkSize(file, next, label) {
     const limit = limitOf(file)
+    const size = codePoints(next)
     // A file already past its size (edited by hand) may still shrink.
-    if (next.length <= limit || next.length <= file.content.length) return
+    if (size <= limit || size <= codePoints(file.content)) return
     throw new Error(file.topic === undefined
-      ? `${capital(label)}'s MEMORY.md would have ${next.length} of its ${limit} characters. Make room first: move details into a topic (remember with topic, then forget them here), merge entries with replace, or forget what no longer matters.`
-      : `The topic [[${file.topic}]] in ${label} would have ${next.length} of its ${limit} characters. Put the rest in another topic, or forget old entries.`)
+      ? `${capital(label)}'s MEMORY.md would have ${size} of its ${limit} characters. Make room first: move details into a topic (remember with topic, then forget them here), merge entries with replace, or forget what no longer matters.`
+      : `The topic [[${file.topic}]] in ${label} would have ${size} of its ${limit} characters. Put the rest in another topic, or forget old entries.`)
   }
   const capital = text => text[0].toUpperCase() + text.slice(1)
-  const sizeNote = (file, size) => `${file.file}: ${size} of ${limitOf(file)} characters`
+  // #1895: both the bound and the number it prints count code points, so a file of emoji is
+  // measured the way a reader sees it.
+  const sizeNote = (file, text) => `${file.file}: ${codePoints(text)} of ${limitOf(file)} characters`
   const allEntries = files => files.flatMap(file => file.entries.map(entry => ({ ...entry, file })))
   const actorOf = actor => ({ actor: actor.id, actorName: actor.name, ...(actor.auto ? { auto: true } : {}) })
 
@@ -589,7 +595,7 @@ export function install(rt) {
       await journal({ op: 'add', scope: scopeKey(scope), file: target.file, ...actorOf(actor), after: line })
       noteNews(scope, actor, 'add', line)
       const where = topic === undefined ? MAIN : fresh ? `the new topic [[${topic}]], linked from MEMORY.md` : `topic [[${topic}]]`
-      return { changed: true, file: target, message: `Saved to ${label}, in ${where} (${sizeNote(target, next.length)}).` }
+      return { changed: true, file: target, message: `Saved to ${label}, in ${where} (${sizeNote(target, next)}).` }
     })
   }
 
@@ -879,7 +885,7 @@ export function install(rt) {
         const topic = cleanTopic(args.topic)
         for (const { label, files } of loaded) {
           const file = files.find(item => item.topic === topic)
-          if (file !== undefined) return `Topic [[${topic}]] in ${label} (${sizeNote(file, file.content.length)}):\n${file.content.trim() || '(empty)'}`
+          if (file !== undefined) return `Topic [[${topic}]] in ${label} (${sizeNote(file, file.content)}):\n${file.content.trim() || '(empty)'}`
         }
         const known = loaded.flatMap(({ tag, files }) => files.filter(file => file.topic).map(file => `[[${file.topic}]] (${tag})`))
         return `No topic [[${String(args.topic).slice(0, 60)}]] there. Topics: ${known.join(', ') || 'none yet'}.`
@@ -904,7 +910,7 @@ export function install(rt) {
         return loaded.map(({ tag, label, files }) => {
           const [main, ...topics] = files
           return [
-            `${capital(label)}: ${MAIN} has ${main.entries.length} entries (${sizeNote(main, main.content.length)})${topics.length > 0 ? `; topics ${topics.map(file => `[[${file.topic}]] (${file.entries.length})`).join(', ')}` : '; no topics'}.`,
+            `${capital(label)}: ${MAIN} has ${main.entries.length} entries (${sizeNote(main, main.content)})${topics.length > 0 ? `; topics ${topics.map(file => `[[${file.topic}]] (${file.entries.length})`).join(', ')}` : '; no topics'}.`,
             ...main.entries.map(entry => `[${where(tag, main)}] ${formatEntry(entry.text, entry.meta).slice(2)}`),
           ].join('\n')
         }).join('\n\n')
@@ -1062,7 +1068,7 @@ export function install(rt) {
     return {
       botId: bot.id,
       entries,
-      sizes: { own: own[0].content.length, team: team[0].content.length },
+      sizes: { own: codePoints(own[0].content), team: codePoints(team[0].content) },
       limits: MEMORY_LIMITS,
       summary: entries.length > 0 && saved !== undefined ? { text: saved.text, at: saved.at } : null,
       fresh: entries.length === 0 || saved?.hash === hash,
@@ -1324,7 +1330,7 @@ export function install(rt) {
       const teamFiles = loadScope(TEAM)
       const said = read.filter(line => line.who === 'user').map(line => line.text).join('\n')
       const { numbered, left } = reviewEntries(ownFiles, teamFiles, REVIEW_MEMORY_MAX, said)
-      const sizes = { own: ownFiles[0].content.length, team: teamFiles[0].content.length }
+      const sizes = { own: codePoints(ownFiles[0].content), team: codePoints(teamFiles[0].content) }
       const earlier = batch.earlier.map(line => reviewLine(line.who, line.text, bot))
       const language = writtenIn(read.filter(line => line.who === 'user').map(line => line.text.replace(/^User: /, '')), languageNow())
       const reply = await botShot(bot, reviewSystem(writesTeam(bot.id)), reviewRequest({ bot, numbered, left, sizes, where: batch.where, earlier, fresh: fresh.texts, more, language }), { sessionId: rt.chatOf(bot.id), kind: 'memory' })

core/src/host/project-assistant.js

@@ -17,6 +17,7 @@
 import { defineTool } from './tool-schema.js'
 import { MAX_ENTRY } from './constants.js'
 import { listProjectState } from './project-state.js'
+import { codePoints } from './text.js'
 import { output } from './tools.js'
 import { inQuietHours, clockIn, quietHoursOf, readQuietHours } from '../shared/quiet-hours.js'
 
@@ -190,7 +191,8 @@ export function install(rt) {
     if (entry === '') throw refusal('A Project memory entry needs text')
     // #1886: count code points, the same unit `appendProjectMemory` counts, so the two writers
     // cannot disagree about which note fits (`String#length` charged an emoji two units).
-    if ([...entry].length > MAX_ENTRY) throw refusal(`A Project memory entry is at most ${MAX_ENTRY} characters`)
+    // #1895: both count through the one `codePoints` helper.
+    if (codePoints(entry) > MAX_ENTRY) throw refusal(`A Project memory entry is at most ${MAX_ENTRY} characters`)
     const ownerId = text(body.ownerId) || ownersOf(project)[0] || ''
     if (text(body.ownerId) !== '' && !ownersOf(project).includes(text(body.ownerId))) throw outsideBoundary(project, body.ownerId)
     rt.appendProjectMemory?.(project.id, kind, { text: entry, at: Date.now(), ownerId, ...(text(body.by) === '' ? {} : { by: text(body.by) }) })

core/src/host/schedules.js

@@ -31,7 +31,7 @@ import { randomUUID } from 'node:crypto'
 import { createUserMessage } from '@deepseek-ai/dsh-llm'
 import { quietHoursOf, inQuietHours, clockIn, quietError } from '../shared/quiet-hours.js'
 import { defineTool } from './tool-schema.js'
-import { textOf } from './text.js'
+import { codePoints, textOf } from './text.js'
 import { output } from './tools.js'
 import { retiredBot } from './store.js'
 
@@ -335,7 +335,8 @@ export function install(rt) {
     const title = String(body.title ?? '').trim()
     const prompt = String(body.prompt ?? '').trim()
     if (title === '') throw invalid('Give the task a name')
-    if (title.length > MAX_TITLE) throw invalid(`Keep the name under ${MAX_TITLE} characters`)
+    // #1895: the bound counts code points, so a name of emoji is measured as a reader counts it.
+    if (codePoints(title) > MAX_TITLE) throw invalid(`Keep the name under ${MAX_TITLE} characters`)
     if (prompt === '') throw invalid('Say what the Bot should do')
     return { sessionId: bot ? rt.chatOf(bot.id) : room.id, title, prompt }
   }

core/src/host/store.js

@@ -8,7 +8,7 @@ import { join } from 'node:path'
 import { LEGACY_THEMES, MAX_ENTRY, MAX_EXCHANGES, ROOM_MODES } from './constants.js'
 import { homeLock } from './home-lock.js'
 import { openJournal } from './journal.js'
-import { currentColor, normalize, unquote } from './text.js'
+import { codePoints, currentColor, normalize, unquote } from './text.js'
 
 // The saved team's format version. Version 2 (#47) adds the identity schema: member
 // records, a per-Bot owner binding, and the owner's designated Chief. A version-1 file
@@ -519,8 +519,9 @@ export function install(rt) {
     const entryText = projectText(entry.text)
     if (entryText === '') throw projectRefusal('A Project memory entry needs text')
     // #1886: the bound counts code points, not UTF-16 units (`MAX_ENTRY` says why), so the emoji
-    // note the assistant writer accepts at 600 characters is accepted here too.
-    if ([...entryText].length > MAX_ENTRY) throw projectRefusal(`A Project memory entry is at most ${MAX_ENTRY} characters`)
+    // note the assistant writer accepts at 600 characters is accepted here too. #1895: both
+    // writers count through the one `codePoints` helper.
+    if (codePoints(entryText) > MAX_ENTRY) throw projectRefusal(`A Project memory entry is at most ${MAX_ENTRY} characters`)
     const table = memoryTable(project)
     const ownerId = projectText(entry.ownerId, 120) || primaryOwner(project)
     const record = ownerRow(table, ownerId, project)

core/src/host/text.js

@@ -23,6 +23,17 @@ export const unquote = text => String(text ?? '').trim().replace(/^["'“”「
 
 export const escapeRegExp = text => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
 
+// The host bounds this change counts go through it (#1895): one character costs one
+// whatever its UTF-16 width, so an emoji inside a 300-character bound is one character,
+// not two.
+export const codePoints = (text) => {
+  const value = String(text)
+  let n = 0
+  // A code point is one or two UTF-16 units; step by that width so no array is built.
+  for (let i = 0; i < value.length; n += 1) i += value.codePointAt(i) > 0xffff ? 2 : 1
+  return n
+}
+
 // The part of a long message around the first match, so a search hit stays visible.
 export const excerpt = (text, wanted, max = OWN_CHAT_LINE_MAX) => {
   if (text.length <= max) return text

core/test/flybot.test.js

@@ -1037,3 +1037,13 @@ test('#587: list_bots marks a Bot whose send waits on the owner with a count, ne
   const after = await tool('list_bots', {}, 'a')
   assert.ok(!after.includes('to approve'), 'a decided hold leaves no tag')
 })
+
+// #1895: a message's 20,000-character bound counts code points, so an emoji message is
+// measured the way a reader sees it instead of at double length.
+test('a message counts code points against its 20,000-character bound (#1895)', async (t) => {
+  const { fb, stop } = await start()
+  t.after(stop)
+  const atCap = '😀'.repeat(20_000)
+  assert.equal((await fb.bot.send({ from: 'a', to: 'b', text: atCap })).action, 'allow', '20,000 emoji is 20,000 characters')
+  await assert.rejects(fb.bot.send({ from: 'a', to: 'b', text: '😀'.repeat(20_001) }), /a message is at most 20000 characters/, 'one code point over is refused')
+})

core/test/memory.test.js

@@ -1176,3 +1176,38 @@ test('recall related lists what a topic links to', async (t) => {
   assert.match(out, /- \[\[memory\]\] — linked/)
   assert.match(await tool('recall', { related: 'nothinghere' }, 'b2'), /Nothing in your memory and the team memory links "nothinghere"/)
 })
+
+// #1895: a character outside the BMP is one character, not two. The entry bound counts
+// code points, and a refusal prints the number a reader would count.
+test('the entry bound counts code points (#1895)', async (t) => {
+  const { tool, stop } = start()
+  t.after(() => stop())
+  const atCap = '😀'.repeat(MEMORY_LIMITS.entry)
+  assert.match(await tool('remember', { text: atCap }, 'b2'), /^Saved to your memory/, '300 emoji is 300 characters')
+  const over = '😀'.repeat(MEMORY_LIMITS.entry + 1)
+  assert.match(await tool('remember', { text: over }, 'b2'), /at most 300 characters; this one has 301\./, 'the refusal counts code points')
+})
+
+// #1895: a MEMORY.md at its 4,000-character size is measured in code points too, and the
+// size note prints what a reader would count — not twice the emoji count.
+test('a MEMORY.md size and its note count code points (#1895)', async (t) => {
+  const { tool, read, stop, home } = start()
+  t.after(() => stop())
+  const dir = join(home, 'memory', 'bots', 'b2')
+  const file = join(dir, 'MEMORY.md')
+  const withEmoji = count => `# Memory\n\n- ${'😀'.repeat(count)}\n`
+  mkdirSync(dir, { recursive: true })
+
+  // A hand-written file of 3,943 code points (about 7,900 UTF-16 units) takes one more entry.
+  writeFileSync(file, withEmoji(3930))
+  const savedMessage = await tool('remember', { text: 'One more' }, 'b2')
+  assert.match(savedMessage, /^Saved to your memory/, 'a file under 4,000 code points takes an entry')
+  const saved = [...read('bots', 'b2', 'MEMORY.md')].length
+  const added = saved - [...withEmoji(3930)].length
+  assert.ok(saved <= 4000, String(saved))
+  assert.match(savedMessage, new RegExp(`MEMORY\\.md: ${saved} of 4000 characters`), 'the note prints the code-point count')
+  // Then one code point more in the file puts the next entry one over the bound, and the
+  // refusal prints that code-point number rather than twice it.
+  writeFileSync(file, withEmoji(4001 - added - 13))
+  assert.match(await tool('remember', { text: 'One more' }, 'b2'), /MEMORY\.md would have 4001 of its 4000 characters/, 'one code point over is refused with the code-point count')
+})

core/test/schedules.test.js

@@ -921,3 +921,19 @@ test('#1321: with nobody signed in the single-human path still copies across Bot
   assert.equal(copied.ok, true, JSON.stringify(copied))
   assert.equal([...schedule.tasks.values()].at(-1).sessionId, 'b1')
 })
+
+// #1895: a task name's 120-character bound counts code points, so a name of emoji is
+// measured the way a reader sees it instead of at double length.
+test('a task name counts code points against its 120-character bound (#1895)', async (t) => {
+  const schedule = scheduleService()
+  const { ctx, home, call } = setup({ schedule })
+  t.after(async () => { await ctx.dispose(); rmSync(home, { recursive: true, force: true, maxRetries: 5 }) })
+  const atCap = '😀'.repeat(120)
+  const made = await call('schedule-create', { target: 'm1', title: atCap, prompt: 'Go', when: { kind: 'every', minutes: 5 } })
+  assert.equal(made.ok, true, JSON.stringify(made))
+  assert.equal([...schedule.tasks.values()].at(-1).record.title, atCap, 'the whole 120-emoji name reaches DSH')
+  const over = '😀'.repeat(121)
+  const refused = await call('schedule-create', { target: 'm1', title: over, prompt: 'Go', when: { kind: 'every', minutes: 5 } })
+  assert.equal(refused.ok, false)
+  assert.match(refused.error.message, /Keep the name under 120 characters/)
+})

core/test/text.test.js

@@ -1,6 +1,18 @@
 import assert from 'node:assert/strict'
 import { test } from 'node:test'
-import { clip, createClock, escapeRegExp, estimateTokens, excerpt, isColor, normalize, splitRef, textOf, unquote } from '../src/host/text.js'
+import { clip, codePoints, createClock, escapeRegExp, estimateTokens, excerpt, isColor, normalize, splitRef, textOf, unquote } from '../src/host/text.js'
+
+// #1895: the one helper every host limit counts with, so a character outside the BMP is
+// one character and never two.
+test('codePoints counts one character as one whatever its UTF-16 width (#1895)', () => {
+  assert.equal(codePoints(''), 0)
+  assert.equal(codePoints('abc'), 3)
+  assert.equal(codePoints('😀'), 1, 'an emoji is one, though its UTF-16 length is 2')
+  assert.equal('😀'.length, 2, 'the unit the limits used to count it in')
+  assert.equal(codePoints('a😀b'), 3)
+  assert.equal(codePoints('旅行'), 2)
+  assert.equal(codePoints(`a${'😀'.repeat(4)}`), 5)
+})
 
 test('textOf joins the text blocks only', () => {
   assert.equal(textOf([{ type: 'text', text: ' a ' }, { type: 'image' }, { type: 'text', text: 'b' }, { type: 'text', text: 3 }]), 'a \nb')