From f026fec49c4f4248ffd527f1419ce298060a8a16 Mon Sep 17 00:00:00 2001 From: kun654 <273006726+kun654@users.noreply.github.com> Date: Sat, 7 Nov 2026 13:21:58 +0800 Subject: [PATCH 1/3] fix(core): truncate guardrail character limits on character boundaries createMaxLengthGuardrail and createInputLengthGuardrail sliced by UTF-16 code units, so a limit landing between a surrogate pair emitted a lone high surrogate. Encoding that to UTF-8 replaces it with U+FFFD, silently corrupting truncated output that contains emoji or astral ideographs. --- .changeset/nervous-pandas-tap.md | 5 ++ .../src/agent/guardrails/defaults.spec.ts | 72 +++++++++++++++++++ .../core/src/agent/guardrails/defaults.ts | 41 +++++++++-- 3 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 .changeset/nervous-pandas-tap.md diff --git a/.changeset/nervous-pandas-tap.md b/.changeset/nervous-pandas-tap.md new file mode 100644 index 000000000..68a0bf9f2 --- /dev/null +++ b/.changeset/nervous-pandas-tap.md @@ -0,0 +1,5 @@ +--- +"@voltagent/core": patch +--- + +fix(core): `createMaxLengthGuardrail` and `createInputLengthGuardrail` now truncate on character boundaries, so cutting at an astral character (emoji, CJK extension ideographs) no longer emits a lone surrogate that UTF-8 encodes as `�` diff --git a/packages/core/src/agent/guardrails/defaults.spec.ts b/packages/core/src/agent/guardrails/defaults.spec.ts index 0b0fe8a82..3ed3ddcae 100644 --- a/packages/core/src/agent/guardrails/defaults.spec.ts +++ b/packages/core/src/agent/guardrails/defaults.spec.ts @@ -370,4 +370,76 @@ describe("built-in guardrails", () => { ]); }); }); + + describe("truncation keeps characters intact", () => { + // A surrogate pair is one character stored as two UTF-16 code units. Cutting between + // them yields a lone surrogate, which encodes as U+FFFD ("?") once written as UTF-8. + const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { + const guardrail = createInputLengthGuardrail({ maxCharacters: 11, mode: "truncate" }); + const result = await guardrail.handler({ + input: EMOJI_TEXT, + inputText: EMOJI_TEXT, + agent, + context: operationContext, + operation: "generateText", + } as any); + + expect(result.pass).toBe(true); + expect(result.action).toBe("modify"); + expect(result.modifiedInput).toEqual(expect.any(String)); + expect(result.modifiedInput).not.toMatch(LONE_SURROGATE); + expect(result.modifiedInput).toBe("Great job "); + }); + + it("does not split an emoji when the max length guardrail truncates", async () => { + const guardrail = createMaxLengthGuardrail({ maxCharacters: 11 }); + const result = await guardrail.handler({ + output: EMOJI_TEXT, + originalOutput: EMOJI_TEXT, + agent, + context: operationContext, + operation: "generateText", + metadata: {}, + } as any); + + expect(result.pass).toBe(true); + expect(result.action).toBe("modify"); + expect(result.modifiedOutput).not.toMatch(LONE_SURROGATE); + expect(result.modifiedOutput).toBe("Great job "); + expect(result.metadata.truncatedTo).toBe(10); + }); + + it("does not split an emoji when the max length guardrail truncates a stream", async () => { + const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 11 })); + const parts: VoltAgentTextStreamPart[] = [ + { type: "text-start", id: "t-1" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "Great job " } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "\u{1F44D} team" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: " and friends" } as VoltAgentTextStreamPart, + { type: "text-end", id: "t-1" } as VoltAgentTextStreamPart, + { type: "finish", finishReason: "stop" } as VoltAgentTextStreamPart, + ]; + + const streamed: string[] = []; + for (const part of parts) { + const processed = await runner.processPart(part); + const text = extractText(processed); + if (text) { + streamed.push(text); + } + } + + await runner.finalize({}); + const sanitized = runner.getSanitizedText(); + + expect(streamed.join("")).not.toMatch(LONE_SURROGATE); + expect(streamed.join("")).toBe("Great job "); + expect(sanitized).toBe("Great job "); + }); + }); }); diff --git a/packages/core/src/agent/guardrails/defaults.ts b/packages/core/src/agent/guardrails/defaults.ts index 95c3a4401..1d17568ae 100644 --- a/packages/core/src/agent/guardrails/defaults.ts +++ b/packages/core/src/agent/guardrails/defaults.ts @@ -93,6 +93,26 @@ const DEFAULT_PROFANITY_WORDS = [ const EMAIL_REGEX = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g; const PHONE_REGEX = /(?= 0xd800 && lastCode <= 0xdbff; + + return value.slice(0, endsWithHighSurrogate ? maxCodeUnits - 1 : maxCodeUnits); +} + /** * Creates a guardrail that redacts long numeric sequences such as account or card numbers. */ @@ -445,9 +465,10 @@ export function createMaxLengthGuardrail( } const latestOutput = - typeof output === "string" ? output : originalOutput.slice(0, maxCharacters); - const sanitizedOutput = - latestOutput.length <= maxCharacters ? latestOutput : latestOutput.slice(0, maxCharacters); + typeof output === "string" + ? output + : truncateToCharacterLimit(originalOutput, maxCharacters); + const sanitizedOutput = truncateToCharacterLimit(latestOutput, maxCharacters); return { pass: true, @@ -484,8 +505,16 @@ export function createMaxLengthGuardrail( } const remaining = maxCharacters - guardState.emitted; - const emitText = chunk.length <= remaining ? chunk : chunk.slice(0, remaining); - guardState.emitted += emitText.length; + const emitText = + chunk.length <= remaining ? chunk : truncateToCharacterLimit(chunk, remaining); + // Pin the counter to the limit whenever we trimmed the chunk: dropping a trailing + // high surrogate would otherwise leave room for the next chunk to emit its now + // orphaned low surrogate. + if (emitText.length === chunk.length) { + guardState.emitted += emitText.length; + } else { + guardState.emitted = maxCharacters; + } guardState.truncated = guardState.truncated || emitText.length !== chunk.length; if (chunk.length > remaining && mode === "block") { @@ -711,7 +740,7 @@ export function createInputLengthGuardrail(options: InputLengthGuardrailOptions) return { pass: true, action: "modify", - modifiedInput: input.slice(0, maxCharacters), + modifiedInput: truncateToCharacterLimit(input, maxCharacters), message, }; } From 50058679be5f3dde1e0e6d7faeae95234f8928c9 Mon Sep 17 00:00:00 2001 From: kun654 <273006726+kun654@users.noreply.github.com> Date: Mon, 9 Nov 2026 08:03:29 +0800 Subject: [PATCH 2/3] fix(core): keep streamed truncation on character boundaries createMaxLengthGuardrail only trimmed inside a single delta, so a delta ending on a high surrogate was streamed alone and a following delta could start with its orphaned low surrogate. Hold the high surrogate until the next chunk completes the pair, and pin the counter once the limit is met. --- .../src/agent/guardrails/defaults.spec.ts | 54 +++++++++++++++++++ .../core/src/agent/guardrails/defaults.ts | 42 +++++++++++---- 2 files changed, 86 insertions(+), 10 deletions(-) diff --git a/packages/core/src/agent/guardrails/defaults.spec.ts b/packages/core/src/agent/guardrails/defaults.spec.ts index 3ed3ddcae..cc2e5bb13 100644 --- a/packages/core/src/agent/guardrails/defaults.spec.ts +++ b/packages/core/src/agent/guardrails/defaults.spec.ts @@ -441,5 +441,59 @@ describe("built-in guardrails", () => { expect(streamed.join("")).toBe("Great job "); expect(sanitized).toBe("Great job "); }); + + it("holds a high surrogate back when a stream delta ends exactly at the limit", async () => { + const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 11 })); + const parts: VoltAgentTextStreamPart[] = [ + { type: "text-start", id: "t-1" } as VoltAgentTextStreamPart, + // 11 code units: "Great job " plus the high surrogate of U+1F44D. + { type: "text-delta", id: "t-1", text: "Great job \ud83d" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "\udc4d team" } as VoltAgentTextStreamPart, + { type: "text-end", id: "t-1" } as VoltAgentTextStreamPart, + { type: "finish", finishReason: "stop" } as VoltAgentTextStreamPart, + ]; + + const streamed: string[] = []; + for (const part of parts) { + const processed = await runner.processPart(part); + const text = extractText(processed); + if (text) { + streamed.push(text); + } + } + + await runner.finalize({}); + + expect(streamed.join("")).not.toMatch(LONE_SURROGATE); + expect(streamed.join("")).toBe("Great job "); + expect(runner.getSanitizedText()).toBe("Great job "); + }); + + it("drops a low surrogate when the previous delta ended at the limit", async () => { + const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 10 })); + const parts: VoltAgentTextStreamPart[] = [ + { type: "text-start", id: "t-1" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "Great job " } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "\udc4d team" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "and friends" } as VoltAgentTextStreamPart, + { type: "text-end", id: "t-1" } as VoltAgentTextStreamPart, + { type: "finish", finishReason: "stop" } as VoltAgentTextStreamPart, + ]; + + const streamed: string[] = []; + for (const part of parts) { + const processed = await runner.processPart(part); + const text = extractText(processed); + if (text) { + streamed.push(text); + } + } + + await runner.finalize({}); + + expect(streamed.join("")).not.toMatch(LONE_SURROGATE); + expect(streamed.join("")).toBe("Great job "); + expect(runner.getSanitizedText()).toBe("Great job "); + }); }); }); diff --git a/packages/core/src/agent/guardrails/defaults.ts b/packages/core/src/agent/guardrails/defaults.ts index 1d17568ae..1303f406e 100644 --- a/packages/core/src/agent/guardrails/defaults.ts +++ b/packages/core/src/agent/guardrails/defaults.ts @@ -491,9 +491,11 @@ export function createMaxLengthGuardrail( return part; } - let guardState = state.maxLength as { emitted: number; truncated: boolean } | undefined; + let guardState = state.maxLength as + | { emitted: number; truncated: boolean; pendingHighSurrogate: string } + | undefined; if (!guardState) { - guardState = { emitted: 0, truncated: false }; + guardState = { emitted: 0, truncated: false, pendingHighSurrogate: "" }; state.maxLength = guardState; } @@ -504,20 +506,40 @@ export function createMaxLengthGuardrail( return null; } + // A delta can split a surrogate pair across its boundary, so the high surrogate held + // back from the previous chunk is re-joined here before the limit is applied. + const candidate = guardState.pendingHighSurrogate + chunk; + guardState.pendingHighSurrogate = ""; const remaining = maxCharacters - guardState.emitted; - const emitText = - chunk.length <= remaining ? chunk : truncateToCharacterLimit(chunk, remaining); - // Pin the counter to the limit whenever we trimmed the chunk: dropping a trailing - // high surrogate would otherwise leave room for the next chunk to emit its now - // orphaned low surrogate. - if (emitText.length === chunk.length) { + let emitText = candidate.length <= remaining ? candidate : candidate.slice(0, remaining); + + const lastCode = emitText.length > 0 ? emitText.charCodeAt(emitText.length - 1) : 0; + if (lastCode >= 0xd800 && lastCode <= 0xdbff) { + const completesPair = + candidate.length > emitText.length && + candidate.charCodeAt(emitText.length) >= 0xdc00 && + candidate.charCodeAt(emitText.length) <= 0xdfff; + if (completesPair && emitText.length < remaining) { + // Room for the low surrogate too: emit the character whole and cut off here. + emitText = candidate.slice(0, emitText.length + 1); + } else { + // The pair is either cut by the limit or spread over two deltas; hold the high + // surrogate so the next chunk can complete it instead of streaming it alone. + guardState.pendingHighSurrogate = emitText.slice(emitText.length - 1); + emitText = emitText.slice(0, emitText.length - 1); + } + } + + if (emitText.length === candidate.length && candidate.length <= remaining) { guardState.emitted += emitText.length; } else { + // Trimmed or held back either way: the limit is reached, so pin the counter to keep + // a later delta from emitting the now orphaned low surrogate. guardState.emitted = maxCharacters; } - guardState.truncated = guardState.truncated || emitText.length !== chunk.length; + guardState.truncated = guardState.truncated || emitText.length !== candidate.length; - if (chunk.length > remaining && mode === "block") { + if (candidate.length > remaining && mode === "block") { abort(`Output blocked. Maximum length of ${maxCharacters} characters exceeded.`); } From 6a5d5371bee298b5e12be56caaac9bb29387540f Mon Sep 17 00:00:00 2001 From: kun654 <273006726+kun654@users.noreply.github.com> Date: Mon, 9 Nov 2026 08:30:14 +0800 Subject: [PATCH 3/3] fix(core): finish split surrogate pairs in stream truncation Holding a high surrogate back stopped the counter, so the delta that supplied its partner was dropped along with the rest of the output. Hold the half only while the partner may still arrive, and drop it when the character cannot fit inside the limit. --- .../src/agent/guardrails/defaults.spec.ts | 32 ++++++++++++- .../core/src/agent/guardrails/defaults.ts | 48 ++++++++++--------- 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/packages/core/src/agent/guardrails/defaults.spec.ts b/packages/core/src/agent/guardrails/defaults.spec.ts index cc2e5bb13..583496065 100644 --- a/packages/core/src/agent/guardrails/defaults.spec.ts +++ b/packages/core/src/agent/guardrails/defaults.spec.ts @@ -442,8 +442,8 @@ describe("built-in guardrails", () => { expect(sanitized).toBe("Great job "); }); - it("holds a high surrogate back when a stream delta ends exactly at the limit", async () => { - const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 11 })); + it("completes a surrogate pair split across two deltas when the limit allows it", async () => { + const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 12 })); const parts: VoltAgentTextStreamPart[] = [ { type: "text-start", id: "t-1" } as VoltAgentTextStreamPart, // 11 code units: "Great job " plus the high surrogate of U+1F44D. @@ -464,6 +464,34 @@ describe("built-in guardrails", () => { await runner.finalize({}); + expect(streamed.join("")).not.toMatch(LONE_SURROGATE); + expect(streamed.join("")).toBe("Great job \u{1F44D}"); + expect(runner.getSanitizedText()).toBe("Great job \u{1F44D}"); + }); + + it("drops a split emoji instead of streaming a lone surrogate", async () => { + const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 11 })); + const parts: VoltAgentTextStreamPart[] = [ + { type: "text-start", id: "t-1" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "Great job \ud83d" } as VoltAgentTextStreamPart, + // The pair needs two code units and only one is left, so neither half is streamed. + { type: "text-delta", id: "t-1", text: "\udc4d team" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: " and friends" } as VoltAgentTextStreamPart, + { type: "text-end", id: "t-1" } as VoltAgentTextStreamPart, + { type: "finish", finishReason: "stop" } as VoltAgentTextStreamPart, + ]; + + const streamed: string[] = []; + for (const part of parts) { + const processed = await runner.processPart(part); + const text = extractText(processed); + if (text) { + streamed.push(text); + } + } + + await runner.finalize({}); + expect(streamed.join("")).not.toMatch(LONE_SURROGATE); expect(streamed.join("")).toBe("Great job "); expect(runner.getSanitizedText()).toBe("Great job "); diff --git a/packages/core/src/agent/guardrails/defaults.ts b/packages/core/src/agent/guardrails/defaults.ts index 1303f406e..e05df7e25 100644 --- a/packages/core/src/agent/guardrails/defaults.ts +++ b/packages/core/src/agent/guardrails/defaults.ts @@ -506,40 +506,42 @@ export function createMaxLengthGuardrail( return null; } - // A delta can split a surrogate pair across its boundary, so the high surrogate held - // back from the previous chunk is re-joined here before the limit is applied. - const candidate = guardState.pendingHighSurrogate + chunk; + // The previous delta may have stopped in the middle of a surrogate pair. Rejoin the + // halves when this delta supplies the low surrogate; if it does not, the held half was + // never valid text and is dropped instead of streamed. + const pending = guardState.pendingHighSurrogate; guardState.pendingHighSurrogate = ""; + const nextCode = chunk.charCodeAt(0); + const candidate = + pending.length > 0 && nextCode >= 0xdc00 && nextCode <= 0xdfff ? pending + chunk : chunk; + const consumed = pending.length + chunk.length; const remaining = maxCharacters - guardState.emitted; - let emitText = candidate.length <= remaining ? candidate : candidate.slice(0, remaining); + const truncatedByLimit = candidate.length > remaining; + let emitText = truncatedByLimit ? candidate.slice(0, remaining) : candidate; - const lastCode = emitText.length > 0 ? emitText.charCodeAt(emitText.length - 1) : 0; + const lastCode = emitText.charCodeAt(emitText.length - 1); if (lastCode >= 0xd800 && lastCode <= 0xdbff) { - const completesPair = - candidate.length > emitText.length && - candidate.charCodeAt(emitText.length) >= 0xdc00 && - candidate.charCodeAt(emitText.length) <= 0xdfff; - if (completesPair && emitText.length < remaining) { - // Room for the low surrogate too: emit the character whole and cut off here. - emitText = candidate.slice(0, emitText.length + 1); - } else { - // The pair is either cut by the limit or spread over two deltas; hold the high - // surrogate so the next chunk can complete it instead of streaming it alone. + if (emitText.length === candidate.length) { + // The delta ran out mid-character: hold the half so the next one can finish it. guardState.pendingHighSurrogate = emitText.slice(emitText.length - 1); - emitText = emitText.slice(0, emitText.length - 1); } + // Otherwise the limit itself cut through the character, so the pair can never be + // completed within the budget and the half is dropped. + emitText = emitText.slice(0, emitText.length - 1); } - if (emitText.length === candidate.length && candidate.length <= remaining) { - guardState.emitted += emitText.length; - } else { - // Trimmed or held back either way: the limit is reached, so pin the counter to keep - // a later delta from emitting the now orphaned low surrogate. + guardState.emitted += emitText.length; + // Once the limit has cut a delta short with nothing pending, the budget is spent: pin + // the counter so a later delta cannot stream the orphaned low surrogate on its own. + if (truncatedByLimit && guardState.pendingHighSurrogate.length === 0) { guardState.emitted = maxCharacters; } - guardState.truncated = guardState.truncated || emitText.length !== candidate.length; + guardState.truncated = + guardState.truncated || + truncatedByLimit || + emitText.length + guardState.pendingHighSurrogate.length !== consumed; - if (candidate.length > remaining && mode === "block") { + if (truncatedByLimit && mode === "block") { abort(`Output blocked. Maximum length of ${maxCharacters} characters exceeded.`); }