diff --git a/.changeset/nervous-pandas-tap.md b/.changeset/nervous-pandas-tap.md new file mode 100644 index 000000000..68a0bf9f2 --- /dev/null +++ b/.changeset/nervous-pandas-tap.md @@ -0,0 +1,5 @@ +--- +"@voltagent/core": patch +--- + +fix(core): `createMaxLengthGuardrail` and `createInputLengthGuardrail` now truncate on character boundaries, so cutting at an astral character (emoji, CJK extension ideographs) no longer emits a lone surrogate that UTF-8 encodes as `�` diff --git a/packages/core/src/agent/guardrails/defaults.spec.ts b/packages/core/src/agent/guardrails/defaults.spec.ts index 0b0fe8a82..583496065 100644 --- a/packages/core/src/agent/guardrails/defaults.spec.ts +++ b/packages/core/src/agent/guardrails/defaults.spec.ts @@ -370,4 +370,158 @@ describe("built-in guardrails", () => { ]); }); }); + + describe("truncation keeps characters intact", () => { + // A surrogate pair is one character stored as two UTF-16 code units. Cutting between + // them yields a lone surrogate, which encodes as U+FFFD ("?") once written as UTF-8. + const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { + const guardrail = createInputLengthGuardrail({ maxCharacters: 11, mode: "truncate" }); + const result = await guardrail.handler({ + input: EMOJI_TEXT, + inputText: EMOJI_TEXT, + agent, + context: operationContext, + operation: "generateText", + } as any); + + expect(result.pass).toBe(true); + expect(result.action).toBe("modify"); + expect(result.modifiedInput).toEqual(expect.any(String)); + expect(result.modifiedInput).not.toMatch(LONE_SURROGATE); + expect(result.modifiedInput).toBe("Great job "); + }); + + it("does not split an emoji when the max length guardrail truncates", async () => { + const guardrail = createMaxLengthGuardrail({ maxCharacters: 11 }); + const result = await guardrail.handler({ + output: EMOJI_TEXT, + originalOutput: EMOJI_TEXT, + agent, + context: operationContext, + operation: "generateText", + metadata: {}, + } as any); + + expect(result.pass).toBe(true); + expect(result.action).toBe("modify"); + expect(result.modifiedOutput).not.toMatch(LONE_SURROGATE); + expect(result.modifiedOutput).toBe("Great job "); + expect(result.metadata.truncatedTo).toBe(10); + }); + + it("does not split an emoji when the max length guardrail truncates a stream", async () => { + const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 11 })); + const parts: VoltAgentTextStreamPart[] = [ + { type: "text-start", id: "t-1" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "Great job " } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "\u{1F44D} team" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: " and friends" } as VoltAgentTextStreamPart, + { type: "text-end", id: "t-1" } as VoltAgentTextStreamPart, + { type: "finish", finishReason: "stop" } as VoltAgentTextStreamPart, + ]; + + const streamed: string[] = []; + for (const part of parts) { + const processed = await runner.processPart(part); + const text = extractText(processed); + if (text) { + streamed.push(text); + } + } + + await runner.finalize({}); + const sanitized = runner.getSanitizedText(); + + expect(streamed.join("")).not.toMatch(LONE_SURROGATE); + expect(streamed.join("")).toBe("Great job "); + expect(sanitized).toBe("Great job "); + }); + + it("completes a surrogate pair split across two deltas when the limit allows it", async () => { + const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 12 })); + const parts: VoltAgentTextStreamPart[] = [ + { type: "text-start", id: "t-1" } as VoltAgentTextStreamPart, + // 11 code units: "Great job " plus the high surrogate of U+1F44D. + { type: "text-delta", id: "t-1", text: "Great job \ud83d" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "\udc4d team" } as VoltAgentTextStreamPart, + { type: "text-end", id: "t-1" } as VoltAgentTextStreamPart, + { type: "finish", finishReason: "stop" } as VoltAgentTextStreamPart, + ]; + + const streamed: string[] = []; + for (const part of parts) { + const processed = await runner.processPart(part); + const text = extractText(processed); + if (text) { + streamed.push(text); + } + } + + await runner.finalize({}); + + expect(streamed.join("")).not.toMatch(LONE_SURROGATE); + expect(streamed.join("")).toBe("Great job \u{1F44D}"); + expect(runner.getSanitizedText()).toBe("Great job \u{1F44D}"); + }); + + it("drops a split emoji instead of streaming a lone surrogate", async () => { + const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 11 })); + const parts: VoltAgentTextStreamPart[] = [ + { type: "text-start", id: "t-1" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "Great job \ud83d" } as VoltAgentTextStreamPart, + // The pair needs two code units and only one is left, so neither half is streamed. + { type: "text-delta", id: "t-1", text: "\udc4d team" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: " and friends" } as VoltAgentTextStreamPart, + { type: "text-end", id: "t-1" } as VoltAgentTextStreamPart, + { type: "finish", finishReason: "stop" } as VoltAgentTextStreamPart, + ]; + + const streamed: string[] = []; + for (const part of parts) { + const processed = await runner.processPart(part); + const text = extractText(processed); + if (text) { + streamed.push(text); + } + } + + await runner.finalize({}); + + expect(streamed.join("")).not.toMatch(LONE_SURROGATE); + expect(streamed.join("")).toBe("Great job "); + expect(runner.getSanitizedText()).toBe("Great job "); + }); + + it("drops a low surrogate when the previous delta ended at the limit", async () => { + const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 10 })); + const parts: VoltAgentTextStreamPart[] = [ + { type: "text-start", id: "t-1" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "Great job " } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "\udc4d team" } as VoltAgentTextStreamPart, + { type: "text-delta", id: "t-1", text: "and friends" } as VoltAgentTextStreamPart, + { type: "text-end", id: "t-1" } as VoltAgentTextStreamPart, + { type: "finish", finishReason: "stop" } as VoltAgentTextStreamPart, + ]; + + const streamed: string[] = []; + for (const part of parts) { + const processed = await runner.processPart(part); + const text = extractText(processed); + if (text) { + streamed.push(text); + } + } + + await runner.finalize({}); + + expect(streamed.join("")).not.toMatch(LONE_SURROGATE); + expect(streamed.join("")).toBe("Great job "); + expect(runner.getSanitizedText()).toBe("Great job "); + }); + }); }); diff --git a/packages/core/src/agent/guardrails/defaults.ts b/packages/core/src/agent/guardrails/defaults.ts index 95c3a4401..e05df7e25 100644 --- a/packages/core/src/agent/guardrails/defaults.ts +++ b/packages/core/src/agent/guardrails/defaults.ts @@ -93,6 +93,26 @@ const DEFAULT_PROFANITY_WORDS = [ const EMAIL_REGEX = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g; const PHONE_REGEX = /(?= 0xd800 && lastCode <= 0xdbff; + + return value.slice(0, endsWithHighSurrogate ? maxCodeUnits - 1 : maxCodeUnits); +} + /** * Creates a guardrail that redacts long numeric sequences such as account or card numbers. */ @@ -445,9 +465,10 @@ export function createMaxLengthGuardrail( } const latestOutput = - typeof output === "string" ? output : originalOutput.slice(0, maxCharacters); - const sanitizedOutput = - latestOutput.length <= maxCharacters ? latestOutput : latestOutput.slice(0, maxCharacters); + typeof output === "string" + ? output + : truncateToCharacterLimit(originalOutput, maxCharacters); + const sanitizedOutput = truncateToCharacterLimit(latestOutput, maxCharacters); return { pass: true, @@ -470,9 +491,11 @@ export function createMaxLengthGuardrail( return part; } - let guardState = state.maxLength as { emitted: number; truncated: boolean } | undefined; + let guardState = state.maxLength as + | { emitted: number; truncated: boolean; pendingHighSurrogate: string } + | undefined; if (!guardState) { - guardState = { emitted: 0, truncated: false }; + guardState = { emitted: 0, truncated: false, pendingHighSurrogate: "" }; state.maxLength = guardState; } @@ -483,12 +506,42 @@ export function createMaxLengthGuardrail( return null; } + // The previous delta may have stopped in the middle of a surrogate pair. Rejoin the + // halves when this delta supplies the low surrogate; if it does not, the held half was + // never valid text and is dropped instead of streamed. + const pending = guardState.pendingHighSurrogate; + guardState.pendingHighSurrogate = ""; + const nextCode = chunk.charCodeAt(0); + const candidate = + pending.length > 0 && nextCode >= 0xdc00 && nextCode <= 0xdfff ? pending + chunk : chunk; + const consumed = pending.length + chunk.length; const remaining = maxCharacters - guardState.emitted; - const emitText = chunk.length <= remaining ? chunk : chunk.slice(0, remaining); + const truncatedByLimit = candidate.length > remaining; + let emitText = truncatedByLimit ? candidate.slice(0, remaining) : candidate; + + const lastCode = emitText.charCodeAt(emitText.length - 1); + if (lastCode >= 0xd800 && lastCode <= 0xdbff) { + if (emitText.length === candidate.length) { + // The delta ran out mid-character: hold the half so the next one can finish it. + guardState.pendingHighSurrogate = emitText.slice(emitText.length - 1); + } + // Otherwise the limit itself cut through the character, so the pair can never be + // completed within the budget and the half is dropped. + emitText = emitText.slice(0, emitText.length - 1); + } + guardState.emitted += emitText.length; - guardState.truncated = guardState.truncated || emitText.length !== chunk.length; + // Once the limit has cut a delta short with nothing pending, the budget is spent: pin + // the counter so a later delta cannot stream the orphaned low surrogate on its own. + if (truncatedByLimit && guardState.pendingHighSurrogate.length === 0) { + guardState.emitted = maxCharacters; + } + guardState.truncated = + guardState.truncated || + truncatedByLimit || + emitText.length + guardState.pendingHighSurrogate.length !== consumed; - if (chunk.length > remaining && mode === "block") { + if (truncatedByLimit && mode === "block") { abort(`Output blocked. Maximum length of ${maxCharacters} characters exceeded.`); } @@ -711,7 +764,7 @@ export function createInputLengthGuardrail(options: InputLengthGuardrailOptions) return { pass: true, action: "modify", - modifiedInput: input.slice(0, maxCharacters), + modifiedInput: truncateToCharacterLimit(input, maxCharacters), message, }; }