Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/nervous-pandas-tap.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@voltagent/core": patch
---

fix(core): `createMaxLengthGuardrail` and `createInputLengthGuardrail` now truncate on character boundaries, so cutting at an astral character (emoji, CJK extension ideographs) no longer emits a lone surrogate that UTF-8 encodes as `�`
154 changes: 154 additions & 0 deletions packages/core/src/agent/guardrails/defaults.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -370,4 +370,158 @@ describe("built-in guardrails", () => {
]);
});
});

describe("truncation keeps characters intact", () => {
// A surrogate pair is one character stored as two UTF-16 code units. Cutting between
// them yields a lone surrogate, which encodes as U+FFFD ("?") once written as UTF-8.
const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/;

// "Great job " + the high surrogate of U+1F44D sits exactly at index 10.
const EMOJI_TEXT = "Great job \u{1F44D} team";

it("does not split an emoji when the input length guardrail truncates", async () => {
const guardrail = createInputLengthGuardrail({ maxCharacters: 11, mode: "truncate" });
const result = await guardrail.handler({
input: EMOJI_TEXT,
inputText: EMOJI_TEXT,
agent,
context: operationContext,
operation: "generateText",
} as any);

expect(result.pass).toBe(true);
expect(result.action).toBe("modify");
expect(result.modifiedInput).toEqual(expect.any(String));
expect(result.modifiedInput).not.toMatch(LONE_SURROGATE);
expect(result.modifiedInput).toBe("Great job ");
});

it("does not split an emoji when the max length guardrail truncates", async () => {
const guardrail = createMaxLengthGuardrail({ maxCharacters: 11 });
const result = await guardrail.handler({
output: EMOJI_TEXT,
originalOutput: EMOJI_TEXT,
agent,
context: operationContext,
operation: "generateText",
metadata: {},
} as any);

expect(result.pass).toBe(true);
expect(result.action).toBe("modify");
expect(result.modifiedOutput).not.toMatch(LONE_SURROGATE);
expect(result.modifiedOutput).toBe("Great job ");
expect(result.metadata.truncatedTo).toBe(10);
});

it("does not split an emoji when the max length guardrail truncates a stream", async () => {
const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 11 }));
const parts: VoltAgentTextStreamPart[] = [
{ type: "text-start", id: "t-1" } as VoltAgentTextStreamPart,
{ type: "text-delta", id: "t-1", text: "Great job " } as VoltAgentTextStreamPart,
{ type: "text-delta", id: "t-1", text: "\u{1F44D} team" } as VoltAgentTextStreamPart,
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
{ type: "text-delta", id: "t-1", text: " and friends" } as VoltAgentTextStreamPart,
{ type: "text-end", id: "t-1" } as VoltAgentTextStreamPart,
{ type: "finish", finishReason: "stop" } as VoltAgentTextStreamPart,
];

const streamed: string[] = [];
for (const part of parts) {
const processed = await runner.processPart(part);
const text = extractText(processed);
if (text) {
streamed.push(text);
}
}

await runner.finalize({});
const sanitized = runner.getSanitizedText();

expect(streamed.join("")).not.toMatch(LONE_SURROGATE);
expect(streamed.join("")).toBe("Great job ");
expect(sanitized).toBe("Great job ");
});

it("completes a surrogate pair split across two deltas when the limit allows it", async () => {
const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 12 }));
const parts: VoltAgentTextStreamPart[] = [
{ type: "text-start", id: "t-1" } as VoltAgentTextStreamPart,
// 11 code units: "Great job " plus the high surrogate of U+1F44D.
{ type: "text-delta", id: "t-1", text: "Great job \ud83d" } as VoltAgentTextStreamPart,
{ type: "text-delta", id: "t-1", text: "\udc4d team" } as VoltAgentTextStreamPart,
{ type: "text-end", id: "t-1" } as VoltAgentTextStreamPart,
{ type: "finish", finishReason: "stop" } as VoltAgentTextStreamPart,
];

const streamed: string[] = [];
for (const part of parts) {
const processed = await runner.processPart(part);
const text = extractText(processed);
if (text) {
streamed.push(text);
}
}

await runner.finalize({});

expect(streamed.join("")).not.toMatch(LONE_SURROGATE);
expect(streamed.join("")).toBe("Great job \u{1F44D}");
expect(runner.getSanitizedText()).toBe("Great job \u{1F44D}");
});

it("drops a split emoji instead of streaming a lone surrogate", async () => {
const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 11 }));
const parts: VoltAgentTextStreamPart[] = [
{ type: "text-start", id: "t-1" } as VoltAgentTextStreamPart,
{ type: "text-delta", id: "t-1", text: "Great job \ud83d" } as VoltAgentTextStreamPart,
// The pair needs two code units and only one is left, so neither half is streamed.
{ type: "text-delta", id: "t-1", text: "\udc4d team" } as VoltAgentTextStreamPart,
{ type: "text-delta", id: "t-1", text: " and friends" } as VoltAgentTextStreamPart,
{ type: "text-end", id: "t-1" } as VoltAgentTextStreamPart,
{ type: "finish", finishReason: "stop" } as VoltAgentTextStreamPart,
];

const streamed: string[] = [];
for (const part of parts) {
const processed = await runner.processPart(part);
const text = extractText(processed);
if (text) {
streamed.push(text);
}
}

await runner.finalize({});

expect(streamed.join("")).not.toMatch(LONE_SURROGATE);
expect(streamed.join("")).toBe("Great job ");
expect(runner.getSanitizedText()).toBe("Great job ");
});

it("drops a low surrogate when the previous delta ended at the limit", async () => {
const runner = createRunner(createMaxLengthGuardrail({ maxCharacters: 10 }));
const parts: VoltAgentTextStreamPart[] = [
{ type: "text-start", id: "t-1" } as VoltAgentTextStreamPart,
{ type: "text-delta", id: "t-1", text: "Great job " } as VoltAgentTextStreamPart,
{ type: "text-delta", id: "t-1", text: "\udc4d team" } as VoltAgentTextStreamPart,
{ type: "text-delta", id: "t-1", text: "and friends" } as VoltAgentTextStreamPart,
{ type: "text-end", id: "t-1" } as VoltAgentTextStreamPart,
{ type: "finish", finishReason: "stop" } as VoltAgentTextStreamPart,
];

const streamed: string[] = [];
for (const part of parts) {
const processed = await runner.processPart(part);
const text = extractText(processed);
if (text) {
streamed.push(text);
}
}

await runner.finalize({});

expect(streamed.join("")).not.toMatch(LONE_SURROGATE);
expect(streamed.join("")).toBe("Great job ");
expect(runner.getSanitizedText()).toBe("Great job ");
});
});
});
71 changes: 62 additions & 9 deletions packages/core/src/agent/guardrails/defaults.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,26 @@ const DEFAULT_PROFANITY_WORDS = [
const EMAIL_REGEX = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
const PHONE_REGEX = /(?<!\w)(?:\+?\d[\d\s\-()]{6,}\d)/g;

/**
* Truncate a string to at most `maxCodeUnits` UTF-16 code units without ever cutting a
* character in half.
*
* Characters outside the Basic Multilingual Plane (emoji such as U+1F44D, most CJK
* extension ideographs) are stored as surrogate pairs. A plain `slice` can end on the
* high surrogate of such a pair, and a lone surrogate is not valid text: encoding it as
* UTF-8 replaces it with U+FFFD ("?"), so the truncated output is silently corrupted.
*/
function truncateToCharacterLimit(value: string, maxCodeUnits: number): string {
if (value.length <= maxCodeUnits) {
return value;
}

const lastCode = value.charCodeAt(maxCodeUnits - 1);
const endsWithHighSurrogate = lastCode >= 0xd800 && lastCode <= 0xdbff;

return value.slice(0, endsWithHighSurrogate ? maxCodeUnits - 1 : maxCodeUnits);
}

/**
* Creates a guardrail that redacts long numeric sequences such as account or card numbers.
*/
Expand Down Expand Up @@ -445,9 +465,10 @@ export function createMaxLengthGuardrail(
}

const latestOutput =
typeof output === "string" ? output : originalOutput.slice(0, maxCharacters);
const sanitizedOutput =
latestOutput.length <= maxCharacters ? latestOutput : latestOutput.slice(0, maxCharacters);
typeof output === "string"
? output
: truncateToCharacterLimit(originalOutput, maxCharacters);
const sanitizedOutput = truncateToCharacterLimit(latestOutput, maxCharacters);

return {
pass: true,
Expand All @@ -470,9 +491,11 @@ export function createMaxLengthGuardrail(
return part;
}

let guardState = state.maxLength as { emitted: number; truncated: boolean } | undefined;
let guardState = state.maxLength as
| { emitted: number; truncated: boolean; pendingHighSurrogate: string }
| undefined;
if (!guardState) {
guardState = { emitted: 0, truncated: false };
guardState = { emitted: 0, truncated: false, pendingHighSurrogate: "" };
state.maxLength = guardState;
}

Expand All @@ -483,12 +506,42 @@ export function createMaxLengthGuardrail(
return null;
}

// The previous delta may have stopped in the middle of a surrogate pair. Rejoin the
// halves when this delta supplies the low surrogate; if it does not, the held half was
// never valid text and is dropped instead of streamed.
const pending = guardState.pendingHighSurrogate;
guardState.pendingHighSurrogate = "";
const nextCode = chunk.charCodeAt(0);
const candidate =
pending.length > 0 && nextCode >= 0xdc00 && nextCode <= 0xdfff ? pending + chunk : chunk;
const consumed = pending.length + chunk.length;
const remaining = maxCharacters - guardState.emitted;
const emitText = chunk.length <= remaining ? chunk : chunk.slice(0, remaining);
const truncatedByLimit = candidate.length > remaining;
let emitText = truncatedByLimit ? candidate.slice(0, remaining) : candidate;

const lastCode = emitText.charCodeAt(emitText.length - 1);
if (lastCode >= 0xd800 && lastCode <= 0xdbff) {
if (emitText.length === candidate.length) {
// The delta ran out mid-character: hold the half so the next one can finish it.
guardState.pendingHighSurrogate = emitText.slice(emitText.length - 1);
}
// Otherwise the limit itself cut through the character, so the pair can never be
// completed within the budget and the half is dropped.
emitText = emitText.slice(0, emitText.length - 1);
}

guardState.emitted += emitText.length;
guardState.truncated = guardState.truncated || emitText.length !== chunk.length;
// Once the limit has cut a delta short with nothing pending, the budget is spent: pin
// the counter so a later delta cannot stream the orphaned low surrogate on its own.
if (truncatedByLimit && guardState.pendingHighSurrogate.length === 0) {
guardState.emitted = maxCharacters;
}
guardState.truncated =
guardState.truncated ||
truncatedByLimit ||
emitText.length + guardState.pendingHighSurrogate.length !== consumed;

if (chunk.length > remaining && mode === "block") {
if (truncatedByLimit && mode === "block") {
abort(`Output blocked. Maximum length of ${maxCharacters} characters exceeded.`);
}

Expand Down Expand Up @@ -711,7 +764,7 @@ export function createInputLengthGuardrail(options: InputLengthGuardrailOptions)
return {
pass: true,
action: "modify",
modifiedInput: input.slice(0, maxCharacters),
modifiedInput: truncateToCharacterLimit(input, maxCharacters),
message,
};
}
Expand Down
Loading