From 3d95ab02587343941320528068b55a87c46caf50 Mon Sep 17 00:00:00 2001 From: Zedong Date: Sun, 15 Nov 2026 22:52:54 +0800 Subject: [PATCH] fix(rag): tokenize special-token spellings as document text --- .changeset/tokenize-literal-document-text.md | 5 +++ .../src/__tests__/literal-token-text.spec.ts | 38 +++++++++++++++++++ packages/rag/src/utils/tokenizer.ts | 2 +- 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 .changeset/tokenize-literal-document-text.md create mode 100644 packages/rag/src/__tests__/literal-token-text.spec.ts diff --git a/.changeset/tokenize-literal-document-text.md b/.changeset/tokenize-literal-document-text.md new file mode 100644 index 000000000..a9f365483 --- /dev/null +++ b/.changeset/tokenize-literal-document-text.md @@ -0,0 +1,5 @@ +--- +"@voltagent/rag": patch +--- + +Treat tokenizer special-token spellings as ordinary document text, so token counting and chunking no longer throw on content such as `<|endoftext|>`. diff --git a/packages/rag/src/__tests__/literal-token-text.spec.ts b/packages/rag/src/__tests__/literal-token-text.spec.ts new file mode 100644 index 000000000..ffc00f1c4 --- /dev/null +++ b/packages/rag/src/__tests__/literal-token-text.spec.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { TokenChunker, createTikTokenizer } from "../index"; + +describe("literal special-token spellings", () => { + const tokenizer = createTikTokenizer(); + + it.each([ + "ordinary research document", + "Text about <|endoftext|>.", + "Text about <|fim_prefix|>.", + ])("counts and tokenizes ordinary document content: %s", (text) => { + const tokens = tokenizer.tokenize(text); + expect(tokens.map((token) => token.value).join("")).toBe(text); + expect(tokenizer.countTokens(text)).toBe(tokens.length); + expect(tokens.at(-1)?.end).toBe(text.length); + }); + + it.each([ + "ordinary research document", + "Text about <|endoftext|>.", + "Text about <|fim_prefix|>.", + ])("preserves literal content through public chunking: %s", (text) => { + const chunks = new TokenChunker().chunk(text, { maxTokens: 4 }); + expect(chunks.map((chunk) => chunk.content).join("")).toBe(text); + for (const chunk of chunks) { + expect(chunk.content).toBe(text.slice(chunk.start, chunk.end)); + expect(chunk.tokens).toBeLessThanOrEqual(4); + } + }); + + it.each(["cl100k_base", "o200k_base"] as const)( + "counts the spelling as multiple ordinary tokens with %s", + (encoding) => { + const tokenizer = createTikTokenizer({ encoding }); + expect(tokenizer.countTokens("<|endoftext|>")).toBe(7); + }, + ); +}); diff --git a/packages/rag/src/utils/tokenizer.ts b/packages/rag/src/utils/tokenizer.ts index 8aa444dcd..e310c0dc2 100644 --- a/packages/rag/src/utils/tokenizer.ts +++ b/packages/rag/src/utils/tokenizer.ts @@ -41,7 +41,7 @@ export function createTikTokenizer(config?: { } })(); - const encode = (text: string): number[] => Array.from(encoding.encode(text)); + const encode = (text: string): number[] => Array.from(encoding.encode(text, [], [])); const decode = (tokens: number[]): string => encoding.decode(tokens); return {