Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/tokenize-literal-document-text.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@voltagent/rag": patch
---

Treat tokenizer special-token spellings as ordinary document text, so token counting and chunking no longer throw on content such as `<|endoftext|>`.
38 changes: 38 additions & 0 deletions packages/rag/src/__tests__/literal-token-text.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
import { describe, expect, it } from "vitest";
import { TokenChunker, createTikTokenizer } from "../index";

describe("literal special-token spellings", () => {
const tokenizer = createTikTokenizer();

it.each([
"ordinary research document",
"Text about <|endoftext|>.",
"Text about <|fim_prefix|>.",
])("counts and tokenizes ordinary document content: %s", (text) => {
const tokens = tokenizer.tokenize(text);
expect(tokens.map((token) => token.value).join("")).toBe(text);
expect(tokenizer.countTokens(text)).toBe(tokens.length);
expect(tokens.at(-1)?.end).toBe(text.length);
});

it.each([
"ordinary research document",
"Text about <|endoftext|>.",
"Text about <|fim_prefix|>.",
])("preserves literal content through public chunking: %s", (text) => {
const chunks = new TokenChunker().chunk(text, { maxTokens: 4 });
expect(chunks.map((chunk) => chunk.content).join("")).toBe(text);
for (const chunk of chunks) {
expect(chunk.content).toBe(text.slice(chunk.start, chunk.end));
expect(chunk.tokens).toBeLessThanOrEqual(4);
}
});

it.each(["cl100k_base", "o200k_base"] as const)(
"counts the spelling as multiple ordinary tokens with %s",
(encoding) => {
const tokenizer = createTikTokenizer({ encoding });
expect(tokenizer.countTokens("<|endoftext|>")).toBe(7);
},
);
});
2 changes: 1 addition & 1 deletion packages/rag/src/utils/tokenizer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ export function createTikTokenizer(config?: {
}
})();

const encode = (text: string): number[] => Array.from(encoding.encode(text));
const encode = (text: string): number[] => Array.from(encoding.encode(text, [], []));
const decode = (tokens: number[]): string => encoding.decode(tokens);

return {
Expand Down