diff --git a/.changeset/steady-neural-chunks.md b/.changeset/steady-neural-chunks.md new file mode 100644 index 000000000..aaa8889e9 --- /dev/null +++ b/.changeset/steady-neural-chunks.md @@ -0,0 +1,5 @@ +--- +"@voltagent/rag": patch +--- + +Assign distinct NeuralChunker IDs to fitting detector segments after earlier token expansion, so returned chunks do not overwrite each other in ID-keyed stores. Fixes #1518. diff --git a/packages/rag/src/chunkers/neural-chunker.spec.ts b/packages/rag/src/chunkers/neural-chunker.spec.ts new file mode 100644 index 000000000..a4d6b22c5 --- /dev/null +++ b/packages/rag/src/chunkers/neural-chunker.spec.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { whitespaceTokenizer } from "../utils/tokenizer"; +import { NeuralChunker } from "./neural-chunker"; + +describe("NeuralChunker output IDs", () => { + it.each([ + { name: "default tokenizer", tokenizer: undefined }, + { name: "whitespace tokenizer", tokenizer: whitespaceTokenizer }, + ])("keeps IDs distinct after token expansion with $name", async ({ tokenizer }) => { + for (const label of ["neural", "custom"]) { + const chunks = await new NeuralChunker(tokenizer).chunk("a b c d e f", { + detector: () => [1, 9], + maxTokens: 2, + label, + }); + expect(chunks).toHaveLength(4); + expect(chunks.map((chunk) => chunk.id)).toEqual([ + `${label}-0`, + `${label}-1`, + `${label}-2`, + `${label}-3`, + ]); + expect(chunks.every((chunk) => (chunk.tokens ?? 0) <= 2)).toBe(true); + } + }); +}); diff --git a/packages/rag/src/chunkers/neural-chunker.ts b/packages/rag/src/chunkers/neural-chunker.ts index b28d04c2f..187951abd 100644 --- a/packages/rag/src/chunkers/neural-chunker.ts +++ b/packages/rag/src/chunkers/neural-chunker.ts @@ -64,6 +64,7 @@ export class NeuralChunker implements Chunker { if ((chunk.tokens ?? 0) <= maxTokens) { finalChunks.push({ ...chunk, + id: `${label}-${finalChunks.length}`, metadata: buildMetadata({ format: "neural", sourceType: "neural",