From 9e825e6ffdadb7061aa98f6a808b173ad14ca6c3 Mon Sep 17 00:00:00 2001 From: Elon-Jiang <191222665+Elon-Jiang@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:14:27 +0800 Subject: [PATCH 1/2] fix(rag): keep NeuralChunker output IDs distinct after expansion Co-authored-by: OpenAI Codex --- .changeset/steady-neural-chunks.md | 5 +++++ .../rag/src/__tests__/neural-chunker.spec.ts | 22 +++++++++++++++++++ packages/rag/src/chunkers/neural-chunker.ts | 1 + 3 files changed, 28 insertions(+) create mode 100644 .changeset/steady-neural-chunks.md diff --git a/.changeset/steady-neural-chunks.md b/.changeset/steady-neural-chunks.md new file mode 100644 index 000000000..aaa8889e9 --- /dev/null +++ b/.changeset/steady-neural-chunks.md @@ -0,0 +1,5 @@ +--- +"@voltagent/rag": patch +--- + +Assign distinct NeuralChunker IDs to fitting detector segments after earlier token expansion, so returned chunks do not overwrite each other in ID-keyed stores. Fixes #1518. diff --git a/packages/rag/src/__tests__/neural-chunker.spec.ts b/packages/rag/src/__tests__/neural-chunker.spec.ts index 5476f5b1c..3a66cbaeb 100644 --- a/packages/rag/src/__tests__/neural-chunker.spec.ts +++ b/packages/rag/src/__tests__/neural-chunker.spec.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { NeuralChunker } from "../chunkers/neural-chunker"; +import { whitespaceTokenizer } from "../utils/tokenizer"; describe("NeuralChunker", () => { it("uses detector boundaries and falls back to token chunking when needed", async () => { @@ -15,6 +16,27 @@ describe("NeuralChunker", () => { expect(chunks[0]?.id.startsWith("neural")).toBe(true); }); + it.each([ + { name: "default tokenizer", tokenizer: undefined }, + { name: "whitespace tokenizer", tokenizer: whitespaceTokenizer }, + ])("keeps IDs distinct after token expansion with $name", async ({ tokenizer }) => { + for (const label of ["neural", "custom"]) { + const chunks = await new NeuralChunker(tokenizer).chunk("a b c d e f", { + detector: () => [1, 9], + maxTokens: 2, + label, + }); + expect(chunks).toHaveLength(4); + expect(chunks.map((chunk) => chunk.id)).toEqual([ + `${label}-0`, + `${label}-1`, + `${label}-2`, + `${label}-3`, + ]); + expect(chunks.every((chunk) => (chunk.tokens ?? 0) <= 2)).toBe(true); + } + }); + it("throws when detector is missing", async () => { const chunker = new NeuralChunker(); await expect(chunker.chunk("text")).rejects.toThrow(); diff --git a/packages/rag/src/chunkers/neural-chunker.ts b/packages/rag/src/chunkers/neural-chunker.ts index b28d04c2f..187951abd 100644 --- a/packages/rag/src/chunkers/neural-chunker.ts +++ b/packages/rag/src/chunkers/neural-chunker.ts @@ -64,6 +64,7 @@ export class NeuralChunker implements Chunker { if ((chunk.tokens ?? 0) <= maxTokens) { finalChunks.push({ ...chunk, + id: `${label}-${finalChunks.length}`, metadata: buildMetadata({ format: "neural", sourceType: "neural", From 15d958dc37e3ab960efc9f95b23222710e53c321 Mon Sep 17 00:00:00 2001 From: Elon-Jiang <191222665+Elon-Jiang@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:44:26 +0800 Subject: [PATCH 2/2] test(rag): colocate NeuralChunker ID regressions Co-authored-by: OpenAI Codex --- .../rag/src/__tests__/neural-chunker.spec.ts | 22 ---------------- .../rag/src/chunkers/neural-chunker.spec.ts | 26 +++++++++++++++++++ 2 files changed, 26 insertions(+), 22 deletions(-) create mode 100644 packages/rag/src/chunkers/neural-chunker.spec.ts diff --git a/packages/rag/src/__tests__/neural-chunker.spec.ts b/packages/rag/src/__tests__/neural-chunker.spec.ts index 3a66cbaeb..5476f5b1c 100644 --- a/packages/rag/src/__tests__/neural-chunker.spec.ts +++ b/packages/rag/src/__tests__/neural-chunker.spec.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from "vitest"; import { NeuralChunker } from "../chunkers/neural-chunker"; -import { whitespaceTokenizer } from "../utils/tokenizer"; describe("NeuralChunker", () => { it("uses detector boundaries and falls back to token chunking when needed", async () => { @@ -16,27 +15,6 @@ describe("NeuralChunker", () => { expect(chunks[0]?.id.startsWith("neural")).toBe(true); }); - it.each([ - { name: "default tokenizer", tokenizer: undefined }, - { name: "whitespace tokenizer", tokenizer: whitespaceTokenizer }, - ])("keeps IDs distinct after token expansion with $name", async ({ tokenizer }) => { - for (const label of ["neural", "custom"]) { - const chunks = await new NeuralChunker(tokenizer).chunk("a b c d e f", { - detector: () => [1, 9], - maxTokens: 2, - label, - }); - expect(chunks).toHaveLength(4); - expect(chunks.map((chunk) => chunk.id)).toEqual([ - `${label}-0`, - `${label}-1`, - `${label}-2`, - `${label}-3`, - ]); - expect(chunks.every((chunk) => (chunk.tokens ?? 0) <= 2)).toBe(true); - } - }); - it("throws when detector is missing", async () => { const chunker = new NeuralChunker(); await expect(chunker.chunk("text")).rejects.toThrow(); diff --git a/packages/rag/src/chunkers/neural-chunker.spec.ts b/packages/rag/src/chunkers/neural-chunker.spec.ts new file mode 100644 index 000000000..a4d6b22c5 --- /dev/null +++ b/packages/rag/src/chunkers/neural-chunker.spec.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { whitespaceTokenizer } from "../utils/tokenizer"; +import { NeuralChunker } from "./neural-chunker"; + +describe("NeuralChunker output IDs", () => { + it.each([ + { name: "default tokenizer", tokenizer: undefined }, + { name: "whitespace tokenizer", tokenizer: whitespaceTokenizer }, + ])("keeps IDs distinct after token expansion with $name", async ({ tokenizer }) => { + for (const label of ["neural", "custom"]) { + const chunks = await new NeuralChunker(tokenizer).chunk("a b c d e f", { + detector: () => [1, 9], + maxTokens: 2, + label, + }); + expect(chunks).toHaveLength(4); + expect(chunks.map((chunk) => chunk.id)).toEqual([ + `${label}-0`, + `${label}-1`, + `${label}-2`, + `${label}-3`, + ]); + expect(chunks.every((chunk) => (chunk.tokens ?? 0) <= 2)).toBe(true); + } + }); +});