diff --git a/.changeset/scope-document-chunk-ids.md b/.changeset/scope-document-chunk-ids.md new file mode 100644 index 000000000..11c32de29 --- /dev/null +++ b/.changeset/scope-document-chunk-ids.md @@ -0,0 +1,5 @@ +--- +"@voltagent/rag": patch +--- + +Scope StructuredDocument chunk IDs to their source document so chunks from different documents remain distinct in the returned collection and link graph. diff --git a/packages/rag/src/__tests__/document-chunk-ids.spec.ts b/packages/rag/src/__tests__/document-chunk-ids.spec.ts new file mode 100644 index 000000000..605458c00 --- /dev/null +++ b/packages/rag/src/__tests__/document-chunk-ids.spec.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { StructuredDocument } from "../index"; + +const documents = [ + { docId: "policy", text: "Refunds are available within thirty days. Contact support for help." }, + { docId: "guide", text: "Keep your receipt. Provide the order number when requesting a refund." }, +]; + +describe("StructuredDocument chunk identity", () => { + it.each(["token", "recursive", "markdown"] as const)( + "keeps chunks from different documents distinct with %s", + (strategy) => { + const doc = new StructuredDocument(documents); + const { chunks } = doc.chunk({ strategy, maxTokens: 5 }); + const byId = new Map(chunks.map((chunk) => [chunk.id, chunk])); + + expect(chunks.length).toBeGreaterThan(documents.length); + expect(byId.size).toBe(chunks.length); + for (const { docId } of documents) { + const ownChunks = chunks.filter((chunk) => chunk.metadata?.docId === docId); + expect(ownChunks.length).toBeGreaterThan(1); + expect(doc.getLinkGraph()[docId]).toEqual(ownChunks.map((chunk) => chunk.id)); + for (const chunk of ownChunks) { + expect(byId.get(chunk.id)?.metadata?.docId).toBe(docId); + expect(byId.get(chunk.id)?.content).toBe(chunk.content); + } + } + }, + ); + + it("uses generated document identities when no docId is supplied", () => { + const doc = new StructuredDocument(documents.map(({ text }) => ({ text }))); + const { chunks } = doc.chunk({ strategy: "token", maxTokens: 100 }); + + expect(new Set(chunks.map((chunk) => chunk.id)).size).toBe(2); + for (const node of doc.getNodes()) { + const ownChunk = chunks.find((chunk) => chunk.metadata?.docId === node.id); + expect(ownChunk).toBeDefined(); + expect(doc.getLinkGraph()[node.id]).toEqual([ownChunk?.id]); + } + }); + + it("keeps IDs stable for an unchanged document and preserves chunk contents", () => { + const doc = new StructuredDocument([documents[0]]); + const first = doc.chunk({ strategy: "token", maxTokens: 100 }).chunks; + const second = doc.chunk({ strategy: "token", maxTokens: 100 }).chunks; + + expect(second.map((chunk) => chunk.id)).toEqual(first.map((chunk) => chunk.id)); + expect(first.map((chunk) => chunk.content)).toEqual([documents[0].text]); + expect(first[0]?.metadata?.docId).toBe("policy"); + }); + + it("keeps chunks distinct even when callers repeat an explicit document ID", () => { + const doc = new StructuredDocument( + documents.map((document) => ({ ...document, docId: "shared" })), + ); + const { chunks } = doc.chunk({ strategy: "token", maxTokens: 100 }); + + expect(new Set(chunks.map((chunk) => chunk.id)).size).toBe(2); + expect(chunks.map((chunk) => chunk.content)).toEqual( + documents.map((document) => document.text), + ); + expect(doc.getLinkGraph().shared).toEqual(chunks.map((chunk) => chunk.id)); + doc.getNodes().forEach((node, index) => { + expect(node.links).toEqual([{ nodeId: chunks[index]?.id, type: "chunk" }]); + }); + }); + + it("keeps empty input empty", () => { + const doc = new StructuredDocument([]); + expect(doc.chunk({ strategy: "token" }).chunks).toEqual([]); + expect(doc.getLinkGraph()).toEqual({}); + }); +}); diff --git a/packages/rag/src/document/document.ts b/packages/rag/src/document/document.ts index 6d8dedf51..d729408dd 100644 --- a/packages/rag/src/document/document.ts +++ b/packages/rag/src/document/document.ts @@ -55,21 +55,24 @@ export class StructuredDocument { chunk(params: { strategy: ChunkStrategy; maxTokens?: number }): { chunks: Chunk[] } { const chunks: Chunk[] = []; - const graph: Record = {}; - this.nodes.forEach((node) => { + this.nodes.forEach((node, documentIndex) => { const cks = chunkByStrategy(params.strategy, node.text, { maxTokens: params.maxTokens, - }).map((chunk) => ({ + }).map((chunk, index) => ({ ...chunk, + id: `${node.id}-${documentIndex}-chunk-${index}`, metadata: { ...chunk.metadata, docId: node.id, }, })); - cks.forEach((ck) => { - this.addLink(node.id, ck.id, "chunk"); - graph[node.id] = [...(graph[node.id] ?? []), ck.id]; - }); + this.nodes[documentIndex] = { + ...node, + links: [ + ...(node.links ?? []), + ...cks.map((ck) => ({ nodeId: ck.id, type: "chunk" as const })), + ], + }; chunks.push(...cks); }); return { chunks };