Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/scope-document-chunk-ids.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@voltagent/rag": patch
---

Scope StructuredDocument chunk IDs to their source document so chunks from different documents remain distinct in the returned collection and link graph.
74 changes: 74 additions & 0 deletions packages/rag/src/__tests__/document-chunk-ids.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import { describe, expect, it } from "vitest";
import { StructuredDocument } from "../index";

const documents = [
{ docId: "policy", text: "Refunds are available within thirty days. Contact support for help." },
{ docId: "guide", text: "Keep your receipt. Provide the order number when requesting a refund." },
];

describe("StructuredDocument chunk identity", () => {
it.each(["token", "recursive", "markdown"] as const)(
"keeps chunks from different documents distinct with %s",
(strategy) => {
const doc = new StructuredDocument(documents);
const { chunks } = doc.chunk({ strategy, maxTokens: 5 });
const byId = new Map(chunks.map((chunk) => [chunk.id, chunk]));
Comment thread
Elon-Jiang marked this conversation as resolved.
Resolved

expect(chunks.length).toBeGreaterThan(documents.length);
expect(byId.size).toBe(chunks.length);
for (const { docId } of documents) {
const ownChunks = chunks.filter((chunk) => chunk.metadata?.docId === docId);
expect(ownChunks.length).toBeGreaterThan(1);
expect(doc.getLinkGraph()[docId]).toEqual(ownChunks.map((chunk) => chunk.id));
for (const chunk of ownChunks) {
expect(byId.get(chunk.id)?.metadata?.docId).toBe(docId);
expect(byId.get(chunk.id)?.content).toBe(chunk.content);
}
}
},
);

it("uses generated document identities when no docId is supplied", () => {
const doc = new StructuredDocument(documents.map(({ text }) => ({ text })));
const { chunks } = doc.chunk({ strategy: "token", maxTokens: 100 });

expect(new Set(chunks.map((chunk) => chunk.id)).size).toBe(2);
for (const node of doc.getNodes()) {
const ownChunk = chunks.find((chunk) => chunk.metadata?.docId === node.id);
expect(ownChunk).toBeDefined();
expect(doc.getLinkGraph()[node.id]).toEqual([ownChunk?.id]);
}
});

it("keeps IDs stable for an unchanged document and preserves chunk contents", () => {
const doc = new StructuredDocument([documents[0]]);
const first = doc.chunk({ strategy: "token", maxTokens: 100 }).chunks;
const second = doc.chunk({ strategy: "token", maxTokens: 100 }).chunks;

expect(second.map((chunk) => chunk.id)).toEqual(first.map((chunk) => chunk.id));
expect(first.map((chunk) => chunk.content)).toEqual([documents[0].text]);
expect(first[0]?.metadata?.docId).toBe("policy");
});

it("keeps chunks distinct even when callers repeat an explicit document ID", () => {
const doc = new StructuredDocument(
documents.map((document) => ({ ...document, docId: "shared" })),
);
const { chunks } = doc.chunk({ strategy: "token", maxTokens: 100 });

expect(new Set(chunks.map((chunk) => chunk.id)).size).toBe(2);
expect(chunks.map((chunk) => chunk.content)).toEqual(
documents.map((document) => document.text),
);
expect(doc.getLinkGraph().shared).toEqual(chunks.map((chunk) => chunk.id));
doc.getNodes().forEach((node, index) => {
expect(node.links).toEqual([{ nodeId: chunks[index]?.id, type: "chunk" }]);
});
});

it("keeps empty input empty", () => {
const doc = new StructuredDocument([]);
expect(doc.chunk({ strategy: "token" }).chunks).toEqual([]);
expect(doc.getLinkGraph()).toEqual({});
});
});
17 changes: 10 additions & 7 deletions packages/rag/src/document/document.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,21 +55,24 @@ export class StructuredDocument {

chunk(params: { strategy: ChunkStrategy; maxTokens?: number }): { chunks: Chunk[] } {
const chunks: Chunk[] = [];
const graph: Record<string, string[]> = {};
this.nodes.forEach((node) => {
this.nodes.forEach((node, documentIndex) => {
const cks = chunkByStrategy(params.strategy, node.text, {
maxTokens: params.maxTokens,
}).map((chunk) => ({
}).map((chunk, index) => ({
...chunk,
id: `${node.id}-${documentIndex}-chunk-${index}`,
metadata: {
...chunk.metadata,
docId: node.id,
},
}));
cks.forEach((ck) => {
this.addLink(node.id, ck.id, "chunk");
graph[node.id] = [...(graph[node.id] ?? []), ck.id];
});
this.nodes[documentIndex] = {
...node,
links: [
...(node.links ?? []),
...cks.map((ck) => ({ nodeId: ck.id, type: "chunk" as const })),
],
};
chunks.push(...cks);
});
return { chunks };
Expand Down