diff --git a/apps/desktop/electron/main/ipc/agent-ipc.ts b/apps/desktop/electron/main/ipc/agent-ipc.ts index ddab845538..9d2d28a791 100644 --- a/apps/desktop/electron/main/ipc/agent-ipc.ts +++ b/apps/desktop/electron/main/ipc/agent-ipc.ts @@ -295,8 +295,12 @@ export function registerAgentIpc({ Boolean(req.attachments?.length), ) : null; + // The inline text the user's draft carried decides both what the message + // shows and where each image block sits in the prompt. + const steerContent = mcpExpansion?.expanded ?? req.content; const prepared = await preparePromptAttachments( - dataDir, req.sessionId, context.projectPath, req.attachments ?? [], context.supportsVision, + dataDir, req.sessionId, context.projectPath, req.attachments ?? [], + context.supportsVision, steerContent, ); const session = await host.call<{ session?: { messages?: UiMessage[] } }>("session.get", { id: req.sessionId, messageLimit: 1, @@ -304,7 +308,7 @@ export function registerAgentIpc({ const message: UiMessage = { id: durableUserMessageId(req.messageId, session.session?.messages ?? []), role: "user", - content: mcpExpansion?.expanded ?? req.content, + content: steerContent, ...(mcpExpansion ? { command: mcpExpansion.command } : {}), status: "complete", createdAt: new Date().toISOString(), @@ -316,11 +320,12 @@ export function registerAgentIpc({ // never turn into a normal prompt or alter the next turn's configuration. return sidecar.call<{ accepted: boolean; turnId: string }>("agent.steer", { sessionId: req.sessionId, expectedTurnId: req.expectedTurnId, message, - content: appendPromptFallbackPaths(mcpExpansion?.expanded ?? req.content, prepared), + content: appendPromptFallbackPaths(steerContent, prepared), ...(mcpExpansion ? { mcpServerIds: mcpExpansion.mcpServerIds, mcpToolNames: mcpExpansion.mcpToolNames } : {}), attachments: prepared.filter((attachment) => attachment.inlineData).map((attachment) => ({ path: attachment.message.ref, name: attachment.message.name, kind: attachment.message.kind, mimeType: attachment.message.mimeType, size: attachment.message.size, data: attachment.inlineData, + ...(attachment.message.inlinePath ? { inlinePath: attachment.message.inlinePath } : {}), })), }); }); @@ -560,6 +565,9 @@ export function registerAgentIpc({ : undefined, req.attachments ?? [], supportsVision, + // The text the durable message will hold: an image chip the Composer + // left inline keeps its place in the prompt and in the transcript. + promptContent, ); } catch (error) { await finishTurn(req.sessionId, "error", (error as any)?.errorCode, { @@ -677,6 +685,9 @@ export function registerAgentIpc({ mimeType: attachment.message.mimeType, size: attachment.message.size, data: attachment.inlineData, + ...(attachment.message.inlinePath + ? { inlinePath: attachment.message.inlinePath } + : {}), })), // A referenced conversation crosses the sidecar as quoted text for // this turn; the durable record above keeps it for later turns. diff --git a/apps/desktop/electron/main/prompt-attachments.ts b/apps/desktop/electron/main/prompt-attachments.ts index 18f829cb8a..81c908c1ff 100644 --- a/apps/desktop/electron/main/prompt-attachments.ts +++ b/apps/desktop/electron/main/prompt-attachments.ts @@ -21,7 +21,9 @@ import { isAbsolute, join, relative, resolve } from "node:path"; import { ErrorCodes, formatFileInsert, + formatPromptPathText, isSvgAttachment, + locateInlinePromptPaths, SVG_MIME_TYPE, MAX_INLINE_IMAGE_BYTES, type AgentPromptAttachment, @@ -228,9 +230,21 @@ export async function preparePromptAttachments( projectPath: string | undefined, attachments: readonly AgentPromptAttachment[], supportsVision: boolean, + /** + * The prompt text this input carries, with the attachment chips the Composer + * left inline. An image named there keeps that text on the durable message and + * in the prompt, instead of being appended after the body. + */ + promptContent: string, ): Promise { const prepared: PreparedPromptAttachment[] = []; - for (const attachment of attachments) { + // Reference order is the user's order, so each entry takes the next matching + // `@path` text the prompt holds. + const inlineSpans = locateInlinePromptPaths( + promptContent, + attachments.map((attachment) => attachment.path), + ); + for (const [index, attachment] of attachments.entries()) { const source = resolvePromptPath(dataRoot, sessionId, projectPath, attachment.path); if (!source) { throw Object.assign(new Error(`Attachment path is outside the session roots: ${attachment.path}`), { @@ -265,6 +279,12 @@ export async function preparePromptAttachments( : bytes ? ensureAttachmentBlob(dataRoot, bytes) : await ensureAttachmentBlobFromFile(dataRoot, source.absolute); + // The user's own position for this image: the draft named its path between + // words, so the text stays in the prompt and the transcript renders the + // image there. + const inlinePath = inlineSpans[index] + ? formatPromptPathText(attachment.path) + : undefined; const fallbackPath = inline ? displayPromptPath(source, projectPath) : await fallbackPathForStoredAttachment( @@ -280,6 +300,7 @@ export async function preparePromptAttachments( ref, mimeType, size, + ...(inlinePath ? { inlinePath } : {}), }, fallbackPath, ...(bytes @@ -318,6 +339,12 @@ export function appendPromptFallbackPaths( ): string { const paths = attachments .filter((attachment) => !attachment.inlineData) + // A path the prompt already names at the user's own position needs no second + // copy at the end; a rewritten fallback (a replayed copy) still travels. + .filter( + (attachment) => + attachment.message.inlinePath !== formatPromptPathText(attachment.fallbackPath), + ) .map((attachment) => formatFileInsert(attachment.fallbackPath, "file")) .join("") .trim(); diff --git a/apps/desktop/src/features/chat/composer/hooks/useComposerSubmit.ts b/apps/desktop/src/features/chat/composer/hooks/useComposerSubmit.ts index 641c1429a7..1e2a50c78c 100644 --- a/apps/desktop/src/features/chat/composer/hooks/useComposerSubmit.ts +++ b/apps/desktop/src/features/chat/composer/hooks/useComposerSubmit.ts @@ -18,7 +18,6 @@ import { resolveSlashDispatch, } from "../slash-dispatch"; import { readEditorValue, setEditorCaret, type ComposerFileReference } from "../editor"; -import { detachImageTokens } from "../image-attachments"; import type { ComposerDraftSnapshot } from "../../../../lib/composer-smart-stop"; import type { ComposerDraftController } from "./useComposerDraft"; @@ -200,14 +199,11 @@ export function useComposerSubmit({ const submit = async (steering = false) => { const rawText = draft.ref.current ? readEditorValue(draft.ref.current) : value; - // Images stay inline chips while editing; the model still receives them as - // the structured attachment, so their tokens leave the prompt text here. - const outgoing = detachImageTokens(rawText, activeFileReferences, 0); - const inlineContent = serializeInlineComposerFileReferences( - outgoing.text, - outgoing.references, - ); - const serializedContent = serializeComposerFileReferences(outgoing.text, outgoing.references); + // An image chip keeps its place in the prompt: main resolves the `@path` it + // serializes to against the attachment it prepared, so the image block + // arrives where the user put it instead of trailing the text. + const inlineContent = serializeInlineComposerFileReferences(rawText, activeFileReferences); + const serializedContent = serializeComposerFileReferences(rawText, activeFileReferences); if (!serializedContent) return; if (sendBlocked) { if (pasting) showToast(t("chat.pasteInProgress"), { variant: "info" }); @@ -262,11 +258,10 @@ export function useComposerSubmit({ visibleCommandEnd === -1 ? "" : visibleDraft.slice(visibleCommandEnd).trim(); - const outgoingBody = detachImageTokens(visibleCommandBody, activeFileReferences, 0); const accepted = await sendPrompt( serializeInlineComposerFileReferences( - outgoingBody.text, - outgoingBody.references, + visibleCommandBody, + activeFileReferences, ), draft.draftSnapshot(visibleCommandBody), activeSessionId ?? undefined, diff --git a/apps/desktop/src/features/chat/composer/image-attachments.ts b/apps/desktop/src/features/chat/composer/image-attachments.ts index b643a41dd8..554522ded2 100644 --- a/apps/desktop/src/features/chat/composer/image-attachments.ts +++ b/apps/desktop/src/features/chat/composer/image-attachments.ts @@ -4,28 +4,6 @@ export function isImageReference(reference: ComposerFileReference): boolean { return reference.kind === "image" || Boolean(reference.mimeType?.toLowerCase().startsWith("image/")); } -/** - * Keep images outside the submitted prompt text. Images stay inline chips while - * the draft is being edited, so the text handed to the model is detached only - * at submission time, where the structured reference already carries the image. - */ -export function detachImageTokens(text: string, references: ComposerFileReference[], caret: number) { - let nextText = text; - let nextCaret = caret; - const nextReferences = references.map((reference) => { - if (!isImageReference(reference) || !reference.token) return reference; - const { token, ...attachment } = reference; - let index = nextText.indexOf(token); - while (index !== -1) { - if (index < nextCaret) nextCaret -= Math.min(token.length, nextCaret - index); - nextText = nextText.slice(0, index) + nextText.slice(index + token.length); - index = nextText.indexOf(token); - } - return attachment; - }); - return { text: nextText, references: nextReferences, caret: nextCaret }; -} - /** * Keep images visible as inline chips when a draft enters the editor. A draft * cached before images became chips, a restored queue entry, or a prefill can diff --git a/apps/desktop/src/features/chat/transcript/MessageRow.tsx b/apps/desktop/src/features/chat/transcript/MessageRow.tsx index 00889e2c7b..a25d04804a 100644 --- a/apps/desktop/src/features/chat/transcript/MessageRow.tsx +++ b/apps/desktop/src/features/chat/transcript/MessageRow.tsx @@ -9,6 +9,7 @@ import { } from "react"; import { useTranslation } from "react-i18next"; import type { UiMessage } from "@pi-desktop/shared"; +import { splitInlineContent } from "@pi-desktop/shared"; import { useOpenChatFileRef } from "../../../hooks/use-preview-target"; import { useAppStore } from "../../../stores/app-store"; import { Markdown } from "../../../components/Markdown"; @@ -115,6 +116,32 @@ export const MessageRow = memo(function MessageRow({ workspaceRoot, ); }, [message.attachments, message.content, workspaceRoot]); + // The user's own order: an image chip the draft named inline renders between + // the text runs it sat between, exactly where the Composer showed it. + const userContentParts = useMemo( + () => splitInlineContent(String(message.content || ""), message.attachments ?? []).parts, + [message.attachments, message.content], + ); + const renderContentPart = ( + part: (typeof userContentParts)[number], + index: number, + keyPrefix: string, + ) => + part.kind === "text" ? ( + + ) : ( + + ); // An attachment the body does not already name inline continues the body // text instead of taking a line of its own above it. const attachmentChips = extraAttachments.length ? ( @@ -280,21 +307,28 @@ export const MessageRow = memo(function MessageRow({ {message.content ? (
{editableUserMessage && message.command ? ( - message.skillMentions?.length ? ( - - ) : ( - // Templates retain the existing whole-invocation chip. - - {message.command} - - ) + <> + {message.skillMentions?.length ? ( + + ) : ( + // Templates retain the existing whole-invocation chip. + + {message.command} + + )} + {userContentParts + .filter((part) => part.kind === "attachment") + .map((part, index) => renderContentPart(part, index, "inline"))} + ) : ( - + userContentParts.map((part, index) => + renderContentPart(part, index, "content"), + ) )} {attachmentChips}
diff --git a/apps/desktop/src/features/chat/transcript/extra-attachments.ts b/apps/desktop/src/features/chat/transcript/extra-attachments.ts index 4389486793..8f917c4649 100644 --- a/apps/desktop/src/features/chat/transcript/extra-attachments.ts +++ b/apps/desktop/src/features/chat/transcript/extra-attachments.ts @@ -1,13 +1,19 @@ -import type { MessageAttachment } from "@pi-desktop/shared"; +import { isRenderableAttachment, splitInlineContent, type MessageAttachment } from "@pi-desktop/shared"; import { splitChatText } from "../../../lib/chat-links.ts"; -/** Attachments already represented by a verified inline file link stay inline. */ +/** + * Attachments that still need their own place in the message: the ones already + * represented by a verified inline file link, and the ones the body names at + * their inline position (`inlinePath`), stay where the user put them. + */ export function getExtraMessageAttachments( content: string, attachments: readonly MessageAttachment[] | undefined, workspaceRoot?: string | null, ): (MessageAttachment & { kind: "file" | "image" })[] { if (!attachments?.length) return []; + const renderable = attachments.filter(isRenderableAttachment); + if (!renderable.length) return []; const inline = new Set( splitChatText(content, workspaceRoot) .map((segment) => segment.kind === "target" && segment.target.kind === "file" @@ -15,8 +21,7 @@ export function getExtraMessageAttachments( : null) .filter((path): path is string => path !== null), ); - return attachments.filter( - (attachment): attachment is MessageAttachment & { kind: "file" | "image" } => - attachment.kind !== "session" && !inline.has(attachment.ref), + return splitInlineContent(content, renderable).trailing.filter( + (attachment) => !inline.has(attachment.ref), ); } diff --git a/apps/desktop/src/features/chat/transcript/shared.tsx b/apps/desktop/src/features/chat/transcript/shared.tsx index 3b0dbcf341..d38a4619b3 100644 --- a/apps/desktop/src/features/chat/transcript/shared.tsx +++ b/apps/desktop/src/features/chat/transcript/shared.tsx @@ -535,9 +535,12 @@ export function SessionLinkChip({ sessionId, ...position }: { sessionId: string export function MessageAttachmentImage({ attachment, onOpenFile, + inline = false, }: { attachment: MessageAttachment; onOpenFile: (path: string, baseDir?: string, mimeType?: string) => void; + /** Inline placement inside the message prose; only the trailing list has a list parent. */ + inline?: boolean; }) { const dataUrl = useReferencedImageDataUrl(attachment.ref, attachment.mimeType); const [anchor, setAnchor] = useState(null); @@ -554,7 +557,7 @@ export function MessageAttachmentImage({ {segments.map((segment, index) => { - const start = offset; + const start = sourceOffset + offset; offset += segment.text.length; - const position = { "data-source-start": start, "data-source-end": offset }; + const position = { "data-source-start": start, "data-source-end": sourceOffset + offset }; return segment.kind === "text" ? ( {segment.text} ) : segment.target.kind === "file" ? ( diff --git a/apps/desktop/src/lib/session-transcript.ts b/apps/desktop/src/lib/session-transcript.ts index 3f97549943..a69ec2b9d5 100644 --- a/apps/desktop/src/lib/session-transcript.ts +++ b/apps/desktop/src/lib/session-transcript.ts @@ -1,5 +1,6 @@ import { preservePlanHistory } from "./plan-history"; import type { AgentEvent, MessageAttachment, UiMessage } from "@pi-desktop/shared"; +import { formatPromptPathText, locateInlinePromptPaths } from "@pi-desktop/shared"; import { getSessionMessageSnapshot, registerSessionMessageAppend, @@ -18,6 +19,7 @@ type OptimisticFileReference = { const OPTIMISTIC_USER_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const MAX_OPTIMISTIC_ECHO_DELAY_MS = 30_000; +const IMAGE_PATH_PATTERN = /\.(avif|bmp|gif|heic|jpe?g|png|tiff?|webp)$/i; /** * The user row shown the moment a prompt is sent, before the host has @@ -31,18 +33,26 @@ export function optimisticUserMessage( fileReferences: readonly OptimisticFileReference[] = [], createdAt: string = new Date().toISOString(), ): UiMessage { - const attachments: MessageAttachment[] = fileReferences - .filter((reference) => !reference.token) - .map((reference) => ({ - kind: - reference.kind ?? - (/\.(avif|bmp|gif|heic|jpe?g|png|tiff?|webp)$/i.test(reference.path) - ? "image" - : "file"), - name: reference.name, - ref: reference.path, - ...(reference.mimeType ? { mimeType: reference.mimeType } : {}), - })); + // A plain file chip travels as `@path` text and needs no attachment row, but + // an image chip is one: it also records where its path sits, so the row shows + // the image in place before the durable echo arrives (D288). + const rendered = fileReferences.flatMap((reference) => { + const kind = + reference.kind ?? + (IMAGE_PATH_PATTERN.test(reference.path) ? ("image" as const) : ("file" as const)); + return reference.token && kind !== "image" ? [] : [{ reference, kind }]; + }); + const spans = locateInlinePromptPaths( + content, + rendered.map(({ reference }) => reference.path), + ); + const attachments: MessageAttachment[] = rendered.map(({ reference, kind }, index) => ({ + kind, + name: reference.name, + ref: reference.path, + ...(reference.mimeType ? { mimeType: reference.mimeType } : {}), + ...(spans[index] ? { inlinePath: formatPromptPathText(reference.path) } : {}), + })); return { id, role: "user", diff --git a/apps/desktop/test/composer-file-reference-display.test.mjs b/apps/desktop/test/composer-file-reference-display.test.mjs index fd18f449df..2b90332812 100644 --- a/apps/desktop/test/composer-file-reference-display.test.mjs +++ b/apps/desktop/test/composer-file-reference-display.test.mjs @@ -79,7 +79,7 @@ test("composer renders atomic inline chips and serializes paths on send", () => assert.match(composer, /chip\.title = origin/); assert.match( composer, - /serializeComposerFileReferences\(outgoing\.text, outgoing\.references\)/, + /serializeComposerFileReferences\(rawText, activeFileReferences\)/, ); assert.match( composer, diff --git a/apps/desktop/test/composer-image-attachments.test.mjs b/apps/desktop/test/composer-image-attachments.test.mjs index 9eac914544..c3e3cb603d 100644 --- a/apps/desktop/test/composer-image-attachments.test.mjs +++ b/apps/desktop/test/composer-image-attachments.test.mjs @@ -1,38 +1,11 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { - attachImageTokens, - detachImageTokens, -} from "../src/features/chat/composer/image-attachments.ts"; +import { attachImageTokens } from "../src/features/chat/composer/image-attachments.ts"; const image = { id: "image", sessionId: "a", path: "/scratch/a.png", name: "a.png", kind: "image", token: "\ue001" }; const tokenlessImage = { id: "inline", sessionId: "a", path: "/scratch/b.png", name: "b.png", kind: "image" }; const file = { id: "file", sessionId: "a", path: "/scratch/a.txt", name: "a.txt", kind: "file", token: "\ue002" }; -test("submission detaches image tokens and image references while text keeps file order", () => { - const result = detachImageTokens(`before ${image.token}${file.token} after`, [image, file], 9); - assert.equal(result.text, `before ${file.token} after`); - assert.equal(result.caret, 8); - const { token, ...detachedImage } = image; - assert.deepEqual(result.references, [detachedImage, file]); - assert.equal(image.token, "\ue001"); -}); - -test("image-only drafts keep metadata and a valid empty-text caret", () => { - const result = detachImageTokens(image.token, [image], 1); - assert.equal(result.text, ""); - assert.equal(result.caret, 0); - assert.equal(result.references[0].path, image.path); - assert.equal(result.references[0].token, undefined); -}); - -test("MIME image references detach without changing earlier text or repeated normalization", () => { - const result = detachImageTokens(`abc${image.token}def${image.token}`, [{ ...image, kind: "file", mimeType: "IMAGE/PNG" }], 2); - assert.equal(result.text, "abcdef"); - assert.equal(result.caret, 2); - assert.deepEqual(detachImageTokens(result.text, result.references, result.caret), result); -}); - test("a restored image without a token becomes an inline chip", () => { const { text, references } = attachImageTokens(`look${file.token} here`, [tokenlessImage, file], () => "\ue101"); assert.equal(text, `look${file.token} here\ue101`); diff --git a/apps/desktop/test/composer-inline-image-chips.test.mjs b/apps/desktop/test/composer-inline-image-chips.test.mjs index 23101a0d57..2fb4cefbc4 100644 --- a/apps/desktop/test/composer-inline-image-chips.test.mjs +++ b/apps/desktop/test/composer-inline-image-chips.test.mjs @@ -47,9 +47,15 @@ test("hovering an image chip reveals a read-only preview card", () => { assert.match(hoverCard, /const target = controller\.preview \? null : hover;/); }); -test("image tokens leave the prompt text only at submission", () => { - assert.match(submitHook, /const outgoing = detachImageTokens\(rawText, activeFileReferences, 0\);/); - assert.match(submitHook, /serializeComposerFileReferences\(outgoing\.text, outgoing\.references\)/); +test("image tokens stay in the prompt so the image keeps its place", () => { + // The chip's `@path` reaches main inline, which records it on the attachment + // and sends the image block at that position; only the bytes travel detached. + assert.match( + submitHook, + /const inlineContent = serializeInlineComposerFileReferences\(rawText, activeFileReferences\);/, + ); + assert.match(submitHook, /const serializedContent = serializeComposerFileReferences\(rawText, activeFileReferences\);/); + assert.doesNotMatch(submitHook, /detachImageTokens/); assert.match(submitHook, /const submittedDraft = draft\.draftSnapshot\(rawText\);/); assert.match(composer, /attachImageTokens\(/); }); diff --git a/apps/desktop/test/composer-paste-files.test.mjs b/apps/desktop/test/composer-paste-files.test.mjs index ff3146b9ff..c3ecf5e4ac 100644 --- a/apps/desktop/test/composer-paste-files.test.mjs +++ b/apps/desktop/test/composer-paste-files.test.mjs @@ -40,10 +40,10 @@ test("composer converts oversized text paste and materializes clipboard files", composer, /createFileReference\(file\.path, file\.name, sessionId, \{[\s\S]*kind: file\.kind/, ); - assert.match(composer, /serializeComposerFileReferences\(outgoing\.text, outgoing\.references\)/); + assert.match(composer, /serializeComposerFileReferences\(rawText, activeFileReferences\)/); assert.match( composer, - /const serializedContent = serializeComposerFileReferences\(outgoing\.text, outgoing\.references\)/, + /const serializedContent = serializeComposerFileReferences\(rawText, activeFileReferences\)/, ); // The draft is a contenteditable rich field: sentinels render as atomic // chips and every caret write goes through the DOM-range helper. diff --git a/apps/desktop/test/composer-send-state.test.mjs b/apps/desktop/test/composer-send-state.test.mjs index 3846f71e35..01b2c3a4e7 100644 --- a/apps/desktop/test/composer-send-state.test.mjs +++ b/apps/desktop/test/composer-send-state.test.mjs @@ -242,7 +242,7 @@ test("send clears the composer before the round trip and restores a rejected dra submit, /const rawText = draft\.ref\.current \? readEditorValue\(draft\.ref\.current\) : value;/, ); - assert.match(submit, /serializeInlineComposerFileReferences\(\s*outgoing\.text,\s*outgoing\.references,\s*\)/); + assert.match(submit, /serializeInlineComposerFileReferences\(rawText, activeFileReferences\)/); // Blocked and not-ready states are said, not swallowed. assert.match(submit, /if \(pasting\) showToast\(t\("chat\.pasteInProgress"\)/); assert.match( @@ -279,7 +279,7 @@ test("mode slash prefixes send the trailing prompt and retain failed drafts", () ); assert.match( submit, - /serializeInlineComposerFileReferences\(\s*outgoingBody\.text,\s*outgoingBody\.references,\s*\)/, + /serializeInlineComposerFileReferences\(\s*visibleCommandBody,\s*activeFileReferences,\s*\)/, ); assert.match( submit, diff --git a/apps/desktop/test/prompt-inline-attachments.test.mjs b/apps/desktop/test/prompt-inline-attachments.test.mjs new file mode 100644 index 0000000000..9dc7e83676 --- /dev/null +++ b/apps/desktop/test/prompt-inline-attachments.test.mjs @@ -0,0 +1,83 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { appendPromptFallbackPaths, preparePromptAttachments } from "../electron/main/prompt-attachments.ts"; + +const png = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a9z8AAAAASUVORK5CYII=", + "base64", +); +const sessionId = "inline-placement"; + +async function fixture(t) { + const root = await mkdtemp(join(tmpdir(), "pi-inline-attachments-")); + t.after(() => rm(root, { recursive: true, force: true })); + const data = join(root, "data"); + const pasted = join(data, "scratch", sessionId, "pasted"); + await mkdir(pasted, { recursive: true }); + const path = join(pasted, "pasted-a.png"); + await writeFile(path, png); + return { data, path }; +} + +test("an image the draft named inline records that text and keeps its place", async (t) => { + const { data, path } = await fixture(t); + const attachment = { path, name: "pasted-a.png", kind: "image", mimeType: "image/png" }; + const content = `compare @${path} with the rest`; + const [prepared] = await preparePromptAttachments( + data, + sessionId, + undefined, + [attachment], + true, + content, + ); + assert.equal(prepared.message.kind, "image"); + assert.equal(prepared.message.inlinePath, `@${path}`); + assert.ok(prepared.inlineData); + // The prompt already names the path where the user put it, so no copy follows. + assert.equal(appendPromptFallbackPaths(content, [prepared]), content); +}); + +test("an image the prompt does not name inline keeps the trailing position", async (t) => { + const { data, path } = await fixture(t); + const attachment = { path, name: "pasted-a.png", kind: "image", mimeType: "image/png" }; + const [prepared] = await preparePromptAttachments( + data, + sessionId, + undefined, + [attachment], + false, + "look at this", + ); + assert.equal(prepared.message.inlinePath, undefined); + assert.equal(prepared.inlineData, undefined); + const text = appendPromptFallbackPaths("look at this", [prepared]); + assert.ok(text.startsWith("look at this")); + assert.ok(text.includes(path)); +}); + +test("a replayed copy still travels for the path the prompt names", async (t) => { + const { data } = await fixture(t); + const ref = `attachments/${"a".repeat(64)}`; + await mkdir(join(data, "attachments"), { recursive: true }); + await writeFile(join(data, ref), png); + const attachment = { path: ref, name: "a.png", kind: "image", mimeType: "image/png" }; + const content = `look @${ref}`; + const [prepared] = await preparePromptAttachments( + data, + sessionId, + undefined, + [attachment], + false, + content, + ); + assert.equal(prepared.message.inlinePath, `@${ref}`); + const text = appendPromptFallbackPaths(content, [prepared]); + assert.ok(text.startsWith(content)); + // The stored blob is replayed into scratch, so that working path is added. + assert.notEqual(prepared.fallbackPath, ref); + assert.ok(text.includes(prepared.fallbackPath)); +}); diff --git a/apps/desktop/test/session-message-presentation.test.mjs b/apps/desktop/test/session-message-presentation.test.mjs index cc1e32dc54..0d20e6c71b 100644 --- a/apps/desktop/test/session-message-presentation.test.mjs +++ b/apps/desktop/test/session-message-presentation.test.mjs @@ -4,7 +4,9 @@ import test from "node:test"; import * as React from "react"; import * as jsxRuntime from "react/jsx-runtime"; import { renderToStaticMarkup } from "react-dom/server"; +import { splitInlineContent } from "@pi-desktop/shared"; import ts from "typescript"; +import { getExtraMessageAttachments } from "../src/features/chat/transcript/extra-attachments.ts"; const t = (key, values) => values?.name ? `${key}: ${values.name}` : key; const store = { @@ -22,7 +24,8 @@ const shared = { CopyButton: ({ label }) => React.createElement("button", { "aria-label": label }), FileRefChip: () => null, LinkifiedText: ({ text }) => text, - MessageAttachmentImage: () => null, + MessageAttachmentImage: ({ attachment }) => + React.createElement("span", { "data-image": attachment.name }), MessageTimestamp: () => null, }; @@ -51,6 +54,8 @@ function loadComponent(name, extras = {}) { }, "./ActionBarSlots": { ActionSlotSide: () => null }, "../../../plugins/renderer-slots/slot-message": { slotMessage: () => undefined }, + // The row renders the user's own order through the shared placement helper. + "@pi-desktop/shared": { splitInlineContent }, ...extras, }; const module = { exports: {} }; @@ -64,7 +69,7 @@ function loadComponent(name, extras = {}) { const origin = loadComponent("SessionMessageOrigin"); const { MessageRow } = loadComponent("MessageRow", { "./SessionMessageOrigin": origin, - "./extra-attachments": { getExtraMessageAttachments: () => [] }, + "./extra-attachments": { getExtraMessageAttachments }, }); const userMessage = { id: "incoming-row", @@ -122,3 +127,41 @@ test("ordinary text cannot forge another session's provenance", () => { assert.match(html, /aria-label="chat.editMessage"/); assert.doesNotMatch(html, /session-message-origin|data-session-message-kind/); }); + +test("a user message keeps the place of an image the draft named inline", () => { + const content = "before @/scratch/pasted/a.png after"; + const html = render({ + ...userMessage, + content, + attachments: [ + { + kind: "image", + name: "a.png", + ref: "attachments/abc", + mimeType: "image/png", + inlinePath: "@/scratch/pasted/a.png", + }, + ], + }); + const before = html.indexOf("before "); + const image = html.indexOf('data-image="a.png"'); + const after = html.indexOf(" after"); + assert.ok(before >= 0 && image > before && after > image, html); + // The message names the image once, in place: the path text becomes the chip + // and the attachment never repeats below the body. + assert.doesNotMatch(html, /scratch\/pasted\/a\.png/); + assert.equal(html.match(/data-image="a\.png"/g)?.length, 1); +}); + +test("an image without a recorded position still follows the body", () => { + const html = render({ + ...userMessage, + content: "look at this", + attachments: [ + { kind: "image", name: "b.png", ref: "attachments/def", mimeType: "image/png" }, + ], + }); + const text = html.indexOf("look at this"); + const image = html.indexOf('data-image="b.png"'); + assert.ok(text >= 0 && image > text, html); +}); diff --git a/apps/desktop/test/svg-attachments.test.mjs b/apps/desktop/test/svg-attachments.test.mjs index c5d1925901..a167428221 100644 --- a/apps/desktop/test/svg-attachments.test.mjs +++ b/apps/desktop/test/svg-attachments.test.mjs @@ -45,7 +45,7 @@ for (const [name, mimeType, expectedName] of [ assert.equal(saved.mimeType, SVG_MIME); assert.equal(saved.name, expectedName); assert.deepEqual(await readFile(saved.path), svg); - const prepared = await preparePromptAttachments(data, sessionId, undefined, [saved], true); + const prepared = await preparePromptAttachments(data, sessionId, undefined, [saved], true, ""); assert.equal(prepared[0].message.kind, "file"); assert.equal(prepared[0].inlineData, undefined); assert.equal(prepared[0].message.mimeType, SVG_MIME); @@ -64,7 +64,7 @@ test("picker import and project-path drop use the same SVG fallback", async (t) assert.deepEqual(await readFile(picked.path), svg); const dropped = { path: source, name: "diagram.SVG", kind: "image", mimeType: "image/png" }; for (const supportsVision of [false, true]) { - const prepared = await preparePromptAttachments(data, sessionId, project, [picked, dropped], supportsVision); + const prepared = await preparePromptAttachments(data, sessionId, project, [picked, dropped], supportsVision, ""); assert.ok(prepared.every((item) => item.message.kind === "file" && item.inlineData === undefined)); assert.ok(prepared.every((item) => item.message.mimeType === SVG_MIME)); assert.ok(appendPromptFallbackPaths("Explain both", prepared).includes("diagram.SVG")); @@ -83,7 +83,7 @@ test("legacy content-store SVG retries produce a session-readable file, not an i ]) { const attachment = { kind: "image", path: ref, ...metadata }; const before = structuredClone(attachment); - const [prepared] = await preparePromptAttachments(data, sessionId, undefined, [attachment], true); + const [prepared] = await preparePromptAttachments(data, sessionId, undefined, [attachment], true, ""); assert.equal(prepared.message.kind, "file"); assert.equal(prepared.message.ref, ref); assert.equal(prepared.message.mimeType, SVG_MIME); @@ -101,7 +101,7 @@ test("mixed SVG and PNG paste keeps PNG image bytes and SVG file references", as { name: "diagram.svg", mimeType: SVG_MIME, data: svg }, { name: "pixel.png", mimeType: "image/png", data: png }, ]); - const prepared = await preparePromptAttachments(data, sessionId, undefined, saved, true); + const prepared = await preparePromptAttachments(data, sessionId, undefined, saved, true, ""); assert.deepEqual(prepared.map((item) => item.message.kind), ["file", "image"]); assert.equal(prepared[0].inlineData, undefined); assert.equal(prepared[1].message.mimeType, "image/png"); @@ -110,7 +110,7 @@ test("mixed SVG and PNG paste keeps PNG image bytes and SVG file references", as assert.ok(text.startsWith("Compare\n")); assert.ok(text.includes(saved[0].path)); assert.ok(!text.includes(saved[1].path)); - const nonVision = await preparePromptAttachments(data, sessionId, undefined, saved, false); + const nonVision = await preparePromptAttachments(data, sessionId, undefined, saved, false, ""); assert.ok(nonVision.every((item) => item.inlineData === undefined)); assert.ok(appendPromptFallbackPaths("Compare", nonVision).includes(saved[1].path)); }); @@ -122,7 +122,7 @@ test("SVG classification does not bypass attachment roots or mutate failed input const attachment = { kind: "image", path: outside, name: "outside.svg", mimeType: SVG_MIME }; const before = structuredClone(attachment); await assert.rejects( - preparePromptAttachments(data, sessionId, project, [attachment], true), + preparePromptAttachments(data, sessionId, project, [attachment], true, ""), { errorCode: "PATH_OUTSIDE_WORKSPACE" }, ); assert.deepEqual(attachment, before); @@ -135,7 +135,7 @@ test("SVG classification does not bypass attachment roots or mutate failed input throw error; } await assert.rejects( - preparePromptAttachments(data, sessionId, project, [{ ...attachment, path: link }], true), + preparePromptAttachments(data, sessionId, project, [{ ...attachment, path: link }], true, ""), { errorCode: "PATH_OUTSIDE_WORKSPACE" }, ); }); @@ -232,7 +232,7 @@ test("existing non-SVG image metadata is not narrowed to a new provider allowlis const bytes = Buffer.from(`transport fixture: ${extension}`); const [saved] = await saveComposerPasteFiles(data, sessionId, [{ name: `fixture.${extension}`, mimeType, data: bytes }]); assert.equal(saved.kind, "image"); - const [prepared] = await preparePromptAttachments(data, sessionId, undefined, [saved], true); + const [prepared] = await preparePromptAttachments(data, sessionId, undefined, [saved], true, ""); assert.equal(prepared.message.kind, "image"); assert.equal(prepared.message.mimeType, mimeType); assert.deepEqual(Buffer.from(prepared.inlineData, "base64"), bytes); diff --git a/crates/host-core/src/data_relocation.rs b/crates/host-core/src/data_relocation.rs index 14e7ab4082..b450924da4 100644 --- a/crates/host-core/src/data_relocation.rs +++ b/crates/host-core/src/data_relocation.rs @@ -74,7 +74,7 @@ impl Roots { // Never rewrite narrative content, shell commands, source // code, credentials, or arbitrary serialized user input. "text" | "content" | "summary" | "command" | "code" | "prompt" - | "markdown" | "instructions" | "env" => {} + | "markdown" | "instructions" | "env" | "inlinePath" => {} _ => changed |= self.structured(item), } } diff --git a/crates/host-core/src/sessions.rs b/crates/host-core/src/sessions.rs index 65ce3b94e4..8669afe644 100644 --- a/crates/host-core/src/sessions.rs +++ b/crates/host-core/src/sessions.rs @@ -159,6 +159,11 @@ pub struct MessageAttachment { /// link. Travels with the user message so the model keeps reading it. #[serde(skip_serializing_if = "Option::is_none")] pub text: Option, + /// The `@path` text this attachment occupies inside the message content when + /// the user placed it between words. The transcript renders the attachment + /// at that position instead of appending it after the body. + #[serde(skip_serializing_if = "Option::is_none")] + pub inline_path: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -456,6 +461,9 @@ pub(crate) fn ui_to_record(message: &UiMessage) -> (MessageRecord, Option UiMessage { .get("text") .and_then(|v| v.as_str()) .map(str::to_string), + inline_path: block + .get("inlinePath") + .and_then(|v| v.as_str()) + .map(str::to_string), }) }) .collect::>(); @@ -4998,6 +5010,7 @@ mod tests { mime_type: Some("image/png".into()), size: Some(42), text: None, + inline_path: Some("@/scratch/pasted/image.png".into()), }]); append_message(&db, &session.id, &user, None).unwrap(); @@ -5014,6 +5027,7 @@ mod tests { let blocks = record.blocks; assert_eq!(blocks[1]["type"], "attachment"); assert_eq!(blocks[1]["ref"], "attachments/abc123"); + assert_eq!(blocks[1]["inlinePath"], "@/scratch/pasted/image.png"); assert!(blocks[1].get("data").is_none()); } diff --git a/docs/spec/03-runtime/01-ipc-protocol.md b/docs/spec/03-runtime/01-ipc-protocol.md index 1024f3081c..36eadf78ae 100644 --- a/docs/spec/03-runtime/01-ipc-protocol.md +++ b/docs/spec/03-runtime/01-ipc-protocol.md @@ -137,6 +137,8 @@ type AgentPromptAttachment = { kind: "image" | "file"; mimeType?: string; size?: number; + /** `@path` text this attachment occupies inline in `content`; main-filled. */ + inlinePath?: string; }; type AgentPromptResponse = { @@ -210,7 +212,12 @@ older images use the existing safe `@path` fallback. Unknown/custom models without an explicit image override, non-vision models, oversized images, and unavailable refs also use the safe fallback. Main uses streamed hashing and file copying for oversized images. The durable user message stores `content` plus -attachment metadata/ref, never base64. +attachment metadata/ref, never base64. An image the draft named inline also +carries that `@path` text as `inlinePath` on the durable message and on the +sidecar attachment: the runtime keeps the prompt content blocks in the user's +order instead of appending every image after the text, and the transcript +renders the image at that position. An attachment without it keeps the trailing +position. Invalid attachment paths fail with `PATH_OUTSIDE_WORKSPACE`. Regenerate history (D109) also uses session channels: diff --git a/docs/spec/03-runtime/02-agent-runtime.md b/docs/spec/03-runtime/02-agent-runtime.md index 306ebdf290..ffc51e6fd5 100644 --- a/docs/spec/03-runtime/02-agent-runtime.md +++ b/docs/spec/03-runtime/02-agent-runtime.md @@ -772,7 +772,9 @@ attachment, scratch, and project roots. Each image remains subject to the 10 MB safety bound, and restored history has a 30 MB aggregate raw-byte budget. The newest refs are considered first; all eligible images remain image blocks when the history fits, while over-budget or oversized images become safe -`@path` fallbacks. Reads are bounded by the admitted file size. Base64 is +`@path` fallbacks. An image whose message names it inline (`inlinePath`) is +spliced back into the prompt at that position, so a restored turn keeps the +order the user wrote. Reads are bounded by the admitted file size. Base64 is transient and never restored into durable UI messages or transcript records. - Failed assistant messages remain durable diagnostic transcript entries but are never restored into pi model context on a later turn. diff --git a/docs/spec/04-ux/08-component-spec.md b/docs/spec/04-ux/08-component-spec.md index 320760b0cf..a06ddb9798 100644 --- a/docs/spec/04-ux/08-component-spec.md +++ b/docs/spec/04-ux/08-component-spec.md @@ -1744,10 +1744,12 @@ Single message render — either user (plaintext) or assistant (markdown streami only trailing/leading composer trim is applied, never internal newline collapse. Serialized `@path` file references render as compact leaf-name chips matching the composer node (icon + ellipsized name; canonical path in - the tooltip and accessible name). Image attachments that are not already - inlined as `@path` chips render as the same compact image chip inside the - body's own text flow, continuing the message text instead of heading it on a - line of its own. Hovering or focusing the chip reveals the bounded data URL + the tooltip and accessible name). An image the draft named inline (`inlinePath` + on the attachment) renders as the same compact image chip at that position in + the body, so a sent message keeps the order the Composer showed. An image + attachment without a recorded position continues the message text instead of + heading it on a line of its own. Hovering or focusing the chip reveals the + bounded data URL (`fs/readImageDataUrl`) in a read-only preview card above it (below it when the chip sits at the top), and an unresolved load leaves the chip alone. A referenced @@ -3412,10 +3414,14 @@ Anatomy: `/scratch//pasted/`, and returns each UUID-backed absolute path with its sanitized original leaf name and kind. The composer displays the leaf name, keeps the structured reference in session-scoped - transient state, and submits it separately from visible text. Main stores + transient state, and serializes its `@path` inline in the prompt text, at the + position the chip held in the draft. Main stores image bytes under `attachments/` and sends visual input only when the selected model's effective binding capability accepts images and the 10 MB - inline bound is met; otherwise it appends a safe `@path` fallback. SVG inputs + inline bound is met; otherwise it appends a safe `@path` fallback. An image the + draft named inline records that text on the attachment (`inlinePath`), which is + what places the model-facing image block at the user's position and what the + transcript and a restored session read back. SVG inputs (`image/svg+xml` or `.svg` extension) are always classified as files, never as model images, regardless of vision capability (see `03-runtime/svg-attachment-input.md`). Removing diff --git a/docs/spec/06-delivery/04-e2e-test-plan.md b/docs/spec/06-delivery/04-e2e-test-plan.md index ff6dc52090..4137d32036 100644 --- a/docs/spec/06-delivery/04-e2e-test-plan.md +++ b/docs/spec/06-delivery/04-e2e-test-plan.md @@ -7994,6 +7994,11 @@ must keep splitting are covered by `markdown-blocks.test.mjs`. text-plus-image send to verify full draft restoration. In a narrow pane, prefill 20 images, confirm 20 inline chips inside the composer, and remove one without losing the others. + Send a prompt that puts text, an image, then more text in that order: + confirm the sent message renders the image chip at that position instead of + after the body, and that an image the draft did not name inline still + follows the text. The provider-facing prompt keeps the same order; the + runtime placement tests assert those content blocks. Inspect the chip and open it with click, Enter, and Space. Confirm a centered modal preview opens, the work panel stays unchanged, and the draft is neither edited nor sent. Check small images @@ -8044,7 +8049,12 @@ must keep splitting are covered by `markdown-blocks.test.mjs`. F (persistence), Quality - **Milestone**: M5 - **Status**: Unit-covered (`composer-paste-files.test.mjs`, - `composer-clipboard.test.mjs`, `composer-native-deletion.test.mjs`); `pnpm test:e2e:composer-paste` mounts the real + `composer-clipboard.test.mjs`, `composer-native-deletion.test.mjs`); + `prompt-inline-attachments.test.mjs` covers where an inline image's `@path` + is recorded and that a replayed fallback copy still travels, + `session-message-presentation.test.mjs` renders the row order, and + `packages/agent-runtime/src/runtime.test.ts` asserts the prompt content + blocks. `pnpm test:e2e:composer-paste` mounts the real ComposerInput, draft/paste hooks, file viewer, production CSS and sandboxed preload. It dispatches Chromium ClipboardEvents with synthetic mixed data and native File objects, exercises the real scratch writer and contained diff --git a/docs/zh-CN/spec/03-runtime/01-ipc-protocol.md b/docs/zh-CN/spec/03-runtime/01-ipc-protocol.md index 86238c7808..d8586829fb 100644 --- a/docs/zh-CN/spec/03-runtime/01-ipc-protocol.md +++ b/docs/zh-CN/spec/03-runtime/01-ipc-protocol.md @@ -186,6 +186,8 @@ Root 用户轮次可能包括 `revisionRootId`、`revisionCount` 和 输入框 附件可供性保持隐藏，直到 main、sidecar、pi 模型 功能和持久性都会消耗有效负载。 +草稿内联命名过位置的图片会带上 `inlinePath`，运行时按它把图片块放回原位置， +而不是统一追加在提示文本之后。 ### 5.1a 向当前回合补充指令 diff --git a/docs/zh-CN/spec/03-runtime/02-agent-runtime.md b/docs/zh-CN/spec/03-runtime/02-agent-runtime.md index 43341d7a94..850e3c1d2f 100644 --- a/docs/zh-CN/spec/03-runtime/02-agent-runtime.md +++ b/docs/zh-CN/spec/03-runtime/02-agent-runtime.md @@ -95,7 +95,9 @@ pi 消费排队输入时保留渲染器提供的消息 id；即使补充输入 `agent_end` 之前不会调用这条路径 6. 在 Electron main 的会话绑定路径边界上校验结构化附件，按 SHA-256 持久化 图片字节，持久用户消息中只保留附件引用。只有处于视觉模型 10 MB 内联上限 - 之内的图片才会被读进内存；更大的图片走流式哈希/复制以及既有的安全路径回退 + 之内的图片才会被读进内存；更大的图片走流式哈希/复制以及既有的安全路径回退。 + 草稿内联命名过位置的图片会把那段 `@path` 文本作为 `inlinePath` 随持久消息和 + sidecar 附件传递；运行时据此把图片块放在用户写下的位置，而不是统一追加在提示文本之后 草稿里的 `pi-desktop://session/` 链接在同一步骤里解析成有界摘录附件：只限 同一项目，当前对话在任何读取之前就被跳过；运行时随后把它作为一个 `` 块排在用户自己的话之前引用给模型 @@ -590,7 +592,8 @@ Goal 批准所承诺的内容与 Plan 批准所承诺的内容完全相同：`mo 同样的重复还会让下一个边界的估算翻倍，并让备用模型拿到的末行无法被 `continue()` 续跑（D620）。 - 视觉运行时只从会话绑定的附件、scratch 与项目根目录中水合持久化的图片引用。 处于 10 MB 内联安全上限之内的图片会成为临时的 pi-ai 图片块；超限或不可用的 - 图片则退化为安全的 `@path` 回退。超限历史的水合会直接复制文件，不会先把内容 + 图片则退化为安全的 `@path` 回退。消息里内联命名过位置的图片（`inlinePath`） + 会按那个位置拼回提示内容，重启后仍是用户写下的顺序。超限历史的水合会直接复制文件，不会先把内容 读进内存。Base64 绝不会被还原进持久的 UI 消息或转录记录。 - 失败的助理消息仍然是持久的诊断记录条目，但 在以后的回合中永远不会恢复到 pi 模型上下文中。 diff --git a/docs/zh-CN/spec/04-ux/08-component-spec.md b/docs/zh-CN/spec/04-ux/08-component-spec.md index c6c58f33ab..48639dd42f 100644 --- a/docs/zh-CN/spec/04-ux/08-component-spec.md +++ b/docs/zh-CN/spec/04-ux/08-component-spec.md @@ -1242,7 +1242,7 @@ row 而不是在转录本中添加树镶边。 聊天轮流而不是全角块。气泡按提示的 max-content 收缩，而不是撑到该上限；内联文件芯片使用确定的 240px 名称上限，因此粘贴文件加上短提示仍保持紧凑。用户正文是明文 保留硬换行符（`white-space: pre-wrap`）； 仅应用 trailing/leading 输入框修剪，从不应用内部换行符 -崩溃。序列化的 `@path` 文件引用画成与输入框一致的叶子名芯片（图标 + 省略号文件名；规范路径放在工具提示和无障碍名称里）。未内联成 `@path` 芯片的图片附件渲染为同样的图片芯片，接在正文文本之后、与正文同一段流里，而不是另起一行排在正文上方；悬停或聚焦该芯片会在其上方（芯片位于顶部时改在下方）浮出只读预览小卡，内容来自有界的 `fs/readImageDataUrl` data URL，无法解析时保留芯片。消息正文里的裸路径 token 识别 Unicode 字母与数字。在用户消息正文中，它们只是候选：现有的 `fs/resolveRef` 查找确认是真实文件后才显示芯片。等待、未找到或失败的查找保留原文，包括 `使用llama.cpp`。显式 `@path` 引用和结构化附件保留现有芯片，不做猜测查询。每条消息最多检查 32 个唯一候选，可见行之间最多 4 个并发查找；其余候选保持纯文本。确认结果绑定到当前消息文本、工作区路径和会话；任一变化都会丢弃旧结果并取消排队的工作。之后新建的文件在消息重新挂载或作用域变化时再考虑，不轮询。非 ASCII 文件名仍受支持。绝对路径与 `~/` 整体匹配，工作区之外（以及一切家目录路径）保持纯文本，而不是渲染一个永远打不开的芯片——包含边界不变（D322）。点击芯片先经 `pi-desktop/fs/resolveRef` 补全引用——该通道搜索整个打开的项目，按项目组文件夹顺序、主文件夹优先（ADR 0263）——再按解析结果打开：项目文件在随应用打包的 `pi.file-manager` 工作面板视图中打开（该视图不可用时退回宿主 `file:` 选项卡），会话临时目录或附件文件在宿主 `file:` 选项卡中打开，主文件夹中的 `.html`/`.htm` 仍由侧边浏览器打开。交给该视图的地址跟随应答的文件夹：主文件夹中的文件用项目内相对路径传递，同一项目的同级文件夹中的文件用绝对路径传递，与会话临时目录和附件文件一致。已解析的图片芯片按同样规则打开。什么都没匹配到的芯片不打开任何东西，而是自己报告出来；系统默认应用不再由这次点击触发。HTTP(S) URL 仍是内联文本链接；裸 URL 保留路径、查询和片段中成对的圆括号，正文包裹网址时多出的右括号不属于链接；紧跟 URL 右括号的句末标点也不属于链接，但 `(draft).html` 这样的后缀仍保留，在侧边浏览器打开；长 URL 在板内换行并保持逻辑开始对齐，而不是继承浏览器的居中按钮文本。 +草稿里内联命名过位置的图片（附件上的 `inlinePath`）渲染为同样的图片芯片，就落在正文里的那个位置，已发送消息因此保持输入框里的顺序；没有记录位置的图片附件接在正文文本之后、与正文同一段流里，而不是另起一行排在正文上方； - 被引用的对话（`kind: "session"`）画成聊天图标芯片，标签是共享的引用标签加被引用 标题；工具提示与无障碍名称来自语言目录，激活它会打开那段对话（§20B）。 - Assistant：透明表面，左对齐，Markdown 完全渲染 @@ -2176,6 +2176,8 @@ MainChat 底部的输入区域，用于撰写和发送提示。支持多行输 ADR 0131） - Main 把图片字节存在 `attachments/`，仅当所选 models.dev 模型接受 图片且不超过 10 MB 内联上限时发送视觉输入；否则追加安全的 `@path` 回退。 + 草稿里内联命名过位置的图片会把那段 `@path` 文本记在附件的 `inlinePath` 上： + 模型看到的图片块因此留在用户写下的位置，转录和恢复的会话也按它把图片放回原位。 SVG 输入（`image/svg+xml` 或 `.svg` 扩展名）始终被分类为文件而非模型图片， 无论视觉能力如何（见 `03-runtime/svg-attachment-input.md`）（D361）。草稿仍是文本；芯片在发送前序列化为规范 `@`， 代理用文件工具跟随路径。 diff --git a/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md b/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md index 568f5d7de9..70dcf834a5 100644 --- a/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md +++ b/docs/zh-CN/spec/06-delivery/04-e2e-test-plan.md @@ -5029,6 +5029,9 @@ eleven-tool-round desktop paths are verified by 内联图片芯片，撤销会恢复；仅图片草稿切换会话后仍保留图片与发送元数据。挂载完整 Composer，验证仅图片时点击发送会交付附件并清空草稿，文字加图片的发送立即失败时 全部恢复。窄面板预填 20 张图片，确认输入框内有 20 个内联芯片，删除其中一个后其余保留。 + 发送一条顺序为「文字、图片、更多文字」的提示：确认已发送消息把图片芯片渲染在该位置， + 而不是接在正文之后；草稿未内联命名的图片仍跟随正文之后。面向模型的提示保持同样的顺序， + 由运行时放置测试断言这些内容块。 点击芯片或用 Enter/空格激活它打开居中图片浮层，工作面板与草稿保持 不变。小图不放大，宽图等比适应窗口；验证缩放、重置、下载原图、多图按钮与方向键 不变。小图不放大，宽图等比适应窗口；验证缩放、重置、下载原图、多图按钮与方向键 @@ -5066,6 +5069,9 @@ eleven-tool-round desktop paths are verified by F（坚持），品质 - **里程碑**：M5 - **状态**：单元覆盖（`composer-paste-files.test.mjs`、`composer-clipboard.test.mjs`）； + `prompt-inline-attachments.test.mjs` 覆盖内联图片记录 `@path` 与重放副本仍随行， + `session-message-presentation.test.mjs` 渲染消息行顺序， + `packages/agent-runtime/src/runtime.test.ts` 断言提示内容块； `pnpm test:e2e:composer-paste` 挂载真实 ComposerInput、草稿/粘贴 hook、文件查看器、 生产 CSS 和沙盒 preload，以 Chromium ClipboardEvent 注入混合数据与原生 File， 调用真实暂存文件写入器和受限文件读取器，核对字节并解码预览图片。 diff --git a/packages/agent-runtime/src/attachment-history.ts b/packages/agent-runtime/src/attachment-history.ts index 930a52b6fb..f5bcdc54fe 100644 --- a/packages/agent-runtime/src/attachment-history.ts +++ b/packages/agent-runtime/src/attachment-history.ts @@ -241,9 +241,14 @@ export async function hydrateAttachmentHistory( const resolved = resolvedHistory[index]; if (!resolved) return message; const fallbackPaths = resolved - .map((item) => item.fallbackPath) - .filter((path): path is string => Boolean(path)) - .map((path) => formatFileInsert(path, "file")) + .filter( + (item) => + Boolean(item.fallbackPath) && + // A path the message already names at the user's own position needs no + // second copy at the end; a rewritten replay copy still travels. + formatFileInsert(item.fallbackPath!, "file").trim() !== item.attachment.inlinePath, + ) + .map((item) => formatFileInsert(item.fallbackPath!, "file")) .join("") .trim(); const content = message.content.trim() diff --git a/packages/agent-runtime/src/runtime.test.ts b/packages/agent-runtime/src/runtime.test.ts index 218a956e01..7e366f6c06 100644 --- a/packages/agent-runtime/src/runtime.test.ts +++ b/packages/agent-runtime/src/runtime.test.ts @@ -3808,7 +3808,11 @@ describe("DesktopAgentRuntime session collaboration provenance", () => { const prompt = vi.spyOn(agent, "prompt").mockResolvedValue(); vi.spyOn(agent, "waitForIdle").mockResolvedValue(); await runtime.prompt({ text: content, sessionMessage: origin }, "user-1", "turn-1"); - expect(prompt).toHaveBeenCalledWith(expected, []); + // The runtime hands pi the built user message, so an image the Composer + // placed inline keeps that position instead of trailing the text. + expect(prompt).toHaveBeenCalledWith( + expect.objectContaining({ role: "user", content: expected }), + ); expect(expected).toContain("not by the user"); expect(expected).toContain("does not grant new user authorization"); const restored = createRuntime({ history: [ @@ -3860,8 +3864,13 @@ describe("DesktopAgentRuntime session collaboration provenance", () => { const agent = (runtime as any).agent; const handle = (runtime as any).handleAgentEvent.bind(runtime); const calls: Array<{ text: string; images: unknown[] }> = []; - const respond = async (text: string, images: unknown[] = []) => { - calls.push({ text, images: images ?? [] }); + // pi receives the built user message: text plus every inline image block. + const respond = async (input: unknown) => { + const text = + typeof input === "string" + ? input + : String((input as { content: unknown }).content); + calls.push({ text, images: [] }); await handle({ type: "agent_start" }); const reply = assistantMessage({ content: [{ type: "text", text: "ok" }] }); agent.state.messages = [{ role: "user", content: text, timestamp: 1 }, reply]; @@ -11006,3 +11015,103 @@ it("does not reuse stale plugin declarations when schema or permission metadata expect(runtimeMatches(runtime, { pluginTools: [{ ...plugin, planSafeActions: ["inspect"] }] })).toBe(false); } finally { await runtime.dispose(); } }); + +describe("inline image placement", () => { + const imageAttachment = (data: string, inlinePath?: string) => ({ + path: "attachments/prepared", + name: "pasted.png", + kind: "image" as const, + mimeType: "image/png", + data, + ...(inlinePath ? { inlinePath } : {}), + }); + + it("keeps every image block where the user placed it in the prompt", async () => { + const runtime = createRuntime(); + const agent = (runtime as unknown as { agent: Agent }).agent; + const prompt = vi.spyOn(agent, "prompt").mockResolvedValue(); + vi.spyOn(agent, "waitForIdle").mockResolvedValue(); + try { + await runtime.prompt({ + text: "compare @/scratch/a.png with @/scratch/b.png now", + attachments: [ + { ...imageAttachment("QUJD", "@/scratch/a.png"), name: "a.png" }, + { ...imageAttachment("REVG", "@/scratch/b.png"), name: "b.png" }, + ], + }); + expect(prompt).toHaveBeenCalledWith( + expect.objectContaining({ + role: "user", + content: [ + { type: "text", text: "compare " }, + { type: "image", data: "QUJD", mimeType: "image/png" }, + { type: "text", text: " with " }, + { type: "image", data: "REVG", mimeType: "image/png" }, + { type: "text", text: " now" }, + ], + }), + ); + } finally { + await runtime.dispose(); + } + }); + + it("follows the text with an image the prompt does not name inline", async () => { + const runtime = createRuntime(); + const agent = (runtime as unknown as { agent: Agent }).agent; + const prompt = vi.spyOn(agent, "prompt").mockResolvedValue(); + vi.spyOn(agent, "waitForIdle").mockResolvedValue(); + try { + await runtime.prompt({ + text: "look at this", + attachments: [imageAttachment("QUJD")], + }); + expect(prompt).toHaveBeenCalledWith( + expect.objectContaining({ + role: "user", + content: [ + { type: "text", text: "look at this" }, + { type: "image", data: "QUJD", mimeType: "image/png" }, + ], + }), + ); + } finally { + await runtime.dispose(); + } + }); + + it("restores a durable inline placement from the session history", async () => { + const runtime = createRuntime({ + history: [ + { + id: "user-image", + role: "user", + content: "look @/scratch/a.png please", + status: "complete", + createdAt: new Date().toISOString(), + attachments: [ + { + kind: "image", + name: "a.png", + ref: "attachments/a", + mimeType: "image/png", + data: "QUJD", + inlinePath: "@/scratch/a.png", + }, + ], + }, + ], + }); + try { + const messages = (runtime as unknown as { agent: Agent }).agent.state.messages; + const user = messages.find((message) => message.role === "user"); + expect(user?.content).toEqual([ + { type: "text", text: "look " }, + { type: "image", data: "QUJD", mimeType: "image/png" }, + { type: "text", text: " please" }, + ]); + } finally { + await runtime.dispose(); + } + }); +}); diff --git a/packages/agent-runtime/src/runtime.ts b/packages/agent-runtime/src/runtime.ts index 4286d505a0..dae46d927e 100644 --- a/packages/agent-runtime/src/runtime.ts +++ b/packages/agent-runtime/src/runtime.ts @@ -108,6 +108,7 @@ import { normalizeSubagentName, proposalKindForMode, resolveSubagentToolNames, + splitInlineContent, subagentModelKey, subagentToolsLabel, type ProposalKind, @@ -316,39 +317,47 @@ function promptText(input: RuntimePrompt): string { : input.text; } -function promptContent(input: string | RuntimePrompt): UserMessage["content"] { - if (typeof input === "string") return input; - const text = promptText(input); - const images = (input.attachments ?? []).filter( +/** One provider image block for an attachment whose bytes crossed this turn. */ +function promptImageBlock(attachment: RuntimePromptAttachment): ImageContent { + return { + type: "image" as const, + data: attachment.data!, + mimeType: attachment.mimeType || "image/png", + }; +} + +/** Attachments carrying image bytes this call may inline for the model. */ +function promptImages(input: RuntimePrompt): RuntimePromptAttachment[] { + return (input.attachments ?? []).filter( (attachment) => attachment.kind === "image" && typeof attachment.data === "string" && attachment.data.length > 0, ); - if (!images.length) return text; - return [ - ...(text.trim() ? [{ type: "text" as const, text }] : []), - ...images.map((attachment) => ({ - type: "image" as const, - data: attachment.data!, - mimeType: attachment.mimeType || "image/png", - })), - ]; } -function promptImages(input: RuntimePrompt): ImageContent[] { - return (input.attachments ?? []) - .filter( - (attachment) => - attachment.kind === "image" && - typeof attachment.data === "string" && - attachment.data.length > 0, - ) - .map((attachment) => ({ - type: "image" as const, - data: attachment.data!, - mimeType: attachment.mimeType || "image/png", - })); +/** + * The prompt the model receives. An image the user placed between words carries + * the `@path` the Composer serialized, so it becomes a block at that position + * instead of following every text block (the pre-placement behavior, which is + * still what an image without an `inlinePath` gets). + */ +function promptContent(input: string | RuntimePrompt): UserMessage["content"] { + if (typeof input === "string") return input; + const text = promptText(input); + const images = promptImages(input); + if (!images.length) return text; + const { parts, trailing } = splitInlineContent(text, images); + const blocks: Array<{ type: "text"; text: string } | ImageContent> = parts.flatMap( + (part): Array<{ type: "text"; text: string } | ImageContent> => + part.kind === "text" + ? part.text + ? [{ type: "text" as const, text: part.text }] + : [] + : [promptImageBlock(part.attachment)], + ); + blocks.push(...trailing.map(promptImageBlock)); + return blocks; } function runtimeAttachmentFromMessage( @@ -362,6 +371,9 @@ function runtimeAttachmentFromMessage( ...(attachment.mimeType ? { mimeType: attachment.mimeType } : {}), ...(attachment.size !== undefined ? { size: attachment.size } : {}), ...(attachment.text ? { text: attachment.text } : {}), + // Recorded placement: the message content names this image at that spot, so + // the restored prompt keeps the block there too. + ...(attachment.inlinePath ? { inlinePath: attachment.inlinePath } : {}), ...(data ? { data } : {}), }; } @@ -8554,7 +8566,10 @@ Do not invent objections or turn speculative risks into blockers. Stop when the if (typeof modelInput === "string") { await this.agent.prompt(modelInput); } else { - await this.agent.prompt(promptText(modelInput), promptImages(modelInput)); + // pi's `(text, images)` form appends every image after the text; the + // message form keeps an inline image block where the user placed it, + // which is the same content `promptContent` already built above. + await this.agent.prompt(incomingUserMessage); } await this.waitForIdleAndSteering(); void this.extensionRunner?.emit("agent_settled", { type: "agent_settled" }); diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 6f02a6b8fe..e1fe347a66 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -8,6 +8,7 @@ export * from "./transcript-truncation.js"; export * from "./keyboard-shortcuts.js"; export * from "./changelog.js"; export * from "./composer-trigger.js"; +export * from "./inline-attachments.js"; export * from "./session-link.js"; export * from "./fuzzy.js"; export * from "./mcp-import.js"; diff --git a/packages/shared/src/inline-attachments.test.ts b/packages/shared/src/inline-attachments.test.ts new file mode 100644 index 0000000000..a0527c0f96 --- /dev/null +++ b/packages/shared/src/inline-attachments.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import { + formatPromptPathText, + locateInlinePromptPaths, + splitInlineContent, +} from "./inline-attachments.js"; + +const image = (path: string, extra: { inlinePath?: string } = {}) => ({ + kind: "image" as const, + name: path.split("/").at(-1) ?? path, + path, + ...extra, +}); + +describe("inline prompt attachment placement", () => { + it("formats the path text a prompt names an attachment with", () => { + expect(formatPromptPathText("/scratch/a.png")).toBe("@/scratch/a.png"); + expect(formatPromptPathText("/scratch/a b.png")).toBe('@"/scratch/a b.png"'); + }); + + it("locates each path from the previous match, in order", () => { + const located = locateInlinePromptPaths("see @/a.png then @/b.png", [ + "/a.png", + "/b.png", + ]); + expect(located).toEqual([ + { start: 4, end: 11 }, + { start: 17, end: 24 }, + ]); + }); + + it("keeps a repeated path's second use unmatched", () => { + const located = locateInlinePromptPaths("@/a.png and @/a.png", ["/a.png", "/a.png"]); + expect(located[0]).toEqual({ start: 0, end: 7 }); + expect(located[1]).toEqual({ start: 12, end: 19 }); + expect( + locateInlinePromptPaths("@/a.png", ["/a.png", "/a.png"])[1], + ).toBeNull(); + }); + + it("reports a path the prompt does not name inline", () => { + expect(locateInlinePromptPaths("no paths here", ["/a.png"])).toEqual([null]); + expect(locateInlinePromptPaths("no paths here", [undefined])).toEqual([null]); + }); + + it("splits a prompt into ordered text runs and attachments", () => { + const first = image("/scratch/a.png", { inlinePath: "@/scratch/a.png" }); + const second = image("/scratch/b.png", { inlinePath: "@/scratch/b.png" }); + const { parts, trailing } = splitInlineContent( + "compare @/scratch/a.png with @/scratch/b.png now", + [first, second], + ); + expect(trailing).toEqual([]); + expect(parts).toEqual([ + { kind: "text", text: "compare ", start: 0, end: 8 }, + { kind: "attachment", attachment: first, start: 8, end: 23 }, + { kind: "text", text: " with ", start: 23, end: 29 }, + { kind: "attachment", attachment: second, start: 29, end: 44 }, + { kind: "text", text: " now", start: 44, end: 48 }, + ]); + }); + + it("keeps an attachment the prompt does not name inline trailing", () => { + const inline = image("/scratch/a.png", { inlinePath: "@/scratch/a.png" }); + const detached = image("/scratch/b.png"); + const { parts, trailing } = splitInlineContent("look @/scratch/a.png", [ + inline, + detached, + ]); + expect(trailing).toEqual([detached]); + expect(parts.map((part) => part.kind)).toEqual(["text", "attachment"]); + }); + + it("keeps the prompt text intact, including an image-only draft", () => { + const only = image("/scratch/a.png", { inlinePath: "@/scratch/a.png" }); + expect(splitInlineContent("@/scratch/a.png", [only]).parts).toEqual([ + { kind: "attachment", attachment: only, start: 0, end: 15 }, + ]); + const { parts } = splitInlineContent("", [image("/scratch/b.png")]); + expect(parts).toEqual([]); + }); + it("splits nothing for a prompt without attachments", () => { + expect(splitInlineContent("plain prompt", []).parts).toEqual([ + { kind: "text", text: "plain prompt", start: 0, end: 12 }, + ]); + expect(splitInlineContent("", []).parts).toEqual([]); + }); +}); diff --git a/packages/shared/src/inline-attachments.ts b/packages/shared/src/inline-attachments.ts new file mode 100644 index 0000000000..3ecebd8bc5 --- /dev/null +++ b/packages/shared/src/inline-attachments.ts @@ -0,0 +1,104 @@ +/** + * Inline prompt attachments: where a draft's attachment chips sit in the text. + * + * The Composer keeps images as chips inside the draft, so the outgoing prompt + * names them at the position the user typed them. Electron main records that + * text as `inlinePath` on the attachment it prepared, and every consumer — the + * transcript renderer, the restored history, and the agent runtime that builds + * the provider request — reads the placement back through the helpers below + * instead of appending every image after the text. + */ +import { formatFileInsert } from "./composer-trigger.js"; + +/** The attachment field placement reads. Both durable and prompt shapes qualify. */ +export type InlinePlacedAttachment = { readonly inlinePath?: string }; + +/** An inclusive-exclusive span of prompt text, in `content` offsets. */ +export type InlinePathSpan = { start: number; end: number }; + +/** The `@path` text a prompt uses for one attachment path. */ +export function formatPromptPathText(path: string): string { + return formatFileInsert(path, "file").trim(); +} + +/** + * Match one marker text per entry, in order, from the previous match. An entry + * without a marker, or one the prompt does not name, yields null and leaves the + * cursor untouched, so a later entry can still claim its own text. + */ +function locateMarkers( + content: string, + markers: ReadonlyArray, +): Array { + let cursor = 0; + return markers.map((marker) => { + const text = marker?.trim() ?? ""; + if (!text) return null; + const start = content.indexOf(text, cursor); + if (start === -1) return null; + cursor = start + text.length; + return { start, end: cursor }; + }); +} + +/** + * Locate the path text of each entry, in order. A producer records a placement + * with this call: it holds attachment paths, not prompt text. + */ +export function locateInlinePromptPaths( + content: string, + paths: ReadonlyArray, +): Array { + return locateMarkers( + content, + paths.map((path) => (path ? formatPromptPathText(path) : undefined)), + ); +} + +/** One ordered piece of a prompt: a run of text, or an attachment inside it. */ +export type InlineContentPart = + | { kind: "text"; text: string; start: number; end: number } + | { kind: "attachment"; attachment: A; start: number; end: number }; + +/** + * Split a prompt into ordered parts: the text runs with every attachment whose + * recorded `inlinePath` sits inside them. Offsets stay relative to the whole + * prompt, so a caller can render one message as several runs without losing + * source positions. + * + * `trailing` reports the attachments the prompt does not name inline; they keep + * the previous behavior of following the text. + */ +export function splitInlineContent( + content: string, + attachments: readonly A[], +): { parts: Array>; trailing: A[] } { + const parts: Array> = []; + const trailing: A[] = []; + const located = locateMarkers( + content, + attachments.map((attachment) => attachment.inlinePath), + ); + let cursor = 0; + attachments.forEach((attachment, index) => { + const span = located[index]; + if (!span) { + trailing.push(attachment); + return; + } + if (span.start > cursor) { + parts.push({ + kind: "text", + text: content.slice(cursor, span.start), + start: cursor, + end: span.start, + }); + } + parts.push({ kind: "attachment", attachment, start: span.start, end: span.end }); + cursor = span.end; + }); + if (cursor < content.length) { + parts.push({ kind: "text", text: content.slice(cursor), start: cursor, end: content.length }); + } + return { parts, trailing }; +} diff --git a/packages/shared/src/types/agent.ts b/packages/shared/src/types/agent.ts index b747fab5ea..1bc2ce424b 100644 --- a/packages/shared/src/types/agent.ts +++ b/packages/shared/src/types/agent.ts @@ -66,6 +66,13 @@ export type AgentPromptAttachment = { kind: "image" | "file"; mimeType?: string; size?: number; + /** + * The `@path` text this attachment occupies inline in `content`. Electron main + * fills it for an attachment the user's draft placed between words, so the + * runtime keeps the image block at that position instead of trailing the + * prompt text. Absent for older callers and formatted messages. + */ + inlinePath?: string; }; export type AgentSteerRequest = Pick< diff --git a/packages/shared/src/types/messages.ts b/packages/shared/src/types/messages.ts index 1a2073130b..683cf8d624 100644 --- a/packages/shared/src/types/messages.ts +++ b/packages/shared/src/types/messages.ts @@ -61,6 +61,13 @@ export type MessageAttachment = { * with the user's message; other attachment kinds never carry it. */ text?: string; + /** + * The `@path` text this attachment occupies inside the message content, when + * the user placed it between words. The transcript renders the attachment at + * that position instead of appending it after the text; absent means the + * attachment follows the body (a formatted, expanded, or legacy message). + */ + inlinePath?: string; /** Sidecar-only hydrated image data; never persisted or sent by the host. */ data?: string; }; diff --git a/scripts/e2e/composer-submission.tsx b/scripts/e2e/composer-submission.tsx index f1de6dc78f..61e0152555 100644 --- a/scripts/e2e/composer-submission.tsx +++ b/scripts/e2e/composer-submission.tsx @@ -8,6 +8,7 @@ import { api } from "../../apps/desktop/src/lib/api"; import { writeComposerDraft, deleteComposerDraft } from "../../apps/desktop/src/lib/composer-draft-cache"; import type { ComposerDraftSnapshot } from "../../apps/desktop/src/lib/composer-smart-stop"; import { readEditorValue } from "../../apps/desktop/src/features/chat/composer/editor"; +import { formatPromptPathText } from "@pi-desktop/shared"; const assert = (value: unknown, message: string) => { if (!value) throw new Error(message); }; const painted = () => new Promise((resolve) => requestAnimationFrame(() => resolve())); @@ -124,8 +125,12 @@ export async function verifyComposerSubmission(imagePath: string, i18n: i18n) { accepted = true; sendButton().click(); await painted(); - assert(sent.length === 2 && sent[1].content === "" && sent[1].draft?.fileReferences[0]?.path === imagePath, - "image-only send must preserve the attachment at the submission boundary"); + assert( + sent.length === 2 && + sent[1].content === formatPromptPathText(imagePath) && + sent[1].draft?.fileReferences[0]?.path === imagePath, + "image-only send must keep the image path at the user's position and the attachment at the submission boundary", + ); assert(imageChips().length === 0 && sendButton().disabled, "accepted image-only submission must clear the composer");