From ec871b73c373cb7dbb3cabd8a501eadc8a688033 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 16 Aug 2026 17:58:19 +0000 Subject: [PATCH 1/3] feat: read replies over IMAP, so the reply gate has an input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #34 added a `conversation_open` gate that refuses to write to a contact who has answered — but nothing could tell it that anyone had. The product sent over SMTP and read no mailbox, so every reply was invisible, the funnel's `replied` count was structurally zero, and the queue kept offering prospects who were already mid-conversation. SMTP cannot close that gap. Its whole vocabulary is MAIL FROM / RCPT TO / DATA; there is no verb for "what arrived". Reading is IMAP, on a different port, with the same credentials the mailbox was already connected with — so this adds no second password, and one provider choice fills in both halves. - `packages/email/src/imap.ts` turns a mailbox into a list of messages. - `packages/pipeline/src/receive-email.ts` decides what they mean. - Presets carry IMAP coordinates for Gmail, Microsoft 365, Fastmail, Forward Email and Zoho; the generic entry lets both be typed. - The worker polls on its own five-minute clock, *before* the send sweep — a reply noticed this tick has to stop a message that would otherwise go out on the same tick. Most of the care is in what must not be recorded. The two failure modes are not symmetric: missing a reply means we mail someone who answered, which is visible and recoverable, while inventing one silently removes a prospect from outreach for good on the strength of an out-of-office. So auto-replies, bounces and bulk mail are classified and skipped, mail from an address we never wrote to is left alone, and migration 0012 adds a Message-ID key so an overlapping poll window cannot double-count. A reply to a shared inbox is recorded against that address, which — because `conversationOpen` matches on address as well as person — protects every colleague written to there. Attributing it to one of them is a guess; that the mailbox answered is not. Co-Authored-By: Claude Opus 5 (1M context) --- apps/api/src/app.ts | 7 + apps/server/src/index.ts | 65 +++++- apps/web/components/mailbox-form.tsx | 41 ++++ apps/web/lib/types.ts | 7 + bun.lock | 51 +++++ migrations/0012_inbound_dedupe.sql | 25 +++ packages/contracts/src/index.ts | 11 + packages/email/package.json | 1 + packages/email/src/imap.test.ts | 90 ++++++++ packages/email/src/imap.ts | 196 ++++++++++++++++++ packages/email/src/index.ts | 8 + packages/email/src/smtp.ts | 35 +++- packages/pipeline/src/email-account.ts | 80 ++++++- packages/pipeline/src/index.ts | 7 + packages/pipeline/src/receive-email.test.ts | 218 ++++++++++++++++++++ packages/pipeline/src/receive-email.ts | 184 +++++++++++++++++ 16 files changed, 1022 insertions(+), 4 deletions(-) create mode 100644 migrations/0012_inbound_dedupe.sql create mode 100644 packages/email/src/imap.test.ts create mode 100644 packages/email/src/imap.ts create mode 100644 packages/pipeline/src/receive-email.test.ts create mode 100644 packages/pipeline/src/receive-email.ts diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 4c6b303..2d95f36 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -1625,6 +1625,13 @@ export function createApp(options: AppOptions): Hono { fromEmail: body.fromEmail, ...(body.fromName ? { fromName: body.fromName } : {}), ...(body.replyTo ? { replyTo: body.replyTo } : {}), + ...(body.imapHost + ? { + imapHost: body.imapHost, + imapPort: body.imapPort ?? 993, + imapSecure: body.imapSecure ?? true, + } + : {}), }, encryptionKey: options.encryptionKey, verify: body.skipVerification !== true, diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index 4afa5ee..9d22506 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -27,7 +27,7 @@ import { OpenAIModel, type FallbackEntry, } from '@outreachgraph/ai'; -import { ResendMailer } from '@outreachgraph/email'; +import { ImapReader, ResendMailer } from '@outreachgraph/email'; import { secretKeyFromEnv } from '@outreachgraph/secrets'; import { createApp } from '../../api/src/app'; import { pruneSessions } from '../../api/src/auth'; @@ -42,9 +42,12 @@ import { runAutopilot, runCrawlJob, runDiscoveryJob, + loadImapCredentials, + receiveReplies, runListening, sendDailyDigest, sendLeadAlerts, + workspacesWithReadableMailbox, type QueuedJob, } from '@outreachgraph/pipeline'; import { @@ -60,6 +63,19 @@ import { const PORT = Number(process.env.PORT ?? 8080); const WEB_PORT = Number(process.env.WEB_PORT ?? 3001); const TICK_MS = Number(process.env.WORKER_TICK_MS ?? 60_000); + +/** + * How often a mailbox is read, as opposed to how often the worker ticks. + * + * Five minutes because a reply is not urgent — the thing it must beat is the + * next outreach send, not the reader's patience — and an IMAP login per + * workspace per minute is a great deal of connection churn for a mailbox that + * receives a handful of replies a day. + */ +const RECEIVE_POLL_MS = Number(process.env.RECEIVE_POLL_MS ?? 300_000); + +/** Last successful poll per workspace, so the slower clock survives a tick. */ +const lastPolledAt = new Map(); const ENVIRONMENT = process.env.NODE_ENV ?? 'development'; const db = getDatabase(); @@ -621,6 +637,53 @@ async function tick(): Promise { } } + // ------------------------------------------------------------- receiving + // + // Replies, read over IMAP. This runs *before* the send sweep on purpose: a + // reply noticed on this tick has to stop a message that would otherwise go + // out on the same tick, and reading afterwards would let exactly the send we + // are trying to prevent slip through every time. + // + // Polled on its own, slower clock. The worker tick is a minute; opening an + // IMAP connection per workspace per minute is a lot of login traffic for a + // mailbox that gets a handful of replies a day. + for (const workspaceId of await workspacesWithReadableMailbox(db)) { + const last = lastPolledAt.get(workspaceId) ?? 0; + if (Date.now() - last < RECEIVE_POLL_MS) continue; + lastPolledAt.set(workspaceId, Date.now()); + + try { + const credentials = await loadImapCredentials(db, workspaceId, encryptionKey); + // Undefined covers "no mailbox", "no IMAP host" and "the key changed so + // the password no longer decrypts". All three mean the same thing here. + if (!credentials) continue; + + const received = await receiveReplies({ + db, + workspaceId, + reader: new ImapReader(credentials), + }); + + if (received.recorded > 0) { + console.log(`replies ${workspaceId}: ${received.recorded} recorded`); + } + + // Worth a line even at zero: an inbox full of skipped auto-replies and + // an inbox nobody answers look identical from the outside, and only one + // of them is a problem. + const skipped = Object.entries(received.automated); + if (skipped.length > 0) { + console.log( + `replies ${workspaceId}: ignored ${skipped.map(([k, n]) => `${n} ${k}`).join(', ')}`, + ); + } + } catch (error) { + // A mailbox that will not open is this workspace's problem, not the + // tick's. Sending still has to happen for everyone else. + console.error(`reading replies failed for ${workspaceId}`, error); + } + } + // ------------------------------------------------------------- autopilot // // Sending, alerting and the digest, per workspace. Each is wrapped diff --git a/apps/web/components/mailbox-form.tsx b/apps/web/components/mailbox-form.tsx index e8f391c..f36487f 100644 --- a/apps/web/components/mailbox-form.tsx +++ b/apps/web/components/mailbox-form.tsx @@ -36,6 +36,8 @@ export function MailboxForm({ initial }: { initial: EmailIntegrationView }) { const [fromEmail, setFromEmail] = useState(account.fromEmail ?? ''); const [fromName, setFromName] = useState(account.fromName ?? ''); const [replyTo, setReplyTo] = useState(account.replyTo ?? ''); + const [imapHost, setImapHost] = useState(account.imapHost ?? ''); + const [imapPort, setImapPort] = useState(account.imapPort ?? 993); const [busy, setBusy] = useState<'save' | 'remove' | undefined>(); const [error, setError] = useState(); @@ -53,6 +55,15 @@ export function MailboxForm({ initial }: { initial: EmailIntegrationView }) { setHost(chosen.host); setPort(chosen.port); setSecure(chosen.secure); + + // One picker fills in both halves, because they are one mailbox on one + // credential. Making someone look up that Forward Email sends on + // smtp.forwardemail.net and is read on imap.forwardemail.net is asking + // them for something we already know. + if (chosen.imapHost) { + setImapHost(chosen.imapHost); + setImapPort(chosen.imapPort ?? 993); + } } async function submit(event: FormEvent): Promise { @@ -75,6 +86,7 @@ export function MailboxForm({ initial }: { initial: EmailIntegrationView }) { fromEmail: fromEmail || username, ...(fromName ? { fromName } : {}), ...(replyTo ? { replyTo } : {}), + ...(imapHost ? { imapHost, imapPort: Number(imapPort), imapSecure: true } : {}), }), }); @@ -214,6 +226,35 @@ export function MailboxForm({ initial }: { initial: EmailIntegrationView }) { TLS on connect (port 465). Uncheck for STARTTLS submission (587). +
+ + + +
+ +

+ Optional, and no second password — IMAP and SMTP are two ports on the same mailbox. Leave + it blank to send only; replies then have to be spotted by hand, and the queue will keep + offering people who have already written back. +

+