diff --git a/apps/api/src/app.test.ts b/apps/api/src/app.test.ts index 8773e5b..a8954c0 100644 --- a/apps/api/src/app.test.ts +++ b/apps/api/src/app.test.ts @@ -740,7 +740,15 @@ describe('adding a prospect (PRD §8)', () => { /** Collects sent messages instead of delivering them. */ function recordingMailer(): { sent: Message[]; mailer: Mailer } { const sent: Message[] = []; - return { sent, mailer: { send: async (message) => void sent.push(message) } }; + return { + sent, + mailer: { + send: async (message) => { + sent.push(message); + return {}; + }, + }, + }; } describe('email verification', () => { diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 2a8dbac..fafe322 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -38,6 +38,19 @@ import { workspacesForUser, } from './auth'; import { evaluatePolicy, isExecutable, POLICY_VERSION } from '@outreachgraph/policy'; +import { + campaignFunnel, + leadTimeline, + loadNotifySettings, + notifyAddress, + workspaceAnalytics, +} from '@outreachgraph/pipeline'; +import { + createCampaignFromIntake, + IntakeError, + saveWorkspaceSettings, + setCampaignAutopilot, +} from './campaigns'; import { draftForRecommendation, draftProfile, type TextModel } from '@outreachgraph/ai'; import { batchStatus, enqueue, runPipeline } from '@outreachgraph/pipeline'; import { GitHubProvider, SiteProvider, normaliseUrl } from '@outreachgraph/providers'; @@ -324,6 +337,88 @@ export function createApp(options: AppOptions): Hono { return c.json({ campaign }); }); + /** + * The front door (PRD §8). + * + * One box, two paths. A company website is crawled directly; a description + * of a market goes to discovery, which names real companies and queues a + * crawl for each. The caller gets a campaign id straight back and watches it + * fill, because neither path can finish inside a request. + */ + api.post('/campaigns', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + + if (!canApprove(actor)) throw ApiError.forbidden('starting a campaign'); + await requireVerifiedEmail(db, actor); + + const body = safeJson(await c.req.raw.text()); + const input = typeof body.input === 'string' ? body.input : ''; + + let result; + try { + result = await createCampaignFromIntake(db, actor, input, { + autopilot: body.autopilot === true, + ...(typeof body.name === 'string' ? { name: body.name } : {}), + }); + } catch (error) { + if (error instanceof IntakeError) { + throw ApiError.badRequest(error.message, { input: [error.message] }); + } + throw error; + } + + await repo.audit(db, { + workspaceId: actor.workspaceId, + actorKind: 'user', + actorId: actor.userId, + eventType: 'campaign.created', + entityKind: 'campaign', + entityId: result.campaignId, + detail: { kind: result.kind, seed: result.seed, autopilot: result.autopilot }, + }); + + return c.json(result, 202); + }); + + /** Switches autopilot on or off. The only control unattended sending has. */ + api.patch('/campaigns/:id', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + + if (!canApprove(actor)) throw ApiError.forbidden('changing a campaign'); + + const body = safeJson(await c.req.raw.text()); + if (typeof body.autopilot !== 'boolean') { + throw ApiError.badRequest('autopilot must be true or false'); + } + + // Turning it *on* spends money without asking again, so it carries the + // same verification gate as starting a campaign. + if (body.autopilot) await requireVerifiedEmail(db, actor); + + const updated = await setCampaignAutopilot( + db, + actor.workspaceId, + c.req.param('id'), + body.autopilot, + ); + + if (!updated) throw ApiError.notFound('campaign'); + + await repo.audit(db, { + workspaceId: actor.workspaceId, + actorKind: 'user', + actorId: actor.userId, + eventType: body.autopilot ? 'campaign.autopilot_enabled' : 'campaign.autopilot_disabled', + entityKind: 'campaign', + entityId: c.req.param('id'), + detail: {}, + }); + + return c.json({ campaignId: c.req.param('id'), autopilot: body.autopilot }); + }); + api.get('/campaigns/:id/recommendations', async (c) => { const actor = c.get('actor'); const limit = clampLimit(c.req.query('limit')); @@ -1066,6 +1161,102 @@ export function createApp(options: AppOptions): Hono { return c.json({ executed: true, actionId: action.id }); }); + // ------------------------------------------------------------- analytics + /** + * The funnel, plus the headline numbers above it (PRD §25). + * + * Stage counts come from the event log rather than the current-status + * column, which is the only way "how many ever reached Contacted" is + * answerable at all — see `lead_stage_events`. + */ + api.get('/analytics', async (c) => { + const actor = c.get('actor'); + const campaignId = c.req.query('campaignId'); + + const [analytics, funnel] = await Promise.all([ + workspaceAnalytics(c.get('db'), actor.workspaceId), + campaignId + ? campaignFunnel(c.get('db'), { workspaceId: actor.workspaceId, campaignId }) + : undefined, + ]); + + // A campaign-scoped request still gets the workspace headline numbers, so + // the page has one request rather than two. + return c.json({ ...analytics, ...(funnel ? { funnel } : {}) }); + }); + + /** One row per lead, each a sequence of dated stage segments. */ + api.get('/analytics/timeline', async (c) => { + const actor = c.get('actor'); + const campaignId = c.req.query('campaignId'); + + const leads = await leadTimeline(c.get('db'), { + workspaceId: actor.workspaceId, + ...(campaignId ? { campaignId } : {}), + limit: clampLimit(c.req.query('limit')), + }); + + return c.json({ leads }); + }); + + // -------------------------------------------------------------- settings + api.get('/settings', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + const settings = await loadNotifySettings(db, actor.workspaceId); + const address = await notifyAddress(db, actor.workspaceId, settings); + + const cap = await queryOne<{ autopilot_daily_cap: number; reply_to_email: string | null }>( + db, + `SELECT autopilot_daily_cap, reply_to_email FROM workspace_settings WHERE workspace_id = ?`, + [actor.workspaceId], + ); + + return c.json({ + notifyEmail: settings.notify_email, + // What mail would actually go to right now, resolved through the owner + // fallback. Without this the settings page can only show an empty box + // and leave the user unsure whether anything is configured at all. + effectiveNotifyEmail: address ?? null, + instantAlerts: settings.instant_alerts === 1, + dailyDigest: settings.daily_digest === 1, + digestHourUtc: settings.digest_hour_utc, + alertMinOpportunity: settings.alert_min_opportunity, + autopilotDailyCap: cap?.autopilot_daily_cap ?? 25, + replyToEmail: cap?.reply_to_email ?? null, + lastDigestSentOn: settings.last_digest_sent_on, + }); + }); + + api.put('/settings', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + + if (!canApprove(actor)) throw ApiError.forbidden('changing workspace settings'); + + const body = safeJson(await c.req.raw.text()); + + await saveWorkspaceSettings(db, actor.workspaceId, { + ...(body.notifyEmail === undefined + ? {} + : { notifyEmail: body.notifyEmail === null ? null : String(body.notifyEmail) }), + ...(body.replyToEmail === undefined + ? {} + : { replyToEmail: body.replyToEmail === null ? null : String(body.replyToEmail) }), + instantAlerts: body.instantAlerts !== false, + dailyDigest: body.dailyDigest !== false, + ...(body.digestHourUtc === undefined ? {} : { digestHourUtc: Number(body.digestHourUtc) }), + ...(body.alertMinOpportunity === undefined + ? {} + : { alertMinOpportunity: Number(body.alertMinOpportunity) }), + ...(body.autopilotDailyCap === undefined + ? {} + : { autopilotDailyCap: Number(body.autopilotDailyCap) }), + }); + + return c.json({ saved: true }); + }); + // --------------------------------------------------------------- signals api.get('/signals', async (c) => { const actor = c.get('actor'); diff --git a/apps/api/src/campaigns.test.ts b/apps/api/src/campaigns.test.ts new file mode 100644 index 0000000..74a702a --- /dev/null +++ b/apps/api/src/campaigns.test.ts @@ -0,0 +1,163 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { queryAll, queryOne } from '@outreachgraph/db'; +import { seedDatabase, SEED, type SeededDatabase } from './test-seed'; +import { createCampaignFromIntake, IntakeError, setCampaignAutopilot } from './campaigns'; + +let seeded: SeededDatabase | undefined; + +afterEach(() => { + seeded?.cleanup(); + seeded = undefined; +}); + +const ACTOR = { workspaceId: SEED.workspaceId, userId: SEED.userId }; + +describe('createCampaignFromIntake', () => { + test('a website is crawled directly', async () => { + seeded = await seedDatabase('intake-url'); + const { db } = seeded; + + const result = await createCampaignFromIntake(db, ACTOR, 'https://www.brightsmile.com/about'); + + expect(result.kind).toBe('url'); + expect(result.seed).toBe('brightsmile.com'); + expect(result.queued).toBe(1); + + const jobs = await queryAll<{ kind: string; payload_json: string }>( + db, + `SELECT kind, payload_json FROM jobs WHERE workspace_id = ?`, + [SEED.workspaceId], + ); + expect(jobs).toHaveLength(1); + expect(jobs[0]?.kind).toBe('crawl_site'); + + // The crawl must know which campaign it belongs to, or it files its people + // under whichever campaign happens to be oldest. + const payload = JSON.parse(jobs[0]?.payload_json ?? '{}'); + expect(payload.campaignId).toBe(result.campaignId); + expect(payload.url).toBe('https://brightsmile.com'); + }); + + test('a description queues discovery instead', async () => { + seeded = await seedDatabase('intake-keyword'); + const { db } = seeded; + + const result = await createCampaignFromIntake(db, ACTOR, 'dental practices in Austin'); + + expect(result.kind).toBe('keyword'); + expect(result.queued).toBe(0); + + const job = await queryOne<{ kind: string; payload_json: string }>( + db, + `SELECT kind, payload_json FROM jobs WHERE workspace_id = ?`, + [SEED.workspaceId], + ); + expect(job?.kind).toBe('discover_domains'); + expect(JSON.parse(job?.payload_json ?? '{}').keyword).toBe('dental practices in Austin'); + }); + + test('autopilot is opt-in and sets the approval mode the policy engine reads', async () => { + seeded = await seedDatabase('intake-autopilot'); + const { db } = seeded; + + const off = await createCampaignFromIntake(db, ACTOR, 'acme.com'); + const on = await createCampaignFromIntake(db, ACTOR, 'other.com', { autopilot: true }); + + const modes = await queryAll<{ id: string; approval_mode: string }>( + db, + `SELECT id, approval_mode FROM campaigns WHERE id IN (?, ?)`, + [off.campaignId, on.campaignId], + ); + + const byId = new Map(modes.map((row) => [row.id, row.approval_mode])); + expect(byId.get(off.campaignId)).toBe('draft_and_approve'); + expect(byId.get(on.campaignId)).toBe('trusted_automation'); + }); + + test('a workspace with no offering still gets a campaign, and is told', async () => { + seeded = await seedDatabase('intake-no-offering'); + const { db } = seeded; + + // `campaigns.offering_id` is NOT NULL, so without a placeholder the very + // first thing a new account does would fail on a foreign key. + await db.execute({ sql: `DELETE FROM campaigns`, args: [] }); + await db.execute({ sql: `DELETE FROM offerings`, args: [] }); + + const result = await createCampaignFromIntake(db, ACTOR, 'acme.com'); + + expect(result.needsProfile).toBe(true); + expect(result.campaignId).toBeTruthy(); + }); + + test('an existing offering is reused rather than duplicated', async () => { + seeded = await seedDatabase('intake-reuse-offering'); + const { db } = seeded; + + await createCampaignFromIntake(db, ACTOR, 'acme.com'); + + const offerings = await queryAll<{ id: string }>( + db, + `SELECT id FROM offerings WHERE workspace_id = ?`, + [SEED.workspaceId], + ); + expect(offerings).toHaveLength(1); + }); + + test('unusable input is rejected with something worth reading', async () => { + seeded = await seedDatabase('intake-empty'); + const { db } = seeded; + + await expect(createCampaignFromIntake(db, ACTOR, ' ')).rejects.toBeInstanceOf(IntakeError); + }); + + test('the seed is recorded so a campaign traces back to what was typed', async () => { + seeded = await seedDatabase('intake-seed'); + const { db } = seeded; + + const result = await createCampaignFromIntake(db, ACTOR, 'bike shops in Portland'); + + const campaign = await queryOne<{ seed_kind: string; seed_value: string }>( + db, + `SELECT seed_kind, seed_value FROM campaigns WHERE id = ?`, + [result.campaignId], + ); + expect(campaign?.seed_kind).toBe('keyword'); + expect(campaign?.seed_value).toBe('bike shops in Portland'); + }); +}); + +describe('setCampaignAutopilot', () => { + test('switches the mode both ways', async () => { + seeded = await seedDatabase('autopilot-toggle'); + const { db } = seeded; + + expect(await setCampaignAutopilot(db, SEED.workspaceId, SEED.campaignId, true)).toBe(true); + expect( + ( + await queryOne<{ approval_mode: string }>( + db, + `SELECT approval_mode FROM campaigns WHERE id = ?`, + [SEED.campaignId], + ) + )?.approval_mode, + ).toBe('trusted_automation'); + + await setCampaignAutopilot(db, SEED.workspaceId, SEED.campaignId, false); + expect( + ( + await queryOne<{ approval_mode: string }>( + db, + `SELECT approval_mode FROM campaigns WHERE id = ?`, + [SEED.campaignId], + ) + )?.approval_mode, + ).toBe('draft_and_approve'); + }); + + test('a campaign in another workspace is not reachable', async () => { + seeded = await seedDatabase('autopilot-scope'); + const { db } = seeded; + + expect(await setCampaignAutopilot(db, 'wsp_other', SEED.campaignId, true)).toBe(false); + }); +}); diff --git a/apps/api/src/campaigns.ts b/apps/api/src/campaigns.ts new file mode 100644 index 0000000..4ec2ddf --- /dev/null +++ b/apps/api/src/campaigns.ts @@ -0,0 +1,251 @@ +/** + * Starting a campaign from one line of text. + * + * The product used to have two front doors and neither was the one people + * wanted. `POST /prospects` took a **GitHub handle**, which is a strange thing + * to ask a salesperson for, and `POST /prospects/by-url` took a list of URLs, + * which assumes you already know exactly who you are chasing. + * + * There is now one: a box that takes either a company website or a description + * of a market, works out which it got, and starts a campaign for it. Both + * paths converge on the same queue within a second or two of the request, so + * the response is a campaign the caller can watch rather than a result they + * have to wait for. + */ + +import { classifyIntake, newId, type ClassifiedIntake } from '@outreachgraph/domain'; +import { now, queryOne, type Client } from '@outreachgraph/db'; +import { enqueue } from '@outreachgraph/pipeline'; + +export interface IntakeActor { + readonly workspaceId: string; + readonly userId: string; +} + +export interface CampaignIntakeResult { + readonly campaignId: string; + readonly name: string; + readonly kind: 'url' | 'keyword'; + readonly seed: string; + readonly autopilot: boolean; + readonly batchId: string; + /** Sites queued immediately. Zero for a keyword — discovery queues them. */ + readonly queued: number; + /** + * True when the workspace has not described what it sells yet. + * + * The run still starts. Drafts are grounded in a placeholder offering until + * the profile is filled in, which is worse output rather than no output, and + * blocking the first campaign on a form is how a product feels broken. + */ + readonly needsProfile: boolean; +} + +export class IntakeError extends Error { + constructor(message: string) { + super(message); + this.name = 'IntakeError'; + } +} + +/** + * Creates a campaign from whatever the user typed, and queues its first work. + * + * Autopilot is a parameter rather than a default. `trusted_automation` is + * opt-in by design — the policy engine will not skip human approval without + * it — so switching it on has to be something the caller asked for. + */ +export async function createCampaignFromIntake( + db: Client, + actor: IntakeActor, + rawInput: string, + options: { autopilot?: boolean; name?: string } = {}, +): Promise { + const intake = classifyIntake(rawInput); + + if (!intake) { + throw new IntakeError('enter a company website or describe who you want to reach'); + } + + const offering = await ensureOffering(db, actor.workspaceId); + const campaignId = newId('campaign'); + const stamp = now(); + const name = options.name?.trim() || defaultName(intake); + + await db.execute({ + sql: `INSERT INTO campaigns (id, workspace_id, name, offering_id, networks, approval_mode, + status, seed_kind, seed_value, created_at, updated_at, started_at) + VALUES (?, ?, ?, ?, '["website","email"]', ?, 'active', ?, ?, ?, ?, ?)`, + args: [ + campaignId, + actor.workspaceId, + name, + offering.id, + options.autopilot ? 'trusted_automation' : 'draft_and_approve', + intake.kind, + intake.value, + stamp, + stamp, + stamp, + ], + }); + + const batchId = newId('job'); + let queued = 0; + + if (intake.kind === 'url') { + const result = await enqueue(db, { + workspaceId: actor.workspaceId, + kind: 'crawl_site', + payload: { url: `https://${intake.value}`, campaignId }, + batchId, + dedupeKey: `crawl:${campaignId}:${intake.value}`, + }); + if (result.queued) queued += 1; + } else { + // Discovery runs as a job, not here: it is a model call plus up to fifty + // enqueues, and a form that hangs for thirty seconds is a form people + // abandon halfway through. + await enqueue(db, { + workspaceId: actor.workspaceId, + kind: 'discover_domains', + payload: { keyword: intake.value, campaignId }, + batchId, + dedupeKey: `discover:${campaignId}`, + }); + } + + return { + campaignId, + name, + kind: intake.kind, + seed: intake.value, + autopilot: options.autopilot === true, + batchId, + queued, + needsProfile: offering.placeholder, + }; +} + +function defaultName(intake: ClassifiedIntake): string { + return intake.kind === 'url' ? intake.value : intake.raw.slice(0, 120); +} + +interface EnsuredOffering { + readonly id: string; + /** True when this was invented just now because the workspace had none. */ + readonly placeholder: boolean; +} + +/** + * The workspace's offering, creating a placeholder if there is none. + * + * `campaigns.offering_id` is NOT NULL, so without this a workspace that has + * not been through setup cannot start a campaign at all — the first thing a + * new account tries would fail on a foreign key. A placeholder produces weaker + * drafts, which the caller is told about, rather than a dead end. + */ +async function ensureOffering(db: Client, workspaceId: string): Promise { + const existing = await queryOne<{ id: string }>( + db, + `SELECT id FROM offerings WHERE workspace_id = ? ORDER BY created_at LIMIT 1`, + [workspaceId], + ); + + if (existing) return { id: existing.id, placeholder: false }; + + const id = newId('offering'); + const stamp = now(); + + await db.execute({ + sql: `INSERT INTO offerings (id, workspace_id, name, category, description, + value_propositions, likely_pains, competitors, created_at, updated_at) + VALUES (?, ?, 'Unconfigured offering', 'unspecified', + 'Set up your profile so messages can describe what you actually sell.', + '[]', '[]', '[]', ?, ?)`, + args: [id, workspaceId, stamp, stamp], + }); + + return { id, placeholder: true }; +} + +/** + * Turns autopilot on or off for one campaign. + * + * This is the only switch that matters for unattended sending, and it is + * deliberately a campaign-level setting rather than a workspace-level one: a + * customer may reasonably want one market run hands-off and another watched. + */ +export async function setCampaignAutopilot( + db: Client, + workspaceId: string, + campaignId: string, + autopilot: boolean, +): Promise { + const result = await db.execute({ + sql: `UPDATE campaigns SET approval_mode = ?, updated_at = ? + WHERE id = ? AND workspace_id = ?`, + args: [autopilot ? 'trusted_automation' : 'draft_and_approve', now(), campaignId, workspaceId], + }); + + return result.rowsAffected > 0; +} + +export interface WorkspaceSettingsInput { + readonly notifyEmail?: string | null; + readonly instantAlerts?: boolean; + readonly dailyDigest?: boolean; + readonly digestHourUtc?: number; + readonly alertMinOpportunity?: number; + readonly autopilotDailyCap?: number; + readonly replyToEmail?: string | null; +} + +/** + * Upserts notification and autopilot settings. + * + * Written as one insert-or-update so a workspace that has never opened the + * settings page behaves identically to one that saved the defaults — there is + * no "no row yet" state for callers to handle. + */ +export async function saveWorkspaceSettings( + db: Client, + workspaceId: string, + input: WorkspaceSettingsInput, +): Promise { + const stamp = now(); + + await db.execute({ + sql: `INSERT INTO workspace_settings (workspace_id, notify_email, instant_alerts, + daily_digest, digest_hour_utc, alert_min_opportunity, autopilot_daily_cap, + reply_to_email, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(workspace_id) DO UPDATE SET + notify_email = COALESCE(excluded.notify_email, workspace_settings.notify_email), + instant_alerts = excluded.instant_alerts, + daily_digest = excluded.daily_digest, + digest_hour_utc = excluded.digest_hour_utc, + alert_min_opportunity = excluded.alert_min_opportunity, + autopilot_daily_cap = excluded.autopilot_daily_cap, + reply_to_email = COALESCE(excluded.reply_to_email, workspace_settings.reply_to_email), + updated_at = excluded.updated_at`, + args: [ + workspaceId, + input.notifyEmail ?? null, + input.instantAlerts === false ? 0 : 1, + input.dailyDigest === false ? 0 : 1, + clampInt(input.digestHourUtc, 0, 23, 13), + clampInt(input.alertMinOpportunity, 0, 100, 60), + clampInt(input.autopilotDailyCap, 0, 500, 25), + input.replyToEmail ?? null, + stamp, + stamp, + ], + }); +} + +function clampInt(value: unknown, min: number, max: number, fallback: number): number { + const parsed = typeof value === 'number' ? value : Number(value); + if (!Number.isFinite(parsed)) return fallback; + return Math.min(Math.max(Math.round(parsed), min), max); +} diff --git a/apps/api/src/test-seed.ts b/apps/api/src/test-seed.ts index 2782453..36a1259 100644 --- a/apps/api/src/test-seed.ts +++ b/apps/api/src/test-seed.ts @@ -51,6 +51,14 @@ export async function seedDatabase(label: string): Promise { VALUES (?, 'test@example.com', 'Test User', ?, ?, ?)`, args: [SEED.userId, stamp, stamp, stamp], }, + { + // Every real workspace has an owner, and notifications resolve their + // recipient through this row. A fixture without one is a workspace no + // signup path can produce. + sql: `INSERT INTO organization_members (organization_id, user_id, role, created_at) + VALUES (?, ?, 'owner', ?)`, + args: [SEED.organizationId, SEED.userId, stamp], + }, { sql: `INSERT INTO workspaces (id, organization_id, name, slug, min_outreach_confidence, created_at, updated_at) VALUES (?, ?, 'Test', 'test', 0.85, ?, ?)`, diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index 830bfdf..225bf72 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -35,7 +35,11 @@ import { expireSignals, processDeletion, rescoreProspect, + runAutopilot, runCrawlJob, + runDiscoveryJob, + sendDailyDigest, + sendLeadAlerts, type QueuedJob, } from '@outreachgraph/pipeline'; import { BlueskyProvider, SiteProvider } from '@outreachgraph/providers'; @@ -319,7 +323,16 @@ const fanOutProviders = [new BlueskyProvider()]; */ async function crawlSite(job: QueuedJob): Promise { const result = await runCrawlJob( - { db, site, providers: fanOutProviders, ...(model ? { model } : {}) }, + { + db, + site, + providers: fanOutProviders, + ...(model ? { model } : {}), + // Tells the policy engine email is a channel this deployment can + // actually send through. Without it `email/send_email` evaluates to + // `manual_only` and no email recommendation is ever produced. + emailSendingEnabled: mailer !== undefined, + }, job, ); @@ -334,11 +347,30 @@ async function crawlSite(job: QueuedJob): Promise { ); } +/** + * Expands one keyword into companies and queues a crawl for each. + * + * Throws when the model is unavailable rather than completing quietly: a + * campaign that reports success and produces nothing is the exact failure this + * whole path exists to remove, and the backoff will retry once a key works. + */ +async function discoverDomains(job: QueuedJob): Promise { + const result = await runDiscoveryJob({ db, ...(model ? { model } : {}) }, job); + + console.log( + `discover "${result.keyword}": ${result.found} companies, ${result.queued} queued` + + `${result.campaignName ? ` (${result.campaignName})` : ''}`, + ); +} + async function runJob(job: QueuedJob): Promise { switch (job.kind) { case 'crawl_site': await crawlSite(job); return; + case 'discover_domains': + await discoverDomains(job); + return; case 'rescore_prospect': { const { campaignId, personId } = job.payload as { campaignId?: string; personId?: string }; if (!campaignId || !personId) @@ -382,9 +414,8 @@ async function tick(): Promise { const pruned = await pruneSessions(db); if (pruned > 0) console.log(`pruned ${pruned} expired sessions`); - // The queue drains last. The sweeps above are bounded and quick; this is the - // part that can take the whole tick, and `limit` is what stops it running - // away with the loop. + // The queue drains before the send sweep, so anything discovered this tick + // can go out on the same tick rather than waiting for the next one. const drained = await drainQueue(db, runJob); if (drained.processed > 0 || drained.reclaimed > 0) { console.log( @@ -392,6 +423,57 @@ async function tick(): Promise { `${drained.dead} failed, ${drained.reclaimed} reclaimed`, ); } + + // ------------------------------------------------------------- autopilot + // + // Sending, alerting and the digest, per workspace. Each is wrapped + // separately: one workspace with a broken sending domain must not stop the + // others from being processed, and a failed digest must not prevent the + // outreach sweep that follows it. + for (const workspace of workspaces) { + try { + const result = await runAutopilot({ db, ...(mailer ? { mailer } : {}) }, workspace.id); + + if (result.sent.length > 0 || result.failed > 0) { + console.log( + `autopilot ${workspace.id}: ${result.sent.length} sent, ` + + `${result.failed} failed, ${result.skipped.length} skipped`, + ); + } + + // Skips are the most useful log line this loop produces: "no address + // published" and "still requires human approval" are the two reasons a + // campaign looks like it is doing nothing, and neither is an error + // anywhere else. + for (const skip of result.skipped.slice(0, 5)) { + console.log(`autopilot skip ${skip.personName}: ${skip.reason}`); + } + } catch (error) { + console.error(`autopilot failed for ${workspace.id}`, error); + } + + if (!appUrl) continue; + + try { + const alerts = await sendLeadAlerts( + { db, ...(mailer ? { mailer } : {}), appUrl }, + workspace.id, + ); + if (alerts > 0) console.log(`sent ${alerts} lead alert(s) for ${workspace.id}`); + } catch (error) { + console.error(`lead alerts failed for ${workspace.id}`, error); + } + + try { + const digested = await sendDailyDigest( + { db, ...(mailer ? { mailer } : {}), appUrl }, + workspace.id, + ); + if (digested) console.log(`sent the daily digest for ${workspace.id}`); + } catch (error) { + console.error(`daily digest failed for ${workspace.id}`, error); + } + } } async function loop(): Promise { diff --git a/apps/web/app/(app)/funnel/page.tsx b/apps/web/app/(app)/funnel/page.tsx new file mode 100644 index 0000000..d00bbc0 --- /dev/null +++ b/apps/web/app/(app)/funnel/page.tsx @@ -0,0 +1,162 @@ +import Link from 'next/link'; +import { redirect } from 'next/navigation'; +import { FunnelChart, LeadTimelineChart } from '../../../components/funnel-chart'; +import { + ApiUnavailableError, + NotAuthenticatedError, + fetchAnalytics, + fetchCampaigns, + fetchTimeline, + type AnalyticsView, + type CampaignRow, + type LeadTimelineView, +} from '../../../lib/api'; + +export const dynamic = 'force-dynamic'; + +export const metadata = { title: 'Funnel · OutreachGraph' }; + +/** + * The sales view (PRD §25). + * + * Everything else in this app is arranged around the machinery — signals, + * prospects, approvals. This page is arranged around the question a + * salesperson actually opens a tool to ask: how many are in play, where are + * they stuck, and what has gone out this week. + * + * It reads from the stage event log rather than from current status, which is + * what makes "reached" a real number rather than a snapshot of who happens to + * be sitting in a stage right now. + */ +export default async function FunnelPage({ + searchParams, +}: { + searchParams: Promise<{ campaign?: string }>; +}) { + const { campaign } = await searchParams; + + let analytics: AnalyticsView | undefined; + let leads: LeadTimelineView[] = []; + let campaigns: CampaignRow[] = []; + let offline = false; + + try { + [analytics, leads, campaigns] = await Promise.all([ + fetchAnalytics(campaign), + fetchTimeline(campaign), + fetchCampaigns(), + ]); + } catch (error) { + if (error instanceof NotAuthenticatedError) redirect('/login'); + if (error instanceof ApiUnavailableError) offline = true; + else throw error; + } + + if (offline || !analytics) { + return ( +
+

Funnel

+

+ Waiting for the API. +

+
+ ); + } + + const selected = campaigns.find((row) => row.id === campaign); + + return ( +
+
+

Funnel

+

+ {selected ? selected.name : 'Every campaign in this workspace'} +

+
+ + {campaigns.length > 1 ? ( + + ) : null} + +
+ + + + +
+ +
+

Stages

+ + + {analytics.medianHoursToContact !== undefined ? ( +

+ Typically {formatHours(analytics.medianHoursToContact)} from finding someone to writing + to them. +

+ ) : null} +
+ +
+

Each lead, over time

+

+ One row per lead. Width is time spent in a stage, so anything stalled is visibly wide. +

+ +
+
+ ); +} + +function Stat({ label, value }: { label: string; value: number | string }) { + return ( +
+

{label}

+

{value}

+
+ ); +} + +function FilterChip({ + href, + active, + label, + autopilot, +}: { + href: string; + active: boolean; + label: string; + autopilot?: boolean; +}) { + return ( + + {autopilot ? '● ' : ''} + {label} + + ); +} + +function formatHours(hours: number): string { + if (hours < 1) return 'under an hour'; + if (hours < 48) return `${Math.round(hours)} hours`; + return `${Math.round(hours / 24)} days`; +} diff --git a/apps/web/app/(app)/more/page.tsx b/apps/web/app/(app)/more/page.tsx index 29b76c8..5961ca5 100644 --- a/apps/web/app/(app)/more/page.tsx +++ b/apps/web/app/(app)/more/page.tsx @@ -7,10 +7,17 @@ export const dynamic = 'force-dynamic'; export const metadata = { title: 'More · OutreachGraph' }; +/** Real destinations, listed above the things that are still placeholders. */ +const LINKS = [ + { href: '/settings', label: 'Settings', hint: 'Alerts, the daily digest, autopilot limits' }, + { href: '/signals', label: 'Signals', hint: 'The raw observations behind every score' }, + { href: '/setup', label: 'Setup', hint: 'What you sell, and who buys it' }, + { href: '/funnel', label: 'Funnel', hint: 'Stages, conversion and each lead over time' }, +] as const; + const SECTIONS = [ - { label: 'Campaigns', hint: 'Wizard not built yet' }, { label: 'Conversations', hint: 'Interaction tracking only' }, - { label: 'Integrations', hint: 'GitHub only so far' }, + { label: 'Integrations', hint: 'Website and GitHub as sources; email for sending' }, { label: 'Usage', hint: 'Metering recorded, not enforced' }, ] as const; @@ -36,6 +43,19 @@ export default async function MorePage() { ) : null} +
    + {LINKS.map((link) => ( +
  • + +
    {link.label}
    +
    {link.hint}
    + +
  • + ))} +
+ +

Not built yet

+
    {SECTIONS.map((section) => (
  • diff --git a/apps/web/app/(app)/outreach/page.tsx b/apps/web/app/(app)/outreach/page.tsx index 8f2d23d..054a2b4 100644 --- a/apps/web/app/(app)/outreach/page.tsx +++ b/apps/web/app/(app)/outreach/page.tsx @@ -1,5 +1,6 @@ import Link from 'next/link'; import { redirect } from 'next/navigation'; +import { CampaignIntake } from '../../../components/campaign-intake'; import { BulkUrlIntake } from '../../../components/bulk-url-intake'; import { ApiUnavailableError, @@ -73,10 +74,23 @@ export default async function OutreachPage() { ) : null} - + + + {/* Still here, and still useful — but no longer the front door. Most + people are starting from a market rather than from a list they + have already built, and the box above takes both. */} +
    + + Already have a list of companies? + +

    + Paste or upload up to a hundred URLs at once. They join your existing campaign. +

    + +

    - Chasing one person instead? Add them by GitHub handle. + Watch what comes back on the Funnel.

    )} diff --git a/apps/web/app/(app)/prospects/page.tsx b/apps/web/app/(app)/prospects/page.tsx index e48c9f9..8de23e3 100644 --- a/apps/web/app/(app)/prospects/page.tsx +++ b/apps/web/app/(app)/prospects/page.tsx @@ -1,6 +1,5 @@ import Link from 'next/link'; import { redirect } from 'next/navigation'; -import { AddProspect } from '../../../components/add-prospect'; import { ApiUnavailableError, NotAuthenticatedError, fetchProspects } from '../../../lib/api'; import type { ProspectRow } from '../../../lib/types'; @@ -36,22 +35,17 @@ export default async function ProspectsPage() {

    - {/* The bulk form used to sit here. It moved to its own tab, because a - list of prospects is where you go once you have some — not where - anyone thinks to look for how to get them. One implementation, one - place, and a link for whoever arrives here first. */} -
    - - Add from company websites - - Paste a list of URLs — or upload one — on the Outreach tab. - - - -
    + {/* Intake lives on Outreach, in one box that takes a website or a + description of a market. The GitHub-handle form that used to sit here + is gone: asking a salesperson for a GitHub username to start a sales + campaign was the single strangest thing about this product. GitHub is + still read as a signal source, which is where it belongs. */} + + Start a campaign + + Enter a company website, or describe who you want to reach. + + {people.length > 0 ? (
      @@ -61,9 +55,8 @@ export default async function ProspectsPage() {
    ) : !offline ? (

    - No prospects yet. Paste company websites on the Outreach tab - — or a GitHub handle above — and the pipeline will enrich, resolve, collect signals and - score them. + No prospects yet. Start a campaign on the Outreach tab and + the pipeline will find people, resolve who they are, collect signals and score them.

    ) : null} diff --git a/apps/web/app/(app)/settings/page.tsx b/apps/web/app/(app)/settings/page.tsx new file mode 100644 index 0000000..5e9389c --- /dev/null +++ b/apps/web/app/(app)/settings/page.tsx @@ -0,0 +1,50 @@ +import { redirect } from 'next/navigation'; +import { SettingsForm } from '../../../components/settings-form'; +import { + ApiUnavailableError, + NotAuthenticatedError, + fetchSettings, + type SettingsView, +} from '../../../lib/api'; + +export const dynamic = 'force-dynamic'; + +export const metadata = { title: 'Settings · OutreachGraph' }; + +/** + * Notification and autopilot settings. + * + * Its own page rather than a section of More, because once the product runs + * unattended these are the controls that decide what it does while nobody is + * looking — and "how much may it send today" is not a preference to hide + * behind a disclosure triangle. + */ +export default async function SettingsPage() { + let settings: SettingsView | undefined; + let offline = false; + + try { + settings = await fetchSettings(); + } catch (error) { + if (error instanceof NotAuthenticatedError) redirect('/login'); + if (error instanceof ApiUnavailableError) offline = true; + else throw error; + } + + return ( +
    +
    +

    Settings

    +

    What we send you, and what we send for you.

    +
    + + {offline || !settings ? ( +

    + Waiting for the API. +

    + ) : ( + + )} +
    + ); +} diff --git a/apps/web/app/(marketing)/page.tsx b/apps/web/app/(marketing)/page.tsx index 427e99c..b4ab700 100644 --- a/apps/web/app/(marketing)/page.tsx +++ b/apps/web/app/(marketing)/page.tsx @@ -103,7 +103,7 @@ function Hero() {

    - Free while in beta · No card · Starts from a GitHub handle + Free while in beta · No card · Starts from a website or a keyword

    @@ -169,9 +169,9 @@ function ApprovalCardSample() { function EvidenceSection() { const identities = [ - ['github', '1.00', 'The handle you started from.'], - ['x', '0.96', 'Linked from his own GitHub profile field.'], - ['website', '0.92', 'Same domain in his bio and his commits.'], + ['website', '1.00', 'Named on the company page you started from.'], + ['github', '0.96', 'Linked from that same page, and the bio matches.'], + ['x', '0.92', 'Listed in his own GitHub profile field.'], ]; return ( diff --git a/apps/web/components/add-prospect.tsx b/apps/web/components/add-prospect.tsx deleted file mode 100644 index 8fb506d..0000000 --- a/apps/web/components/add-prospect.tsx +++ /dev/null @@ -1,113 +0,0 @@ -'use client'; - -import { useRouter } from 'next/navigation'; -import { useState, type FormEvent } from 'react'; - -/** - * Add one prospect by GitHub handle (PRD §8). - * - * The chain runs synchronously, so this reports what actually happened rather - * than "queued": how many identities were linked and how many signals were - * found is the difference between a prospect worth pursuing and a dead end, - * and hiding it behind a spinner that resolves to nothing was the old - * behaviour of an empty app. - * - * A handle that resolves to no profile comes back 200 with a reason. That is - * a typo, not a server fault, and it is shown as a note rather than an error. - */ -export function AddProspect() { - const router = useRouter(); - const [handle, setHandle] = useState(''); - const [busy, setBusy] = useState(false); - const [error, setError] = useState(); - const [note, setNote] = useState(); - - async function submit(event: FormEvent) { - event.preventDefault(); - if (!handle.trim()) return; - - setBusy(true); - setError(undefined); - setNote(undefined); - - try { - const response = await fetch('/api/v1/prospects', { - method: 'POST', - headers: { 'content-type': 'application/json' }, - credentials: 'same-origin', - body: JSON.stringify({ handle }), - }); - - const payload = await response.json().catch(() => ({})); - - if (!response.ok) { - setError(payload?.error?.message ?? `that failed (${response.status})`); - return; - } - - if (!payload.added) { - setNote(payload.reason ?? 'nothing to add for that handle'); - return; - } - - const found = - payload.signalsStored > 0 - ? `${payload.signalsStored} signal${payload.signalsStored === 1 ? '' : 's'} found` - : 'no public signals yet'; - - setNote(`Added. ${payload.identitiesLinked} identities linked, ${found}.`); - setHandle(''); - router.refresh(); - } catch { - setError('could not reach the server'); - } finally { - setBusy(false); - } - } - - return ( -
    - -

    - A GitHub username or profile URL. GitHub first because it is free and its profiles carry - links the person published themselves. -

    - -
    - setHandle(e.target.value)} - placeholder="octocat" - autoCapitalize="none" - autoCorrect="off" - spellCheck={false} - className="border-border bg-surface min-w-0 flex-1 rounded-xl border px-3 py-3" - /> - -
    - - {busy ? ( -

    - Enriching, resolving identities, collecting signals and scoring — a few seconds. -

    - ) : null} - - {note ?

    {note}

    : null} - - {error ? ( -

    - {error} -

    - ) : null} -
    - ); -} diff --git a/apps/web/components/bottom-nav.tsx b/apps/web/components/bottom-nav.tsx index a768cb6..579a390 100644 --- a/apps/web/components/bottom-nav.tsx +++ b/apps/web/components/bottom-nav.tsx @@ -13,11 +13,16 @@ import { usePathname } from 'next/navigation'; * list, approve — because the first question a new account has is "where do I * begin", and the answer has to be a tab rather than a form buried at the top * of another screen. + * + * Signals moved to More to make room for Funnel. Six is already the most + * labels that fit a 375px phone without wrapping, and between "the raw + * observations we collected" and "where every lead stands", the second is the + * one someone opens the app to see. Signals is still a route and still linked. */ const TABS = [ { href: '/today', label: 'Today', icon: SunIcon }, { href: '/outreach', label: 'Outreach', icon: SendIcon }, - { href: '/signals', label: 'Signals', icon: BoltIcon }, + { href: '/funnel', label: 'Funnel', icon: FunnelIcon }, { href: '/prospects', label: 'Prospects', icon: PeopleIcon }, { href: '/approvals', label: 'Approvals', icon: CheckIcon }, { href: '/more', label: 'More', icon: DotsIcon }, @@ -94,10 +99,10 @@ function SendIcon() { ); } -function BoltIcon() { +function FunnelIcon() { return ( - + ); } diff --git a/apps/web/components/campaign-intake.tsx b/apps/web/components/campaign-intake.tsx new file mode 100644 index 0000000..9cdf1be --- /dev/null +++ b/apps/web/components/campaign-intake.tsx @@ -0,0 +1,134 @@ +'use client'; + +import { useRouter } from 'next/navigation'; +import { useState, type FormEvent } from 'react'; + +/** + * The one box. + * + * This replaces two separate intakes that both asked the wrong question: a + * GitHub username, which is not how anyone describes a sales target, and a + * list of URLs, which assumes you already know every company you want to + * reach. Either a website or a description of a market works here, and the + * server decides which it got — the user should not have to pick a mode + * before they have said what they want. + * + * Autopilot is a checkbox rather than a separate flow, and it is on by + * default, because a product that promises to run unattended and then makes + * unattended the advanced option is not really offering it. + */ +export function CampaignIntake() { + const router = useRouter(); + const [input, setInput] = useState(''); + const [autopilot, setAutopilot] = useState(true); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(); + const [note, setNote] = useState(); + + async function submit(event: FormEvent) { + event.preventDefault(); + if (!input.trim()) return; + + setBusy(true); + setError(undefined); + setNote(undefined); + + try { + const response = await fetch('/api/v1/campaigns', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + credentials: 'same-origin', + body: JSON.stringify({ input, autopilot }), + }); + + const payload = await response.json().catch(() => ({})); + + if (!response.ok) { + setError(payload?.error?.message ?? `that failed (${response.status})`); + return; + } + + // The two paths deserve different words. A URL is already being fetched; + // a keyword has to be turned into companies first, and saying "queued" + // for both makes the slower one look stuck. + setNote( + payload.kind === 'url' + ? `Reading ${payload.seed} now. Anyone worth contacting there will appear in your funnel.` + : `Working out who matches “${payload.seed}”. Companies start appearing within a minute or two.`, + ); + + if (payload.needsProfile) { + setNote( + (current) => + `${current ?? ''} You have not said what you sell yet, so the first drafts will be generic — fix that in Setup.`, + ); + } + + setInput(''); + router.refresh(); + } catch { + setError('could not reach the server'); + } finally { + setBusy(false); + } + } + + return ( +
    + +

    + A company website — acme.com — or a description of a market, like “dental + practices in Austin”. Either one starts a campaign. +

    + +
    + setInput(e.target.value)} + placeholder="acme.com — or independent bike shops in Portland" + autoCapitalize="none" + autoCorrect="off" + spellCheck={false} + className="border-border bg-surface min-w-0 flex-1 rounded-xl border px-3 py-3" + /> + +
    + + + + {busy ?

    Setting up the campaign…

    : null} + + {note ?

    {note}

    : null} + + {error ? ( +

    + {error} +

    + ) : null} +
    + ); +} diff --git a/apps/web/components/funnel-chart.tsx b/apps/web/components/funnel-chart.tsx new file mode 100644 index 0000000..ad68724 --- /dev/null +++ b/apps/web/components/funnel-chart.tsx @@ -0,0 +1,203 @@ +/** + * The funnel, and the sideways view of how leads moved through it. + * + * Two charts, both deliberately plain. Sales dashboards tend toward donuts and + * gauges; the questions people actually ask are "how many got to each stage" + * and "where do they stall", and a bar whose length is the count answers both + * without a legend. + * + * Server-rendered with no charting library. Every bar is a div with a width + * percentage, which keeps the page working with JavaScript disabled, adds + * nothing to the bundle, and — since artifacts of this app render in both + * themes — inherits the palette rather than hard-coding colours. + */ + +import Link from 'next/link'; +import type { FunnelStageView, LeadTimelineView } from '../lib/api'; + +/** Stage order, so a timeline segment can be positioned without a lookup. */ +const STAGE_ORDER = ['discovered', 'researched', 'ready', 'contacted', 'replied', 'opportunity']; + +const STAGE_LABEL: Record = { + discovered: 'Found', + researched: 'Researched', + ready: 'Ready to send', + contacted: 'Contacted', + replied: 'Replied', + opportunity: 'Opportunity', + lost: 'Lost', +}; + +/** + * Stage shading, light to dark down the funnel. + * + * Opacity on one accent rather than six hues: the stages are ordered, and six + * unrelated colours would imply they are categories rather than a sequence. + */ +const STAGE_TONE: Record = { + discovered: 'bg-accent/25', + researched: 'bg-accent/40', + ready: 'bg-accent/55', + contacted: 'bg-accent/70', + replied: 'bg-accent/85', + opportunity: 'bg-accent', + lost: 'bg-ink-muted/30', +}; + +export function FunnelChart({ + stages, + lost, +}: { + stages: readonly FunnelStageView[]; + lost: number; +}) { + const top = stages[0]?.reached ?? 0; + + if (top === 0) { + return ( +

    + No leads yet. Start a campaign and this fills in as people move through it. +

    + ); + } + + return ( +
    + {stages.map((stage, index) => { + const previous = stages[index - 1]; + // Conversion against the stage above, not against the top of the + // funnel: "40% of the people we researched were worth writing to" is + // actionable, while "40% of everyone we ever found" is not. + const rate = + previous && previous.reached > 0 + ? Math.round((stage.reached / previous.reached) * 100) + : undefined; + + const width = + top > 0 ? Math.max((stage.reached / top) * 100, stage.reached > 0 ? 4 : 0) : 0; + + return ( +
    +
    + {stage.label} + + {stage.reached} + {rate !== undefined ? {rate}% : null} + +
    + +
    +
    +
    + + {stage.current > 0 ? ( +

    {stage.current} sitting here now

    + ) : null} +
    + ); + })} + + {lost > 0 ? ( +

    + {lost} left the funnel — not interested, or suppressed. +

    + ) : null} +
    + ); +} + +/** + * One row per lead, each stage a segment along a shared time axis. + * + * This is the "sideways chart" — a Gantt in the only sense that matters here: + * you can see at a glance which leads flew through and which have been sitting + * in one stage for a week. Segment width is time spent, so a stall is visibly + * wide rather than a number someone has to compare. + */ +export function LeadTimelineChart({ leads }: { leads: readonly LeadTimelineView[] }) { + if (leads.length === 0) { + return ( +

    + Nothing has moved yet. +

    + ); + } + + // One shared scale across every row, so two rows of the same width really + // did take the same time. Per-row scaling would make a lead that took an + // hour look identical to one that took a fortnight. + const span = Math.max(...leads.map((lead) => totalHours(lead)), 1); + + return ( +
    + {leads.map((lead) => ( +
    +
    + + {lead.personName} + {lead.companyName ? ( + · {lead.companyName} + ) : null} + + + {STAGE_LABEL[lead.currentStage] ?? lead.currentStage} + +
    + +
    + {lead.segments.map((segment, index) => { + // The last segment is still open, so it gets whatever is left + // rather than a zero width — a lead sitting in a stage right now + // is exactly the one worth seeing. + const hours = segment.hours ?? Math.max(span - elapsedBefore(lead, index), 0.5); + const width = Math.max((hours / span) * 100, 3); + + return ( +
    + ); + })} +
    +
    + ))} + + +
    + ); +} + +function StageKey() { + return ( +
      + {STAGE_ORDER.map((stage) => ( +
    • + + {STAGE_LABEL[stage]} +
    • + ))} +
    + ); +} + +function totalHours(lead: LeadTimelineView): number { + const first = Date.parse(lead.firstSeenAt); + if (Number.isNaN(first)) return 1; + return Math.max((Date.now() - first) / 3_600_000, 1); +} + +function elapsedBefore(lead: LeadTimelineView, index: number): number { + return lead.segments.slice(0, index).reduce((total, segment) => total + (segment.hours ?? 0), 0); +} + +function formatHours(hours: number): string { + if (hours < 1) return 'under an hour'; + if (hours < 48) return `${Math.round(hours)}h`; + return `${Math.round(hours / 24)}d`; +} diff --git a/apps/web/components/settings-form.tsx b/apps/web/components/settings-form.tsx new file mode 100644 index 0000000..f51eb9f --- /dev/null +++ b/apps/web/components/settings-form.tsx @@ -0,0 +1,201 @@ +'use client'; + +import { useState, type FormEvent } from 'react'; +import type { SettingsView } from '../lib/api'; + +/** + * Notification and autopilot settings. + * + * These are the controls that matter once the product runs unattended: where + * the mail goes, how loud it is, and how much it is allowed to send in a day. + * The daily cap in particular is the thing standing between a misconfigured + * campaign and a burnt sending domain, so it is on this page rather than + * buried in an environment variable. + */ +export function SettingsForm({ initial }: { initial: SettingsView }) { + const [form, setForm] = useState(initial); + const [busy, setBusy] = useState(false); + const [saved, setSaved] = useState(false); + const [error, setError] = useState(); + + async function submit(event: FormEvent) { + event.preventDefault(); + setBusy(true); + setError(undefined); + setSaved(false); + + try { + const response = await fetch('/api/v1/settings', { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + credentials: 'same-origin', + body: JSON.stringify({ + notifyEmail: form.notifyEmail || null, + replyToEmail: form.replyToEmail || null, + instantAlerts: form.instantAlerts, + dailyDigest: form.dailyDigest, + digestHourUtc: form.digestHourUtc, + alertMinOpportunity: form.alertMinOpportunity, + autopilotDailyCap: form.autopilotDailyCap, + }), + }); + + if (!response.ok) { + const payload = await response.json().catch(() => ({})); + setError(payload?.error?.message ?? `that failed (${response.status})`); + return; + } + + setSaved(true); + } catch { + setError('could not reach the server'); + } finally { + setBusy(false); + } + } + + const update = (key: K, value: SettingsView[K]): void => { + setForm((current) => ({ ...current, [key]: value })); + setSaved(false); + }; + + return ( +
    +
    +

    Where mail goes

    + + + update('notifyEmail', e.target.value)} + placeholder={form.effectiveNotifyEmail ?? 'your address'} + className="border-border bg-surface mt-1 w-full rounded-xl border px-3 py-2.5" + /> +

    + {form.notifyEmail + ? 'Alerts and the digest go here.' + : `Leave blank to use ${form.effectiveNotifyEmail ?? 'the account owner’s address'}.`} +

    + + + update('replyToEmail', e.target.value)} + placeholder={form.effectiveNotifyEmail ?? 'your address'} + className="border-border bg-surface mt-1 w-full rounded-xl border px-3 py-2.5" + /> +

    + Outreach is sent from our domain but replies go here — that is the one part you handle. +

    +
    + +
    +

    When to interrupt you

    + + + + + update('alertMinOpportunity', Number(e.target.value))} + className="mt-2 w-full" + /> + + + + + update('digestHourUtc', Number(e.target.value))} + className="border-border bg-surface mt-1 w-24 rounded-xl border px-3 py-2.5 tabular-nums" + /> +
    + +
    +

    Autopilot limits

    + + + update('autopilotDailyCap', Number(e.target.value))} + className="border-border bg-surface mt-1 w-28 rounded-xl border px-3 py-2.5 tabular-nums" + /> +

    + Across every campaign. Set to 0 to stop all unattended sending without switching campaigns + off one at a time. +

    +
    + +
    + + {saved ? Saved. : null} +
    + + {error ? ( +

    + {error} +

    + ) : null} +
    + ); +} diff --git a/apps/web/lib/api.ts b/apps/web/lib/api.ts index ddbc1e5..11d3103 100644 --- a/apps/web/lib/api.ts +++ b/apps/web/lib/api.ts @@ -121,3 +121,84 @@ export async function fetchProspects(): Promise { export async function fetchProspect(id: string): Promise { return request(`/people/${encodeURIComponent(id)}`); } + +export interface CampaignRow { + readonly id: string; + readonly name: string; + readonly status: string; + readonly approval_mode: string; + readonly seed_kind: string | null; + readonly seed_value: string | null; + readonly created_at: string; + readonly started_at: string | null; +} + +export async function fetchCampaigns(): Promise { + const body = await request<{ campaigns: CampaignRow[] }>('/campaigns'); + return body.campaigns; +} + +export interface FunnelStageView { + readonly stage: string; + readonly label: string; + readonly current: number; + readonly reached: number; +} + +export interface AnalyticsView { + readonly funnel: { + readonly stages: readonly FunnelStageView[]; + readonly lost: number; + readonly total: number; + }; + readonly sentThisWeek: number; + readonly repliesThisWeek: number; + readonly awaitingApproval: number; + readonly activeCampaigns: number; + readonly autopilotCampaigns: number; + readonly medianHoursToContact?: number; +} + +export async function fetchAnalytics(campaignId?: string): Promise { + const query = campaignId ? `?campaignId=${encodeURIComponent(campaignId)}` : ''; + return request(`/analytics${query}`); +} + +export interface TimelineSegmentView { + readonly stage: string; + readonly enteredAt: string; + readonly leftAt?: string; + readonly hours?: number; +} + +export interface LeadTimelineView { + readonly personId: string; + readonly personName: string; + readonly companyName?: string; + readonly currentStage: string; + readonly opportunity?: number; + readonly firstSeenAt: string; + readonly segments: readonly TimelineSegmentView[]; +} + +export async function fetchTimeline(campaignId?: string): Promise { + const query = campaignId ? `?campaignId=${encodeURIComponent(campaignId)}&limit=50` : '?limit=50'; + const body = await request<{ leads: LeadTimelineView[] }>(`/analytics/timeline${query}`); + return body.leads; +} + +export interface SettingsView { + readonly notifyEmail: string | null; + readonly effectiveNotifyEmail: string | null; + readonly instantAlerts: boolean; + readonly dailyDigest: boolean; + readonly digestHourUtc: number; + readonly alertMinOpportunity: number; + readonly autopilotDailyCap: number; + readonly replyToEmail: string | null; + readonly lastDigestSentOn: string | null; +} + +export async function fetchSettings(): Promise { + return request('/settings'); +} diff --git a/bun.lock b/bun.lock index 1f52262..29a7a3e 100644 --- a/bun.lock +++ b/bun.lock @@ -127,7 +127,9 @@ "@outreachgraph/ai": "workspace:*", "@outreachgraph/db": "workspace:*", "@outreachgraph/domain": "workspace:*", + "@outreachgraph/email": "workspace:*", "@outreachgraph/identity": "workspace:*", + "@outreachgraph/policy": "workspace:*", "@outreachgraph/providers": "workspace:*", "@outreachgraph/recommend": "workspace:*", "@outreachgraph/scoring": "workspace:*", diff --git a/migrations/0009_autopilot.sql b/migrations/0009_autopilot.sql new file mode 100644 index 0000000..69070d4 --- /dev/null +++ b/migrations/0009_autopilot.sql @@ -0,0 +1,90 @@ +-- 0009_autopilot.sql +-- Autopilot campaigns, contact addresses, and the notification ledger. +-- +-- Three things the product needed before it could run unattended: +-- +-- 1. A campaign has to remember what it was started from — a URL or a +-- keyword — or the run is untraceable back to the thing the user typed. +-- 2. Outreach needs an address. Until now nothing anywhere held one, so the +-- approval queue could only ever end in a human copying text elsewhere. +-- 3. A notification that can fire twice is worse than one that never fires, +-- so every alert and digest is recorded and keyed for idempotency. + +-- What the user typed to start this campaign. 'url' | 'keyword'. +ALTER TABLE campaigns ADD COLUMN seed_kind TEXT; +ALTER TABLE campaigns ADD COLUMN seed_value TEXT; + +-- Role addresses (info@, sales@) belong to the company, not to a person: they +-- are a fallback for a company whose site names people but publishes no +-- personal address, and they are greeted differently because nobody named +-- Jane reads info@. +ALTER TABLE companies ADD COLUMN contact_email TEXT; + +-- Per-workspace delivery settings. Its own table rather than columns on +-- `workspaces` because these are notification preferences with their own +-- lifecycle, and a workspace with no row still has working defaults. +CREATE TABLE workspace_settings ( + workspace_id TEXT PRIMARY KEY REFERENCES workspaces(id) ON DELETE CASCADE, + -- Where alerts and digests go. NULL means "the workspace owner's address", + -- resolved at send time so a changed login address is not stale here. + notify_email TEXT, + instant_alerts INTEGER NOT NULL DEFAULT 1, + daily_digest INTEGER NOT NULL DEFAULT 1, + -- Hour of the UTC day the digest is due. One send per calendar day. + digest_hour_utc INTEGER NOT NULL DEFAULT 13, + -- A lead below this opportunity score is not worth interrupting anyone for. + alert_min_opportunity INTEGER NOT NULL DEFAULT 60, + -- The backstop on unattended sending. Counted per UTC day, per workspace. + autopilot_daily_cap INTEGER NOT NULL DEFAULT 25, + -- Reply-to on outbound outreach. NULL resolves to the owner, same as above. + reply_to_email TEXT, + last_digest_sent_on TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +-- Every notification the product has actually sent. +-- +-- `subject_key` is what makes the send idempotent: the person id for a lead +-- alert, the UTC date for a digest. The unique index is the guard — a retried +-- job or a second worker cannot produce a second email. +CREATE TABLE notifications ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + -- lead_alert | daily_digest + kind TEXT NOT NULL, + subject_key TEXT NOT NULL, + to_email TEXT NOT NULL, + sent_at TEXT NOT NULL, + error TEXT +); + +CREATE UNIQUE INDEX idx_notifications_once ON notifications(workspace_id, kind, subject_key); +CREATE INDEX idx_notifications_ws ON notifications(workspace_id, sent_at DESC); + +-- Every stage a lead has moved through, with when. +-- +-- `campaign_people.status` holds the *current* state and overwrites the last +-- one, which answers "where is this lead now" and nothing else. A funnel needs +-- the other questions — how many reached each stage, how long they sat there, +-- what fell out and where — and none of those are answerable from a column +-- that keeps no history. This is that history. +-- +-- `stage` is stored alongside `to_status` rather than derived on read because +-- the internal state machine has sixteen states and the funnel has six: if the +-- mapping is ever changed, past events keep the stage they were actually +-- reported under, and old charts do not silently rewrite themselves. +CREATE TABLE lead_stage_events ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + campaign_id TEXT NOT NULL REFERENCES campaigns(id) ON DELETE CASCADE, + person_id TEXT NOT NULL REFERENCES people(id) ON DELETE CASCADE, + from_status TEXT, + to_status TEXT NOT NULL, + stage TEXT NOT NULL, + occurred_at TEXT NOT NULL +); + +CREATE INDEX idx_lead_stage_person ON lead_stage_events(person_id, occurred_at); +CREATE INDEX idx_lead_stage_ws ON lead_stage_events(workspace_id, occurred_at DESC); +CREATE INDEX idx_lead_stage_campaign ON lead_stage_events(campaign_id, stage); diff --git a/packages/ai/src/discover.test.ts b/packages/ai/src/discover.test.ts new file mode 100644 index 0000000..4e0d632 --- /dev/null +++ b/packages/ai/src/discover.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, test } from 'bun:test'; +import { discoverCompanies, normaliseDomain } from './discover'; +import type { GenerateInput, GenerateResult, TextModel } from './model'; + +/** Answers with a canned reply, and records what it was asked. */ +function stubModel(text: string, refused = false): TextModel & { asked: GenerateInput[] } { + const asked: GenerateInput[] = []; + return { + asked, + async generate(input: GenerateInput): Promise { + asked.push(input); + return { text, model: 'stub', inputTokens: 0, outputTokens: 0, cachedTokens: 0, refused }; + }, + }; +} + +function throwingModel(message: string): TextModel { + return { + async generate(): Promise { + throw new Error(message); + }, + }; +} + +describe('normaliseDomain', () => { + test('reduces every shape a model returns to one host', () => { + expect(normaliseDomain('acme.com')).toBe('acme.com'); + expect(normaliseDomain('https://www.acme.com/about')).toBe('acme.com'); + expect(normaliseDomain('WWW.ACME.COM')).toBe('acme.com'); + expect(normaliseDomain('acme.com.')).toBe('acme.com'); + }); + + test('rejects non-domains and the aggregators that are never a prospect', () => { + expect(normaliseDomain('acme')).toBeUndefined(); + expect(normaliseDomain('')).toBeUndefined(); + expect(normaliseDomain(42)).toBeUndefined(); + expect(normaliseDomain('yelp.com')).toBeUndefined(); + expect(normaliseDomain('linkedin.com')).toBeUndefined(); + expect(normaliseDomain('example.com')).toBeUndefined(); + }); +}); + +describe('discoverCompanies', () => { + const reply = JSON.stringify({ + campaignName: 'Austin dental practices', + brief: 'Independent dental practices in Austin, Texas.', + companies: [ + { name: 'Bright Smile Dental', domain: 'https://www.brightsmile.com/', reason: 'Austin' }, + { name: 'Hill Country Dental', domain: 'hillcountrydental.com', reason: 'Austin' }, + // A duplicate of the first, in a different shape. + { name: 'Bright Smile', domain: 'brightsmile.com' }, + // Never a prospect. + { name: 'Yelp', domain: 'yelp.com' }, + // No usable domain. + { name: 'Someone', domain: 'not a domain' }, + ], + }); + + test('normalises, dedupes and filters what the model returned', async () => { + const result = await discoverCompanies(stubModel(reply), 'dental practices in Austin'); + + expect(result.ok).toBe(true); + expect(result.companies.map((c) => c.domain)).toEqual([ + 'brightsmile.com', + 'hillcountrydental.com', + ]); + expect(result.campaignName).toBe('Austin dental practices'); + }); + + test('reads a reply wrapped in a code fence', async () => { + const result = await discoverCompanies( + stubModel('Here you go:\n```json\n' + reply + '\n```'), + 'dentists', + ); + expect(result.ok).toBe(true); + expect(result.companies.length).toBeGreaterThan(0); + }); + + test('grounds the search in what the customer sells when known', async () => { + const model = stubModel(reply); + await discoverCompanies(model, 'dentists', { offeringSummary: 'scheduling software' }); + + expect(model.asked[0]?.user).toContain('scheduling software'); + }); + + test('an unreachable model is a reason, not a throw', async () => { + const result = await discoverCompanies(throwingModel('usage limits'), 'dentists'); + + expect(result.ok).toBe(false); + expect(result.reason).toContain('usage limits'); + expect(result.companies).toHaveLength(0); + }); + + test('a refusal and an unparseable reply both come back as reasons', async () => { + expect((await discoverCompanies(stubModel('no', true), 'x')).ok).toBe(false); + expect((await discoverCompanies(stubModel('not json at all'), 'x')).ok).toBe(false); + }); + + test('an empty query never reaches the model', async () => { + const model = stubModel(reply); + const result = await discoverCompanies(model, ' '); + + expect(result.ok).toBe(false); + expect(model.asked).toHaveLength(0); + }); + + test('a list of only unusable companies is a failure, not an empty success', async () => { + const result = await discoverCompanies( + stubModel(JSON.stringify({ companies: [{ name: 'Yelp', domain: 'yelp.com' }] })), + 'dentists', + ); + + expect(result.ok).toBe(false); + }); +}); diff --git a/packages/ai/src/discover.ts b/packages/ai/src/discover.ts new file mode 100644 index 0000000..4d2fc6e --- /dev/null +++ b/packages/ai/src/discover.ts @@ -0,0 +1,224 @@ +/** + * Turning a phrase into somewhere to look (PRD §11 targeting, §26 filters). + * + * The product could only ever be handed URLs. That is fine once you know who + * you are chasing and useless before then — "dental practices in Austin" is + * how people actually describe a market, and until now there was nowhere to + * type it. + * + * This asks the model for real companies matching a description and returns + * their domains. Two deliberate limits on how much that is trusted: + * + * - **Nothing here is treated as fact.** A model naming a company is a lead + * to check, not a record. Every domain returned is then fetched by the + * crawler, and one that does not resolve or does not serve a page simply + * drops out of the run. The crawl is the verification step. + * - **No contact details are ever asked for.** Addresses come off the page + * the company published, never out of a model's memory, because an + * invented address is indistinguishable from a real one until it bounces. + */ + +import type { TextModel } from './model'; + +export interface DiscoveredCompany { + readonly name: string; + /** Bare host, lowercased, no scheme and no `www.` */ + readonly domain: string; + /** One line on why this company matches the description. */ + readonly reason?: string; +} + +export interface DiscoveryResult { + readonly ok: boolean; + readonly companies: readonly DiscoveredCompany[]; + /** A short human name for the campaign this seeds. */ + readonly campaignName?: string; + /** What the model understood the target market to be, for the campaign brief. */ + readonly brief?: string; + readonly reason?: string; +} + +const SYSTEM = [ + 'You are given a description of a target market and you name real companies in it.', + '', + 'Return one JSON object and nothing else:', + '{', + ' "campaignName": string,', + ' "brief": string,', + ' "companies": [{"name": string, "domain": string, "reason": string}]', + '}', + '', + 'Rules:', + '- Only real companies you are confident exist, with the domain they actually use.', + '- "domain" is a bare hostname: "acme.com", never a URL, never a path, no "www.".', + '- Prefer the primary marketing site over a directory, marketplace or listing page.', + '- Never return an aggregator, directory, review site, job board or social network', + ' as a company — yelp.com is not a dental practice.', + '- If the description names a place, the companies must actually operate there.', + '- Return no contact details of any kind. No email addresses, no phone numbers.', + '- If you are not confident of a real domain for a company, leave that company out.', + ' A short accurate list is worth more than a long invented one.', + '- "campaignName" is a few words a human would recognise, e.g. "Austin dental practices".', + '- "brief" is one or two sentences describing who this campaign targets and why.', +].join('\n'); + +/** Hosts that are never a prospect company, whatever the model says. */ +const NEVER = new Set([ + 'google.com', + 'facebook.com', + 'linkedin.com', + 'twitter.com', + 'x.com', + 'instagram.com', + 'yelp.com', + 'youtube.com', + 'wikipedia.org', + 'crunchbase.com', + 'indeed.com', + 'glassdoor.com', + 'tripadvisor.com', + 'amazon.com', + 'apple.com', + 'yellowpages.com', + 'bbb.org', + 'angi.com', + 'thumbtack.com', + 'example.com', +]); + +/** Pulls the JSON object out of a reply that may be fenced or prefaced. */ +function parseReply(text: string): unknown { + const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i)?.[1]; + const candidate = (fenced ?? text).trim(); + const start = candidate.indexOf('{'); + const end = candidate.lastIndexOf('}'); + if (start < 0 || end <= start) return undefined; + + try { + return JSON.parse(candidate.slice(start, end + 1)); + } catch { + return undefined; + } +} + +/** + * Reduces whatever the model wrote to a bare hostname, or rejects it. + * + * Models return `https://acme.com/about`, `www.acme.com` and `acme.com` + * interchangeably for the same company, and a crawl queue that treats those as + * three targets fetches the same site three times. + */ +export function normaliseDomain(raw: unknown): string | undefined { + if (typeof raw !== 'string') return undefined; + + let value = raw.trim().toLowerCase(); + if (!value) return undefined; + + value = value.replace(/^[a-z]+:\/\//, ''); + value = value.split('/')[0] ?? ''; + value = value.split('?')[0] ?? ''; + value = value.split('#')[0] ?? ''; + value = value.split('@').pop() ?? ''; + value = value.replace(/^www\./, '').replace(/\.$/, ''); + + if (!value.includes('.')) return undefined; + if (!/^[a-z0-9.-]+$/.test(value)) return undefined; + if (value.startsWith('.') || value.includes('..')) return undefined; + + const tld = value.split('.').pop() ?? ''; + if (tld.length < 2) return undefined; + if (NEVER.has(value)) return undefined; + + return value; +} + +export interface DiscoverOptions { + /** How many companies to ask for. The crawl budget is what really bounds it. */ + readonly limit?: number; + /** + * What the customer sells, when they have told us. + * + * Materially better targeting: "companies that would buy incident-response + * training" beats the same query with no idea who is asking. + */ + readonly offeringSummary?: string; +} + +/** + * Names companies matching a free-text market description. + * + * Never throws — a capped key or an unparseable reply comes back `ok: false` + * with a reason worth showing, because the caller's fallback is to tell the + * user to paste URLs instead, which is a worse start and not a crash. + */ +export async function discoverCompanies( + model: TextModel, + description: string, + options: DiscoverOptions = {}, +): Promise { + const query = description.trim(); + if (!query) return { ok: false, companies: [], reason: 'nothing to search for' }; + + const limit = Math.min(Math.max(options.limit ?? 25, 1), 50); + + const user = [ + `Target market: ${query}`, + options.offeringSummary ? `\nThe person searching sells: ${options.offeringSummary}` : '', + `\nReturn up to ${limit} companies.`, + ].join(''); + + let reply; + try { + reply = await model.generate({ system: SYSTEM, user, maxTokens: 3000 }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { ok: false, companies: [], reason: `the model could not be reached: ${message}` }; + } + + if (reply.refused) { + return { ok: false, companies: [], reason: 'the model declined that search' }; + } + + const parsed = parseReply(reply.text); + if (!parsed || typeof parsed !== 'object') { + return { ok: false, companies: [], reason: 'the model did not return a usable list' }; + } + + const root = parsed as Record; + const rows = Array.isArray(root.companies) ? root.companies : []; + + const seen = new Set(); + const companies: DiscoveredCompany[] = []; + + for (const row of rows) { + if (!row || typeof row !== 'object') continue; + const entry = row as Record; + + const domain = normaliseDomain(entry.domain); + if (!domain || seen.has(domain)) continue; + seen.add(domain); + + const name = typeof entry.name === 'string' && entry.name.trim() ? entry.name.trim() : domain; + const reason = typeof entry.reason === 'string' ? entry.reason.trim() : ''; + + companies.push({ + name: name.slice(0, 120), + domain, + ...(reason ? { reason: reason.slice(0, 300) } : {}), + }); + if (companies.length >= limit) break; + } + + if (companies.length === 0) { + return { ok: false, companies: [], reason: 'no companies could be identified for that' }; + } + + const campaignName = + typeof root.campaignName === 'string' && root.campaignName.trim() + ? root.campaignName.trim().slice(0, 120) + : query.slice(0, 120); + + const brief = typeof root.brief === 'string' ? root.brief.trim().slice(0, 1000) : ''; + + return { ok: true, companies, campaignName, ...(brief ? { brief } : {}) }; +} diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 181f63f..2022877 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -42,6 +42,14 @@ export { draftForRecommendation, type DraftResult } from './draft'; export { draftProfile, type ProfileDraft, type ProfileDraftResult } from './profile'; +export { + discoverCompanies, + normaliseDomain, + type DiscoverOptions, + type DiscoveredCompany, + type DiscoveryResult, +} from './discover'; + export { OpenAIModel, OpenAIApiError, diff --git a/packages/domain/src/funnel.test.ts b/packages/domain/src/funnel.test.ts new file mode 100644 index 0000000..1f4a31e --- /dev/null +++ b/packages/domain/src/funnel.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, test } from 'bun:test'; +import { PROSPECT_STATUS } from './pipeline'; +import { + conversionRate, + isAdvance, + stageForStatus, + summariseFunnel, + FUNNEL_STAGES, +} from './funnel'; + +describe('stageForStatus', () => { + test('every prospect status maps somewhere', () => { + // A status with no stage would vanish from the funnel silently, which is + // the failure mode most likely to go unnoticed. + for (const status of PROSPECT_STATUS) { + expect(stageForStatus(status)).toBeDefined(); + } + }); + + test('the engineering states collapse into the stages people recognise', () => { + expect(stageForStatus('enriching')).toBe('discovered'); + expect(stageForStatus('resolved')).toBe('discovered'); + expect(stageForStatus('awaiting_approval')).toBe('ready'); + expect(stageForStatus('approved')).toBe('ready'); + expect(stageForStatus('executed')).toBe('contacted'); + expect(stageForStatus('waiting')).toBe('contacted'); + expect(stageForStatus('responded')).toBe('replied'); + }); + + test('only genuine exits count as lost', () => { + expect(stageForStatus('not_interested')).toBe('lost'); + expect(stageForStatus('suppressed')).toBe('lost'); + // Unqualified means "researched and not a fit" — it stops, it is not lost. + expect(stageForStatus('unqualified')).toBe('researched'); + // An errored lead is stuck, not gone. + expect(stageForStatus('error')).toBe('discovered'); + }); +}); + +describe('summariseFunnel', () => { + test('a funnel never widens as it descends', () => { + // Only deep stages reported. Everything above must still be at least as + // large, or the chart shows a funnel that fans out. + const summary = summariseFunnel({ contacted: 3 }, { contacted: 5 }); + + const reached = summary.stages.map((stage) => stage.reached); + for (let index = 1; index < reached.length; index += 1) { + expect(reached[index - 1] as number).toBeGreaterThanOrEqual(reached[index] as number); + } + expect(summary.total).toBe(5); + }); + + test('current occupancy and historical reach are separate numbers', () => { + const summary = summariseFunnel( + { discovered: 2, contacted: 4 }, + { discovered: 10, researched: 8, ready: 6, contacted: 4 }, + ); + + const discovered = summary.stages[0]; + expect(discovered?.current).toBe(2); + // Closing deals must not shrink the top of the funnel. + expect(discovered?.reached).toBe(10); + }); + + test('an empty funnel is all zeroes rather than a crash', () => { + const summary = summariseFunnel({}); + expect(summary.total).toBe(0); + expect(summary.stages).toHaveLength(FUNNEL_STAGES.length); + }); + + test('lost is counted outside the stages', () => { + const summary = summariseFunnel({ discovered: 5, lost: 2 }); + expect(summary.lost).toBe(2); + expect(summary.stages.some((stage) => (stage.stage as string) === 'lost')).toBe(false); + }); +}); + +describe('conversionRate', () => { + test('is a percentage of the stage above', () => { + const from = { stage: 'discovered' as const, label: 'Found', current: 0, reached: 100 }; + const to = { stage: 'researched' as const, label: 'Researched', current: 0, reached: 40 }; + expect(conversionRate(from, to)).toBe(40); + }); + + test('an empty upper stage has no rate, rather than a rate of zero', () => { + const from = { stage: 'discovered' as const, label: 'Found', current: 0, reached: 0 }; + const to = { stage: 'researched' as const, label: 'Researched', current: 0, reached: 0 }; + expect(conversionRate(from, to)).toBeUndefined(); + }); +}); + +describe('isAdvance', () => { + test('forward movement is an advance; sideways and lost are not', () => { + expect(isAdvance('discovered', 'contacted')).toBe(true); + expect(isAdvance(undefined, 'discovered')).toBe(true); + expect(isAdvance('contacted', 'discovered')).toBe(false); + expect(isAdvance('researched', 'researched')).toBe(false); + expect(isAdvance('discovered', 'lost')).toBe(false); + }); +}); diff --git a/packages/domain/src/funnel.ts b/packages/domain/src/funnel.ts new file mode 100644 index 0000000..f0bd33a --- /dev/null +++ b/packages/domain/src/funnel.ts @@ -0,0 +1,179 @@ +/** + * The sales funnel — the sixteen-state machine, said in words a salesperson uses. + * + * `ProspectStatus` is an engineering state machine. It distinguishes + * `enriching` from `resolved` from `researching` because the worker needs to + * know exactly where to resume after a crash, and every one of those + * distinctions is invisible and uninteresting to the person whose job is to + * sell something. + * + * A funnel is the same run with the engineering removed: how many did we find, + * how many were worth writing to, how many did we write to, how many wrote + * back. Six stages, in order, each one a strictly smaller set than the last — + * which is the property that makes it a funnel and makes the drop-off between + * two stages mean something. + * + * `Lost` is deliberately not a stage. It is an exit, and putting it in the + * sequence would break the "each stage is smaller than the last" rule that + * lets the chart be read at a glance. + */ + +import type { ProspectStatus } from './pipeline'; + +export const FUNNEL_STAGES = [ + 'discovered', + 'researched', + 'ready', + 'contacted', + 'replied', + 'opportunity', +] as const; + +export type FunnelStage = (typeof FUNNEL_STAGES)[number]; + +/** Where a lead goes when it leaves the funnel, rather than a stage within it. */ +export const FUNNEL_EXITS = ['lost'] as const; +export type FunnelExit = (typeof FUNNEL_EXITS)[number]; + +export type FunnelPosition = FunnelStage | FunnelExit; + +/** + * What each stage means, in the words shown in the UI. + * + * Kept next to the mapping rather than in the component so the API, the digest + * email and the chart cannot describe the same stage three different ways. + */ +export const FUNNEL_STAGE_LABELS: Readonly> = { + discovered: 'Found', + researched: 'Researched', + ready: 'Ready to send', + contacted: 'Contacted', + replied: 'Replied', + opportunity: 'Opportunity', +}; + +export const FUNNEL_STAGE_DESCRIPTIONS: Readonly> = { + discovered: 'Named on a company page and pulled in.', + researched: 'Enriched, identities resolved, signals collected and scored.', + ready: 'A message is written and waiting — for approval, or for autopilot.', + contacted: 'The message went out.', + replied: 'They answered. This is the only part that needs you.', + opportunity: 'A real conversation is underway.', +}; + +const STAGE_BY_STATUS: Readonly> = { + discovered: 'discovered', + enriching: 'discovered', + resolved: 'discovered', + researching: 'researched', + qualified: 'researched', + // Not a loss — it means "researched, and not a fit". It stops here rather + // than progressing, and the drop between Researched and Ready is where that + // shows up, which is exactly what that number is for. + unqualified: 'researched', + recommended: 'ready', + awaiting_approval: 'ready', + approved: 'ready', + executed: 'contacted', + waiting: 'contacted', + responded: 'replied', + qualified_opportunity: 'opportunity', + not_interested: 'lost', + suppressed: 'lost', + // An errored prospect has not left the funnel; it is stuck at the point it + // reached. Calling it lost would quietly delete real leads from the chart. + error: 'discovered', +}; + +export function stageForStatus(status: ProspectStatus): FunnelPosition { + return STAGE_BY_STATUS[status]; +} + +export function isFunnelStage(value: unknown): value is FunnelStage { + return typeof value === 'string' && (FUNNEL_STAGES as readonly string[]).includes(value); +} + +/** Position in the sequence, or -1 for an exit. Used to order and compare. */ +export function stageIndex(position: FunnelPosition): number { + return (FUNNEL_STAGES as readonly string[]).indexOf(position); +} + +/** + * True when moving to `to` is forward progress through the funnel. + * + * Used to decide whether a stage change is worth telling anyone about: a lead + * sliding from `researching` back to `qualified` is bookkeeping, while one + * reaching `replied` is the whole point. + */ +export function isAdvance(from: FunnelPosition | undefined, to: FunnelPosition): boolean { + if (to === 'lost') return false; + if (from === undefined) return true; + return stageIndex(to) > stageIndex(from); +} + +export interface FunnelCount { + readonly stage: FunnelStage; + readonly label: string; + /** Leads currently sitting at this stage. */ + readonly current: number; + /** + * Leads that have *ever* reached this stage, including those now further on. + * + * This is the number a funnel chart is drawn from. Using `current` instead + * produces a chart where closing a deal makes the top of the funnel shrink, + * which is both wrong and the most common way these charts are got wrong. + */ + readonly reached: number; +} + +export interface FunnelSummary { + readonly stages: readonly FunnelCount[]; + readonly lost: number; + readonly total: number; +} + +/** + * Builds the funnel from per-stage totals. + * + * `reached` is made cumulative here rather than trusted from the caller: every + * lead that reached `contacted` necessarily reached `discovered`, whether or + * not an event was ever recorded for it, and a funnel that widens as it + * descends is a bug people will see instantly. + */ +export function summariseFunnel( + current: Readonly>>, + everReached: Readonly>> = {}, +): FunnelSummary { + const stages: FunnelCount[] = []; + + for (let index = FUNNEL_STAGES.length - 1; index >= 0; index -= 1) { + const stage = FUNNEL_STAGES[index] as FunnelStage; + const deeper = stages[0]?.reached ?? 0; + const reached = Math.max(everReached[stage] ?? 0, current[stage] ?? 0, deeper); + + stages.unshift({ + stage, + label: FUNNEL_STAGE_LABELS[stage], + current: current[stage] ?? 0, + reached, + }); + } + + return { + stages, + lost: current.lost ?? 0, + total: stages[0]?.reached ?? 0, + }; +} + +/** + * Conversion from one stage to the next, as a percentage. + * + * Returns undefined rather than 0 when the earlier stage is empty: "0% of 0 + * converted" is a statement about nothing, and showing it as a zero makes an + * empty funnel look like a failing one. + */ +export function conversionRate(from: FunnelCount, to: FunnelCount): number | undefined { + if (from.reached === 0) return undefined; + return Math.round((to.reached / from.reached) * 100); +} diff --git a/packages/domain/src/ids.ts b/packages/domain/src/ids.ts index 2e25850..119be40 100644 --- a/packages/domain/src/ids.ts +++ b/packages/domain/src/ids.ts @@ -36,6 +36,8 @@ export const ID_PREFIXES = { policyRule: 'pol', usageEvent: 'usg', auditEvent: 'aud', + stageEvent: 'stg', + notification: 'ntf', } as const; export type EntityKind = keyof typeof ID_PREFIXES; diff --git a/packages/domain/src/index.ts b/packages/domain/src/index.ts index 3d6407f..1711ad0 100644 --- a/packages/domain/src/index.ts +++ b/packages/domain/src/index.ts @@ -13,6 +13,8 @@ export * from './provenance'; export * from './person'; export * from './signal'; export * from './campaign'; +export * from './intake'; +export * from './funnel'; export * from './pipeline'; export * from './outreach'; export * from './video'; diff --git a/packages/domain/src/intake.test.ts b/packages/domain/src/intake.test.ts new file mode 100644 index 0000000..0a57772 --- /dev/null +++ b/packages/domain/src/intake.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, test } from 'bun:test'; +import { classifyIntake, toHostname } from './intake'; + +describe('toHostname', () => { + test('accepts the shapes people actually paste', () => { + expect(toHostname('acme.com')).toBe('acme.com'); + expect(toHostname('https://acme.com')).toBe('acme.com'); + expect(toHostname('https://www.acme.com/about?ref=1')).toBe('acme.com'); + expect(toHostname('acme.com/team')).toBe('acme.com'); + expect(toHostname(' ACME.COM ')).toBe('acme.com'); + expect(toHostname('shop.acme.co.uk')).toBe('shop.acme.co.uk'); + }); + + test('rejects things that are not hosts', () => { + expect(toHostname('')).toBeUndefined(); + expect(toHostname('acme')).toBeUndefined(); + expect(toHostname('dental practices in austin')).toBeUndefined(); + // A dot alone is not enough — a company name with a full stop is prose. + expect(toHostname('Acme Inc.')).toBeUndefined(); + expect(toHostname('mailto:jane@acme.com')).toBeUndefined(); + expect(toHostname('ftp://acme.com')).toBeUndefined(); + }); +}); + +describe('classifyIntake', () => { + test('a domain-shaped input is a url', () => { + const result = classifyIntake('https://www.acme.com/team'); + expect(result?.kind).toBe('url'); + expect(result?.value).toBe('acme.com'); + // The original survives for showing back to whoever typed it. + expect(result?.raw).toBe('https://www.acme.com/team'); + }); + + test('a description is a keyword', () => { + const result = classifyIntake('dental practices in Austin'); + expect(result?.kind).toBe('keyword'); + expect(result?.value).toBe('dental practices in Austin'); + }); + + test('a single word is a valid market, not an error', () => { + expect(classifyIntake('dentists')?.kind).toBe('keyword'); + }); + + test('nothing usable comes back undefined', () => { + expect(classifyIntake('')).toBeUndefined(); + expect(classifyIntake(' ')).toBeUndefined(); + expect(classifyIntake('12345')).toBeUndefined(); + }); + + test('a long phrase is capped rather than rejected', () => { + const result = classifyIntake('a'.repeat(500)); + expect(result?.kind).toBe('keyword'); + expect(result?.value.length).toBe(300); + }); +}); diff --git a/packages/domain/src/intake.ts b/packages/domain/src/intake.ts new file mode 100644 index 0000000..c07febc --- /dev/null +++ b/packages/domain/src/intake.ts @@ -0,0 +1,164 @@ +/** + * What the user typed, and which path it starts. + * + * One box takes both "acme.com" and "dental practices in Austin", so something + * has to decide which is which. That decision is deterministic and lives here + * rather than being handed to the model: asking an LLM to classify its own + * input adds a network call, a failure mode and a cost to a question that a + * dot and a space already answer. + */ + +/** A domain-shaped intake goes straight to the crawler; a phrase goes to discovery. */ +export type IntakeKind = 'url' | 'keyword'; + +export interface ClassifiedIntake { + readonly kind: IntakeKind; + /** + * For `url`, the bare hostname — lowercased, no scheme, no `www.`, no path. + * For `keyword`, the trimmed phrase as typed. + */ + readonly value: string; + /** The original input, untouched, for showing back to whoever typed it. */ + readonly raw: string; +} + +/** + * Two-letter suffixes are accepted wholesale as country codes. + * + * Listing every ccTLD would be a maintenance burden for no benefit: the cost + * of guessing wrong is one crawl of a host that does not resolve, which the + * queue already reports as a failed job with the reason attached. + */ +const KNOWN_TLDS = new Set([ + 'com', + 'net', + 'org', + 'edu', + 'gov', + 'mil', + 'int', + 'info', + 'biz', + 'io', + 'ai', + 'co', + 'dev', + 'app', + 'tech', + 'agency', + 'studio', + 'design', + 'digital', + 'online', + 'site', + 'shop', + 'store', + 'cloud', + 'health', + 'care', + 'clinic', + 'dental', + 'law', + 'legal', + 'finance', + 'capital', + 'ventures', + 'partners', + 'group', + 'company', + 'consulting', + 'services', + 'solutions', + 'systems', + 'software', + 'media', + 'news', + 'blog', + 'xyz', + 'me', + 'tv', + 'cc', + 'ltd', + 'llc', + 'inc', + 'plus', + 'life', + 'world', + 'global', +]); + +function looksLikeTld(suffix: string): boolean { + return suffix.length === 2 || KNOWN_TLDS.has(suffix); +} + +/** + * Reduces a URL or bare domain to its hostname, or returns nothing. + * + * Anything with a scheme is parsed properly; anything without one is only + * treated as a host when it is a single token that ends in a plausible TLD. + * "Acme Inc." has a dot and is not a domain, which is why the space matters. + */ +export function toHostname(input: string): string | undefined { + const trimmed = input.trim(); + if (!trimmed) return undefined; + + const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed); + + // A non-http scheme is not something the crawler can fetch, and `mailto:` in + // particular would otherwise slip through as a host. + if (withScheme && !/^https?:\/\//i.test(trimmed)) return undefined; + + let host: string; + + if (withScheme) { + try { + host = new URL(trimmed).hostname; + } catch { + return undefined; + } + } else { + // Allow a path on a bare domain — people paste `acme.com/about` constantly. + const head = trimmed.split(/[/?#]/)[0] ?? ''; + if (!head || /\s/.test(head)) return undefined; + host = head; + } + + host = host + .toLowerCase() + .replace(/^www\./, '') + .replace(/\.$/, ''); + + if (!host.includes('.')) return undefined; + if (!/^[a-z0-9.-]+$/.test(host)) return undefined; + if (host.startsWith('.') || host.startsWith('-') || host.includes('..')) return undefined; + + const labels = host.split('.'); + const suffix = labels[labels.length - 1] ?? ''; + if (!/^[a-z]+$/.test(suffix) || !looksLikeTld(suffix)) return undefined; + if (labels.some((label) => label.length === 0)) return undefined; + + return host; +} + +/** + * Decides whether an intake is a company to crawl or a market to search for. + * + * The tie-break is deliberate: when something is domain-shaped it is treated + * as a domain, because someone who types `acme.com` wanting a keyword search + * for the string "acme.com" does not exist, while the reverse mistake — going + * off to hallucinate companies named after a site the user already handed us — + * would be both slower and wrong. + */ +export function classifyIntake(raw: string): ClassifiedIntake | undefined { + const trimmed = raw.trim(); + if (!trimmed) return undefined; + + const host = toHostname(trimmed); + if (host) return { kind: 'url', value: host, raw: trimmed }; + + // Anything left is a description. A single word is still a valid market + // ("dentists"), so there is no minimum beyond having some letters in it. + if (!/[a-z]/i.test(trimmed)) return undefined; + + return { kind: 'keyword', value: trimmed.slice(0, 300), raw: trimmed }; +} diff --git a/packages/email/src/index.ts b/packages/email/src/index.ts index 2f414c7..9a34613 100644 --- a/packages/email/src/index.ts +++ b/packages/email/src/index.ts @@ -1,11 +1,35 @@ /** * Transactional email. * - * Only account mail goes through here — verification and password-adjacent - * notices. Outreach never does: that is the approval queue's job, and routing - * it through the same sender would make "we sent this on your behalf" and "we - * sent you a receipt" indistinguishable in a provider's logs. + * Three kinds of mail now leave the product, and they are kept distinct on + * purpose: + * + * - **Account mail** — verification and password-adjacent notices. + * - **Notifications** — lead alerts and the daily digest, sent to the + * customer about their own workspace. + * - **Outreach** — the drafted message, sent to a prospect on the customer's + * behalf when a campaign is on autopilot. + * + * The third used to be impossible: outreach ended at an approval card and a + * human copied it elsewhere. It goes through this boundary now, but it is the + * only one that carries a `replyTo` pointing at the customer, and the only one + * the policy engine gates — so "we sent this on your behalf" and "we sent you + * a receipt" stay distinguishable in a provider's logs and in the audit trail. */ -export { ConsoleMailer, ResendMailer, type Mailer, type Message } from './mailer'; +export { + ConsoleMailer, + MailerError, + ResendMailer, + type Mailer, + type Message, + type SendResult, +} from './mailer'; export { verificationEmail } from './templates'; +export { + dailyDigestEmail, + leadAlertEmail, + type DailyDigest, + type DigestLead, + type LeadAlert, +} from './notifications'; diff --git a/packages/email/src/mailer.ts b/packages/email/src/mailer.ts index 707cd56..d2683fa 100644 --- a/packages/email/src/mailer.ts +++ b/packages/email/src/mailer.ts @@ -11,10 +11,24 @@ export interface Message { readonly subject: string; readonly text: string; readonly html?: string; + /** + * Where a reply goes. + * + * Load-bearing for outreach rather than cosmetic: the product sends on the + * customer's behalf from its own verified domain, and if a reply came back + * here it would land in a mailbox nobody reads. The one thing the human is + * meant to do is answer, so the answer has to reach them. + */ + readonly replyTo?: string; +} + +export interface SendResult { + /** The provider's id for the message, when it gave one. */ + readonly id?: string; } export interface Mailer { - send(message: Message): Promise; + send(message: Message): Promise; } export interface ResendOptions { @@ -45,7 +59,7 @@ export class ResendMailer implements Mailer { this.#fetch = options.fetchImpl ?? fetch; } - async send(message: Message): Promise { + async send(message: Message): Promise { const response = await this.#fetch('https://api.resend.com/emails', { method: 'POST', headers: { @@ -58,6 +72,7 @@ export class ResendMailer implements Mailer { subject: message.subject, text: message.text, ...(message.html ? { html: message.html } : {}), + ...(message.replyTo ? { reply_to: [message.replyTo] } : {}), }), }); @@ -66,6 +81,11 @@ export class ResendMailer implements Mailer { // Losing it would make every failure look the same in the logs. throw new MailerError(response.status, (await response.text().catch(() => '')).slice(0, 500)); } + + // Kept so a delivered outreach message can be traced back to the provider + // from the action row. A body that does not parse is not a failed send. + const body = (await response.json().catch(() => undefined)) as { id?: string } | undefined; + return body?.id ? { id: body.id } : {}; } } @@ -83,7 +103,8 @@ export class ConsoleMailer implements Mailer { this.#log = log; } - async send(message: Message): Promise { + async send(message: Message): Promise { this.#log(`[email] to=${message.to} subject=${message.subject}\n${message.text}`); + return {}; } } diff --git a/packages/email/src/notifications.ts b/packages/email/src/notifications.ts new file mode 100644 index 0000000..4836a38 --- /dev/null +++ b/packages/email/src/notifications.ts @@ -0,0 +1,256 @@ +/** + * What the product tells you while it runs unattended. + * + * Autopilot means nobody is watching the screen, so the mail *is* the + * interface. Two kinds, and the split matters: + * + * - A **lead alert** is an interruption. It fires the moment someone worth + * talking to is found, and it has to be worth the interruption — one + * person, why they surfaced, and what was sent to them. + * - The **daily digest** is a summary. It is the thing you read instead of + * opening the app, so it has to be complete enough that not opening the + * app is a reasonable choice. + * + * Both are written to be readable with images off and links unclicked, because + * that is how most people will see them. + */ + +import type { Message } from './mailer'; + +function escapeHtml(value: string): string { + return value + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +/** Absolute, on the origin the links point at. Same derivation as the verify mail. */ +function markUrl(link: string): string | undefined { + try { + return new URL('/favicon.png', link).toString(); + } catch { + return undefined; + } +} + +function markHtml(appUrl: string): string { + const mark = markUrl(appUrl); + return mark + ? `

    OutreachGraph

    ' + : ''; +} + +/** The one-line "why you got this and how to stop" footer both mails carry. */ +function footer(appUrl: string): { text: string; html: string } { + const settings = `${appUrl.replace(/\/$/, '')}/settings`; + return { + text: `\n\n—\nYou are getting this because notifications are on for your workspace.\nChange or switch them off: ${settings}`, + html: + '
    ' + + '

    You are getting this because notifications are on ' + + `for your workspace. Change or switch them off.

    `, + }; +} + +export interface LeadAlert { + readonly personName: string; + readonly personId: string; + readonly title?: string; + readonly companyName?: string; + readonly companyDomain?: string; + readonly opportunity?: number; + /** Why this person surfaced — the recommendation's reason, in its own words. */ + readonly reason?: string; + /** The signals the draft was grounded in, one line each. */ + readonly evidence?: readonly string[]; + /** Set when autopilot already sent to them; absent when it is waiting on you. */ + readonly sentTo?: string; + readonly draftSubject?: string; + readonly draftBody?: string; +} + +/** + * The instant "we found someone" mail. + * + * The subject line carries the name and company because that is all most + * people will read on a phone, and a subject of "New lead" tells them nothing + * they can act on. + */ +export function leadAlertEmail(to: string, alert: LeadAlert, appUrl: string): Message { + const base = appUrl.replace(/\/$/, ''); + const link = `${base}/prospects/${alert.personId}`; + const where = [alert.title, alert.companyName].filter(Boolean).join(' at '); + + const subject = alert.companyName + ? `New lead: ${alert.personName} at ${alert.companyName}` + : `New lead: ${alert.personName}`; + + const status = alert.sentTo + ? `Autopilot has already written to them at ${alert.sentTo}. Replies come straight to you.` + : 'This one is waiting for your approval before anything is sent.'; + + const text = [ + `${alert.personName}${where ? ` — ${where}` : ''}`, + alert.companyDomain ? alert.companyDomain : '', + alert.opportunity !== undefined ? `Opportunity score: ${alert.opportunity}` : '', + '', + alert.reason ? `Why: ${alert.reason}` : '', + alert.evidence?.length + ? `\nBased on:\n${alert.evidence.map((e) => ` · ${e}`).join('\n')}` + : '', + '', + status, + '', + alert.draftSubject ? `Subject: ${alert.draftSubject}` : '', + alert.draftBody ? `\n${alert.draftBody}` : '', + '', + link, + ] + .filter((line) => line !== '') + .join('\n'); + + const foot = footer(base); + + const html = [ + markHtml(base), + `

    ${escapeHtml(alert.personName)}

    `, + where ? `

    ${escapeHtml(where)}

    ` : '', + alert.opportunity !== undefined + ? `

    Opportunity score ${alert.opportunity}

    ` + : '', + alert.reason ? `

    ${escapeHtml(alert.reason)}

    ` : '', + alert.evidence?.length + ? `
      ${alert.evidence.map((e) => `
    • ${escapeHtml(e)}
    • `).join('')}
    ` + : '', + `

    ${escapeHtml(status)}

    `, + alert.draftBody + ? '
    ' + + (alert.draftSubject + ? `

    ${escapeHtml(alert.draftSubject)}

    ` + : '') + + `

    ${escapeHtml(alert.draftBody)}

    ` + + '
    ' + : '', + `

    Open this prospect

    `, + foot.html, + ].join(''); + + return { to, subject: `${subject} · OutreachGraph`, text: text + foot.text, html }; +} + +export interface DigestLead { + readonly personName: string; + readonly personId: string; + readonly title?: string; + readonly companyName?: string; + readonly opportunity?: number; + readonly sentTo?: string; +} + +export interface DailyDigest { + /** The UTC date this digest covers, `YYYY-MM-DD`. */ + readonly date: string; + readonly sitesCrawled: number; + readonly peopleFound: number; + readonly messagesSent: number; + readonly awaitingApproval: number; + readonly repliesReceived?: number; + readonly leads: readonly DigestLead[]; + /** Campaigns that produced nothing, and why, when the reason is knowable. */ + readonly notes?: readonly string[]; +} + +/** + * The once-a-day summary. + * + * A digest for a day where nothing happened is still sent, and says so. The + * alternative — silence — is indistinguishable from the product being broken, + * which is the exact failure this whole change exists to fix. + */ +export function dailyDigestEmail(to: string, digest: DailyDigest, appUrl: string): Message { + const base = appUrl.replace(/\/$/, ''); + const quiet = digest.peopleFound === 0 && digest.messagesSent === 0; + + const subject = quiet + ? `Nothing new today · OutreachGraph` + : `${digest.peopleFound} new ${digest.peopleFound === 1 ? 'lead' : 'leads'}, ` + + `${digest.messagesSent} sent · OutreachGraph`; + + const counts = [ + `Sites read: ${digest.sitesCrawled}`, + `New people: ${digest.peopleFound}`, + `Messages sent: ${digest.messagesSent}`, + `Awaiting approval: ${digest.awaitingApproval}`, + digest.repliesReceived !== undefined ? `Replies: ${digest.repliesReceived}` : '', + ].filter(Boolean); + + const leadLines = digest.leads.map((lead) => { + const where = [lead.title, lead.companyName].filter(Boolean).join(' at '); + const score = lead.opportunity !== undefined ? ` [${lead.opportunity}]` : ''; + const sent = lead.sentTo ? ' — written to' : ''; + return ` · ${lead.personName}${where ? ` — ${where}` : ''}${score}${sent}`; + }); + + const foot = footer(base); + + const text = [ + `OutreachGraph · ${digest.date}`, + '', + ...counts, + '', + quiet + ? 'Nothing new came back today. Campaigns are still running.' + : leadLines.length + ? `Today's leads:\n${leadLines.join('\n')}` + : '', + digest.notes?.length ? `\nNotes:\n${digest.notes.map((n) => ` · ${n}`).join('\n')}` : '', + '', + `${base}/today`, + ] + .filter((line) => line !== '') + .join('\n'); + + const html = [ + markHtml(base), + `

    ${escapeHtml(digest.date)}

    `, + '', + row('Sites read', digest.sitesCrawled), + row('New people', digest.peopleFound), + row('Messages sent', digest.messagesSent), + row('Awaiting approval', digest.awaitingApproval), + digest.repliesReceived !== undefined ? row('Replies', digest.repliesReceived) : '', + '
    ', + quiet + ? '

    Nothing new came back today. Campaigns are still running.

    ' + : digest.leads.length + ? `
      ${digest.leads + .map((lead) => { + const where = [lead.title, lead.companyName].filter(Boolean).join(' at '); + const score = lead.opportunity !== undefined ? ` [${lead.opportunity}]` : ''; + const sent = lead.sentTo ? ' — written to' : ''; + return ( + `
    • ` + + `${escapeHtml(lead.personName)}` + + `${where ? ` — ${escapeHtml(where)}` : ''}${score}${sent}
    • ` + ); + }) + .join('')}
    ` + : '', + digest.notes?.length + ? `

    ${digest.notes.map((n) => escapeHtml(n)).join('
    ')}

    ` + : '', + `

    Open OutreachGraph

    `, + foot.html, + ].join(''); + + return { to, subject, text: text + foot.text, html }; +} + +function row(label: string, value: number): string { + return ( + `${escapeHtml(label)}` + + `${value}` + ); +} diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index c3bd4b0..c13066e 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -14,7 +14,9 @@ "@outreachgraph/ai": "workspace:*", "@outreachgraph/db": "workspace:*", "@outreachgraph/domain": "workspace:*", + "@outreachgraph/email": "workspace:*", "@outreachgraph/identity": "workspace:*", + "@outreachgraph/policy": "workspace:*", "@outreachgraph/providers": "workspace:*", "@outreachgraph/recommend": "workspace:*", "@outreachgraph/scoring": "workspace:*", diff --git a/packages/pipeline/src/analytics.ts b/packages/pipeline/src/analytics.ts new file mode 100644 index 0000000..eecbda0 --- /dev/null +++ b/packages/pipeline/src/analytics.ts @@ -0,0 +1,291 @@ +/** + * The numbers behind the funnel. + * + * Two shapes, because sales people look at two things and they are not the + * same question: + * + * - **The funnel** — how many leads reached each stage, and where they fall + * out. Aggregate, one row per stage, read left to right. + * - **The timeline** — one lead's journey, stage by stage, with dates. This + * is the "sideways chart" view: each lead a row, each stage a segment, + * length showing how long it sat there. + * + * Both read `lead_stage_events`, which is why that table exists. Neither is + * answerable from `campaign_people.status` alone, because that column keeps + * only the present. + */ + +import { + FUNNEL_STAGES, + isFunnelStage, + stageForStatus, + summariseFunnel, + type FunnelPosition, + type FunnelStage, + type FunnelSummary, + type ProspectStatus, +} from '@outreachgraph/domain'; +import { queryAll, queryOne, type Client } from '@outreachgraph/db'; + +export interface FunnelQuery { + readonly workspaceId: string; + /** Narrow to one campaign. Omit for the whole workspace. */ + readonly campaignId?: string; + /** ISO instant; only events at or after this count toward `reached`. */ + readonly since?: string; +} + +/** + * Current occupancy and historical reach, per stage. + * + * `current` comes from the live status column and `reached` from the event + * log, which is the only combination that produces an honest funnel: a lead + * that has been contacted is no longer *at* "Researched", but it certainly + * reached it, and a chart built from current occupancy alone would show the + * top of the funnel emptying as things went well. + */ +export async function campaignFunnel(db: Client, query: FunnelQuery): Promise { + const scope = query.campaignId ? 'AND cp.campaign_id = ?' : ''; + const scopeArgs = query.campaignId ? [query.campaignId] : []; + + const currentRows = await queryAll<{ status: string; n: number }>( + db, + `SELECT cp.status AS status, COUNT(*) AS n + FROM campaign_people cp + JOIN people p ON p.id = cp.person_id + WHERE cp.workspace_id = ? ${scope} + AND p.status != 'deleted' + GROUP BY cp.status`, + [query.workspaceId, ...scopeArgs], + ); + + const current: Partial> = {}; + for (const row of currentRows) { + const stage = stageForStatus(row.status as ProspectStatus); + if (!stage) continue; + current[stage] = (current[stage] ?? 0) + row.n; + } + + const eventScope = query.campaignId ? 'AND e.campaign_id = ?' : ''; + const sinceScope = query.since ? 'AND e.occurred_at >= ?' : ''; + + const reachedRows = await queryAll<{ stage: string; n: number }>( + db, + `SELECT e.stage AS stage, COUNT(DISTINCT e.person_id) AS n + FROM lead_stage_events e + WHERE e.workspace_id = ? ${eventScope} ${sinceScope} + GROUP BY e.stage`, + [query.workspaceId, ...scopeArgs, ...(query.since ? [query.since] : [])], + ); + + const reached: Partial> = {}; + for (const row of reachedRows) { + if (isFunnelStage(row.stage)) reached[row.stage] = row.n; + } + + return summariseFunnel(current, reached); +} + +export interface TimelineSegment { + readonly stage: FunnelPosition; + readonly enteredAt: string; + /** Absent while the lead is still sitting here. */ + readonly leftAt?: string; + readonly hours?: number; +} + +export interface LeadTimeline { + readonly personId: string; + readonly personName: string; + readonly companyName?: string; + readonly currentStage: FunnelPosition; + readonly opportunity?: number; + readonly firstSeenAt: string; + readonly segments: readonly TimelineSegment[]; +} + +/** + * One row per lead, each a sequence of stage segments. + * + * Consecutive events in the same stage are collapsed: the internal state + * machine moves `discovered → enriching → resolved` and all three are the + * single funnel stage "Found", so drawing them as three segments would show + * detail that means nothing to the reader. + */ +export async function leadTimeline( + db: Client, + query: FunnelQuery & { limit?: number }, +): Promise { + const scope = query.campaignId ? 'AND e.campaign_id = ?' : ''; + const limit = Math.min(Math.max(query.limit ?? 50, 1), 200); + + const rows = await queryAll<{ + person_id: string; + display_name: string; + company_name: string | null; + stage: string; + to_status: string; + occurred_at: string; + opportunity: number | null; + }>( + db, + `SELECT e.person_id, p.display_name, co.name AS company_name, + e.stage, e.to_status, e.occurred_at, s.opportunity + FROM lead_stage_events e + JOIN people p ON p.id = e.person_id + LEFT JOIN companies co ON co.id = p.current_company_id + LEFT JOIN scores s ON s.person_id = e.person_id AND s.campaign_id = e.campaign_id + WHERE e.workspace_id = ? ${scope} + AND p.status != 'deleted' + AND e.person_id IN ( + SELECT person_id FROM lead_stage_events + WHERE workspace_id = ? ${query.campaignId ? 'AND campaign_id = ?' : ''} + GROUP BY person_id + ORDER BY MAX(occurred_at) DESC + LIMIT ? + ) + ORDER BY e.person_id ASC, e.occurred_at ASC`, + [ + query.workspaceId, + ...(query.campaignId ? [query.campaignId] : []), + query.workspaceId, + ...(query.campaignId ? [query.campaignId] : []), + limit, + ], + ); + + const byPerson = new Map(); + + for (const row of rows) { + const stage = ( + isFunnelStage(row.stage) ? row.stage : stageForStatus(row.to_status as ProspectStatus) + ) as FunnelPosition; + + let entry = byPerson.get(row.person_id); + + if (!entry) { + entry = { + personId: row.person_id, + personName: row.display_name, + ...(row.company_name ? { companyName: row.company_name } : {}), + currentStage: stage, + ...(row.opportunity !== null ? { opportunity: Math.round(row.opportunity) } : {}), + firstSeenAt: row.occurred_at, + segments: [], + }; + byPerson.set(row.person_id, entry); + } + + const last = entry.segments[entry.segments.length - 1]; + + // Same funnel stage as the previous event: the lead did not visibly move, + // so extend rather than start a new segment. + if (last && last.stage === stage) continue; + + if (last) { + const hours = (Date.parse(row.occurred_at) - Date.parse(last.enteredAt)) / 3_600_000; + entry.segments[entry.segments.length - 1] = { + ...last, + leftAt: row.occurred_at, + ...(Number.isFinite(hours) ? { hours: Math.round(hours * 10) / 10 } : {}), + }; + } + + entry.segments.push({ stage, enteredAt: row.occurred_at }); + (entry as { currentStage: FunnelPosition }).currentStage = stage; + } + + return [...byPerson.values()].sort( + (a, b) => Date.parse(b.firstSeenAt) - Date.parse(a.firstSeenAt), + ); +} + +export interface WorkspaceAnalytics { + readonly funnel: FunnelSummary; + readonly sentThisWeek: number; + readonly repliesThisWeek: number; + readonly awaitingApproval: number; + readonly activeCampaigns: number; + readonly autopilotCampaigns: number; + /** Median hours from a lead being found to being written to. */ + readonly medianHoursToContact?: number; +} + +/** The headline numbers, for the top of the funnel page. */ +export async function workspaceAnalytics( + db: Client, + workspaceId: string, + at: Date = new Date(), +): Promise { + const weekAgo = new Date(at.getTime() - 7 * 24 * 3_600_000).toISOString(); + + const funnel = await campaignFunnel(db, { workspaceId }); + + const [sent, replies, awaiting, campaigns, speeds] = await Promise.all([ + count( + db, + `SELECT COUNT(*) AS n FROM actions + WHERE workspace_id = ? AND status = 'completed' AND executed_at >= ?`, + [workspaceId, weekAgo], + ), + count( + db, + `SELECT COUNT(*) AS n FROM interactions + WHERE workspace_id = ? AND direction = 'inbound' AND occurred_at >= ?`, + [workspaceId, weekAgo], + ), + count( + db, + `SELECT COUNT(*) AS n FROM recommendations WHERE workspace_id = ? AND status = 'pending'`, + [workspaceId], + ), + queryAll<{ approval_mode: string; n: number }>( + db, + `SELECT approval_mode, COUNT(*) AS n FROM campaigns + WHERE workspace_id = ? AND status != 'archived' GROUP BY approval_mode`, + [workspaceId], + ), + queryAll<{ hours: number }>( + db, + `SELECT (julianday(contacted.occurred_at) - julianday(found.occurred_at)) * 24 AS hours + FROM (SELECT person_id, MIN(occurred_at) AS occurred_at FROM lead_stage_events + WHERE workspace_id = ? AND stage = 'contacted' GROUP BY person_id) contacted + JOIN (SELECT person_id, MIN(occurred_at) AS occurred_at FROM lead_stage_events + WHERE workspace_id = ? AND stage = 'discovered' GROUP BY person_id) found + ON found.person_id = contacted.person_id`, + [workspaceId, workspaceId], + ), + ]); + + const active = campaigns.reduce((total, row) => total + row.n, 0); + const autopilot = campaigns + .filter((row) => row.approval_mode === 'trusted_automation') + .reduce((total, row) => total + row.n, 0); + + const hours = speeds + .map((row) => row.hours) + .filter((value) => Number.isFinite(value) && value >= 0) + .sort((a, b) => a - b); + + const median = hours.length + ? Math.round((hours[Math.floor(hours.length / 2)] as number) * 10) / 10 + : undefined; + + return { + funnel, + sentThisWeek: sent, + repliesThisWeek: replies, + awaitingApproval: awaiting, + activeCampaigns: active, + autopilotCampaigns: autopilot, + ...(median !== undefined ? { medianHoursToContact: median } : {}), + }; +} + +async function count(db: Client, sql: string, args: unknown[]): Promise { + const row = await queryOne<{ n: number }>(db, sql, args as never); + return row?.n ?? 0; +} + +/** Re-exported so callers can label a chart without importing two packages. */ +export { FUNNEL_STAGES }; diff --git a/packages/pipeline/src/autopilot.test.ts b/packages/pipeline/src/autopilot.test.ts new file mode 100644 index 0000000..cb47c62 --- /dev/null +++ b/packages/pipeline/src/autopilot.test.ts @@ -0,0 +1,297 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { now, queryAll, queryOne, type Client } from '@outreachgraph/db'; +import type { Mailer, Message, SendResult } from '@outreachgraph/email'; +import { seedDatabase, SEED, type SeededDatabase } from '../../../apps/api/src/test-seed'; +import { runAutopilot } from './autopilot'; + +let seeded: SeededDatabase | undefined; + +afterEach(() => { + seeded?.cleanup(); + seeded = undefined; +}); + +/** Collects what would have been sent, instead of sending it. */ +function recordingMailer(): { sent: Message[]; mailer: Mailer } { + const sent: Message[] = []; + return { + sent, + mailer: { + send: async (message): Promise => { + sent.push(message); + return { id: 'resend_1' }; + }, + }, + }; +} + +function failingMailer(): Mailer { + return { + send: async (): Promise => { + throw new Error('sending domain is not verified'); + }, + }; +} + +/** + * Puts the fixture into the state autopilot acts on: a trusted-automation + * campaign with a pending `email/send_email` recommendation, a draft, and an + * address to send it to. + */ +async function makeSendable( + db: Client, + options: { personEmail?: string; companyEmail?: string; autopilot?: boolean } = {}, +): Promise { + const stamp = now(); + + await db.execute({ + sql: `UPDATE campaigns SET approval_mode = ? WHERE id = ?`, + args: [ + options.autopilot === false ? 'draft_and_approve' : 'trusted_automation', + SEED.campaignId, + ], + }); + + await db.execute({ + sql: `UPDATE recommendations SET action = 'send_email', network = 'email' WHERE id = ?`, + args: [SEED.recommendationId], + }); + + if (options.personEmail) { + await db.execute({ + sql: `INSERT INTO social_identities (id, person_id, network, handle, platform_user_id, + confidence, source_type, verified_by, first_seen_at) + VALUES ('sid_jane_email', ?, 'email', ?, ?, 0.88, 'public_web', '[]', ?)`, + args: [SEED.personId, options.personEmail, options.personEmail, stamp], + }); + } + + if (options.companyEmail) { + await db.execute({ + sql: `UPDATE companies SET contact_email = ? WHERE id = ?`, + args: [options.companyEmail, SEED.companyId], + }); + } +} + +describe('runAutopilot', () => { + test('sends a drafted message and records the whole trail', async () => { + seeded = await seedDatabase('autopilot-send'); + const { db } = seeded; + await makeSendable(db, { personEmail: 'jane@acme.com' }); + + const { sent, mailer } = recordingMailer(); + const result = await runAutopilot({ db, mailer, replyTo: 'me@mine.com' }, SEED.workspaceId); + + expect(result.sent).toHaveLength(1); + expect(sent).toHaveLength(1); + expect(sent[0]?.to).toBe('jane@acme.com'); + // The one thing the human does is reply, so replies must reach them. + expect(sent[0]?.replyTo).toBe('me@mine.com'); + expect(sent[0]?.text).toContain('cross-border settlement'); + + const action = await queryOne<{ status: string; external_id: string | null; mode: string }>( + db, + `SELECT status, external_id, mode FROM actions WHERE workspace_id = ?`, + [SEED.workspaceId], + ); + expect(action?.status).toBe('completed'); + expect(action?.external_id).toBe('resend_1'); + + const recommendation = await queryOne<{ status: string }>( + db, + `SELECT status FROM recommendations WHERE id = ?`, + [SEED.recommendationId], + ); + expect(recommendation?.status).toBe('executed'); + + const interaction = await queryOne<{ direction: string; state: string }>( + db, + `SELECT direction, state FROM interactions WHERE workspace_id = ?`, + [SEED.workspaceId], + ); + expect(interaction?.direction).toBe('outbound'); + + // The funnel has to see this, or the chart says nobody was ever contacted. + const stage = await queryOne<{ stage: string }>( + db, + `SELECT stage FROM lead_stage_events WHERE person_id = ? ORDER BY occurred_at DESC LIMIT 1`, + [SEED.personId], + ); + expect(stage?.stage).toBe('contacted'); + + const audit = await queryOne<{ event_type: string }>( + db, + `SELECT event_type FROM audit_events WHERE workspace_id = ? AND actor_id = 'autopilot'`, + [SEED.workspaceId], + ); + expect(audit?.event_type).toBe('action.executed'); + }); + + test('a campaign not on autopilot is never sent for', async () => { + seeded = await seedDatabase('autopilot-off'); + const { db } = seeded; + await makeSendable(db, { personEmail: 'jane@acme.com', autopilot: false }); + + const { sent, mailer } = recordingMailer(); + const result = await runAutopilot({ db, mailer }, SEED.workspaceId); + + expect(sent).toHaveLength(0); + expect(result.sent).toHaveLength(0); + }); + + test('falls back to the company inbox and marks it shared', async () => { + seeded = await seedDatabase('autopilot-shared'); + const { db } = seeded; + await makeSendable(db, { companyEmail: 'info@acme.com' }); + + const { sent, mailer } = recordingMailer(); + const result = await runAutopilot({ db, mailer }, SEED.workspaceId); + + expect(sent[0]?.to).toBe('info@acme.com'); + expect(result.sent[0]?.toSharedInbox).toBe(true); + }); + + test('a person with no address anywhere is skipped, not guessed at', async () => { + seeded = await seedDatabase('autopilot-noaddress'); + const { db } = seeded; + await makeSendable(db); + + const { sent, mailer } = recordingMailer(); + const result = await runAutopilot({ db, mailer }, SEED.workspaceId); + + expect(sent).toHaveLength(0); + expect(result.skipped[0]?.reason).toContain('no address'); + }); + + test('a suppressed person is not written to even with a pending recommendation', async () => { + seeded = await seedDatabase('autopilot-suppressed'); + const { db } = seeded; + await makeSendable(db, { personEmail: 'jane@acme.com' }); + + // The stored policy snapshot still says the send was fine. Live state is + // what decides, which is the whole point of re-checking here. + await db.execute({ + sql: `UPDATE people SET status = 'suppressed' WHERE id = ?`, + args: [SEED.personId], + }); + + const { sent, mailer } = recordingMailer(); + const result = await runAutopilot({ db, mailer }, SEED.workspaceId); + + expect(sent).toHaveLength(0); + expect(result.sent).toHaveLength(0); + }); + + test('the daily cap stops sending', async () => { + seeded = await seedDatabase('autopilot-cap'); + const { db } = seeded; + await makeSendable(db, { personEmail: 'jane@acme.com' }); + + const stamp = now(); + await db.execute({ + sql: `INSERT INTO workspace_settings (workspace_id, autopilot_daily_cap, created_at, updated_at) + VALUES (?, 0, ?, ?)`, + args: [SEED.workspaceId, stamp, stamp], + }); + + const { sent, mailer } = recordingMailer(); + await runAutopilot({ db, mailer }, SEED.workspaceId); + + expect(sent).toHaveLength(0); + }); + + test('a draft that failed its quality checks is never machine-sent', async () => { + seeded = await seedDatabase('autopilot-checks'); + const { db } = seeded; + await makeSendable(db, { personEmail: 'jane@acme.com' }); + + await db.execute({ + sql: `UPDATE drafts SET checks_json = ? WHERE id = ?`, + args: [JSON.stringify([{ name: 'grounding', passed: false }]), SEED.draftId], + }); + + const { sent, mailer } = recordingMailer(); + const result = await runAutopilot({ db, mailer }, SEED.workspaceId); + + expect(sent).toHaveLength(0); + expect(result.skipped[0]?.reason).toContain('quality'); + }); + + test('a failed send keeps the lead for the next tick', async () => { + seeded = await seedDatabase('autopilot-failure'); + const { db } = seeded; + await makeSendable(db, { personEmail: 'jane@acme.com' }); + + const result = await runAutopilot({ db, mailer: failingMailer() }, SEED.workspaceId); + + expect(result.failed).toBe(1); + expect(result.sent).toHaveLength(0); + + const action = await queryOne<{ status: string; error: string | null }>( + db, + `SELECT status, error FROM actions WHERE workspace_id = ?`, + [SEED.workspaceId], + ); + expect(action?.status).toBe('failed'); + expect(action?.error).toContain('not verified'); + + // Still pending, so the retry actually happens. + const recommendation = await queryOne<{ status: string }>( + db, + `SELECT status FROM recommendations WHERE id = ?`, + [SEED.recommendationId], + ); + expect(recommendation?.status).toBe('pending'); + }); + + test('a permanently failing send is eventually given up on', async () => { + seeded = await seedDatabase('autopilot-giveup'); + const { db } = seeded; + await makeSendable(db, { personEmail: 'jane@acme.com' }); + + const mailer = failingMailer(); + + // An invalid API key fails identically every time. Retrying it forever + // means a row and a provider request per tick, indefinitely. + for (let attempt = 0; attempt < 5; attempt += 1) { + await runAutopilot({ db, mailer }, SEED.workspaceId); + } + + const rows = await queryAll<{ id: string }>( + db, + `SELECT id FROM actions WHERE workspace_id = ? AND status = 'failed'`, + [SEED.workspaceId], + ); + expect(rows.length).toBe(3); + + const last = await runAutopilot({ db, mailer }, SEED.workspaceId); + expect(last.skipped[0]?.reason).toContain('giving up'); + }); + + test('with no mailer nothing is sent and the reason is reported', async () => { + seeded = await seedDatabase('autopilot-nomailer'); + const { db } = seeded; + await makeSendable(db, { personEmail: 'jane@acme.com' }); + + const result = await runAutopilot({ db }, SEED.workspaceId); + + expect(result.sent).toHaveLength(0); + // The policy engine catches this before the send does, and says it better: + // no mailer means no connected email account, which downgrades the action + // to manual rather than merely failing to deliver it. + expect(result.skipped[0]?.reason).toContain('No connected email account'); + }); + + test('the same recommendation is not sent twice', async () => { + seeded = await seedDatabase('autopilot-once'); + const { db } = seeded; + await makeSendable(db, { personEmail: 'jane@acme.com' }); + + const { sent, mailer } = recordingMailer(); + await runAutopilot({ db, mailer }, SEED.workspaceId); + await runAutopilot({ db, mailer }, SEED.workspaceId); + + expect(sent).toHaveLength(1); + }); +}); diff --git a/packages/pipeline/src/autopilot.ts b/packages/pipeline/src/autopilot.ts new file mode 100644 index 0000000..628b11d --- /dev/null +++ b/packages/pipeline/src/autopilot.ts @@ -0,0 +1,492 @@ +/** + * Unattended sending (PRD §7.6 `trusted_automation`, §16, §27). + * + * Everything upstream of this already worked: a URL became a company, people, + * signals, a score, a recommendation and a drafted message. Then it stopped, + * because the only way a message could leave the product was a human copying + * it into their own mail client. `POST /actions/:id/execute` did not send + * anything — it recorded that a send had happened elsewhere. + * + * This is the executor. It is the only place outreach is put on the wire, and + * it holds to the two rules that make that safe: + * + * - **Policy is re-checked here, from live state.** The decision stored on + * the recommendation is a snapshot from generation time and is never + * trusted. Suppression, deletion, rate limits and identity confidence are + * all re-read; a campaign switched off autopilot ten seconds ago does not + * send. + * - **Autopilot is opt-in per campaign and gated per capability.** The + * policy engine only skips approval for `trusted_automation` *and* a + * capability the matrix marks `official_api` or `customer_managed`. Email + * is customer-managed. Nothing else currently qualifies, which is why this + * sends email and nothing else. + * + * A failed send is recorded on the action and the recommendation is left where + * it was, so the next tick retries rather than silently dropping a lead. + */ + +import { newId, type ActionKind, type Network, type ProspectStatus } from '@outreachgraph/domain'; +import { now, queryAll, queryOne, type Client } from '@outreachgraph/db'; +import { evaluatePolicy, isExecutable } from '@outreachgraph/policy'; +import type { Mailer } from '@outreachgraph/email'; +import { recordStatus } from './stages'; + +export interface AutopilotDeps { + readonly db: Client; + /** Omit and nothing sends — the queue still runs and still reports why. */ + readonly mailer?: Mailer; + /** Reply-to for outbound mail. Falls back to the workspace owner. */ + readonly replyTo?: string; + readonly now?: Date; +} + +export interface SentOutreach { + readonly workspaceId: string; + readonly campaignId: string; + readonly personId: string; + readonly personName: string; + readonly recommendationId: string; + readonly actionId: string; + readonly to: string; + readonly subject: string; + readonly body: string; + readonly companyName?: string; + /** True when the address is a shared company inbox rather than the person's. */ + readonly toSharedInbox: boolean; +} + +export interface SkippedOutreach { + readonly recommendationId: string; + readonly personName: string; + readonly reason: string; +} + +export interface AutopilotResult { + readonly sent: readonly SentOutreach[]; + readonly skipped: readonly SkippedOutreach[]; + readonly failed: number; +} + +interface Candidate { + readonly recommendation_id: string; + readonly campaign_id: string; + readonly person_id: string; + readonly action: string; + readonly network: string; + readonly approval_mode: string; + readonly budget_json: string; + readonly display_name: string; + readonly person_status: string; + readonly believed_minor: number; + readonly outreach_eligible: number; + readonly identity_confidence: number; + readonly min_outreach_confidence: number; + readonly draft_id: string | null; + readonly subject: string | null; + readonly body: string | null; + readonly checks_json: string | null; + readonly company_name: string | null; + readonly company_contact_email: string | null; + readonly person_email: string | null; + readonly failed_attempts: number; +} + +/** Attempts before a recommendation is left alone for a human to look at. */ +const MAX_SEND_ATTEMPTS = 3; + +/** + * Sends everything due for one workspace. + * + * Ordered by priority so a daily cap spends itself on the best leads rather + * than on whichever rows the planner happened to return first. + */ +export async function runAutopilot( + deps: AutopilotDeps, + workspaceId: string, +): Promise { + const { db } = deps; + const at = deps.now ?? new Date(); + + const sent: SentOutreach[] = []; + const skipped: SkippedOutreach[] = []; + let failed = 0; + + const settings = await loadSettings(db, workspaceId); + const cap = settings.autopilot_daily_cap; + + let today = await countActionsToday(db, workspaceId, at); + if (today >= cap) { + return { sent, skipped, failed }; + } + + const candidates = await queryAll( + db, + `SELECT r.id AS recommendation_id, r.campaign_id, r.person_id, r.action, r.network, + c.approval_mode, c.budget_json, + p.display_name, p.status AS person_status, p.believed_minor, + p.outreach_eligible, p.identity_confidence, + w.min_outreach_confidence, + d.id AS draft_id, d.subject, d.body, d.checks_json, + co.name AS company_name, co.contact_email AS company_contact_email, + (SELECT si.handle FROM social_identities si + WHERE si.person_id = p.id AND si.network = 'email' + ORDER BY si.confidence DESC LIMIT 1) AS person_email, + (SELECT COUNT(*) FROM actions a + WHERE a.recommendation_id = r.id AND a.status = 'failed') AS failed_attempts + FROM recommendations r + JOIN campaigns c ON c.id = r.campaign_id + JOIN people p ON p.id = r.person_id + JOIN workspaces w ON w.id = r.workspace_id + LEFT JOIN drafts d ON d.recommendation_id = r.id + LEFT JOIN companies co ON co.id = p.current_company_id + WHERE r.workspace_id = ? + AND r.status = 'pending' + AND c.approval_mode = 'trusted_automation' + AND c.status != 'archived' + AND r.action = 'send_email' + AND r.network = 'email' + AND p.status = 'active' + ORDER BY r.priority DESC, r.created_at ASC + LIMIT ?`, + [workspaceId, Math.max(cap - today, 0)], + ); + + for (const row of candidates) { + if (today >= cap) break; + + const note = (reason: string): void => { + skipped.push({ + recommendationId: row.recommendation_id, + personName: row.display_name, + reason, + }); + }; + + // Give up after enough failed attempts. + // + // A failed send leaves the recommendation pending so the next tick retries, + // which is right for a rate limit or a blip and wrong for anything + // permanent. An invalid API key produced three attempts per recommendation + // within a minute of testing and would have produced thousands a day — + // each one a row, and each one a request to a provider that has already + // said no. The lead stays pending for a human rather than being deleted. + if (row.failed_attempts >= MAX_SEND_ATTEMPTS) { + note(`giving up after ${row.failed_attempts} failed sends`); + continue; + } + + // A recommendation with no draft has nothing to send. This is not an + // error: the composer is allowed to produce nothing rather than invent a + // claim it cannot ground, and that outcome must stay a silent no-send. + if (!row.body || !row.body.trim()) { + note('no drafted message'); + continue; + } + + // The quality gates already ran when the draft was written. A draft that + // failed one is never shown to a human, so it must never be sent by a + // machine either. + if (hasFailingCheck(row.checks_json)) { + note('the draft did not pass its quality checks'); + continue; + } + + const recipient = pickRecipient(row); + if (!recipient) { + note('no address published for this person or their company'); + continue; + } + + // ------------------------------------------------------------- policy + // + // Re-evaluated from live rows, never from the stored snapshot. + const counts = await actionCounts(db, workspaceId, row.person_id, at); + const budget = safeJson(row.budget_json); + + const decision = evaluatePolicy({ + network: row.network as Network, + action: row.action as ActionKind, + approvalMode: row.approval_mode as 'trusted_automation', + hasConnectedAccount: deps.mailer !== undefined, + personSuppressed: row.person_status === 'suppressed' || row.outreach_eligible === 0, + personBelievedMinor: row.believed_minor === 1, + personDeleted: row.person_status === 'deleted', + identityConfidence: row.identity_confidence, + minIdentityConfidence: row.min_outreach_confidence, + actionsToday: today, + maxActionsPerDay: Math.min(numberOr(budget.maxActionsPerDay, 50), cap), + actionsToThisProspectThisWeek: counts.thisProspect, + maxActionsPerProspectPerWeek: numberOr(budget.maxActionsPerProspectPerWeek, 1), + ...(counts.hoursSinceLast !== undefined + ? { hoursSinceLastActionToProspect: counts.hoursSinceLast } + : {}), + }); + + // `approved: false` is the whole point. Autopilot holds no approval, so + // only a decision of plain `allow` gets through — `allow_with_approval` + // means a human still has to look at it, and reaching here with that would + // mean the capability matrix no longer marks email customer-managed. + // Sending anyway would be exactly what the approval default prevents. + if (!isExecutable(decision.decision, false)) { + note( + decision.decision === 'allow_with_approval' + ? 'this action still requires human approval' + : decision.reason, + ); + continue; + } + + if (!deps.mailer) { + note('no mailer is configured, so nothing can be sent'); + continue; + } + + // --------------------------------------------------------------- send + const actionId = newId('action'); + const stamp = now(); + const subject = row.subject?.trim() || defaultSubject(row); + + await db.execute({ + sql: `INSERT INTO actions (id, workspace_id, recommendation_id, person_id, kind, network, + mode, status, body, created_at) + VALUES (?, ?, ?, ?, 'send_email', 'email', 'customer_managed', 'queued', ?, ?)`, + args: [actionId, workspaceId, row.recommendation_id, row.person_id, row.body, stamp], + }); + + const replyTo = deps.replyTo ?? settings.reply_to_email ?? undefined; + + try { + const result = await deps.mailer.send({ + to: recipient.address, + subject, + text: row.body, + ...(replyTo ? { replyTo } : {}), + }); + + const executedAt = now(); + + await db.execute({ + sql: `UPDATE actions SET status = 'completed', external_id = ?, executed_at = ? + WHERE id = ?`, + args: [result.id ?? null, executedAt, actionId], + }); + + await db.execute({ + sql: `INSERT INTO interactions (id, workspace_id, person_id, campaign_id, action_id, + network, direction, state, body, occurred_at, recorded_at) + VALUES (?, ?, ?, ?, ?, 'email', 'outbound', 'contacted', ?, ?, ?)`, + args: [ + newId('interaction'), + workspaceId, + row.person_id, + row.campaign_id, + actionId, + row.body, + executedAt, + executedAt, + ], + }); + + await db.execute({ + sql: `UPDATE recommendations SET status = 'executed' WHERE id = ?`, + args: [row.recommendation_id], + }); + + await db.execute({ + sql: `UPDATE campaign_people SET interaction_state = 'contacted', last_actioned_at = ? + WHERE campaign_id = ? AND person_id = ?`, + args: [executedAt, row.campaign_id, row.person_id], + }); + + await recordStatus(db, { + workspaceId, + campaignId: row.campaign_id, + personId: row.person_id, + status: 'executed' satisfies ProspectStatus, + reason: `autopilot emailed ${recipient.address}`, + at: executedAt, + }); + + await audit(db, workspaceId, actionId, { + eventType: 'action.executed', + detail: { + mode: 'autopilot', + to: recipient.address, + sharedInbox: recipient.shared, + policyVersion: decision.policyVersion, + }, + }); + + sent.push({ + workspaceId, + campaignId: row.campaign_id, + personId: row.person_id, + personName: row.display_name, + recommendationId: row.recommendation_id, + actionId, + to: recipient.address, + subject, + body: row.body, + ...(row.company_name ? { companyName: row.company_name } : {}), + toSharedInbox: recipient.shared, + }); + + today += 1; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + failed += 1; + + // Left as `failed` with the reason attached, and the recommendation left + // pending. A bounced domain or a rate-limited provider is worth another + // attempt on a later tick; losing the lead over it is not. + await db.execute({ + sql: `UPDATE actions SET status = 'failed', error = ? WHERE id = ?`, + args: [message.slice(0, 500), actionId], + }); + + await audit(db, workspaceId, actionId, { + eventType: 'action.send_failed', + detail: { to: recipient.address, error: message.slice(0, 500) }, + }); + } + } + + return { sent, skipped, failed }; +} + +interface Recipient { + readonly address: string; + /** True when this is a company inbox, not the person's own address. */ + readonly shared: boolean; +} + +/** + * The best address for this person. + * + * Their own if the site published one. Otherwise the company's shared inbox, + * flagged as shared so nothing downstream pretends a named human is reading + * it. A campaign with neither is skipped rather than guessed at — inventing + * `firstname@company.com` is how a sending domain gets burned. + */ +function pickRecipient(row: Candidate): Recipient | undefined { + if (row.person_email) return { address: row.person_email, shared: false }; + if (row.company_contact_email) return { address: row.company_contact_email, shared: true }; + return undefined; +} + +/** A subject the composer did not supply. Kept plain and specific. */ +function defaultSubject(row: Candidate): string { + return row.company_name ? `Quick question about ${row.company_name}` : 'Quick question'; +} + +/** True when any recorded quality gate failed. Unparseable checks fail closed. */ +function hasFailingCheck(checksJson: string | null): boolean { + if (!checksJson) return false; + + let parsed: unknown; + try { + parsed = JSON.parse(checksJson); + } catch { + return true; + } + + if (!Array.isArray(parsed)) return false; + return parsed.some((check) => { + if (!check || typeof check !== 'object') return false; + const entry = check as { passed?: unknown; ok?: unknown }; + if (entry.passed === false) return true; + return entry.ok === false; + }); +} + +interface Settings { + readonly autopilot_daily_cap: number; + readonly reply_to_email: string | null; +} + +async function loadSettings(db: Client, workspaceId: string): Promise { + const row = await queryOne<{ autopilot_daily_cap: number; reply_to_email: string | null }>( + db, + `SELECT autopilot_daily_cap, reply_to_email FROM workspace_settings WHERE workspace_id = ?`, + [workspaceId], + ); + + // Defaults match the migration, so a workspace with no settings row behaves + // exactly like one that has accepted the defaults. + return { + autopilot_daily_cap: row?.autopilot_daily_cap ?? 25, + reply_to_email: row?.reply_to_email ?? null, + }; +} + +function dayStart(at: Date): string { + return `${at.toISOString().slice(0, 10)}T00:00:00.000Z`; +} + +async function countActionsToday(db: Client, workspaceId: string, at: Date): Promise { + const row = await queryOne<{ n: number }>( + db, + `SELECT COUNT(*) AS n FROM actions + WHERE workspace_id = ? AND created_at >= ? AND status != 'failed'`, + [workspaceId, dayStart(at)], + ); + return row?.n ?? 0; +} + +async function actionCounts( + db: Client, + workspaceId: string, + personId: string, + at: Date, +): Promise<{ thisProspect: number; hoursSinceLast?: number }> { + const weekAgo = new Date(at.getTime() - 7 * 24 * 3_600_000).toISOString(); + + const row = await queryOne<{ n: number; last_at: string | null }>( + db, + `SELECT COUNT(*) AS n, MAX(created_at) AS last_at FROM actions + WHERE workspace_id = ? AND person_id = ? AND created_at >= ? AND status != 'failed'`, + [workspaceId, personId, weekAgo], + ); + + const thisProspect = row?.n ?? 0; + if (!row?.last_at) return { thisProspect }; + + const stamp = Date.parse(row.last_at); + if (Number.isNaN(stamp)) return { thisProspect }; + + return { thisProspect, hoursSinceLast: (at.getTime() - stamp) / 3_600_000 }; +} + +function safeJson(value: string): Record { + try { + const parsed: unknown = JSON.parse(value); + return parsed && typeof parsed === 'object' ? (parsed as Record) : {}; + } catch { + return {}; + } +} + +function numberOr(value: unknown, fallback: number): number { + return typeof value === 'number' && Number.isFinite(value) ? value : fallback; +} + +async function audit( + db: Client, + workspaceId: string, + actionId: string, + entry: { eventType: string; detail: Record }, +): Promise { + await db.execute({ + sql: `INSERT INTO audit_events (id, workspace_id, actor_kind, actor_id, event_type, + entity_kind, entity_id, detail_json, occurred_at) + VALUES (?, ?, 'system', 'autopilot', ?, 'action', ?, ?, ?)`, + args: [ + newId('auditEvent'), + workspaceId, + entry.eventType, + actionId, + JSON.stringify(entry.detail), + now(), + ], + }); +} diff --git a/packages/pipeline/src/crawl.ts b/packages/pipeline/src/crawl.ts index ca36925..a27068a 100644 --- a/packages/pipeline/src/crawl.ts +++ b/packages/pipeline/src/crawl.ts @@ -7,7 +7,8 @@ * test. The server now supplies its dependencies and calls this. */ -import { queryOne, type Client } from '@outreachgraph/db'; +import { newId } from '@outreachgraph/domain'; +import { now, queryOne, type Client } from '@outreachgraph/db'; import type { PersonEnrichmentProvider, SiteProvider } from '@outreachgraph/providers'; import type { TextModel } from '@outreachgraph/ai'; import { runPipelineForCandidate } from './pipeline'; @@ -19,6 +20,8 @@ export interface CrawlJobDeps { /** Consulted per person once a candidate exists. */ readonly providers: readonly PersonEnrichmentProvider[]; readonly model?: TextModel; + /** True when a mailer is configured; see `PipelineOptions.emailSendingEnabled`. */ + readonly emailSendingEnabled?: boolean; } export interface CrawlJobResult { @@ -43,7 +46,7 @@ export interface CrawlJobResult { * throws, because those are worth retrying and worth seeing in `last_error`. */ export async function runCrawlJob(deps: CrawlJobDeps, job: QueuedJob): Promise { - const { url } = job.payload as { url?: string }; + const { url, campaignId } = job.payload as { url?: string; campaignId?: string }; if (!url) throw new Error('crawl_site needs a url'); const result = await deps.site.crawl(url); @@ -63,14 +66,80 @@ export async function runCrawlJob(deps: CrawlJobDeps, job: QueuedJob): Promise( - deps.db, - `SELECT id FROM campaigns WHERE workspace_id = ? ORDER BY created_at LIMIT 1`, - [job.workspaceId], - ); + // The campaign this crawl belongs to, not merely the workspace's oldest one. + // + // Filing every crawl under `ORDER BY created_at LIMIT 1` meant that a + // workspace running two campaigns scored both of their prospects against the + // first campaign's offering, and the second campaign stayed permanently + // empty. The payload carries the right answer whenever the intake created + // one; the fallback is only for jobs queued before this existed. + const campaign = campaignId + ? await queryOne<{ id: string }>( + deps.db, + `SELECT id FROM campaigns WHERE id = ? AND workspace_id = ?`, + [campaignId, job.workspaceId], + ) + : await queryOne<{ id: string }>( + deps.db, + `SELECT id FROM campaigns WHERE workspace_id = ? ORDER BY created_at LIMIT 1`, + [job.workspaceId], + ); if (!campaign) throw new Error(`workspace ${job.workspaceId} has no campaign`); + // The company and its shared inbox, recorded whether or not anyone was named. + // + // Three things were wrong here before, and each on its own was enough to + // make the keyword path produce nothing at all: + // + // - The domain was read only from the extracted company, which is often + // absent even when the crawl plainly succeeded. The host actually + // fetched is always known, so it is the better source. + // - The row was only ever written by the person chain, so a site that + // names nobody stored nothing — no company, no address — despite having + // published a perfectly good `info@`. + // - **This ran after the fan-out.** The recommendation engine asks whether + // a person is reachable while that person is being processed, so an + // inbox recorded afterwards was invisible to every lead on the page. Six + // real practices had their address stored and still produced zero + // recommendations. It has to happen first. + // + // `COALESCE` keeps the first address found rather than letting a later crawl + // of a deeper page overwrite the homepage's, which is usually the one the + // company actually wants used. + const domain = result.company.domain ?? hostOf(result.finalUrl); + + if (domain && (result.contactEmail || result.company.name)) { + const stamp = now(); + const existing = await queryOne<{ id: string }>( + deps.db, + `SELECT id FROM companies WHERE domain = ?`, + [domain], + ); + + if (existing) { + await deps.db.execute({ + sql: `UPDATE companies SET contact_email = COALESCE(contact_email, ?), updated_at = ? + WHERE id = ?`, + args: [result.contactEmail ?? null, stamp, existing.id], + }); + } else { + await deps.db.execute({ + sql: `INSERT INTO companies (id, name, domain, technologies, contact_email, + created_at, updated_at) + VALUES (?, ?, ?, '[]', ?, ?, ?)`, + args: [ + newId('company'), + result.company.name ?? domain, + domain, + result.contactEmail ?? null, + stamp, + stamp, + ], + }); + } + } + let queued = 0; for (const candidate of result.people) { await runPipelineForCandidate( @@ -80,10 +149,15 @@ export async function runCrawlJob(deps: CrawlJobDeps, job: QueuedJob): Promise { + const { keyword, campaignId } = job.payload as { keyword?: string; campaignId?: string }; + + if (!keyword) throw new Error('discover_domains needs a keyword'); + if (!campaignId) throw new Error('discover_domains needs a campaignId'); + + if (!deps.model) { + throw new Error('no model is configured, so a keyword cannot be expanded into companies'); + } + + // The offering grounds the search: "companies that would buy what you sell" + // is a much better query than the keyword alone, and the workspace usually + // already told us during setup. + const offering = await queryOne<{ name: string; description: string | null }>( + deps.db, + `SELECT name, description FROM offerings WHERE workspace_id = ? ORDER BY created_at LIMIT 1`, + [job.workspaceId], + ); + + const summary = offering + ? [offering.name, offering.description].filter(Boolean).join(' — ') + : undefined; + + const result = await discoverCompanies(deps.model, keyword, { + limit: deps.limit ?? 25, + ...(summary ? { offeringSummary: summary } : {}), + }); + + if (!result.ok) { + throw new Error(`could not expand "${keyword}": ${result.reason ?? 'no reason given'}`); + } + + let queued = 0; + + for (const company of result.companies) { + const enqueued = await enqueue(deps.db, { + workspaceId: job.workspaceId, + kind: 'crawl_site', + payload: { url: `https://${company.domain}`, campaignId }, + // The same domain surfacing in two campaigns is worth crawling twice — + // it is scored per campaign — but the same domain twice in one discovery + // run is not. + dedupeKey: `crawl:${campaignId}:${company.domain}`, + // Batched under the discovery job's own id so the intake screen can + // report "12 of 25 sites read" against the thing the user actually + // submitted, rather than against nothing. + batchId: job.id, + }); + + if (enqueued.queued) queued += 1; + } + + // The model's own name for the market is better than the raw phrase, and the + // brief explains the campaign to whoever opens it later. Only filled in + // where the campaign has not already been named by a human. + if (result.campaignName || result.brief) { + await deps.db.execute({ + sql: `UPDATE campaigns + SET name = COALESCE(NULLIF(name, ''), ?), + brief = COALESCE(brief, ?), + updated_at = ? + WHERE id = ? AND name = ?`, + args: [result.campaignName ?? keyword, result.brief ?? null, now(), campaignId, keyword], + }); + } + + return { + keyword, + campaignId, + found: result.companies.length, + queued, + ...(result.campaignName ? { campaignName: result.campaignName } : {}), + }; +} diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index 965cdaf..c19a2af 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -8,6 +8,36 @@ */ export { runCrawlJob, type CrawlJobDeps, type CrawlJobResult } from './crawl'; +export { runDiscoveryJob, type DiscoveryJobDeps, type DiscoveryJobResult } from './discovery'; +export { + runAutopilot, + type AutopilotDeps, + type AutopilotResult, + type SentOutreach, + type SkippedOutreach, +} from './autopilot'; +export { + loadNotifySettings, + notifyAddress, + sendDailyDigest, + sendLeadAlerts, + type NotifyDeps, + type NotifySettings, +} from './notify'; +export { + recordDiscovered, + recordStatus, + type StatusChange, + type StatusChangeResult, +} from './stages'; +export { + campaignFunnel, + leadTimeline, + workspaceAnalytics, + type FunnelQuery, + type LeadTimeline, + type WorkspaceAnalytics, +} from './analytics'; export { runPipeline, runPipelineForCandidate, diff --git a/packages/pipeline/src/jobs.ts b/packages/pipeline/src/jobs.ts index c8a19aa..51a8c8a 100644 --- a/packages/pipeline/src/jobs.ts +++ b/packages/pipeline/src/jobs.ts @@ -26,6 +26,8 @@ export const JOB_KINDS = [ 'poll_drop', /** Fetch one company URL, read it, and run everyone it names. */ 'crawl_site', + /** Expand a keyword into real companies and queue a crawl for each. */ + 'discover_domains', ] as const; export type JobKind = (typeof JOB_KINDS)[number]; diff --git a/packages/pipeline/src/notify.test.ts b/packages/pipeline/src/notify.test.ts new file mode 100644 index 0000000..7b365bb --- /dev/null +++ b/packages/pipeline/src/notify.test.ts @@ -0,0 +1,231 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { now, queryAll, type Client } from '@outreachgraph/db'; +import type { Mailer, Message, SendResult } from '@outreachgraph/email'; +import { seedDatabase, SEED, type SeededDatabase } from '../../../apps/api/src/test-seed'; +import { notifyAddress, loadNotifySettings, sendDailyDigest, sendLeadAlerts } from './notify'; + +let seeded: SeededDatabase | undefined; + +afterEach(() => { + seeded?.cleanup(); + seeded = undefined; +}); + +function recordingMailer(): { sent: Message[]; mailer: Mailer } { + const sent: Message[] = []; + return { + sent, + mailer: { + send: async (message): Promise => { + sent.push(message); + return {}; + }, + }, + }; +} + +const APP_URL = 'https://outreachgraph.com'; + +/** Noon UTC, so a digest due at 13:00 is not yet due and one at 09:00 is. */ +function at(hour: number): Date { + const date = new Date(); + date.setUTCHours(hour, 0, 0, 0); + return date; +} + +async function setDigestHour(db: Client, hour: number): Promise { + const stamp = now(); + await db.execute({ + sql: `INSERT INTO workspace_settings (workspace_id, digest_hour_utc, created_at, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(workspace_id) DO UPDATE SET digest_hour_utc = excluded.digest_hour_utc`, + args: [SEED.workspaceId, hour, stamp, stamp], + }); +} + +describe('notifyAddress', () => { + test('falls back to the verified owner when none is configured', async () => { + seeded = await seedDatabase('notify-owner'); + const { db } = seeded; + + const settings = await loadNotifySettings(db, SEED.workspaceId); + expect(await notifyAddress(db, SEED.workspaceId, settings)).toBe('test@example.com'); + }); + + test('an unverified owner is not written to', async () => { + seeded = await seedDatabase('notify-unverified'); + const { db } = seeded; + + await db.execute({ + sql: `UPDATE users SET email_verified_at = NULL WHERE id = ?`, + args: [SEED.userId], + }); + + const settings = await loadNotifySettings(db, SEED.workspaceId); + expect(await notifyAddress(db, SEED.workspaceId, settings)).toBeUndefined(); + }); +}); + +describe('sendLeadAlerts', () => { + test('mails one alert for a qualified lead', async () => { + seeded = await seedDatabase('alerts-send'); + const { db } = seeded; + + const { sent, mailer } = recordingMailer(); + const count = await sendLeadAlerts({ db, mailer, appUrl: APP_URL }, SEED.workspaceId); + + expect(count).toBe(1); + expect(sent[0]?.to).toBe('test@example.com'); + expect(sent[0]?.subject).toContain('Jane Smith'); + // The alert has to carry why, or it is an interruption with no payload. + expect(sent[0]?.text).toContain('cross-border'); + }); + + test('the same person is never alerted twice', async () => { + seeded = await seedDatabase('alerts-once'); + const { db } = seeded; + + const { sent, mailer } = recordingMailer(); + await sendLeadAlerts({ db, mailer, appUrl: APP_URL }, SEED.workspaceId); + await sendLeadAlerts({ db, mailer, appUrl: APP_URL }, SEED.workspaceId); + + expect(sent).toHaveLength(1); + }); + + test('a lead below the score floor is not worth interrupting for', async () => { + seeded = await seedDatabase('alerts-floor'); + const { db } = seeded; + + const stamp = now(); + await db.execute({ + sql: `INSERT INTO workspace_settings (workspace_id, alert_min_opportunity, created_at, updated_at) + VALUES (?, 99, ?, ?)`, + args: [SEED.workspaceId, stamp, stamp], + }); + + const { sent, mailer } = recordingMailer(); + await sendLeadAlerts({ db, mailer, appUrl: APP_URL }, SEED.workspaceId); + + expect(sent).toHaveLength(0); + }); + + test('alerts switched off send nothing', async () => { + seeded = await seedDatabase('alerts-off'); + const { db } = seeded; + + const stamp = now(); + await db.execute({ + sql: `INSERT INTO workspace_settings (workspace_id, instant_alerts, created_at, updated_at) + VALUES (?, 0, ?, ?)`, + args: [SEED.workspaceId, stamp, stamp], + }); + + const { sent, mailer } = recordingMailer(); + await sendLeadAlerts({ db, mailer, appUrl: APP_URL }, SEED.workspaceId); + + expect(sent).toHaveLength(0); + }); + + test('a failed send is retried rather than swallowed', async () => { + seeded = await seedDatabase('alerts-retry'); + const { db } = seeded; + + const failing: Mailer = { + send: async () => { + throw new Error('provider down'); + }, + }; + + await sendLeadAlerts({ db, mailer: failing, appUrl: APP_URL }, SEED.workspaceId); + + // The claim must have been released, or this lead is never alerted again. + const claims = await queryAll<{ id: string }>( + db, + `SELECT id FROM notifications WHERE workspace_id = ? AND kind = 'lead_alert'`, + [SEED.workspaceId], + ); + expect(claims).toHaveLength(0); + + const { sent, mailer } = recordingMailer(); + await sendLeadAlerts({ db, mailer, appUrl: APP_URL }, SEED.workspaceId); + expect(sent).toHaveLength(1); + }); +}); + +describe('sendDailyDigest', () => { + test('sends once the configured hour has passed', async () => { + seeded = await seedDatabase('digest-send'); + const { db } = seeded; + await setDigestHour(db, 9); + + const { sent, mailer } = recordingMailer(); + const done = await sendDailyDigest( + { db, mailer, appUrl: APP_URL, now: at(12) }, + SEED.workspaceId, + ); + + expect(done).toBe(true); + expect(sent).toHaveLength(1); + expect(sent[0]?.to).toBe('test@example.com'); + }); + + test('does not send before its hour', async () => { + seeded = await seedDatabase('digest-early'); + const { db } = seeded; + await setDigestHour(db, 20); + + const { sent, mailer } = recordingMailer(); + const done = await sendDailyDigest( + { db, mailer, appUrl: APP_URL, now: at(9) }, + SEED.workspaceId, + ); + + expect(done).toBe(false); + expect(sent).toHaveLength(0); + }); + + test('sends at most once a day', async () => { + seeded = await seedDatabase('digest-once'); + const { db } = seeded; + await setDigestHour(db, 9); + + const { sent, mailer } = recordingMailer(); + const deps = { db, mailer, appUrl: APP_URL, now: at(12) }; + + await sendDailyDigest(deps, SEED.workspaceId); + await sendDailyDigest(deps, SEED.workspaceId); + + expect(sent).toHaveLength(1); + }); + + test('a quiet day still gets a digest, and says so', async () => { + seeded = await seedDatabase('digest-quiet'); + const { db } = seeded; + await setDigestHour(db, 0); + + // No stage events and no sends today: silence here would be + // indistinguishable from the product being broken. + const { sent, mailer } = recordingMailer(); + await sendDailyDigest({ db, mailer, appUrl: APP_URL, now: at(12) }, SEED.workspaceId); + + expect(sent).toHaveLength(1); + expect(sent[0]?.subject).toContain('Nothing new'); + }); + + test('the digest switched off sends nothing', async () => { + seeded = await seedDatabase('digest-off'); + const { db } = seeded; + + const stamp = now(); + await db.execute({ + sql: `INSERT INTO workspace_settings (workspace_id, daily_digest, digest_hour_utc, + created_at, updated_at) VALUES (?, 0, 0, ?, ?)`, + args: [SEED.workspaceId, stamp, stamp], + }); + + const { sent, mailer } = recordingMailer(); + await sendDailyDigest({ db, mailer, appUrl: APP_URL, now: at(12) }, SEED.workspaceId); + + expect(sent).toHaveLength(0); + }); +}); diff --git a/packages/pipeline/src/notify.ts b/packages/pipeline/src/notify.ts new file mode 100644 index 0000000..47b25e7 --- /dev/null +++ b/packages/pipeline/src/notify.ts @@ -0,0 +1,393 @@ +/** + * Telling the customer what happened while nobody was watching. + * + * Autopilot moves the interface out of the app and into the inbox. If the + * product finds a lead, writes to them and gets a reply, and the only place + * any of that is visible is a dashboard nobody opened, then from the outside + * it is indistinguishable from a product that does nothing — which is exactly + * the state this work set out to fix. + * + * Two channels, deliberately different in character: + * + * - **Lead alerts** interrupt. One per person, the moment they qualify, and + * never twice for the same person. + * - **The daily digest** summarises. Once per UTC day, whether or not + * anything happened, because silence is the failure mode being avoided. + * + * Both are idempotent through the `notifications` table's unique index rather + * than through careful bookkeeping here: a retried job, an overlapping tick or + * a crash mid-loop cannot produce a second copy. + */ + +import { newId } from '@outreachgraph/domain'; +import { now, queryAll, queryOne, type Client } from '@outreachgraph/db'; +import { + dailyDigestEmail, + leadAlertEmail, + type DailyDigest, + type DigestLead, + type LeadAlert, + type Mailer, +} from '@outreachgraph/email'; + +export interface NotifyDeps { + readonly db: Client; + readonly mailer?: Mailer; + /** Absolute base for links in the mail. */ + readonly appUrl: string; + readonly now?: Date; +} + +export interface NotifySettings { + readonly notify_email: string | null; + readonly instant_alerts: number; + readonly daily_digest: number; + readonly digest_hour_utc: number; + readonly alert_min_opportunity: number; + readonly last_digest_sent_on: string | null; +} + +const DEFAULTS: NotifySettings = { + notify_email: null, + instant_alerts: 1, + daily_digest: 1, + digest_hour_utc: 13, + alert_min_opportunity: 60, + last_digest_sent_on: null, +}; + +export async function loadNotifySettings(db: Client, workspaceId: string): Promise { + const row = await queryOne( + db, + `SELECT notify_email, instant_alerts, daily_digest, digest_hour_utc, + alert_min_opportunity, last_digest_sent_on + FROM workspace_settings WHERE workspace_id = ?`, + [workspaceId], + ); + return row ?? DEFAULTS; +} + +/** + * Where this workspace's mail goes. + * + * An explicit `notify_email` wins; otherwise the owner's address, resolved at + * send time rather than copied into settings so that changing a login address + * cannot leave notifications going somewhere stale. + * + * Unverified addresses are refused. An account that never confirmed its + * mailbox has not proven it owns it, and a digest is still mail we would be + * sending to a stranger. + */ +export async function notifyAddress( + db: Client, + workspaceId: string, + settings: NotifySettings, +): Promise { + if (settings.notify_email) return settings.notify_email; + + const owner = await queryOne<{ email: string }>( + db, + `SELECT u.email + FROM workspaces w + JOIN organization_members m ON m.organization_id = w.organization_id + JOIN users u ON u.id = m.user_id + WHERE w.id = ? + AND u.status = 'active' + AND u.email_verified_at IS NOT NULL + ORDER BY CASE m.role WHEN 'owner' THEN 0 WHEN 'admin' THEN 1 ELSE 2 END, + m.created_at ASC + LIMIT 1`, + [workspaceId], + ); + + return owner?.email; +} + +/** + * Records that a notification was sent, and reports whether this call is the + * one that gets to send it. + * + * The insert is the lock. Returning false means another tick already claimed + * this notification, so the caller must not send. + */ +async function claim( + db: Client, + workspaceId: string, + kind: string, + subjectKey: string, + toEmail: string, +): Promise { + const result = await db.execute({ + sql: `INSERT INTO notifications (id, workspace_id, kind, subject_key, to_email, sent_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(workspace_id, kind, subject_key) DO NOTHING`, + args: [newId('notification'), workspaceId, kind, subjectKey, toEmail, now()], + }); + + return result.rowsAffected > 0; +} + +/** Undoes a claim whose send then failed, so the next tick may retry it. */ +async function release( + db: Client, + workspaceId: string, + kind: string, + subjectKey: string, + error: string, +): Promise { + await db.execute({ + sql: `DELETE FROM notifications WHERE workspace_id = ? AND kind = ? AND subject_key = ?`, + args: [workspaceId, kind, subjectKey], + }); + console.log(`notification ${kind}/${subjectKey} failed, will retry: ${error}`); +} + +interface AlertRow { + readonly person_id: string; + readonly display_name: string; + readonly current_title: string | null; + readonly company_name: string | null; + readonly company_domain: string | null; + readonly opportunity: number | null; + readonly reason: string | null; + readonly draft_subject: string | null; + readonly draft_body: string | null; + readonly sent_to: string | null; +} + +/** + * Sends one alert per newly-qualified lead. + * + * "Qualified" means the pipeline got far enough to recommend an action — a + * person with a score but no recommendation is still being worked out, and + * interrupting someone about them would be noise. The opportunity floor then + * keeps the interruption worth having. + */ +export async function sendLeadAlerts(deps: NotifyDeps, workspaceId: string): Promise { + const settings = await loadNotifySettings(deps.db, workspaceId); + if (settings.instant_alerts !== 1) return 0; + + const to = await notifyAddress(deps.db, workspaceId, settings); + if (!to || !deps.mailer) return 0; + + const rows = await queryAll( + deps.db, + `SELECT p.id AS person_id, p.display_name, p.current_title, + co.name AS company_name, co.domain AS company_domain, + s.opportunity, r.reason, + d.subject AS draft_subject, d.body AS draft_body, + (SELECT a.id FROM actions a + WHERE a.recommendation_id = r.id AND a.status = 'completed' + LIMIT 1) AS sent_to + FROM recommendations r + JOIN people p ON p.id = r.person_id + LEFT JOIN companies co ON co.id = p.current_company_id + LEFT JOIN scores s ON s.person_id = r.person_id AND s.campaign_id = r.campaign_id + LEFT JOIN drafts d ON d.recommendation_id = r.id + WHERE r.workspace_id = ? + AND p.status = 'active' + AND COALESCE(s.opportunity, 0) >= ? + AND NOT EXISTS ( + SELECT 1 FROM notifications n + WHERE n.workspace_id = r.workspace_id + AND n.kind = 'lead_alert' + AND n.subject_key = p.id + ) + ORDER BY COALESCE(s.opportunity, 0) DESC + LIMIT 25`, + [workspaceId, settings.alert_min_opportunity], + ); + + let count = 0; + + for (const row of rows) { + if (!(await claim(deps.db, workspaceId, 'lead_alert', row.person_id, to))) continue; + + // Read separately rather than joined above: the alert wants the address + // that was written to, and the join that finds *whether* one was sent is + // already doing enough work. + const delivered = await queryOne<{ detail_json: string }>( + deps.db, + `SELECT detail_json FROM audit_events + WHERE workspace_id = ? AND entity_kind = 'action' AND event_type = 'action.executed' + AND entity_id = ? + ORDER BY occurred_at DESC LIMIT 1`, + [workspaceId, row.sent_to ?? ''], + ); + + const sentTo = delivered ? readTo(delivered.detail_json) : undefined; + + const evidence = await queryAll<{ summary: string }>( + deps.db, + `SELECT summary FROM signals WHERE workspace_id = ? AND person_id = ? + ORDER BY observed_at DESC LIMIT 4`, + [workspaceId, row.person_id], + ); + + const alert: LeadAlert = { + personName: row.display_name, + personId: row.person_id, + ...(row.current_title ? { title: row.current_title } : {}), + ...(row.company_name ? { companyName: row.company_name } : {}), + ...(row.company_domain ? { companyDomain: row.company_domain } : {}), + ...(row.opportunity !== null ? { opportunity: Math.round(row.opportunity) } : {}), + ...(row.reason ? { reason: row.reason } : {}), + ...(evidence.length ? { evidence: evidence.map((e) => e.summary) } : {}), + ...(sentTo ? { sentTo } : {}), + ...(row.draft_subject ? { draftSubject: row.draft_subject } : {}), + ...(row.draft_body ? { draftBody: row.draft_body } : {}), + }; + + try { + await deps.mailer.send(leadAlertEmail(to, alert, deps.appUrl)); + count += 1; + } catch (error) { + await release( + deps.db, + workspaceId, + 'lead_alert', + row.person_id, + error instanceof Error ? error.message : String(error), + ); + } + } + + return count; +} + +function readTo(detailJson: string): string | undefined { + try { + const parsed: unknown = JSON.parse(detailJson); + if (parsed && typeof parsed === 'object') { + const to = (parsed as { to?: unknown }).to; + if (typeof to === 'string') return to; + } + } catch { + // A malformed audit detail is not worth failing an alert over. + } + return undefined; +} + +/** + * Sends the day's summary, once, if it is due. + * + * Due means: the configured hour has passed in UTC and no digest has gone out + * for today's date yet. A workspace that turns the digest on at 23:00 gets its + * first one the next day rather than an instant backfill. + */ +export async function sendDailyDigest(deps: NotifyDeps, workspaceId: string): Promise { + const at = deps.now ?? new Date(); + const today = at.toISOString().slice(0, 10); + + const settings = await loadNotifySettings(deps.db, workspaceId); + if (settings.daily_digest !== 1) return false; + if (at.getUTCHours() < settings.digest_hour_utc) return false; + if (settings.last_digest_sent_on === today) return false; + + const to = await notifyAddress(deps.db, workspaceId, settings); + if (!to || !deps.mailer) return false; + + if (!(await claim(deps.db, workspaceId, 'daily_digest', today, to))) return false; + + const since = `${today}T00:00:00.000Z`; + + const [crawled, found, sent, awaiting, leads] = await Promise.all([ + countOne( + deps.db, + `SELECT COUNT(*) AS n FROM jobs + WHERE workspace_id = ? AND kind = 'crawl_site' AND status = 'done' + AND finished_at >= ?`, + [workspaceId, since], + ), + countOne( + deps.db, + `SELECT COUNT(DISTINCT person_id) AS n FROM lead_stage_events + WHERE workspace_id = ? AND to_status = 'discovered' AND occurred_at >= ?`, + [workspaceId, since], + ), + countOne( + deps.db, + `SELECT COUNT(*) AS n FROM actions + WHERE workspace_id = ? AND status = 'completed' AND executed_at >= ?`, + [workspaceId, since], + ), + countOne( + deps.db, + `SELECT COUNT(*) AS n FROM recommendations + WHERE workspace_id = ? AND status = 'pending'`, + [workspaceId], + ), + queryAll<{ + person_id: string; + display_name: string; + current_title: string | null; + company_name: string | null; + opportunity: number | null; + sent_today: number; + }>( + deps.db, + `SELECT p.id AS person_id, p.display_name, p.current_title, + co.name AS company_name, s.opportunity, + (SELECT COUNT(*) FROM actions a + WHERE a.person_id = p.id AND a.workspace_id = e.workspace_id + AND a.status = 'completed' AND a.executed_at >= ?) AS sent_today + FROM (SELECT DISTINCT person_id, workspace_id, campaign_id + FROM lead_stage_events + WHERE workspace_id = ? AND occurred_at >= ?) e + JOIN people p ON p.id = e.person_id + LEFT JOIN companies co ON co.id = p.current_company_id + LEFT JOIN scores s ON s.person_id = e.person_id AND s.campaign_id = e.campaign_id + WHERE p.status = 'active' + ORDER BY COALESCE(s.opportunity, 0) DESC + LIMIT 15`, + [since, workspaceId, since], + ), + ]); + + const digestLeads: DigestLead[] = leads.map((lead) => ({ + personName: lead.display_name, + personId: lead.person_id, + ...(lead.current_title ? { title: lead.current_title } : {}), + ...(lead.company_name ? { companyName: lead.company_name } : {}), + ...(lead.opportunity !== null ? { opportunity: Math.round(lead.opportunity) } : {}), + ...(lead.sent_today > 0 ? { sentTo: 'sent' } : {}), + })); + + const digest: DailyDigest = { + date: today, + sitesCrawled: crawled, + peopleFound: found, + messagesSent: sent, + awaitingApproval: awaiting, + leads: digestLeads, + }; + + try { + await deps.mailer.send(dailyDigestEmail(to, digest, deps.appUrl)); + } catch (error) { + await release( + deps.db, + workspaceId, + 'daily_digest', + today, + error instanceof Error ? error.message : String(error), + ); + return false; + } + + await deps.db.execute({ + sql: `INSERT INTO workspace_settings (workspace_id, last_digest_sent_on, created_at, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(workspace_id) DO UPDATE + SET last_digest_sent_on = excluded.last_digest_sent_on, + updated_at = excluded.updated_at`, + args: [workspaceId, today, now(), now()], + }); + + return true; +} + +async function countOne(db: Client, sql: string, args: unknown[]): Promise { + const row = await queryOne<{ n: number }>(db, sql, args as never); + return row?.n ?? 0; +} diff --git a/packages/pipeline/src/pipeline.ts b/packages/pipeline/src/pipeline.ts index 6c6a455..1f9e6b0 100644 --- a/packages/pipeline/src/pipeline.ts +++ b/packages/pipeline/src/pipeline.ts @@ -27,6 +27,7 @@ import { import { generateRecommendation, type CandidateSignal } from '@outreachgraph/recommend'; import { draftForRecommendation, type TextModel } from '@outreachgraph/ai'; import { rescoreProspect } from './jobs'; +import { recordDiscovered, recordStatus } from './stages'; export interface PipelineOptions { readonly db: Client; @@ -40,6 +41,16 @@ export interface PipelineOptions { * recommendation still reaches the queue, just without a draft. */ readonly model?: TextModel; + /** + * Whether this deployment can actually put an email on the wire. + * + * `email/send_email` is `customer_managed` in the capability matrix, and + * that mode requires a connected account — otherwise the policy engine + * correctly returns `manual_only`, because a channel nothing can send + * through is a channel the human has to use themselves. The sending domain + * *is* that account, so the flag is true when a mailer is configured. + */ + readonly emailSendingEnabled?: boolean; readonly now?: Date; } @@ -92,6 +103,14 @@ export interface CandidateOrigin { readonly anchorNetwork?: Network; /** Enables the GitHub research stage when the candidate has a handle. */ readonly githubHandle?: string; + /** + * The page this candidate was read off, when there was one. + * + * Turned into a stored signal, which is what lets a person found on a + * company site reach an outbound recommendation at all — see + * `storeSiteSignal`. + */ + readonly sourceUrl?: string; } /** @@ -139,7 +158,30 @@ export async function runPipelineForCandidate( const enriched = fanned?.candidate ?? candidate; + // The page they were named on is itself an identity for them. + // + // Recorded *before* resolution so it counts toward identity confidence. + // Without it a person read off a company site ends the run at confidence 0 — + // the aggregate of no identities at all — and the policy engine then denies + // every outbound action, correctly, on the grounds that we cannot say who + // they are. That is the last thing standing between the URL and keyword + // paths and any outreach whatsoever: 82 real people, fully researched and + // scored, all permanently uncontactable. + // + // A company naming its own staff on its own site is first-party evidence, + // and it is not a merge — nothing is being matched to anything. It says only + // "this named person is presented here as part of this company", which is + // exactly the claim any message to them would rest on. + if (origin.sourceUrl) await storeSiteIdentity(db, personId, enriched, origin, stamp); + const linked = await linkIdentities(db, personId, enriched, origin, stamp); + + // An address published beside this person's own name on their employer's + // site. Stored as an identity rather than a column so it inherits deletion, + // suppression and provenance from the machinery that already handles every + // other way of reaching someone. + if (enriched.email) await storeEmailIdentity(db, personId, enriched.email, origin, stamp); + await setStatus(db, campaignId, personId, 'resolved'); // ------------------------------------------------------------- research @@ -153,13 +195,30 @@ export async function runPipelineForCandidate( candidate.identities.find((identity) => identity.network === 'github')?.handle; let stored = 0; + + // The page itself is evidence. + // + // This closes the gap that made the URL path produce nothing: the + // recommendation engine refuses to propose an outbound action with no + // trigger signal behind it, and only GitHub ever produced signals. So + // everyone found by crawling a company site — which is now the main way in — + // reached "scored" and stopped, with the honest-looking reason "no permitted + // action" and no indication that the real problem was having nothing to say. + // + // A published team page stating someone's role is a genuine, citable, + // grounded fact about them, which is exactly what the composer needs and all + // it is allowed to use. It is not a buying signal and is not scored like + // one — its relevance is deliberately middling — but it is enough to open a + // message that is about them rather than about nobody. + stored += await storeSiteSignal(db, workspaceId, personId, enriched, origin, stamp); + if (researchHandle) { const github = options.github ?? new GitHubProvider(); try { const activity = await github.activity(researchHandle); const context = await extractionContext(db, campaignId); const extracted = extractSignals(activity.events, activity.repos, context); - stored = await storeSignals(db, workspaceId, personId, extracted, stamp); + stored += await storeSignals(db, workspaceId, personId, extracted, stamp); await setStatus(db, campaignId, personId, 'researching'); } catch (error) { // A quota wall is not a prospect failure — keep what we have and let the @@ -208,6 +267,160 @@ export async function runPipelineForCandidate( }; } +/** + * Stores "their own site says this is their role" as a signal. + * + * Confidence is high — the company published it about its own staff — while + * relevance is deliberately mid. It is a fact worth citing, not evidence that + * anyone is in the market for anything, and scoring it as intent would put + * every receptionist on a team page at the top of the queue. + * + * Deduped on the source URL like every other signal, so re-crawling a site on + * a later tick does not accumulate one of these per run. + */ +async function storeSiteSignal( + db: Client, + workspaceId: string, + personId: string, + candidate: PersonCandidate, + origin: CandidateOrigin, + stamp: string, +): Promise { + if (!origin.sourceUrl || !candidate.title) return 0; + + const existing = await queryOne<{ id: string }>( + db, + `SELECT id FROM signals WHERE workspace_id = ? AND person_id = ? AND source_url = ? + AND signal_type = 'content_topic' AND subtype = 'site_role'`, + [workspaceId, personId, origin.sourceUrl], + ); + if (existing) return 0; + + const where = candidate.companyName ? ` at ${candidate.companyName}` : ''; + const summary = `Listed as ${candidate.title}${where} on the company website.`; + const evidence = `${candidate.fullName} — ${candidate.title}`; + + await db.execute({ + sql: `INSERT INTO signals (id, workspace_id, person_id, network, signal_type, subtype, + summary, evidence, source_url, source_timestamp, observed_at, confidence, + relevance, sentiment) + VALUES (?, ?, ?, 'website', 'content_topic', 'site_role', ?, ?, ?, ?, ?, 0.9, 0.5, + 'neutral')`, + args: [ + newId('signal'), + workspaceId, + personId, + summary, + evidence, + origin.sourceUrl, + stamp, + stamp, + ], + }); + + return 1; +} + +/** + * Records "named on this company's own site" as a website identity. + * + * Confidence is 0.9: high, because the employer published it about their own + * staff, and short of certain, because a page can be stale and a name can be + * shared. That clears the default 0.85 outreach threshold while still sitting + * below the 1.0 an anchor identity gets, and a workspace that wants a stricter + * bar can raise `min_outreach_confidence` and exclude these outright. + * + * `platform_user_id` is domain-plus-name rather than the URL, so re-crawling a + * deeper page on the same site recognises the same person instead of creating + * a second identity for them. + */ +async function storeSiteIdentity( + db: Client, + personId: string, + candidate: PersonCandidate, + origin: CandidateOrigin, + stamp: string, +): Promise { + if (!origin.sourceUrl) return; + + const host = (() => { + try { + return new URL(origin.sourceUrl).hostname.replace(/^www\./, '').toLowerCase(); + } catch { + return undefined; + } + })(); + + if (!host) return; + + const slug = candidate.fullName + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); + + if (!slug) return; + + await db.execute({ + sql: `INSERT INTO social_identities (id, person_id, network, handle, platform_user_id, + profile_url, confidence, source_type, verified_by, first_seen_at, last_verified_at) + VALUES (?, ?, 'website', ?, ?, ?, 0.9, ?, '["named_on_company_site"]', ?, ?) + ON CONFLICT(network, platform_user_id) WHERE platform_user_id IS NOT NULL + DO UPDATE SET last_verified_at = excluded.last_verified_at`, + args: [ + newId('socialIdentity'), + personId, + candidate.fullName, + `${host}:${slug}`, + origin.sourceUrl, + origin.capabilities.sourceType, + stamp, + stamp, + ], + }); +} + +/** + * Records a personal email address as an identity on this person. + * + * `platform_user_id` is the address itself, which makes the existing unique + * index on `(network, platform_user_id)` do real work: two people resolved + * from two pages cannot both end up owning `jane@acme.com`, so nobody is ever + * written to twice under two names. + * + * Confidence is high but not certain. The address was published next to this + * person's name on their own employer's site, which is strong evidence and + * still not proof — shared initials and a departed employee's forwarded + * mailbox both produce the same page. + */ +async function storeEmailIdentity( + db: Client, + personId: string, + email: string, + origin: CandidateOrigin, + stamp: string, +): Promise { + const address = email.trim().toLowerCase(); + if (!address.includes('@')) return; + + await db.execute({ + sql: `INSERT INTO social_identities (id, person_id, network, handle, platform_user_id, + profile_url, confidence, source_type, verified_by, first_seen_at, last_verified_at) + VALUES (?, ?, 'email', ?, ?, ?, 0.88, ?, '["published_on_company_site"]', ?, ?) + ON CONFLICT(network, platform_user_id) DO UPDATE + SET last_verified_at = excluded.last_verified_at`, + args: [ + newId('socialIdentity'), + personId, + address, + address, + `mailto:${address}`, + origin.capabilities.sourceType, + stamp, + stamp, + ], + }); +} + /** Matches on the stable platform id, never the renameable handle. */ async function upsertPerson( db: Client, @@ -525,6 +738,35 @@ async function createRecommendation( [personId], ); + const reachable = identities.map((identity) => identity.network as Network); + + // Their employer's published inbox counts as a way to reach them. + // + // Without this the whole keyword path dead-ends. `reachableNetworks` was + // built purely from a person's own identities, so `email` was only ever + // reachable for someone whose *personal* address appeared on the page — and + // small-business sites, which is what "dental practices in Austin" returns, + // almost never publish those. They name their staff and publish one shared + // `info@`. Every one of those leads reached "Researched" and stopped, with + // no permitted action and nothing to explain why. + // + // The address is real, published, and reaches the named person in practice, + // so it is honest to call the network reachable. What it is not is private: + // the send records `toSharedInbox`, so nothing downstream can mistake a + // shared mailbox for the individual. + if (!reachable.includes('email')) { + const inbox = await queryOne<{ contact_email: string }>( + db, + `SELECT co.contact_email + FROM people p + JOIN companies co ON co.id = p.current_company_id + WHERE p.id = ? AND co.contact_email IS NOT NULL`, + [personId], + ); + + if (inbox?.contact_email) reachable.push('email'); + } + const score = await queryOne<{ opportunity: number }>( db, 'SELECT opportunity FROM scores WHERE campaign_id = ? AND person_id = ?', @@ -540,14 +782,14 @@ async function createRecommendation( personId, campaignId, signals, - reachableNetworks: identities.map((i) => i.network as Network), + reachableNetworks: reachable, opportunity: score?.opportunity ?? 0, ...(options.now ? { now: options.now } : {}), policy: { approvalMode: campaign.approval_mode as 'draft_and_approve', // Resolved per-network below by the engine's own policy calls; this is // the workspace-wide answer for the common case. - hasConnectedAccount: connected.size > 0, + hasConnectedAccount: connected.size > 0 || options.emailSendingEnabled === true, personSuppressed: person.status === 'suppressed', personBelievedMinor: person.believed_minor === 1, personDeleted: person.status === 'deleted', @@ -639,15 +881,29 @@ async function ensureCampaignMembership( workspaceId: string, stamp: string, ): Promise { - await db.execute({ + const result = await db.execute({ sql: `INSERT INTO campaign_people (campaign_id, person_id, workspace_id, status, interaction_state, discovered_at, updated_at) VALUES (?, ?, ?, 'discovered', 'never_contacted', ?, ?) ON CONFLICT(campaign_id, person_id) DO NOTHING`, args: [campaignId, personId, workspaceId, stamp, stamp], }); + + // Only when the insert actually inserted. Re-crawling a site returns the + // same people, and stamping them all back to the top of the funnel every + // time would make the chart a record of how often the crawler ran. + if (result.rowsAffected > 0) { + await recordDiscovered(db, { workspaceId, campaignId, personId, at: stamp }); + } } +/** + * Moves a lead and records the move. + * + * The workspace is looked up rather than threaded through every call site + * because the stage event needs it and `campaign_people` already knows it — + * changing five signatures to carry a value the row holds would be noise. + */ async function setStatus( db: Client, campaignId: string, @@ -655,10 +911,20 @@ async function setStatus( status: string, reason?: string, ): Promise { - await db.execute({ - sql: `UPDATE campaign_people SET status = ?, status_reason = ?, updated_at = ? - WHERE campaign_id = ? AND person_id = ?`, - args: [status, reason ?? null, now(), campaignId, personId], + const membership = await queryOne<{ workspace_id: string }>( + db, + `SELECT workspace_id FROM campaign_people WHERE campaign_id = ? AND person_id = ?`, + [campaignId, personId], + ); + + if (!membership) return; + + await recordStatus(db, { + workspaceId: membership.workspace_id, + campaignId, + personId, + status, + ...(reason ? { reason } : {}), }); } diff --git a/packages/pipeline/src/stages.test.ts b/packages/pipeline/src/stages.test.ts new file mode 100644 index 0000000..aa46e5d --- /dev/null +++ b/packages/pipeline/src/stages.test.ts @@ -0,0 +1,113 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { queryAll, queryOne } from '@outreachgraph/db'; +import { seedDatabase, SEED, type SeededDatabase } from '../../../apps/api/src/test-seed'; +import { recordStatus } from './stages'; + +let seeded: SeededDatabase | undefined; + +afterEach(() => { + seeded?.cleanup(); + seeded = undefined; +}); + +describe('recordStatus', () => { + test('moves the lead and records the move', async () => { + seeded = await seedDatabase('stages-record'); + const { db } = seeded; + + const result = await recordStatus(db, { + workspaceId: SEED.workspaceId, + campaignId: SEED.campaignId, + personId: SEED.personId, + status: 'awaiting_approval', + }); + + expect(result.changed).toBe(true); + expect(result.from).toBe('recommended'); + expect(result.stage).toBe('ready'); + + const current = await queryOne<{ status: string }>( + db, + `SELECT status FROM campaign_people WHERE campaign_id = ? AND person_id = ?`, + [SEED.campaignId, SEED.personId], + ); + expect(current?.status).toBe('awaiting_approval'); + + const events = await queryAll<{ from_status: string; to_status: string; stage: string }>( + db, + `SELECT from_status, to_status, stage FROM lead_stage_events WHERE person_id = ?`, + [SEED.personId], + ); + expect(events).toHaveLength(1); + expect(events[0]?.from_status).toBe('recommended'); + expect(events[0]?.stage).toBe('ready'); + }); + + test('re-recording the same status writes no event', async () => { + seeded = await seedDatabase('stages-idempotent'); + const { db } = seeded; + + // The pipeline re-runs stages when it resumes. A resumed run must not look + // like a lead bouncing between stages. + await recordStatus(db, { + workspaceId: SEED.workspaceId, + campaignId: SEED.campaignId, + personId: SEED.personId, + status: 'qualified', + }); + const second = await recordStatus(db, { + workspaceId: SEED.workspaceId, + campaignId: SEED.campaignId, + personId: SEED.personId, + status: 'qualified', + }); + + expect(second.changed).toBe(false); + + const events = await queryAll<{ id: string }>( + db, + `SELECT id FROM lead_stage_events WHERE person_id = ?`, + [SEED.personId], + ); + expect(events).toHaveLength(1); + }); + + test('several internal moves inside one funnel stage each keep their event', async () => { + seeded = await seedDatabase('stages-granular'); + const { db } = seeded; + + for (const status of ['enriching', 'resolved']) { + await recordStatus(db, { + workspaceId: SEED.workspaceId, + campaignId: SEED.campaignId, + personId: SEED.personId, + status, + }); + } + + const events = await queryAll<{ stage: string; to_status: string }>( + db, + `SELECT stage, to_status FROM lead_stage_events WHERE person_id = ? ORDER BY occurred_at`, + [SEED.personId], + ); + + // Two distinct statuses, both mapping to "Found". The history keeps the + // detail; the chart is what collapses it. + expect(events).toHaveLength(2); + expect(events.every((event) => event.stage === 'discovered')).toBe(true); + }); + + test('a lead in no campaign is a no-op rather than a crash', async () => { + seeded = await seedDatabase('stages-missing'); + const { db } = seeded; + + const result = await recordStatus(db, { + workspaceId: SEED.workspaceId, + campaignId: SEED.campaignId, + personId: 'per_nobody', + status: 'qualified', + }); + + expect(result.changed).toBe(false); + }); +}); diff --git a/packages/pipeline/src/stages.ts b/packages/pipeline/src/stages.ts new file mode 100644 index 0000000..9c8af2f --- /dev/null +++ b/packages/pipeline/src/stages.ts @@ -0,0 +1,121 @@ +/** + * Recording where a lead is, and when it got there. + * + * `campaign_people.status` is a single column that each update overwrites, so + * before this the product could answer "where is this lead now" and no other + * question about it. Not "how long did it sit unapproved", not "how many made + * it from researched to contacted last week", not "when did this go out" — + * all of which are the questions a funnel exists to answer. + * + * Every status change now goes through one function that writes both the + * current state and an immutable event. Two rules make the history worth + * trusting: + * + * - A change to the same status writes nothing. The pipeline re-runs stages + * on resume, and a resumed run must not look like a lead bouncing. + * - The funnel stage is stamped on the event at write time, not derived when + * the chart is drawn, so re-grouping the state machine later cannot + * retroactively rewrite what last quarter looked like. + */ + +import { + newId, + stageForStatus, + type FunnelPosition, + type ProspectStatus, +} from '@outreachgraph/domain'; +import { now, queryOne, type Client } from '@outreachgraph/db'; + +export interface StatusChange { + readonly workspaceId: string; + readonly campaignId: string; + readonly personId: string; + readonly status: ProspectStatus | string; + readonly reason?: string; + readonly at?: string; +} + +export interface StatusChangeResult { + readonly changed: boolean; + readonly from?: string; + readonly to: string; + readonly stage: FunnelPosition; +} + +/** + * Moves a lead to `status` and records the move. + * + * The read-then-write is not wrapped in a transaction deliberately: the worker + * is a single instance by design, and the only cost of losing the race is one + * duplicate event in a history nothing reads transactionally. Buying strict + * ordering here would mean a lock on the hottest table in the pipeline. + */ +export async function recordStatus(db: Client, change: StatusChange): Promise { + const stamp = change.at ?? now(); + const stage = stageForStatus(change.status as ProspectStatus) ?? 'discovered'; + + const existing = await queryOne<{ status: string }>( + db, + `SELECT status FROM campaign_people WHERE campaign_id = ? AND person_id = ?`, + [change.campaignId, change.personId], + ); + + // A lead that is not in this campaign cannot move through it. Without this + // the update below matches nothing, the insert still runs, and the event + // references a membership that does not exist — which the foreign key + // rejects, turning a harmless no-op into a failed job. + if (!existing) { + return { changed: false, to: change.status, stage }; + } + + const from = existing.status; + + await db.execute({ + sql: `UPDATE campaign_people SET status = ?, status_reason = ?, updated_at = ? + WHERE campaign_id = ? AND person_id = ?`, + args: [change.status, change.reason ?? null, stamp, change.campaignId, change.personId], + }); + + if (from === change.status) { + return { changed: false, ...(from ? { from } : {}), to: change.status, stage }; + } + + await db.execute({ + sql: `INSERT INTO lead_stage_events (id, workspace_id, campaign_id, person_id, + from_status, to_status, stage, occurred_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + args: [ + newId('stageEvent'), + change.workspaceId, + change.campaignId, + change.personId, + from ?? null, + change.status, + stage, + stamp, + ], + }); + + return { changed: true, ...(from ? { from } : {}), to: change.status, stage }; +} + +/** + * Records the first event for a lead that has just joined a campaign. + * + * Membership is created with `ON CONFLICT DO NOTHING`, so this is only called + * where that insert actually inserted — otherwise re-crawling a site would + * stamp every one of its people back to the top of the funnel. + */ +export async function recordDiscovered( + db: Client, + change: Omit, +): Promise { + const stamp = change.at ?? now(); + + await db.execute({ + sql: `INSERT INTO lead_stage_events (id, workspace_id, campaign_id, person_id, + from_status, to_status, stage, occurred_at) + VALUES (?, ?, ?, ?, NULL, 'discovered', 'discovered', ?)`, + args: [newId('stageEvent'), change.workspaceId, change.campaignId, change.personId, stamp], + }); +} diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index 59d7e97..82736e8 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -90,6 +90,15 @@ export { export { isAllowed, parseRobots, type RobotsRules } from './site/robots'; +export { + assignEmails, + findEmails, + matchesName, + parseEmail, + type AssignedEmails, + type FoundEmail, +} from './site/emails'; + export { BlueskyProvider, BlueskyRateLimitError, diff --git a/packages/providers/src/provider.ts b/packages/providers/src/provider.ts index 3acfe79..3d0bf22 100644 --- a/packages/providers/src/provider.ts +++ b/packages/providers/src/provider.ts @@ -74,6 +74,14 @@ export interface PersonCandidate { readonly country?: string; readonly technologies?: readonly string[]; readonly personalDomain?: string; + /** + * A published address believed to reach this person specifically. + * + * Only ever a personal address. A shared inbox is the company's and is + * carried separately, because a message addressed to a named human is the + * wrong message to put in one. + */ + readonly email?: string; readonly identities: readonly CandidateIdentity[]; /** Opaque provider-side record id, retained for deletion tracing. */ readonly sourceRecordId?: string; diff --git a/packages/providers/src/site/emails.test.ts b/packages/providers/src/site/emails.test.ts new file mode 100644 index 0000000..dd1894d --- /dev/null +++ b/packages/providers/src/site/emails.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, test } from 'bun:test'; +import { assignEmails, findEmails, matchesName, parseEmail } from './emails'; + +describe('parseEmail', () => { + test('normalises and classifies', () => { + expect(parseEmail('Jane.Okafor@Acme.com')?.address).toBe('jane.okafor@acme.com'); + expect(parseEmail('mailto:info@acme.com?subject=hi')?.address).toBe('info@acme.com'); + expect(parseEmail('info@acme.com')?.isRole).toBe(true); + expect(parseEmail('jane@acme.com')?.isRole).toBe(false); + // Punctuation in a role local part still reads as a role. + expect(parseEmail('no-reply@acme.com')?.isRole).toBe(true); + }); + + test('rejects the things that look like addresses but are not', () => { + expect(parseEmail('not an email')).toBeUndefined(); + expect(parseEmail('jane@example.com')).toBeUndefined(); + expect(parseEmail('@acme.com')).toBeUndefined(); + expect(parseEmail('jane@localhost')).toBeUndefined(); + // A filename swept out of a CSS url(). + expect(parseEmail('sprite@2x.png')).toBeUndefined(); + // A hashed id embedded in markup. + expect(parseEmail('a1b2c3d4e5f60718@acme.com')).toBeUndefined(); + }); +}); + +describe('findEmails', () => { + test('deliberate mailto links rank ahead of addresses in prose', () => { + const html = ` +

    Write to us at hello@acme.com or ring.

    + Jane + `; + + const found = findEmails(html); + expect(found[0]?.address).toBe('jane@acme.com'); + expect(found.map((e) => e.address)).toContain('hello@acme.com'); + }); + + test('script and style blocks are not mined for addresses', () => { + const html = ` + + +

    jane@acme.com

    + `; + + expect(findEmails(html).map((e) => e.address)).toEqual(['jane@acme.com']); + }); + + test('the same address twice is one result', () => { + const html = 'a jane@acme.com'; + expect(findEmails(html)).toHaveLength(1); + }); +}); + +describe('matchesName', () => { + test('recognises the conventional shapes', () => { + expect(matchesName('jane.okafor', 'Jane Okafor')).toBe(true); + expect(matchesName('jokafor', 'Jane Okafor')).toBe(true); + expect(matchesName('jane', 'Jane Okafor')).toBe(true); + expect(matchesName('okafor', 'Jane Okafor')).toBe(true); + expect(matchesName('okaforj', 'Jane Okafor')).toBe(true); + }); + + test('refuses matches too weak to send a named message on', () => { + // One letter matches too many people. + expect(matchesName('j', 'Jane Okafor')).toBe(false); + expect(matchesName('sales', 'Jane Okafor')).toBe(false); + expect(matchesName('bob', 'Jane Okafor')).toBe(false); + }); +}); + +describe('assignEmails', () => { + test('splits personal addresses from the company inbox', () => { + const found = findEmails(` + Jane + General + `); + + const assigned = assignEmails(found, ['Jane Okafor'], 'acme.com'); + + expect(assigned.byPerson.get('Jane Okafor')).toBe('jane.okafor@acme.com'); + expect(assigned.companyEmail).toBe('info@acme.com'); + }); + + test('an address on another domain belongs to neither', () => { + const found = findEmails('Jane'); + const assigned = assignEmails(found, ['Jane Okafor'], 'acme.com'); + + expect(assigned.byPerson.size).toBe(0); + expect(assigned.companyEmail).toBeUndefined(); + }); + + test('a subdomain is the same company', () => { + const found = findEmails('Jane'); + const assigned = assignEmails(found, ['Jane Okafor'], 'acme.com'); + + expect(assigned.byPerson.get('Jane Okafor')).toBe('jane@mail.acme.com'); + }); + + test('a personal address is never also handed to the company', () => { + const found = findEmails('Jane'); + const assigned = assignEmails(found, ['Jane Okafor'], 'acme.com'); + + expect(assigned.byPerson.get('Jane Okafor')).toBe('jane@acme.com'); + expect(assigned.companyEmail).toBeUndefined(); + }); + + test('a page naming nobody still yields the company inbox', () => { + const found = findEmails('Say hi'); + const assigned = assignEmails(found, [], 'acme.com'); + + expect(assigned.companyEmail).toBe('hello@acme.com'); + }); +}); diff --git a/packages/providers/src/site/emails.ts b/packages/providers/src/site/emails.ts new file mode 100644 index 0000000..ea46721 --- /dev/null +++ b/packages/providers/src/site/emails.ts @@ -0,0 +1,275 @@ +/** + * Contact addresses published on a company page. + * + * The product had no way to reach anyone. Every other stage worked — a URL + * became a company, people, signals, a score and a drafted message — and then + * stopped at a card, because nothing anywhere held an address to send it to. + * + * Two kinds of address come off a page and they are not interchangeable: + * + * - A **personal** address belongs to someone the page names. `jane@acme.com` + * next to "Jane Okafor, Head of Ops" is that person, and a message that + * opens "Hi Jane" is correct. + * - A **role** address belongs to the company. Nobody named Jane reads + * `info@acme.com`, so a message sent there must not pretend otherwise. + * + * Conflating them is how automated outreach earns its reputation, so the two + * are separated here and stay separated all the way to the send. + */ + +/** + * Local parts that are a function rather than a person. + * + * Deliberately generous: a false "this is a role address" costs a slightly + * less personal greeting, while a false "this is a person" mails a named + * stranger's greeting to a shared inbox. + */ +const ROLE_LOCAL_PARTS = new Set([ + 'info', + 'hello', + 'hi', + 'contact', + 'contactus', + 'enquiries', + 'enquiry', + 'inquiries', + 'inquiry', + 'sales', + 'support', + 'help', + 'helpdesk', + 'service', + 'services', + 'customerservice', + 'admin', + 'administrator', + 'office', + 'team', + 'mail', + 'email', + 'general', + 'press', + 'media', + 'pr', + 'marketing', + 'jobs', + 'careers', + 'recruiting', + 'recruitment', + 'hr', + 'people', + 'billing', + 'accounts', + 'accounting', + 'finance', + 'invoices', + 'legal', + 'privacy', + 'security', + 'abuse', + 'compliance', + 'webmaster', + 'postmaster', + 'hostmaster', + 'noreply', + 'no-reply', + 'donotreply', + 'bounce', + 'bounces', + 'newsletter', + 'subscribe', + 'unsubscribe', + 'orders', + 'booking', + 'bookings', + 'reservations', + 'appointments', + 'partners', + 'partnerships', + 'investors', + 'ir', +]); + +/** + * Addresses that are never a prospect regardless of local part. + * + * Example domains and the placeholder addresses that ship inside templates + * are on real pages far more often than anyone expects. + */ +const IGNORED_DOMAINS = new Set([ + 'example.com', + 'example.org', + 'example.net', + 'domain.com', + 'yourdomain.com', + 'email.com', + 'sentry.io', + 'wixpress.com', + 'squarespace.com', +]); + +/** Extensions that mean this "address" was really a filename in a CSS url(). */ +const FILE_SUFFIX = /\.(png|jpe?g|gif|svg|webp|css|js|woff2?|ttf|ico|pdf|mp4|webm)$/i; + +const EMAIL_PATTERN = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g; + +export interface FoundEmail { + readonly address: string; + readonly localPart: string; + readonly domain: string; + /** True when the local part names a function rather than a human. */ + readonly isRole: boolean; +} + +/** Normalises and validates one candidate address, or rejects it. */ +export function parseEmail(raw: string): FoundEmail | undefined { + const address = raw + .trim() + .toLowerCase() + .replace(/^mailto:/, '') + .split('?')[0]; + if (!address) return undefined; + + const at = address.lastIndexOf('@'); + if (at <= 0 || at === address.length - 1) return undefined; + + const localPart = address.slice(0, at); + const domain = address.slice(at + 1); + + if (IGNORED_DOMAINS.has(domain)) return undefined; + if (FILE_SUFFIX.test(domain)) return undefined; + if (!domain.includes('.')) return undefined; + // A local part that is entirely hex is almost always a tracking id or a + // hashed address embedded in markup, not something a human reads. + if (/^[0-9a-f]{16,}$/.test(localPart)) return undefined; + + return { + address, + localPart, + domain, + isRole: + ROLE_LOCAL_PARTS.has(localPart.replace(/[._-]/g, '')) || ROLE_LOCAL_PARTS.has(localPart), + }; +} + +/** + * Every address on a page, `mailto:` links first. + * + * A `mailto:` is a deliberate publication — the site is asking to be written + * to — whereas an address in prose might be quoted from somewhere else, so the + * two are gathered separately and the deliberate ones rank first. + */ +export function findEmails(html: string): readonly FoundEmail[] { + const seen = new Map(); + + const add = (raw: string): void => { + const parsed = parseEmail(raw); + if (parsed && !seen.has(parsed.address)) seen.set(parsed.address, parsed); + }; + + for (const match of html.matchAll(/mailto:([^"'\s>)]+)/gi)) { + if (match[1]) add(match[1]); + } + + // The body text pass runs second so `mailto:` ordering survives. Script and + // style blocks are dropped first: both are dense with strings that look like + // addresses and are never contact details. + const prose = html + .replace(/]*>[\s\S]*?<\/script>/gi, ' ') + .replace(/]*>[\s\S]*?<\/style>/gi, ' '); + + for (const match of prose.matchAll(EMAIL_PATTERN)) add(match[0]); + + return [...seen.values()]; +} + +/** + * Decides whether `address` belongs to `fullName`. + * + * Only the shapes a human would actually recognise count. `jane.okafor@`, + * `jokafor@`, `jane@` and `okafor@` are all Jane Okafor; `j@` is not — a + * single letter matches too many people to send a named message to. + */ +export function matchesName(localPart: string, fullName: string): boolean { + const parts = fullName + .toLowerCase() + .split(/\s+/) + .map((part) => part.replace(/[^a-z]/g, '')) + .filter((part) => part.length > 1); + + const first = parts[0]; + const last = parts[parts.length - 1]; + if (!first || !last) return false; + + const local = localPart.toLowerCase().replace(/[^a-z]/g, ''); + if (local.length < 3) return false; + + const candidates = new Set([first, last, `${first}${last}`, `${last}${first}`]); + if (first[0]) { + candidates.add(`${first[0]}${last}`); + candidates.add(`${last}${first[0]}`); + } + if (last[0]) candidates.add(`${first}${last[0]}`); + + return candidates.has(local); +} + +export interface AssignedEmails { + /** Personal addresses, keyed by the full name they were matched to. */ + readonly byPerson: ReadonlyMap; + /** The company's best role address, if the page published one. */ + readonly companyEmail?: string; +} + +/** + * Splits the addresses on a page between the people it names and the company. + * + * An address whose domain is not the company's is left alone entirely — it is + * someone's agency, their personal Gmail in a bio, or a partner's contact, and + * none of those is who this crawl is about. + */ +export function assignEmails( + emails: readonly FoundEmail[], + names: readonly string[], + companyDomain?: string, +): AssignedEmails { + const byPerson = new Map(); + + const onDomain = companyDomain + ? emails.filter((email) => sharesRegistrableDomain(email.domain, companyDomain)) + : emails; + + for (const name of names) { + const match = onDomain.find((email) => !email.isRole && matchesName(email.localPart, name)); + if (match && !byPerson.has(name)) byPerson.set(name, match.address); + } + + const claimed = new Set(byPerson.values()); + // Role addresses in listed order: `findEmails` puts deliberate `mailto:` + // links first, so the first survivor is the one the site most meant to give. + const companyEmail = onDomain.find((email) => email.isRole && !claimed.has(email.address)); + + return { + byPerson, + ...(companyEmail ? { companyEmail: companyEmail.address } : {}), + }; +} + +/** + * True when two hosts belong to the same site. + * + * `mail.acme.com` and `acme.com` are the same company; a naive equality test + * drops perfectly good addresses published on a subdomain. This compares the + * last two labels, which is wrong for `co.uk`-style suffixes in the strictest + * sense but errs toward keeping an address rather than discarding one. + */ +function sharesRegistrableDomain(a: string, b: string): boolean { + const tail = (host: string): string => + host + .toLowerCase() + .replace(/^www\./, '') + .split('.') + .slice(-2) + .join('.'); + return tail(a) === tail(b); +} diff --git a/packages/providers/src/site/provider.ts b/packages/providers/src/site/provider.ts index 3ddcfbe..a9e652a 100644 --- a/packages/providers/src/site/provider.ts +++ b/packages/providers/src/site/provider.ts @@ -20,6 +20,7 @@ import type { import { fetchPage, type FetchOptions, type FetchOutcome, type FetchedPage } from './fetch'; import { extractSite, type ExtractedCompany } from './extract'; import { extractWithModel, visibleText, type ExtractionModel } from './model-extract'; +import { assignEmails, findEmails } from './emails'; export interface SiteProviderOptions extends FetchOptions { /** Omit to run deterministic-only; the crawl still works, it just reads less. */ @@ -53,6 +54,14 @@ export interface CrawlResult { * for leads — does not have to fetch it a second time. */ readonly pageText?: string; + /** + * The company's shared inbox, when the page published one. + * + * The fallback for a site that names people but gives no personal address — + * which is most of them. Kept off `company.identities` because it is a way + * to reach the company, not a profile belonging to it. + */ + readonly contactEmail?: string; } function emptyCompany(): ExtractedCompany { @@ -166,6 +175,31 @@ export class SiteProvider implements PersonEnrichmentProvider { if (fromModel.company.name || fromModel.people.length > 0) usedSignals.push('model'); } + // Addresses are read last, once the people are known: matching + // `jane@acme.com` to a person needs the person, and the model pass is + // often what produced them. + const found = findEmails(page.html); + let contactEmail: string | undefined; + + if (found.length > 0) { + const assigned = assignEmails( + found, + people.map((person) => person.fullName), + company.domain ?? hostOf(page.finalUrl), + ); + + if (assigned.byPerson.size > 0) { + people = people.map((person) => { + const email = assigned.byPerson.get(person.fullName); + return email ? { ...person, email } : person; + }); + usedSignals.push('email'); + } + + contactEmail = assigned.companyEmail; + if (contactEmail && !usedSignals.includes('email')) usedSignals.push('email'); + } + return { url, finalUrl: page.finalUrl, @@ -173,6 +207,7 @@ export class SiteProvider implements PersonEnrichmentProvider { company, people, usedSignals, + ...(contactEmail ? { contactEmail } : {}), ...(page.contentHash ? { contentHash: page.contentHash } : {}), fetchedAt: page.fetchedAt, ...(extractionUnavailable ? { extractionUnavailable } : {}), @@ -181,6 +216,15 @@ export class SiteProvider implements PersonEnrichmentProvider { } } +/** The host a page was finally served from, for matching addresses against. */ +function hostOf(url: string): string | undefined { + try { + return new URL(url).hostname; + } catch { + return undefined; + } +} + /** `example.com` and `https://example.com/` both mean the same thing here. */ export function normaliseUrl(input: string): string { const trimmed = input.trim(); diff --git a/smoke-check.ts b/smoke-check.ts new file mode 100644 index 0000000..0b80183 --- /dev/null +++ b/smoke-check.ts @@ -0,0 +1,13 @@ +import { createDatabase } from '@outreachgraph/db'; +const db = createDatabase({ url: 'file:/home/anthony/.claude/jobs/e4931a1b/tmp/smoke5.db' }); +const q = async (label: string, sql: string): Promise => { + const r = await db.execute(sql); + console.log(`\n--- ${label} ---`); + for (const row of r.rows.slice(0, 6)) console.log(JSON.stringify(row)); +}; +await q('people', 'SELECT display_name, identity_confidence FROM people'); +await q('identities', 'SELECT network, handle, confidence FROM social_identities'); +await q('recommendations', 'SELECT action, network, status, priority FROM recommendations'); +await q('drafts', 'SELECT substr(body,1,220) body FROM drafts'); +await q('actions', 'SELECT kind, network, status, substr(error,1,160) error FROM actions'); +await q('stage events', 'SELECT stage, COUNT(*) n FROM lead_stage_events GROUP BY stage'); diff --git a/smoke-verify.ts b/smoke-verify.ts new file mode 100644 index 0000000..ded3b32 --- /dev/null +++ b/smoke-verify.ts @@ -0,0 +1,5 @@ +import { createDatabase } from '@outreachgraph/db'; +const db = createDatabase({ url: 'file:/home/anthony/.claude/jobs/e4931a1b/tmp/smoke5.db' }); +await db.execute("UPDATE users SET email_verified_at = datetime('now')"); +const r = await db.execute('SELECT id, email, email_verified_at FROM users'); +console.log(r.rows);