From 7d4f93428ab3ca4124e35ee4704bb7cfc689bbdc Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Tue, 6 Oct 2026 19:05:05 +0000 Subject: [PATCH 1/3] Add leads to a running campaign, a per-row import report, and lead screening Append: POST /api/v1/autogtm/campaigns/{id}/leads takes up to 5,000 leads (JSON or the CSV itself), skips people already in the campaign or its project and anyone on a suppress list, and shares one code path with the new-campaign import. UI on the product page, `og leads add`, MCP add_leads_to_campaign. Report: every row that did not simply become a lead is stored with what happened (rejected, skipped, flagged) and the column or check behind it; returned inline, readable at .../import/{task}/report, downloadable as CSV. Screening: generated names (Borer853), relay mailboxes (passmail, aleeas, simplelogin...), temp-mail domains (known, or one domain shared by 3+ unrelated leads), bot/agent/test accounts and role inboxes are flagged at import and held back from cold outreach through the policy engine (personScreenedOut), on every send path, until a human allows them. Webmail is never taken as a company domain. Migration 0051. Co-Authored-By: Claude Opus 5.5 --- apps/api/src/app.ts | 38 +- apps/api/src/append-leads.test.ts | 222 ++++++++ apps/api/src/autogtm-docs.ts | 136 ++++- apps/api/src/autogtm.ts | 587 +++++++++++++++++----- apps/api/src/import-report.ts | 142 ++++++ apps/cli/src/cli.test.ts | 56 +++ apps/cli/src/commands.ts | 113 +++++ apps/mcp/src/mcp.test.ts | 36 ++ apps/mcp/src/tools.ts | 86 ++++ apps/web/app/(app)/products/[id]/page.tsx | 8 + apps/web/components/add-leads.tsx | 270 ++++++++++ apps/web/components/contact-import.tsx | 27 +- apps/web/components/screened-leads.tsx | 118 +++++ apps/web/lib/csv.ts | 135 +---- migrations-pg/0051_lead_screening.sql | 16 + migrations/0051_lead_screening.sql | 29 ++ packages/domain/src/contact-import.ts | 11 +- packages/domain/src/csv.ts | 149 ++++++ packages/domain/src/index.ts | 2 + packages/domain/src/lead-screen.test.ts | 153 ++++++ packages/domain/src/lead-screen.ts | 445 ++++++++++++++++ packages/pipeline/src/autopilot.ts | 9 + packages/pipeline/src/cadence-runner.ts | 3 + packages/pipeline/src/contact-import.ts | 224 +++++++-- packages/pipeline/src/index.ts | 10 + packages/pipeline/src/lead-screen.ts | 136 +++++ packages/pipeline/src/pipeline.ts | 5 + packages/policy/src/engine.test.ts | 30 ++ packages/policy/src/engine.ts | 14 + 29 files changed, 2929 insertions(+), 281 deletions(-) create mode 100644 apps/api/src/append-leads.test.ts create mode 100644 apps/api/src/import-report.ts create mode 100644 apps/web/components/add-leads.tsx create mode 100644 apps/web/components/screened-leads.tsx create mode 100644 migrations-pg/0051_lead_screening.sql create mode 100644 migrations/0051_lead_screening.sql create mode 100644 packages/domain/src/csv.ts create mode 100644 packages/domain/src/lead-screen.test.ts create mode 100644 packages/domain/src/lead-screen.ts create mode 100644 packages/pipeline/src/lead-screen.ts diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index e7648fd..abac45f 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -86,6 +86,7 @@ import { connectBlueskyAccount, budgetStatus, crawlDedupeKey, + screenHold, startContactImport, importContactChunk, intakeSocialPeople, @@ -248,6 +249,7 @@ import { UnknownProductError, } from './workspace-profile'; import { autogtmRoutes } from './autogtm'; +import { importReport, importReportCsv } from './import-report'; import { inboxRoutes } from './inbox'; import { crmRoutes, webhookRoutes } from './webhooks'; import { audienceRoutes } from './audience'; @@ -3299,7 +3301,7 @@ export function createApp(options: AppOptions): Hono { const batch = await queryOne>( db, `SELECT id, filename, consent_basis, consent_source, total_rows, imported, merged, - updated, rejected, status, created_at + updated, rejected, skipped, flagged, status, created_at FROM contact_imports WHERE id = ? AND workspace_id = ?`, [importId, actor.workspaceId], ); @@ -3308,22 +3310,44 @@ export function createApp(options: AppOptions): Hono { // Grouped, plus a sample. "We dropped 900 rows" is not actionable; "412 of // them were placeholders, here are twenty" lets somebody fix the export. + // The whole list, row by row, is GET .../report (CSV with ?format=csv). const [summary, sample] = await Promise.all([ - queryAll<{ reason: string; n: number }>( + queryAll<{ outcome: string; reason: string; n: number }>( db, - `SELECT reason, count(*) AS n FROM contact_import_rejects - WHERE import_id = ? GROUP BY reason ORDER BY n DESC`, + `SELECT outcome, reason, count(*) AS n FROM contact_import_rejects + WHERE import_id = ? GROUP BY outcome, reason ORDER BY n DESC`, [importId], ), queryAll<{ row_number: number; email: string; reason: string; detail: string }>( db, `SELECT row_number, email, reason, detail FROM contact_import_rejects - WHERE import_id = ? ORDER BY row_number LIMIT 50`, + WHERE import_id = ? AND outcome = 'rejected' ORDER BY row_number LIMIT 50`, [importId], ), ]); - return c.json({ import: batch, rejectsByReason: summary, rejectSample: sample }); + const by = (outcome: string) => + summary + .filter((row) => (row.outcome ?? 'rejected') === outcome) + .map((row) => ({ reason: row.reason, n: Number(row.n) })); + + return c.json({ + import: batch, + rejectsByReason: by('rejected'), + skippedByReason: by('skipped'), + flaggedByReason: by('flagged'), + rejectSample: sample, + reportUrl: `/api/v1/contacts/imports/${importId}/report?format=csv`, + }); + }); + + /** Every row that did not simply become a lead, and why: JSON, or CSV with ?format=csv. */ + api.get('/contacts/imports/:id/report', async (c) => { + const actor = c.get('actor'); + const report = await importReport(c.get('db'), actor.workspaceId, c.req.param('id')); + if (!report) throw ApiError.notFound('import'); + if (c.req.query('format') === 'csv') return importReportCsv(c, report); + return c.json(report); }); /** @@ -6043,6 +6067,7 @@ async function recheckPolicy( const matchKeys = await repo.suppressionKeysForPerson(db, recommendation.person_id); const suppressed = person.status === 'suppressed' || (await repo.isSuppressed(db, actor.workspaceId, matchKeys)); + const screenedOut = await screenHold(db, actor.workspaceId, recommendation.person_id); // Counted against the mailbox rather than the person. Skipped entirely when // no address resolves — there is nothing to protect and nothing to count. @@ -6067,6 +6092,7 @@ async function recheckPolicy( approvalMode: campaign.approval_mode as 'draft_and_approve', hasConnectedAccount: connected || (recommendation.network === 'email' && platformEmailEnabled), personSuppressed: suppressed, + ...(screenedOut ? { personScreenedOut: screenedOut } : {}), personBelievedMinor: person.believed_minor === 1, personDeleted: person.status === 'deleted', identityConfidence: person.identity_confidence, diff --git a/apps/api/src/append-leads.test.ts b/apps/api/src/append-leads.test.ts new file mode 100644 index 0000000..e37c8ea --- /dev/null +++ b/apps/api/src/append-leads.test.ts @@ -0,0 +1,222 @@ +/** + * Adding leads to a campaign that already exists, with nothing dropped + * silently: the per-row report, the dedupe against the campaign, the project + * and suppress lists, and screening holding the junk back from sending. + */ + +import { afterEach, describe, expect, test } from 'bun:test'; +import type { Hono } from 'hono'; +import { now, queryAll, queryOne } from '@outreachgraph/db'; +import { createApp } from './app'; +import type { AppEnv, RequestActor } from './context'; +import { seedDatabase, SEED, type SeededDatabase } from './test-seed'; + +const ACTOR: RequestActor = { + userId: SEED.userId, + workspaceId: SEED.workspaceId, + organizationId: SEED.organizationId, + role: 'owner', + credential: 'api_key', +}; + +let active: SeededDatabase | undefined; + +afterEach(() => { + active?.cleanup(); + active = undefined; +}); + +async function harness(label: string): Promise<{ app: Hono; seeded: SeededDatabase }> { + const seeded = await seedDatabase(label); + active = seeded; + const app = createApp({ db: seeded.db, authenticate: async () => ACTOR }); + return { app, seeded }; +} + +const send = (app: Hono, method: string, path: string, body?: unknown) => + app.request(`/api/v1${path}`, { + method, + headers: { 'content-type': 'application/json' }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + +interface AppendResult { + task_id: string; + received: number; + added: number; + imported: number; + rejected: number; + skipped: number; + flagged: number; + report: { row: number; email: string | null; outcome: string; reason: string; why: string }[]; + report_url: string; +} + +const T = 60_000; + +describe('POST /autogtm/campaigns/:id/leads', () => { + test( + 'appends, dedupes, reports every row and holds screened leads back', + async () => { + const { app, seeded } = await harness('append-leads'); + const { db } = seeded; + + // A suppress list, and a second campaign in the same project with a member. + await send(app, 'POST', '/autogtm/suppress-list/people', { + list_name: 'Do not contact', + emails: ['blocked@northwind.io'], + }); + const other = await send(app, 'POST', '/autogtm/campaigns/import', { + name: 'Earlier list', + project_id: SEED.offeringId, + leads: [{ email: 'old.friend@northwind.io', first_name: 'Old', last_name: 'Friend' }], + }); + expect(other.status).toBe(201); + + const first = await send(app, 'POST', `/autogtm/campaigns/${SEED.campaignId}/leads`, { + leads: [ + { + email: 'ana.reyes@northwind.io', + first_name: 'Ana', + last_name: 'Reyes', + company_domain: 'northwind.io', + job_title: 'Head of Ops', + }, + { email: 'not-an-email', first_name: 'Nope' }, + { first_name: 'No', last_name: 'Address' }, + { email: 'blocked@northwind.io' }, + { email: 'old.friend@northwind.io' }, + { email: 'hide@passmail.net', first_name: 'Jalen', last_name: 'Borer853' }, + { email: 'info@northwind.io' }, + ], + }); + expect(first.status).toBe(201); + const body = (await first.json()) as AppendResult; + + expect(body.received).toBe(7); + expect(body.rejected).toBe(2); + expect(body.skipped).toBe(2); + // ana, the relay and the role inbox join; two of them are held. + expect(body.added).toBe(3); + expect(body.flagged).toBe(2); + + const byRow = new Map(body.report.map((row) => [`${row.row}:${row.outcome}`, row])); + expect(byRow.get('2:rejected')?.reason).toBe('malformed_email'); + expect(byRow.get('3:rejected')?.reason).toBe('no_email'); + expect(byRow.get('3:rejected')?.why).toContain('"email"'); + expect(byRow.get('4:skipped')?.reason).toBe('suppressed'); + expect(byRow.get('5:skipped')?.reason).toBe('already_in_project'); + expect(byRow.get('6:flagged')?.reason).toBe('generated_name+relay_address'); + expect(byRow.get('7:flagged')?.reason).toBe('role_address'); + + // Re-running the same leads adds nobody and says why for each. + const again = (await ( + await send(app, 'POST', `/autogtm/campaigns/${SEED.campaignId}/leads`, { + leads: [{ email: 'ana.reyes@northwind.io', job_title: 'COO' }], + }) + ).json()) as AppendResult; + expect(again.added).toBe(0); + expect(again.report[0]?.reason).toBe('already_in_campaign'); + // ...but the newer data still landed on the person. + const ana = await queryOne<{ current_title: string }>( + db, + `SELECT p.current_title FROM people p JOIN person_emails pe ON pe.person_id = p.id + WHERE pe.address = 'ana.reyes@northwind.io'`, + ); + expect(ana?.current_title).toBe('COO'); + + // The full report downloads as CSV. + const csv = await app.request(body.report_url); + expect(csv.headers.get('content-type')).toContain('text/csv'); + const text = await csv.text(); + expect(text.split('\n')[0]).toBe('row,email,outcome,reason,why,detail'); + expect(text).toContain('blocked@northwind.io,skipped,suppressed'); + + // The screened list names them, and an override clears the hold. + const screened = (await ( + await app.request(`/api/v1/autogtm/campaigns/${SEED.campaignId}/screened`) + ).json()) as { held: number; leads: { person_id: string; flags: string[] }[] }; + expect(screened.held).toBe(2); + const relay = screened.leads.find((lead) => lead.flags.includes('relay_address')); + expect(relay).toBeDefined(); + + const allowed = await send(app, 'POST', `/autogtm/leads/${relay!.person_id}/screening`, { + allow: true, + }); + expect(allowed.status).toBe(200); + const after = (await ( + await app.request(`/api/v1/autogtm/campaigns/${SEED.campaignId}/screened`) + ).json()) as { held: number }; + expect(after.held).toBe(1); + + const rows = await queryAll<{ outcome: string }>( + db, + `SELECT outcome FROM contact_import_rejects WHERE import_id = ?`, + [body.task_id], + ); + expect(rows.length).toBe(6); + }, + T, + ); + + test( + 'takes the CSV itself and names a missing email column', + async () => { + const { app } = await harness('append-csv'); + + const missing = await send(app, 'POST', `/autogtm/campaigns/${SEED.campaignId}/leads`, { + csv: 'Name,Company\nAda,Acme\n', + }); + expect(missing.status).toBe(400); + expect(await missing.text()).toContain('no email column'); + + const ok = await send(app, 'POST', `/autogtm/campaigns/${SEED.campaignId}/leads`, { + csv: 'Email,First Name,Last Name,Company Domain,Job Title\nada.l@acme.dev,Ada,L,acme.dev,CTO\n', + }); + expect(ok.status).toBe(201); + const body = (await ok.json()) as AppendResult; + expect(body.added).toBe(1); + expect(body.report).toEqual([]); + }, + T, + ); + + test( + 'a held lead is refused at approval with the screening reason', + async () => { + const { app, seeded } = await harness('append-hold'); + const { db } = seeded; + + // Jane (the seeded prospect with a pending card) is screened as a role inbox. + await db.execute({ + sql: `INSERT INTO lead_screens (workspace_id, person_id, findings, screened_at) + VALUES (?, ?, ?, ?)`, + args: [ + SEED.workspaceId, + SEED.personId, + JSON.stringify([{ flag: 'role_address', detail: 'info@ is a team inbox, not a person' }]), + now(), + ], + }); + + const refused = await send( + app, + 'POST', + `/recommendations/${SEED.recommendationId}/approve`, + {}, + ); + expect(refused.status).toBe(409); + expect(await refused.text()).toContain('Held by lead screening'); + + await send(app, 'POST', `/autogtm/leads/${SEED.personId}/screening`, { allow: true }); + const retried = await send( + app, + 'POST', + `/recommendations/${SEED.recommendationId}/approve`, + {}, + ); + expect(await retried.text()).not.toContain('Held by lead screening'); + }, + T, + ); +}); diff --git a/apps/api/src/autogtm-docs.ts b/apps/api/src/autogtm-docs.ts index f72c667..4ac2594 100644 --- a/apps/api/src/autogtm-docs.ts +++ b/apps/api/src/autogtm-docs.ts @@ -466,10 +466,141 @@ export const OPERATIONS: readonly Operation[] = [ 'Of the merged, how many people this import changed. Newer data wins: a value in the import replaces a different stored one, unless the row is dated older.', }, rejected: { type: 'integer' }, + skipped: { type: 'integer' }, + flagged: { type: 'integer' }, crawls_queued: { type: 'integer' }, + report: { type: 'array', description: 'The import report, first 500 rows.' }, + report_url: { type: 'string' }, }, }, }, + { + method: 'post', + path: '/autogtm/campaigns/{campaign_id}/leads', + id: 'appendLeads', + tag: 'Campaigns', + summary: 'Add leads to an existing campaign', + description: + 'Up to 5,000 leads per request, as `leads` (the same fields as an import, plus `name` and ' + + '`updated_at`) or as `csv` (the file itself; headers like Email, First Name, Company Domain ' + + 'and Job Title are recognised). Skips people already in this campaign, already in another ' + + 'campaign of the project (`skip_project_duplicates`, default true) and anyone on a suppress ' + + 'list. Screening flags generated names, relay and temp-mail addresses, agent and test ' + + 'accounts and role inboxes, and holds them back from sending until you allow them ' + + '(`allow_flagged: true` sends to them anyway). No row is dropped silently: the response ' + + 'carries a per-row report with the column or check behind every reject, skip and flag.', + params: [idParam('campaign_id', 'The campaign to add to.')], + body: { + type: 'object', + properties: { + leads: { + type: 'array', + maxItems: 5000, + items: { + type: 'object', + properties: { + email: { type: 'string' }, + name: { type: 'string' }, + first_name: { type: 'string' }, + last_name: { type: 'string' }, + company_domain: { type: 'string' }, + company: { type: 'string' }, + job_title: { type: 'string' }, + location: { type: 'string' }, + linkedin_url: { type: 'string' }, + updated_at: { type: 'string' }, + }, + }, + }, + csv: { type: 'string', description: 'A CSV file with a header row. Send this or leads.' }, + filename: { type: 'string' }, + consent_basis: { type: 'string' }, + consent_source: { type: 'string' }, + allow_flagged: { type: 'boolean', description: 'Send to screened leads too.' }, + skip_project_duplicates: { type: 'boolean' }, + }, + }, + status: 201, + response: { + type: 'object', + properties: { + task_id: { type: 'string' }, + campaign_id: { type: 'string' }, + received: { type: 'integer' }, + added: { type: 'integer', description: 'New members of the campaign.' }, + imported: { type: 'integer', description: 'People new to the workspace.' }, + merged: { type: 'integer' }, + updated: { type: 'integer' }, + rejected: { type: 'integer', description: 'Unusable rows.' }, + skipped: { + type: 'integer', + description: 'Already in the campaign/project, or suppressed.', + }, + flagged: { type: 'integer', description: 'Imported, held back by screening.' }, + report: { + type: 'array', + items: { + type: 'object', + properties: { + row: { type: 'integer' }, + email: { type: ['string', 'null'] }, + outcome: { type: 'string', enum: ['rejected', 'skipped', 'flagged'] }, + reason: { type: 'string' }, + why: { type: 'string' }, + detail: { type: ['string', 'null'] }, + }, + }, + }, + report_url: { type: 'string', description: 'The whole report as CSV.' }, + }, + }, + }, + { + method: 'get', + path: '/autogtm/campaigns/import/{task_id}/report', + id: 'getImportReport', + tag: 'Campaigns', + summary: 'Every row an import rejected, skipped or flagged, and why', + description: 'JSON by default; `?format=csv` downloads it as a file.', + params: [ + idParam('task_id', 'The task_id an import returned.'), + { + name: 'format', + in: 'query', + description: 'csv for a download.', + schema: { type: 'string' }, + }, + ], + }, + { + method: 'get', + path: '/autogtm/campaigns/{campaign_id}/screened', + id: 'listScreened', + tag: 'Campaigns', + summary: 'Leads screening is holding back from sending, with the reasons', + params: [ + idParam('campaign_id', 'The campaign.'), + { + name: 'include_allowed', + in: 'query', + description: 'true to list the ones you already allowed too.', + schema: { type: 'string' }, + }, + ], + }, + { + method: 'post', + path: '/autogtm/leads/{person_id}/screening', + id: 'overrideScreening', + tag: 'Campaigns', + summary: 'Send to a screened lead anyway, or hold them back again', + params: [idParam('person_id', 'The lead.')], + body: { + type: 'object', + required: ['allow'], + properties: { allow: { type: 'boolean' } }, + }, + }, { method: 'get', path: '/autogtm/campaigns/import/{task_id}', @@ -488,6 +619,9 @@ export const OPERATIONS: readonly Operation[] = [ merged: { type: 'integer' }, updated: { type: 'integer' }, rejected: { type: 'integer' }, + skipped: { type: 'integer' }, + flagged: { type: 'integer' }, + report_url: { type: 'string' }, }, }, }, @@ -904,7 +1038,7 @@ export function llmsText(baseUrl: string): string { '6. `POST /autogtm/suppress-list/people` `{ "list_name", "emails" }` or `/companies` `{ "list_name", "domains" }` — never write to these. Halts anything queued for them.', ); lines.push( - '7. `POST /autogtm/campaigns/import` `{ "name", "leads": [{ "email", "first_name", "last_name", "company_domain", "job_title" }] }` — a campaign from your own list, up to 5,000 per call.', + '7. `POST /autogtm/campaigns/import` `{ "name", "leads": [{ "email", "first_name", "last_name", "company_domain", "job_title" }] }` — a campaign from your own list, up to 5,000 per call. `POST /autogtm/campaigns/{id}/leads` `{ "leads" }` or `{ "csv" }` adds to an existing one, deduped, with a per-row report.', ); lines.push( '8. `GET /autogtm/billing/balance` — allowance, credits, and whether sending is exhausted.', diff --git a/apps/api/src/autogtm.ts b/apps/api/src/autogtm.ts index 0d0ad37..21af8f3 100644 --- a/apps/api/src/autogtm.ts +++ b/apps/api/src/autogtm.ts @@ -26,17 +26,24 @@ import { Hono } from 'hono'; import { z } from 'zod'; import { + applyMapping, autogtmStatus, CONTACT_PRICE_USD, + isConsumerMailDomain, + mapHeaders, newId, + parseCsv, replyRate, + screenContext, usdForContacts, + type RawContact, } from '@outreachgraph/domain'; import { now, queryAll, queryOne, type Client } from '@outreachgraph/db'; import { applyProjectBudget, budgetStatus, campaignBudgetFrom, + countScreenedHeld, crawlDedupeKey, domainMatchKey, emailMatchKey, @@ -46,8 +53,11 @@ import { normaliseDomain, peopleMatchingKeys, recordDiscovered, + screenedLeads, setCampaignDailyBudget, + setScreenAllowed, startContactImport, + type StoredRow, } from '@outreachgraph/pipeline'; import { POLICY_VERSION } from '@outreachgraph/policy'; import { @@ -57,6 +67,7 @@ import { setCampaignStatus, } from './campaigns'; import { ApiError, canApprove, type AppEnv, type RequestActor } from './context'; +import { importReport, importReportCsv, reasonText } from './import-report'; import * as repo from './repository'; import { UnknownProductError } from './workspace-profile'; @@ -164,8 +175,50 @@ const importBody = z.object({ autopilot: z.boolean().optional(), consent_basis: z.string().max(200).optional(), consent_source: z.string().max(500).optional(), + allow_flagged: z.boolean().optional(), }); +/** + * A lead as appended to an existing campaign. + * + * Looser than `importLead` on purpose: nothing here is validated by the + * schema, because a schema failure answers 400 for the whole request and + * says nothing about the other 4,999 rows. Every row is judged on its own by + * the importer and the bad ones come back in the report with the column or + * check that failed. + */ +const appendLead = z.object({ + email: z.string().max(320).optional(), + name: z.string().max(200).optional(), + first_name: z.string().max(100).optional(), + last_name: z.string().max(100).optional(), + company_domain: z.string().max(253).optional(), + company: z.string().max(200).optional(), + job_title: z.string().max(200).optional(), + location: z.string().max(200).optional(), + linkedin_url: z.string().max(500).optional(), + updated_at: z.string().max(64).optional(), +}); + +const appendBody = z + .object({ + leads: z.array(appendLead).min(1).max(IMPORT_MAX_LEADS).optional(), + /** The file itself, for callers that would rather not parse it. */ + csv: z.string().min(1).max(10_000_000).optional(), + filename: z.string().max(300).optional(), + consent_basis: z.string().max(200).optional(), + consent_source: z.string().max(500).optional(), + /** Send to screened leads too, instead of holding them back. */ + allow_flagged: z.boolean().optional(), + /** Skip people already in another campaign of the same project (default true). */ + skip_project_duplicates: z.boolean().optional(), + }) + .refine((body) => Boolean(body.leads) !== Boolean(body.csv), { + message: 'send exactly one of leads or csv', + }); + +const allowBody = z.object({ allow: z.boolean() }); + // ------------------------------------------------------------------ rows interface CampaignRow { @@ -759,104 +812,20 @@ export function autogtmRoutes(deps: AutogtmDeps): Hono { }); await inheritFilters(db, campaignId, actor.workspaceId, offering.id, stamp); - const importId = await startContactImport(db, { - workspaceId: actor.workspaceId, - userId: actor.userId, + const outcome = await importIntoCampaign(db, actor, { campaignId, + projectId: offering.id, + rows: body.leads.map(rawFromLead), filename: `autogtm:${body.name}`, consentBasis: body.consent_basis ?? 'opt_in', - ...(body.consent_source ? { consentSource: body.consent_source } : {}), - }); - - let imported = 0; - let merged = 0; - let updated = 0; - let rejected = 0; - const personIds = new Set(); - - for (let offset = 0; offset < body.leads.length; offset += IMPORT_CHUNK) { - const rows = body.leads.slice(offset, offset + IMPORT_CHUNK).map((lead) => ({ - email: lead.email, - ...(lead.first_name ? { firstName: lead.first_name } : {}), - ...(lead.last_name ? { lastName: lead.last_name } : {}), - ...(lead.company ? { company: lead.company } : {}), - ...(lead.job_title ? { title: lead.job_title } : {}), - ...(lead.location ? { location: lead.location } : {}), - ...(lead.company_domain ? { companyDomain: lead.company_domain } : {}), - ...(lead.linkedin_url ? { linkedinUrl: lead.linkedin_url } : {}), - })); - - const result = await importContactChunk(db, importId, rows, { startRow: offset }); - imported += result.imported; - merged += result.merged; - updated += result.updated; - rejected += result.rejected; - for (const id of result.personIds) personIds.add(id); - } - - // The importer makes people; a campaign is a membership. Without this the - // leads exist and the campaign is empty, which is the bug the old - // `/contacts/imports` path shipped with. - for (const personId of personIds) { - const inserted = await db.execute({ - sql: `INSERT OR IGNORE INTO campaign_people (campaign_id, person_id, workspace_id, status, - interaction_state, discovered_at, updated_at) - VALUES (?, ?, ?, 'discovered', 'never_contacted', ?, ?)`, - args: [campaignId, personId, actor.workspaceId, stamp, stamp], - }); - if (inserted.rowsAffected > 0) { - await recordDiscovered(db, { workspaceId: actor.workspaceId, campaignId, personId }); - } - } - - // Research is what makes a lead sendable: every message is grounded in - // something read about them, so their company site is queued under this - // campaign. One crawl per domain however many colleagues were listed. - const domains = new Set(); - for (const lead of body.leads) { - const domain = lead.company_domain - ? normaliseDomain(lead.company_domain) - : normaliseDomain(lead.email.split('@')[1] ?? ''); - if (domain && !FREEMAIL.has(domain)) domains.add(domain); - } - let crawls = 0; - for (const domain of domains) { - const url = `https://${domain}`; - const queued = await enqueue(db, { - workspaceId: actor.workspaceId, - kind: 'crawl_site', - payload: { url, campaignId }, - dedupeKey: crawlDedupeKey(url), - }); - if (queued.queued) crawls += 1; - } - - await finishContactImport(db, importId); - - await repo.audit(db, { - workspaceId: actor.workspaceId, - actorKind: 'user', - actorId: actor.userId, - eventType: 'autogtm.campaign_imported', - entityKind: 'campaign', - entityId: campaignId, - detail: { importId, imported, merged, rejected, crawls }, + consentSource: body.consent_source, + allowFlagged: body.allow_flagged === true, + // A new campaign's whole point may be to re-approach part of a project. + skipProjectDuplicates: false, + event: 'autogtm.campaign_imported', }); - return c.json( - { - task_id: importId, - campaign_id: campaignId, - project_id: offering.id, - status: 'completed', - imported, - merged, - updated, - rejected, - crawls_queued: crawls, - }, - 201, - ); + return c.json({ ...outcome, project_id: offering.id }, 201); }); r.get('/campaigns/import/:task_id', async (c) => { @@ -870,12 +839,14 @@ export function autogtmRoutes(deps: AutogtmDeps): Hono { merged: number; updated: number; rejected: number; + skipped: number | null; + flagged: number | null; created_at: string; updated_at: string; }>( c.get('db'), - `SELECT id, campaign_id, status, total_rows, imported, merged, updated, rejected, created_at, - updated_at + `SELECT id, campaign_id, status, total_rows, imported, merged, updated, rejected, skipped, + flagged, created_at, updated_at FROM contact_imports WHERE id = ? AND workspace_id = ?`, [c.req.param('task_id'), actor.workspaceId], ); @@ -891,11 +862,128 @@ export function autogtmRoutes(deps: AutogtmDeps): Hono { merged: row.merged, updated: Number(row.updated ?? 0), rejected: row.rejected, + skipped: Number(row.skipped ?? 0), + flagged: Number(row.flagged ?? 0), + report_url: `/api/v1/autogtm/campaigns/import/${row.id}/report?format=csv`, created_at: row.created_at, updated_at: row.updated_at, }); }); + r.get('/campaigns/import/:task_id/report', async (c) => { + const actor = c.get('actor'); + const report = await importReport(c.get('db'), actor.workspaceId, c.req.param('task_id')); + if (!report) throw ApiError.notFound('import'); + if (c.req.query('format') === 'csv') return importReportCsv(c, report); + return c.json({ + ...report, + rows: report.rows.map((row) => ({ ...row, why: reasonText(row.reason) })), + }); + }); + + // ------------------------------------------------------- leads, screening + + r.post('/campaigns/:id/leads', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + requireWriter(actor, 'adding leads'); + await deps.requireVerifiedEmail(db, actor); + + const campaign = await ownedCampaign(db, actor.workspaceId, c.req.param('id')); + if (campaign.status === 'archived') { + throw ApiError.badRequest('an archived campaign takes no new leads'); + } + const body = await parse(c.req.raw, appendBody); + + let rows: RawContact[]; + if (body.csv) { + const [head, ...data] = parseCsv(body.csv); + if (!head) throw ApiError.badRequest('the csv has no rows'); + const mapping = mapHeaders(head); + if (mapping.email === undefined) { + throw ApiError.badRequest( + `the csv has no email column (columns seen: ${head.join(', ')}); ` + + 'rename one to "email"', + ); + } + if (data.length > IMPORT_MAX_LEADS) { + throw ApiError.badRequest( + `${data.length} rows is more than ${IMPORT_MAX_LEADS} per request; split the file`, + ); + } + rows = applyMapping(data, mapping); + } else { + rows = (body.leads ?? []).map(rawFromLead); + } + + const outcome = await importIntoCampaign(db, actor, { + campaignId: campaign.id, + projectId: campaign.offering_id, + rows, + filename: body.filename ?? `autogtm:${campaign.name}:append`, + consentBasis: body.consent_basis ?? 'opt_in', + consentSource: body.consent_source, + allowFlagged: body.allow_flagged === true, + skipProjectDuplicates: body.skip_project_duplicates !== false, + event: 'autogtm.leads_appended', + }); + + return c.json({ ...outcome, project_id: campaign.offering_id }, 201); + }); + + r.get('/campaigns/:id/screened', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + const campaign = await ownedCampaign(db, actor.workspaceId, c.req.param('id')); + const leads = await screenedLeads(db, { + workspaceId: actor.workspaceId, + campaignId: campaign.id, + includeAllowed: c.req.query('include_allowed') === 'true', + limit: clampPage(c.req.query('limit')), + }); + return c.json({ + campaign_id: campaign.id, + held: await countScreenedHeld(db, actor.workspaceId, campaign.id), + leads: leads.map((lead) => ({ + person_id: lead.personId, + name: lead.name, + email: lead.email, + flags: lead.findings.map((finding) => finding.flag), + reasons: lead.findings, + screened_at: lead.screenedAt, + allowed_at: lead.allowedAt, + held: lead.allowedAt === null, + })), + }); + }); + + r.post('/leads/:person_id/screening', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + requireWriter(actor, 'overriding lead screening'); + const personId = c.req.param('person_id'); + const body = await parse(c.req.raw, allowBody); + const changed = await setScreenAllowed(db, { + workspaceId: actor.workspaceId, + personId, + allow: body.allow, + userId: actor.userId, + }); + if (!changed) throw ApiError.notFound('screened lead'); + + await repo.audit(db, { + workspaceId: actor.workspaceId, + actorKind: 'user', + actorId: actor.userId, + eventType: body.allow ? 'lead.screening_allowed' : 'lead.screening_held', + entityKind: 'person', + entityId: personId, + detail: {}, + }); + + return c.json({ person_id: personId, held: !body.allow }); + }); + r.get('/campaigns/:id', async (c) => { const actor = c.get('actor'); const db = c.get('db'); @@ -1541,23 +1629,298 @@ export function autogtmRoutes(deps: AutogtmDeps): Hono { // ----------------------------------------------------------------- helpers -/** Addresses that name a mailbox provider, not a company worth crawling. */ -const FREEMAIL = new Set([ - 'gmail.com', - 'googlemail.com', - 'yahoo.com', - 'hotmail.com', - 'outlook.com', - 'live.com', - 'icloud.com', - 'me.com', - 'aol.com', - 'protonmail.com', - 'proton.me', - 'mail.com', - 'gmx.com', - 'yandex.com', -]); +type LeadFields = { + readonly email?: string | undefined; + readonly name?: string | undefined; + readonly first_name?: string | undefined; + readonly last_name?: string | undefined; + readonly company?: string | undefined; + readonly company_domain?: string | undefined; + readonly job_title?: string | undefined; + readonly location?: string | undefined; + readonly linkedin_url?: string | undefined; + readonly updated_at?: string | undefined; +}; + +/** An API lead (Explee's column names) as the importer's row. */ +function rawFromLead(lead: LeadFields): RawContact { + return { + ...(lead.email ? { email: lead.email } : {}), + ...(lead.name ? { name: lead.name } : {}), + ...(lead.first_name ? { firstName: lead.first_name } : {}), + ...(lead.last_name ? { lastName: lead.last_name } : {}), + ...(lead.company ? { company: lead.company } : {}), + ...(lead.job_title ? { title: lead.job_title } : {}), + ...(lead.location ? { location: lead.location } : {}), + ...(lead.company_domain ? { companyDomain: lead.company_domain } : {}), + ...(lead.linkedin_url ? { linkedinUrl: lead.linkedin_url } : {}), + ...(lead.updated_at ? { updatedAt: lead.updated_at } : {}), + }; +} + +/** Rows of the report a response carries inline; the rest are one GET away. */ +const INLINE_REPORT_ROWS = 500; + +interface ImportIntoCampaignInput { + readonly campaignId: string; + readonly projectId: string; + readonly rows: readonly RawContact[]; + readonly filename: string; + readonly consentBasis: string; + readonly consentSource?: string | undefined; + readonly allowFlagged: boolean; + /** Also skip people already in another campaign of the project. */ + readonly skipProjectDuplicates: boolean; + readonly event: string; +} + +/** + * The one way leads enter a campaign through this API, new or existing. + * + * In order: the importer cleans every row and makes or patches the person + * (newest data wins), screening flags the ones not worth a message, people + * already in the campaign (or, appending, anywhere in the project) and anyone + * on a suppress list are skipped, and the rest become members with their + * company queued for research. Every row that did not simply become a lead is + * written to the import's report with the reason. + */ +async function importIntoCampaign(db: Client, actor: RequestActor, input: ImportIntoCampaignInput) { + const importId = await startContactImport(db, { + workspaceId: actor.workspaceId, + userId: actor.userId, + campaignId: input.campaignId, + filename: input.filename, + consentBasis: input.consentBasis, + ...(input.consentSource ? { consentSource: input.consentSource } : {}), + }); + + // Screened against the whole list, not a chunk: a temp-mail domain is + // only visible once its signups are counted together. + const screen = screenContext(input.rows); + let imported = 0; + let merged = 0; + let updated = 0; + let rejected = 0; + let flagged = 0; + const stored: StoredRow[] = []; + + for (let offset = 0; offset < input.rows.length; offset += IMPORT_CHUNK) { + const result = await importContactChunk( + db, + importId, + input.rows.slice(offset, offset + IMPORT_CHUNK), + { startRow: offset, screen, allowFlagged: input.allowFlagged, userId: actor.userId }, + ); + imported += result.imported; + merged += result.merged; + updated += result.updated; + rejected += result.rejected; + flagged += result.flagged; + stored.push(...result.stored); + } + + const skips = await skipReasons(db, actor.workspaceId, input, stored); + const stamp = now(); + const statements: { sql: string; args: (string | number | null)[] }[] = []; + const added: StoredRow[] = []; + const seenPeople = new Set(); + + for (const entry of stored) { + const skip = skips.get(entry.personId); + if (skip || seenPeople.has(entry.personId)) { + statements.push({ + sql: `INSERT INTO contact_import_rejects (id, import_id, row_number, email, reason, detail, + outcome, created_at) VALUES (?, ?, ?, ?, ?, ?, 'skipped', ?)`, + args: [ + newId('contactImportReject'), + importId, + entry.row, + entry.email, + skip?.reason ?? 'duplicate', + skip?.detail ?? 'the same person appears earlier in the file', + stamp, + ], + }); + continue; + } + seenPeople.add(entry.personId); + added.push(entry); + } + + // The importer makes people; a campaign is a membership. Without this the + // leads exist and the campaign is empty, which is the bug the old + // `/contacts/imports` path shipped with. + for (const entry of added) { + statements.push({ + sql: `INSERT OR IGNORE INTO campaign_people (campaign_id, person_id, workspace_id, status, + interaction_state, discovered_at, updated_at) + VALUES (?, ?, ?, 'discovered', 'never_contacted', ?, ?)`, + args: [input.campaignId, entry.personId, actor.workspaceId, stamp, stamp], + }); + } + const skipped = stored.length - added.length; + statements.push({ + sql: `UPDATE contact_imports SET skipped = skipped + ?, updated_at = ? WHERE id = ?`, + args: [skipped, stamp, importId], + }); + for (let offset = 0; offset < statements.length; offset += IMPORT_CHUNK) { + await db.batch(statements.slice(offset, offset + IMPORT_CHUNK)); + } + for (const entry of added) { + await recordDiscovered(db, { + workspaceId: actor.workspaceId, + campaignId: input.campaignId, + personId: entry.personId, + }); + } + + // Research is what makes a lead sendable: every message is grounded in + // something read about them, so their company site is queued under this + // campaign. One crawl per domain however many colleagues were listed, and + // never a mailbox provider's homepage. + const byRow = new Map(input.rows.map((row, index) => [index + 1, row] as const)); + const domains = new Set(); + for (const entry of added) { + const raw = byRow.get(entry.row); + const domain = normaliseDomain(raw?.companyDomain ?? entry.email.split('@')[1] ?? ''); + if (domain && domain.includes('.') && !isConsumerMailDomain(domain)) domains.add(domain); + } + let crawls = 0; + for (const domain of domains) { + const url = `https://${domain}`; + const queued = await enqueue(db, { + workspaceId: actor.workspaceId, + kind: 'crawl_site', + payload: { url, campaignId: input.campaignId }, + dedupeKey: crawlDedupeKey(url), + }); + if (queued.queued) crawls += 1; + } + + await finishContactImport(db, importId); + + await repo.audit(db, { + workspaceId: actor.workspaceId, + actorKind: 'user', + actorId: actor.userId, + eventType: input.event, + entityKind: 'campaign', + entityId: input.campaignId, + detail: { importId, imported, merged, rejected, skipped, flagged, added: added.length, crawls }, + }); + + const report = await importReport(db, actor.workspaceId, importId); + const reportRows = report?.rows ?? []; + + return { + task_id: importId, + campaign_id: input.campaignId, + status: 'completed' as const, + received: input.rows.length, + added: added.length, + imported, + merged, + updated, + rejected, + skipped, + flagged, + flagged_held: input.allowFlagged ? 0 : flagged, + crawls_queued: crawls, + report: reportRows + .slice(0, INLINE_REPORT_ROWS) + .map((row) => ({ ...row, why: reasonText(row.reason) })), + report_truncated: reportRows.length > INLINE_REPORT_ROWS, + report_url: `/api/v1/autogtm/campaigns/import/${importId}/report?format=csv`, + }; +} + +/** + * Why each stored person is not added, keyed by person id: already in this + * campaign, already in the project, or suppressed. A handful of reads for + * the whole import rather than a few per row. + */ +async function skipReasons( + db: Client, + workspaceId: string, + input: ImportIntoCampaignInput, + stored: readonly StoredRow[], +): Promise> { + const reasons = new Map(); + const ids = [...new Set(stored.map((entry) => entry.personId))]; + const chunks = (list: readonly string[]) => + Array.from({ length: Math.ceil(list.length / 400) }, (_, i) => + list.slice(i * 400, i * 400 + 400), + ); + + // Suppression first: it is the reason that matters most if several apply. + const keysFor = new Map(); + for (const entry of stored) { + const domain = entry.email.split('@')[1] ?? ''; + const keys = [`person:${entry.personId}`, emailMatchKey(entry.email)]; + if (domain && !isConsumerMailDomain(domain)) keys.push(domainMatchKey(domain)); + keysFor.set(entry.personId, [...(keysFor.get(entry.personId) ?? []), ...keys]); + } + const allKeys = [...new Set([...keysFor.values()].flat())]; + const hits = new Map(); + for (const slice of chunks(allKeys)) { + const rows = await queryAll<{ match_key: string; name: string | null; reason: string | null }>( + db, + `SELECT k.match_key, e.name, e.reason + FROM suppression_keys k LEFT JOIN suppression_entries e ON e.id = k.suppression_id + WHERE k.match_key IN (${slice.map(() => '?').join(', ')}) + AND (k.scope = 'global' OR k.workspace_id = ?)`, + [...slice, workspaceId], + ); + for (const row of rows) { + hits.set( + row.match_key, + row.name ? `on the suppress list "${row.name}"` : `suppressed (${row.reason ?? 'opt-out'})`, + ); + } + } + for (const slice of chunks(ids)) { + const rows = await queryAll<{ id: string; status: string }>( + db, + `SELECT id, status FROM people + WHERE id IN (${slice.map(() => '?').join(', ')}) AND status IN ('suppressed', 'deleted')`, + slice, + ); + for (const row of rows) hits.set(`person:${row.id}`, `the person is ${row.status}`); + } + for (const [personId, keys] of keysFor) { + const hit = keys.find((key) => hits.has(key)); + if (hit) reasons.set(personId, { reason: 'suppressed', detail: hits.get(hit)! }); + } + + // Then membership: this campaign, then (appending) the rest of the project. + for (const slice of chunks(ids)) { + const rows = await queryAll<{ person_id: string; campaign_id: string; name: string }>( + db, + `SELECT cp.person_id, cp.campaign_id, c.name + FROM campaign_people cp JOIN campaigns c ON c.id = cp.campaign_id + WHERE cp.workspace_id = ? AND c.offering_id = ? + AND cp.person_id IN (${slice.map(() => '?').join(', ')})`, + [workspaceId, input.projectId, ...slice], + ); + for (const row of rows) { + if (reasons.has(row.person_id)) continue; + if (row.campaign_id === input.campaignId) { + reasons.set(row.person_id, { + reason: 'already_in_campaign', + detail: 'already a lead in this campaign (their details were updated from the file)', + }); + } else if (input.skipProjectDuplicates) { + reasons.set(row.person_id, { + reason: 'already_in_project', + detail: `already a lead in "${row.name}" in this project`, + }); + } + } + } + // An earlier row of the same import already claimed "this campaign"; the + // first occurrence wins, which the caller handles by order. + return reasons; +} interface LeadMember { readonly person_id: string; diff --git a/apps/api/src/import-report.ts b/apps/api/src/import-report.ts new file mode 100644 index 0000000..901a854 --- /dev/null +++ b/apps/api/src/import-report.ts @@ -0,0 +1,142 @@ +/** + * The per-row import report. + * + * An import that silently drops rows is an import nobody trusts twice: the + * list went in with 124 leads, 122 came out, and nobody could say which two or + * why. So every row that did not simply become a lead is kept with what + * happened to it, and this reads them back, all of them, as JSON or as a CSV + * a person can open beside their own file: + * + * - `rejected`: unusable (no address, a malformed one, a placeholder...), + * with the column or check that failed; + * - `skipped`: usable, but already in this campaign or project, or on a + * suppress list; + * - `flagged`: imported, and held back from sending by lead screening until + * a human allows it. + */ + +import type { Context } from 'hono'; +import { toCsv } from '@outreachgraph/domain'; +import { queryAll, queryOne, type Client } from '@outreachgraph/db'; + +export interface ImportReportRow { + readonly row: number | null; + readonly email: string | null; + readonly outcome: 'rejected' | 'skipped' | 'flagged'; + readonly reason: string; + readonly detail: string | null; +} + +export interface ImportReport { + readonly task_id: string; + readonly campaign_id: string | null; + readonly filename: string | null; + readonly total_rows: number; + readonly imported: number; + readonly merged: number; + readonly updated: number; + readonly rejected: number; + readonly skipped: number; + readonly flagged: number; + readonly rows: readonly ImportReportRow[]; +} + +/** Column names and what fails them, for the human reading a report. */ +const REASON_TEXT: Readonly> = { + no_email: 'required column "email" is empty', + malformed_email: 'the "email" column is not an address', + undeliverable_domain: 'the address domain cannot receive mail', + disposable_domain: 'the address is a throwaway mailbox', + role_address: 'nobody reads that mailbox', + placeholder_address: 'a placeholder or test address', + duplicate: 'the same address appears earlier in the file', + already_in_campaign: 'already a lead in this campaign', + already_in_project: 'already a lead in another campaign of this project', + suppressed: 'on a suppress list', +}; + +/** A reason as words, for the UI and the CSV `why` column. */ +export function reasonText(reason: string): string { + return reason + .split('+') + .map((part) => REASON_TEXT[part] ?? part.replace(/_/g, ' ')) + .join('; '); +} + +export async function importReport( + db: Client, + workspaceId: string, + importId: string, +): Promise { + const batch = await queryOne<{ + id: string; + campaign_id: string | null; + filename: string | null; + total_rows: number; + imported: number; + merged: number; + updated: number | null; + rejected: number; + skipped: number | null; + flagged: number | null; + }>( + db, + `SELECT id, campaign_id, filename, total_rows, imported, merged, updated, rejected, skipped, + flagged + FROM contact_imports WHERE id = ? AND workspace_id = ?`, + [importId, workspaceId], + ); + if (!batch) return undefined; + + const rows = await queryAll<{ + row_number: number | null; + email: string | null; + outcome: string | null; + reason: string; + detail: string | null; + }>( + db, + `SELECT row_number, email, outcome, reason, detail FROM contact_import_rejects + WHERE import_id = ? ORDER BY row_number, outcome`, + [importId], + ); + + return { + task_id: batch.id, + campaign_id: batch.campaign_id, + filename: batch.filename, + total_rows: Number(batch.total_rows), + imported: Number(batch.imported), + merged: Number(batch.merged), + updated: Number(batch.updated ?? 0), + rejected: Number(batch.rejected), + skipped: Number(batch.skipped ?? 0), + flagged: Number(batch.flagged ?? 0), + rows: rows.map((row) => ({ + row: row.row_number === null ? null : Number(row.row_number), + email: row.email, + outcome: (row.outcome ?? 'rejected') as ImportReportRow['outcome'], + reason: row.reason, + detail: row.detail, + })), + }; +} + +/** The report as a download: one line per row, in file order. */ +export function importReportCsv(c: Context, report: ImportReport): Response { + const body = toCsv( + report.rows.map((row) => ({ + row: row.row, + email: row.email, + outcome: row.outcome, + reason: row.reason, + why: reasonText(row.reason), + detail: row.detail, + })), + ['row', 'email', 'outcome', 'reason', 'why', 'detail'], + ); + return c.body(body, 200, { + 'content-type': 'text/csv; charset=utf-8', + 'content-disposition': `attachment; filename="import-report-${report.task_id}.csv"`, + }); +} diff --git a/apps/cli/src/cli.test.ts b/apps/cli/src/cli.test.ts index f07520b..0d05c7d 100644 --- a/apps/cli/src/cli.test.ts +++ b/apps/cli/src/cli.test.ts @@ -588,3 +588,59 @@ describe('audience', () => { ).rejects.toThrow('og audience list'); }); }); + +describe('og leads', () => { + test('add sends the file to the append route and prints the report', async () => { + const { mkdtempSync, writeFileSync, rmSync } = await import('node:fs'); + const dir = mkdtempSync(`${import.meta.dir}/../.test-leads-`); + const file = `${dir}/leads.csv`; + writeFileSync(file, 'email,first_name\nada@acme.dev,Ada\ninfo@acme.dev,\n'); + + try { + const { client: api, calls } = client({ + task_id: 'cim_1', + received: 2, + added: 2, + skipped: 0, + rejected: 0, + flagged: 1, + flagged_held: 1, + report: [ + { + row: 2, + email: 'info@acme.dev', + outcome: 'flagged', + reason: 'role_address', + why: 'role address', + detail: 'role_address: info@ is a team inbox, not a person', + }, + ], + }); + const out = await commandByName('leads')!.run({ + client: api, + args: ['add', 'cmp_1', file], + flags: { 'consent-source': 'signups' }, + }); + + expect(calls[0]?.url).toBe('https://api.test/api/v1/autogtm/campaigns/cmp_1/leads'); + expect(calls[0]?.body).toMatchObject({ + csv: 'email,first_name\nada@acme.dev,Ada\ninfo@acme.dev,\n', + consent_source: 'signups', + allow_flagged: false, + skip_project_duplicates: true, + }); + expect(out).toContain('2 of 2 added'); + expect(out).toContain('held back from sending'); + expect(out).toContain('info@ is a team inbox'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test('allow clears a screening hold', async () => { + const { client: api, calls } = client({ person_id: 'per_1', held: false }); + await commandByName('leads')!.run({ client: api, args: ['allow', 'per_1'], flags: {} }); + expect(calls[0]?.url).toBe('https://api.test/api/v1/autogtm/leads/per_1/screening'); + expect(calls[0]?.body).toEqual({ allow: true }); + }); +}); diff --git a/apps/cli/src/commands.ts b/apps/cli/src/commands.ts index c3528ef..3c46cbf 100644 --- a/apps/cli/src/commands.ts +++ b/apps/cli/src/commands.ts @@ -180,6 +180,112 @@ function jobPostDetail(post: Record): string { * worker reads each posting and searches for its people; `resolve` does that * now and prints them. */ +function csvCell(value: unknown): string { + const text = value === null || value === undefined ? '' : String(value); + return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; +} + +/** An import report as CSV, the same columns the API's ?format=csv download has. */ +export function reportCsv(report: readonly Record[]): string { + const columns = ['row', 'email', 'outcome', 'reason', 'why', 'detail']; + return `${[columns.join(','), ...report.map((r) => columns.map((c) => csvCell(r[c])).join(','))].join('\n')}\n`; +} + +/** One line per reported row: what happened to it and why. */ +export function reportLines(report: readonly Record[]): string[] { + return report.map( + (row) => + ` row ${pad(text(row, 'row', '?'), 6)} ${pad(text(row, 'outcome'), 9)} ` + + `${pad(text(row, 'email', '-'), 32)} ${text(row, 'detail', text(row, 'why'))}`, + ); +} + +/** + * `og leads`: append a CSV to a running campaign, read an import's report, + * and review the leads screening is holding back. + */ +async function runLeads({ client, args, flags }: CommandContext): Promise { + const [verb, target, file] = args; + const fs = await import('node:fs'); + + if (verb === 'add') { + if (!target || !file) { + throw new Error('usage: og leads add --consent-source ""'); + } + const result = (await client.post(`/autogtm/campaigns/${encodeURIComponent(target)}/leads`, { + csv: fs.readFileSync(file, 'utf8'), + filename: file.split('/').pop(), + ...(flagString(flags, 'consent-source') + ? { consent_source: flagString(flags, 'consent-source') } + : {}), + allow_flagged: flags['allow-flagged'] === true, + skip_project_duplicates: flags['keep-project-duplicates'] !== true, + })) as Record; + + const report = rows(result, 'report'); + const out = flagString(flags, 'report'); + if (out) fs.writeFileSync(out, reportCsv(report)); + + return [ + `${text(result, 'added', '0')} of ${text(result, 'received', '0')} added to ${target} ` + + `(task ${text(result, 'task_id')}): ${text(result, 'skipped', '0')} skipped, ` + + `${text(result, 'rejected', '0')} not usable, ${text(result, 'flagged', '0')} flagged by screening` + + (Number(result.flagged_held ?? 0) > 0 ? ' and held back from sending' : ''), + ...reportLines(report), + ...(result.report_truncated ? [' … the rest: og leads report --csv'] : []), + ...(out ? [`Report written to ${out}`] : []), + ].join('\n'); + } + + if (verb === 'report') { + if (!target) throw new Error('usage: og leads report [--csv]'); + const result = (await client.get( + `/autogtm/campaigns/import/${encodeURIComponent(target)}/report`, + )) as Record; + const report = rows(result, 'rows'); + if (flags.csv === true) return reportCsv(report).trimEnd(); + return [ + `${text(result, 'total_rows', '0')} rows: ${text(result, 'imported', '0')} new, ` + + `${text(result, 'merged', '0')} known, ${text(result, 'rejected', '0')} not usable, ` + + `${text(result, 'skipped', '0')} skipped, ${text(result, 'flagged', '0')} flagged`, + ...reportLines(report), + ].join('\n'); + } + + if (verb === 'screened') { + if (!target) throw new Error('usage: og leads screened [--all]'); + const result = (await client.get( + `/autogtm/campaigns/${encodeURIComponent(target)}/screened`, + flags.all === true ? { include_allowed: 'true' } : {}, + )) as Record; + const leads = rows(result, 'leads'); + if (leads.length === 0) return 'Screening is holding nobody back in this campaign.'; + return [ + `${text(result, 'held', '0')} held back from sending:`, + ...leads.map((lead) => { + const reasons = (lead.reasons as { flag: string; detail: string }[] | undefined) ?? []; + return ( + `${pad(text(lead, 'person_id'), 30)} ${lead.held ? 'held ' : 'allowed'} ` + + `${pad(text(lead, 'email', '-'), 32)} ${reasons.map((r) => r.detail).join('; ')}` + ); + }), + 'Send to one anyway: og leads allow ', + ].join('\n'); + } + + if (verb === 'allow' || verb === 'hold') { + if (!target) throw new Error(`usage: og leads ${verb} `); + await client.post(`/autogtm/leads/${encodeURIComponent(target)}/screening`, { + allow: verb === 'allow', + }); + return verb === 'allow' + ? `${target} allowed: screening no longer holds them back.` + : `${target} held back from sending again.`; + } + + throw new Error('usage: og leads add|report|screened|allow|hold … (og help)'); +} + async function runJobs({ client, args, flags }: CommandContext): Promise { const [verb = 'list', ...rest] = args; const target = rest[0]; @@ -1553,6 +1659,13 @@ export const COMMANDS: readonly Command[] = [ summary: 'Job postings by URL or keyword, and the real people behind each one.', run: runJobs, }, + { + name: 'leads', + usage: + 'og leads add --consent-source "" [--allow-flagged] [--keep-project-duplicates] [--report out.csv] | report [--csv] | screened [--all] | allow | hold ', + summary: 'Add a CSV of leads to a running campaign, with a per-row report and screening.', + run: runLeads, + }, { name: 'webhooks', usage: diff --git a/apps/mcp/src/mcp.test.ts b/apps/mcp/src/mcp.test.ts index 29d2fd2..1055fda 100644 --- a/apps/mcp/src/mcp.test.ts +++ b/apps/mcp/src/mcp.test.ts @@ -154,6 +154,8 @@ describe('tools', () => { urls: ['https://apply.workable.com/raydar/j/C39C58F585/'], jobPostId: 'jbp_1', contactId: 'jpc_1', + campaignId: 'cmp_1', + leads: [{ email: 'ada@acme.dev' }], }) .catch(() => undefined); @@ -323,3 +325,37 @@ describe('inbox tools', () => { expect(runTool(tool, client, { personId: 'per_1' })).rejects.toThrow('text is required'); }); }); + +describe('lead tools', () => { + test('add_leads_to_campaign appends through the AutoGTM route', async () => { + const { fetchImpl, calls } = recorder(ok({ added: 1, report: [] })); + const client = createClient(CONFIG, fetchImpl); + + await runTool(toolByName('add_leads_to_campaign')!, client, { + campaignId: 'cmp_1', + csv: 'email\nada@acme.dev\n', + consentSource: 'signups', + }); + expect(calls[0]?.method).toBe('POST'); + expect(calls[0]?.url).toBe('https://api.test/api/v1/autogtm/campaigns/cmp_1/leads'); + expect(calls[0]?.body).toEqual({ + csv: 'email\nada@acme.dev\n', + consent_source: 'signups', + allow_flagged: false, + }); + }); + + test('screening tools read the held list and override one lead', async () => { + const { fetchImpl, calls } = recorder(ok({ held: 0, leads: [] })); + const client = createClient(CONFIG, fetchImpl); + + await runTool(toolByName('list_screened_leads')!, client, { campaignId: 'cmp_1' }); + await runTool(toolByName('allow_screened_lead')!, client, { personId: 'per_1', allow: true }); + await runTool(toolByName('get_import_report')!, client, { taskId: 'cim_1' }); + + expect(calls.map((c) => c.method)).toEqual(['GET', 'POST', 'GET']); + expect(calls[0]?.url).toBe('https://api.test/api/v1/autogtm/campaigns/cmp_1/screened'); + expect(calls[1]?.body).toEqual({ allow: true }); + expect(calls[2]?.url).toBe('https://api.test/api/v1/autogtm/campaigns/import/cim_1/report'); + }); +}); diff --git a/apps/mcp/src/tools.ts b/apps/mcp/src/tools.ts index 41cf3d3..f5950c2 100644 --- a/apps/mcp/src/tools.ts +++ b/apps/mcp/src/tools.ts @@ -920,6 +920,92 @@ export const TOOLS: readonly ToolDefinition[] = [ reason: str(args, 'reason') ?? 'requested by an agent', }), }, + { + name: 'add_leads_to_campaign', + title: 'Add leads to an existing campaign', + description: + 'Append up to 5,000 leads to a campaign that is already running, as `leads` (email, ' + + 'first_name, last_name, company_domain, job_title, linkedin_url...) or as the text of a ' + + 'CSV file. People already in the campaign or its project, and anyone on a suppress list, ' + + 'are skipped. Screening flags generated names, relay and temp-mail addresses, bot and test ' + + 'accounts and role inboxes, and holds them back from sending unless allowFlagged. The ' + + 'result is a per-row report: every rejected, skipped and flagged row with the reason. ' + + 'Report those to the user rather than summarising them away.', + readOnly: false, + inputSchema: { + type: 'object', + properties: { + campaignId: { type: 'string' }, + leads: { type: 'array', items: { type: 'object' } }, + csv: { type: 'string', description: 'A CSV with a header row. Send this or leads.' }, + consentSource: { type: 'string', description: 'Where these people came from.' }, + allowFlagged: { type: 'boolean', description: 'Send to screened leads too.' }, + }, + required: ['campaignId'], + }, + run: (client, args) => + client.post(`/autogtm/campaigns/${encodeURIComponent(require(args, 'campaignId'))}/leads`, { + ...(Array.isArray(args.leads) ? { leads: args.leads } : {}), + // Untrimmed: the file's own line endings are part of the CSV. + ...(typeof args.csv === 'string' && args.csv.trim() ? { csv: args.csv } : {}), + ...(str(args, 'consentSource') ? { consent_source: str(args, 'consentSource') } : {}), + allow_flagged: args.allowFlagged === true, + }), + }, + { + name: 'get_import_report', + title: 'Read an import report', + description: + 'Every row an import rejected, skipped or flagged, with the column or check behind it, ' + + 'for the task_id an import or add_leads_to_campaign returned.', + readOnly: true, + inputSchema: { + type: 'object', + properties: { taskId: { type: 'string' } }, + required: ['taskId'], + }, + run: (client, args) => + client.get(`/autogtm/campaigns/import/${encodeURIComponent(require(args, 'taskId'))}/report`), + }, + { + name: 'list_screened_leads', + title: 'List leads held back by screening', + description: + 'A campaign’s leads that lead screening is holding back from sending, each with its ' + + 'reasons (generated name, relay address, temp-mail domain, bot or test account, role ' + + 'inbox). Nothing is sent to them until a human allows them.', + readOnly: true, + inputSchema: { + type: 'object', + properties: { + campaignId: { type: 'string' }, + includeAllowed: { type: 'boolean' }, + }, + required: ['campaignId'], + }, + run: (client, args) => + client.get( + `/autogtm/campaigns/${encodeURIComponent(require(args, 'campaignId'))}/screened`, + args.includeAllowed === true ? { include_allowed: 'true' } : {}, + ), + }, + { + name: 'allow_screened_lead', + title: 'Send to a screened lead anyway', + description: + 'Clears (allow: true) or restores (allow: false) the screening hold on one lead. Only do ' + + 'this when the user has looked at the reason and decided the person is real.', + readOnly: false, + inputSchema: { + type: 'object', + properties: { personId: { type: 'string' }, allow: { type: 'boolean' } }, + required: ['personId', 'allow'], + }, + run: (client, args) => + client.post(`/autogtm/leads/${encodeURIComponent(require(args, 'personId'))}/screening`, { + allow: args.allow === true, + }), + }, { name: 'list_inbox', title: 'List conversations', diff --git a/apps/web/app/(app)/products/[id]/page.tsx b/apps/web/app/(app)/products/[id]/page.tsx index 14a5ea7..ee3f527 100644 --- a/apps/web/app/(app)/products/[id]/page.tsx +++ b/apps/web/app/(app)/products/[id]/page.tsx @@ -1,7 +1,9 @@ import Link from 'next/link'; import { notFound, redirect } from 'next/navigation'; import { Avatar } from '../../../../components/avatar'; +import { AddLeads } from '../../../../components/add-leads'; import { CampaignControls } from '../../../../components/campaign-controls'; +import { ScreenedLeads } from '../../../../components/screened-leads'; import { ApiUnavailableError, NotAuthenticatedError, @@ -135,6 +137,12 @@ export default async function ProductPage({ params }: { params: Promise<{ id: st + {campaign ? ( + <> + + + + ) : null} {leads.length === 0 ? ( {campaign && campaign.jobs_pending > 0 diff --git a/apps/web/components/add-leads.tsx b/apps/web/components/add-leads.tsx new file mode 100644 index 0000000..5b43249 --- /dev/null +++ b/apps/web/components/add-leads.tsx @@ -0,0 +1,270 @@ +'use client'; + +import { useRouter } from 'next/navigation'; +import { useState, type ChangeEvent } from 'react'; +import { mapHeaders } from '@outreachgraph/domain'; +import { applyMapping, parseCsv, type MappedRow } from '../lib/csv'; + +/** + * Adding leads from a CSV to a campaign that is already running. + * + * The thing Explee cannot do (every import there is a new campaign), and the + * thing that burned us there: rows vanishing without a word. So the result + * is the report, not a count. Every row that did not become a lead is listed + * with the column or check behind it, the whole list downloads as a CSV to + * open beside the original file, and the leads screening is holding back are + * named as such rather than quietly sent to or quietly dropped. + */ + +const MAX_ROWS = 5_000; + +interface ReportRow { + readonly row: number | null; + readonly email: string | null; + readonly outcome: 'rejected' | 'skipped' | 'flagged'; + readonly reason: string; + readonly why: string; + readonly detail: string | null; +} + +interface AppendResult { + readonly received: number; + readonly added: number; + readonly imported: number; + readonly merged: number; + readonly updated: number; + readonly rejected: number; + readonly skipped: number; + readonly flagged: number; + readonly flagged_held: number; + readonly report: readonly ReportRow[]; + readonly report_truncated: boolean; + readonly report_url: string; +} + +const OUTCOME_LABEL: Readonly> = { + rejected: 'Not usable', + skipped: 'Skipped', + flagged: 'Held by screening', +}; + +export function AddLeads({ campaignId }: { campaignId: string }) { + const router = useRouter(); + const [open, setOpen] = useState(false); + const [rows, setRows] = useState([]); + const [mapping, setMapping] = useState>({}); + const [headers, setHeaders] = useState([]); + const [filename, setFilename] = useState(''); + const [consentSource, setConsentSource] = useState(''); + const [allowFlagged, setAllowFlagged] = useState(false); + const [busy, setBusy] = useState(false); + const [result, setResult] = useState(); + const [error, setError] = useState(); + + function onFile(event: ChangeEvent): void { + const file = event.target.files?.[0]; + if (!file) return; + setError(undefined); + setResult(undefined); + setFilename(file.name); + + const reader = new FileReader(); + reader.onload = () => { + const [head, ...body] = parseCsv(String(reader.result ?? '')); + if (!head) { + setError('That file has no rows.'); + return; + } + const found = mapHeaders(head); + setHeaders(head); + if (found.email === undefined) { + setRows([]); + setError(`No email column. Columns seen: ${head.join(', ')}. Rename one to "email".`); + return; + } + if (body.length > MAX_ROWS) { + setRows([]); + setError(`${body.length.toLocaleString()} rows: split the file into ${MAX_ROWS} or fewer.`); + return; + } + setMapping(found); + setRows(applyMapping(body, found)); + }; + reader.readAsText(file); + } + + async function run(): Promise { + setBusy(true); + setError(undefined); + try { + const response = await fetch( + `/api/v1/autogtm/campaigns/${encodeURIComponent(campaignId)}/leads`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + credentials: 'same-origin', + body: JSON.stringify({ + filename, + consent_source: consentSource, + allow_flagged: allowFlagged, + leads: rows.map((row) => ({ + email: row.email, + name: row.name, + first_name: row.firstName, + last_name: row.lastName, + company: row.company, + company_domain: row.companyDomain, + job_title: row.title, + location: row.location, + linkedin_url: row.linkedinUrl, + updated_at: row.updatedAt, + })), + }), + }, + ); + const payload = (await response.json().catch(() => ({}))) as AppendResult & { + error?: { message?: string }; + }; + if (!response.ok) { + setError(payload.error?.message ?? `That failed (${response.status}).`); + return; + } + setResult(payload); + setRows([]); + router.refresh(); + } catch { + setError('Lost the connection. Re-running is safe: leads already added are skipped.'); + } finally { + setBusy(false); + } + } + + if (!open) { + return ( + + ); + } + + return ( +
+

+ A CSV with an email column; names, company domain, job title and LinkedIn are used when + present. People already in this product, and anyone on a suppress list, are skipped. Every + row that does not become a lead is reported with the reason. +

+ + + + {rows.length > 0 ? ( + <> +

+ {rows.length.toLocaleString()} rows in {filename}. +

+

+ {Object.entries(mapping) + .map(([field, index]) => `${field} ← ${headers[index]}`) + .join(' · ')} +

+ setConsentSource(event.target.value)} + placeholder="Where these people came from (e.g. app signups)" + disabled={busy} + className="border-border bg-surface mt-2 w-full rounded-xl border p-2 text-sm" + /> + + + + ) : null} + + {result ? : null} + + {error ? ( +

+ {error} +

+ ) : null} +
+ ); +} + +function ImportReportView({ result }: { result: AppendResult }) { + const groups = new Map(); + for (const row of result.report) { + const key = `${row.outcome}:${row.reason}`; + const group = groups.get(key) ?? { outcome: row.outcome, why: row.why, n: 0 }; + group.n += 1; + groups.set(key, group); + } + + return ( +
+

+ {result.added.toLocaleString()} of{' '} + {result.received.toLocaleString()} added + {result.skipped > 0 ? `, ${result.skipped.toLocaleString()} skipped` : ''} + {result.rejected > 0 ? `, ${result.rejected.toLocaleString()} not usable` : ''} + {result.flagged > 0 + ? `, ${result.flagged.toLocaleString()} flagged by screening${ + result.flagged_held > 0 ? ' and held back from sending' : ' (sending anyway)' + }` + : ''} + . +

+ {groups.size > 0 ? ( +
    + {[...groups.values()].map((group) => ( +
  • + {group.n.toLocaleString()}{' '} + {OUTCOME_LABEL[group.outcome].toLowerCase()}: {group.why} +
    • + ))} +
+ ) : null} + {result.report.length > 0 ? ( +
+ Every row ({result.report.length}) +
    + {result.report.map((row) => ( +
  • + Row {row.row ?? '?'} {row.email ? `(${row.email})` : ''}:{' '} + {OUTCOME_LABEL[row.outcome]}, {row.detail ?? row.why} +
    • + ))} +
+
+ ) : null} + + Download the full report (CSV) + +
+ ); +} diff --git a/apps/web/components/contact-import.tsx b/apps/web/components/contact-import.tsx index ab2f6ca..1c5653d 100644 --- a/apps/web/components/contact-import.tsx +++ b/apps/web/components/contact-import.tsx @@ -60,6 +60,8 @@ export function ContactImport() { const [progress, setProgress] = useState(); const [summary, setSummary] = useState(); const [groups, setGroups] = useState([]); + const [flagged, setFlagged] = useState([]); + const [reportUrl, setReportUrl] = useState(); const [error, setError] = useState(); function onFile(event: ChangeEvent): void { @@ -153,8 +155,14 @@ export function ContactImport() { const detail = await fetch(`/api/v1/contacts/imports/${importId}`); if (detail.ok) { - const body = (await detail.json()) as { rejectsByReason: RejectGroup[] }; + const body = (await detail.json()) as { + rejectsByReason: RejectGroup[]; + flaggedByReason?: RejectGroup[]; + reportUrl?: string; + }; setGroups(body.rejectsByReason); + setFlagged(body.flaggedByReason ?? []); + setReportUrl(body.reportUrl); } setProgress(100); @@ -258,6 +266,23 @@ export function ContactImport() { ) : null} + {flagged.length > 0 ? ( +
    + {flagged.map((group) => ( +
  • + {group.n.toLocaleString()} imported but held + back from sending: {group.reason.replace(/\+/g, ', ').replace(/_/g, ' ')} +
    • + ))} +
+ ) : null} + + {reportUrl && (summary.rejected > 0 || flagged.length > 0) ? ( + + Download every row and why (CSV) + + ) : null} + {progress === 100 ? (

Looking up published profiles for each of them now — a GitHub or Mastodon account diff --git a/apps/web/components/screened-leads.tsx b/apps/web/components/screened-leads.tsx new file mode 100644 index 0000000..cd7834a --- /dev/null +++ b/apps/web/components/screened-leads.tsx @@ -0,0 +1,118 @@ +'use client'; + +import { useEffect, useState } from 'react'; + +/** + * The leads screening is holding back, each with its reason and an Allow. + * + * Held, not hidden: a relay mailbox or a role inbox can still be the right + * person, and the owner is the one who knows. Allowing one clears the hold + * for every send path at once, because the hold lives in the policy engine. + */ + +interface ScreenedLead { + readonly person_id: string; + readonly name: string; + readonly email: string | null; + readonly reasons: readonly { flag: string; detail: string }[]; + readonly held: boolean; +} + +const FLAG_LABEL: Readonly> = { + generated_name: 'generated name', + relay_address: 'relay address', + temp_mail_domain: 'temp-mail domain', + agent_account: 'bot or test account', + role_address: 'role inbox', +}; + +export function ScreenedLeads({ campaignId }: { campaignId: string }) { + const [leads, setLeads] = useState(); + const [held, setHeld] = useState(0); + const [busy, setBusy] = useState(); + const [error, setError] = useState(); + + useEffect(() => { + let cancelled = false; + void fetch(`/api/v1/autogtm/campaigns/${encodeURIComponent(campaignId)}/screened?limit=50`, { + credentials: 'same-origin', + }) + .then(async (response) => { + if (!response.ok || cancelled) return; + const body = (await response.json()) as { held: number; leads: ScreenedLead[] }; + setLeads(body.leads); + setHeld(body.held); + }) + .catch(() => undefined); + return () => { + cancelled = true; + }; + }, [campaignId]); + + async function allow(personId: string): Promise { + setBusy(personId); + setError(undefined); + try { + const response = await fetch( + `/api/v1/autogtm/leads/${encodeURIComponent(personId)}/screening`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + credentials: 'same-origin', + body: JSON.stringify({ allow: true }), + }, + ); + if (!response.ok) { + setError(`Could not allow that lead (${response.status}).`); + return; + } + setLeads((current) => current?.filter((lead) => lead.person_id !== personId)); + setHeld((n) => Math.max(0, n - 1)); + } finally { + setBusy(undefined); + } + } + + if (!leads || held === 0) return null; + + return ( +

+ + {held.toLocaleString()} {held === 1 ? 'lead' : 'leads'} held back by screening + +

+ Imported, researched, not written to. Allow one to send to them anyway. +

+
    + {leads.map((lead) => ( +
  • +
    +
    + {lead.name} + {lead.email ? · {lead.email} : null} +
    +
    + {lead.reasons + .map((reason) => `${FLAG_LABEL[reason.flag] ?? reason.flag}: ${reason.detail}`) + .join('; ')} +
    +
    + +
    • + ))} +
+ {error ? ( +

+ {error} +

+ ) : null} +
+ ); +} diff --git a/apps/web/lib/csv.ts b/apps/web/lib/csv.ts index 576e809..5b678a7 100644 --- a/apps/web/lib/csv.ts +++ b/apps/web/lib/csv.ts @@ -1,134 +1,7 @@ /** - * A CSV reader, in the browser. - * - * Written rather than installed because the alternative is shipping a parsing - * library to every page of a PWA to read one file on one screen, and because - * the hard part of CSV is small and well-specified: quoted fields may contain - * commas, newlines and doubled quotes, and everything else is splitting. - * - * Parsing happens client-side deliberately. The file is already on the user's - * machine; uploading several megabytes so a server can split it on commas - * spends bandwidth and a request timeout to learn nothing the browser did not - * already know. What the browser must not do is *decide* anything — every row - * it produces is still cleaned server-side, because a client can be told to - * lie about which addresses are real. + * The CSV reader lives in `@outreachgraph/domain`, so the CLI and the API read + * a file exactly as this screen does. Re-exported here for the components that + * already import it from this path. */ -/** - * Splits CSV text into rows of fields. - * - * Handles the three things that break naive splitting: `"a,b"` (a comma inside - * a field), `"line one\nline two"` (a newline inside a field), and `"say ""hi"""` - * (an escaped quote). CRLF is normalised, and a trailing newline does not - * produce a phantom empty row. - */ -export function parseCsv(text: string): string[][] { - const rows: string[][] = []; - let row: string[] = []; - let field = ''; - let quoted = false; - let index = 0; - - // A byte-order mark survives Excel exports and would otherwise become part - // of the first header, so `Email` never matches `email`. - const input = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text; - - const endField = (): void => { - row.push(field); - field = ''; - }; - - const endRow = (): void => { - endField(); - // A blank trailing line is not a record. - if (row.length > 1 || row[0] !== '') rows.push(row); - row = []; - }; - - while (index < input.length) { - const char = input[index]; - - if (quoted) { - if (char === '"') { - // A doubled quote inside a quoted field is one literal quote. - if (input[index + 1] === '"') { - field += '"'; - index += 2; - continue; - } - quoted = false; - index += 1; - continue; - } - - field += char; - index += 1; - continue; - } - - if (char === '"') { - quoted = true; - index += 1; - continue; - } - - if (char === ',') { - endField(); - index += 1; - continue; - } - - if (char === '\r') { - index += 1; - continue; - } - - if (char === '\n') { - endRow(); - index += 1; - continue; - } - - field += char; - index += 1; - } - - // Whatever is left after the last newline is a row, unless the file ended - // cleanly on one. - if (field !== '' || row.length > 0) endRow(); - - return rows; -} - -/** A row keyed by the fields the importer understands. */ -export interface MappedRow { - email?: string; - name?: string; - firstName?: string; - lastName?: string; - company?: string; - title?: string; - location?: string; - companyDomain?: string; - linkedinUrl?: string; - updatedAt?: string; -} - -/** Applies a header mapping to the data rows. */ -export function applyMapping( - rows: readonly string[][], - mapping: Readonly>, -): MappedRow[] { - return rows.map((cells) => { - const mapped: MappedRow = {}; - - for (const [field, index] of Object.entries(mapping)) { - const value = cells[index]; - if (value !== undefined && value.trim() !== '') { - mapped[field as keyof MappedRow] = value; - } - } - - return mapped; - }); -} +export { applyMapping, parseCsv, toCsv, type MappedRow } from '@outreachgraph/domain'; diff --git a/migrations-pg/0051_lead_screening.sql b/migrations-pg/0051_lead_screening.sql new file mode 100644 index 0000000..b39301a --- /dev/null +++ b/migrations-pg/0051_lead_screening.sql @@ -0,0 +1,16 @@ +-- 0051_lead_screening.sql (Postgres). See migrations/0051_lead_screening.sql for the reasoning. +-- A new, empty table and three ADD COLUMNs with constant defaults: no rewrite, no heavy index. + +create table if not exists lead_screens ( + workspace_id text NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + person_id text NOT NULL REFERENCES people(id) ON DELETE CASCADE, + findings text NOT NULL DEFAULT '[]', + screened_at text NOT NULL, + allowed_at text, + allowed_by text, + PRIMARY KEY (workspace_id, person_id) +); + +ALTER TABLE contact_import_rejects ADD COLUMN IF NOT EXISTS outcome text NOT NULL DEFAULT 'rejected'; +ALTER TABLE contact_imports ADD COLUMN IF NOT EXISTS skipped bigint NOT NULL DEFAULT 0; +ALTER TABLE contact_imports ADD COLUMN IF NOT EXISTS flagged bigint NOT NULL DEFAULT 0; diff --git a/migrations/0051_lead_screening.sql b/migrations/0051_lead_screening.sql new file mode 100644 index 0000000..b667003 --- /dev/null +++ b/migrations/0051_lead_screening.sql @@ -0,0 +1,29 @@ +-- 0051_lead_screening.sql +-- +-- Two things an import could not say before: which leads are not worth a +-- message, and what happened to every row that did not become a lead. +-- +-- lead_screens holds, per workspace and person, the reasons screening found +-- (a generated name, a relay mailbox, a temp-mail domain, an agent or test +-- account, a role inbox). A row with findings and no allowed_at holds the +-- person back from cold outreach through the policy engine; allowed_at is the +-- human saying "send to them anyway", and it survives a re-import. +-- +-- contact_import_rejects grows an outcome so one table is the whole per-row +-- report: 'rejected' (unusable row, as before), 'skipped' (usable, but already +-- in the campaign or project, or suppressed) and 'flagged' (imported, held by +-- screening). The batch keeps a count of each. + +CREATE TABLE IF NOT EXISTS lead_screens ( + workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + person_id TEXT NOT NULL REFERENCES people(id) ON DELETE CASCADE, + findings TEXT NOT NULL DEFAULT '[]', + screened_at TEXT NOT NULL, + allowed_at TEXT, + allowed_by TEXT, + PRIMARY KEY (workspace_id, person_id) +); + +ALTER TABLE contact_import_rejects ADD COLUMN outcome TEXT NOT NULL DEFAULT 'rejected'; +ALTER TABLE contact_imports ADD COLUMN skipped INTEGER NOT NULL DEFAULT 0; +ALTER TABLE contact_imports ADD COLUMN flagged INTEGER NOT NULL DEFAULT 0; diff --git a/packages/domain/src/contact-import.ts b/packages/domain/src/contact-import.ts index 7438cf0..c9a27fb 100644 --- a/packages/domain/src/contact-import.ts +++ b/packages/domain/src/contact-import.ts @@ -434,7 +434,9 @@ export function cleanContact(raw: RawContact, seen?: ReadonlySet): Clean ...(tidy(raw.company) ? { company: tidy(raw.company) } : {}), ...(tidy(raw.title) ? { title: tidy(raw.title) } : {}), ...(tidy(raw.location) ? { location: tidy(raw.location) } : {}), - ...optional('companyDomain', companyDomainFrom(tidy(raw.companyDomain) || undefined)), + // A webmail host in the company column is a mistake in the export, not + // an employer: gmail.com is never anybody's company. + ...optional('companyDomain', employerDomain(tidy(raw.companyDomain) || undefined)), ...optional('linkedinUrl', linkedinProfile(raw.linkedinUrl)), ...optional('updatedAt', isoTime(raw.updatedAt)), domain, @@ -488,6 +490,13 @@ export function mapHeaders(headers: readonly string[]): Record { return mapping; } +function employerDomain(raw: string | undefined): string | undefined { + const domain = companyDomainFrom(raw); + return domain && !FREEMAIL_DOMAINS.has(domain) && !DISPOSABLE_DOMAINS.has(domain) + ? domain + : undefined; +} + const optional = (key: K, value: string | undefined) => (value ? { [key]: value } : {}) as Partial>; diff --git a/packages/domain/src/csv.ts b/packages/domain/src/csv.ts new file mode 100644 index 0000000..b4dca75 --- /dev/null +++ b/packages/domain/src/csv.ts @@ -0,0 +1,149 @@ +/** + * A CSV reader, shared by the browser, the CLI and the API. + * + * Written rather than installed because the alternative is shipping a parsing + * library to every page of a PWA to read one file on one screen, and because + * the hard part of CSV is small and well-specified: quoted fields may contain + * commas, newlines and doubled quotes, and everything else is splitting. + * + * Parsing happens client-side deliberately. The file is already on the user's + * machine; uploading several megabytes so a server can split it on commas + * spends bandwidth and a request timeout to learn nothing the browser did not + * already know. What the browser must not do is *decide* anything — every row + * it produces is still cleaned server-side, because a client can be told to + * lie about which addresses are real. + */ + +/** + * Splits CSV text into rows of fields. + * + * Handles the three things that break naive splitting: `"a,b"` (a comma inside + * a field), `"line one\nline two"` (a newline inside a field), and `"say ""hi"""` + * (an escaped quote). CRLF is normalised, and a trailing newline does not + * produce a phantom empty row. + */ +export function parseCsv(text: string): string[][] { + const rows: string[][] = []; + let row: string[] = []; + let field = ''; + let quoted = false; + let index = 0; + + // A byte-order mark survives Excel exports and would otherwise become part + // of the first header, so `Email` never matches `email`. + const input = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text; + + const endField = (): void => { + row.push(field); + field = ''; + }; + + const endRow = (): void => { + endField(); + // A blank trailing line is not a record. + if (row.length > 1 || row[0] !== '') rows.push(row); + row = []; + }; + + while (index < input.length) { + const char = input[index]; + + if (quoted) { + if (char === '"') { + // A doubled quote inside a quoted field is one literal quote. + if (input[index + 1] === '"') { + field += '"'; + index += 2; + continue; + } + quoted = false; + index += 1; + continue; + } + + field += char; + index += 1; + continue; + } + + if (char === '"') { + quoted = true; + index += 1; + continue; + } + + if (char === ',') { + endField(); + index += 1; + continue; + } + + if (char === '\r') { + index += 1; + continue; + } + + if (char === '\n') { + endRow(); + index += 1; + continue; + } + + field += char; + index += 1; + } + + // Whatever is left after the last newline is a row, unless the file ended + // cleanly on one. + if (field !== '' || row.length > 0) endRow(); + + return rows; +} + +/** A row keyed by the fields the importer understands. */ +export interface MappedRow { + email?: string; + name?: string; + firstName?: string; + lastName?: string; + company?: string; + title?: string; + location?: string; + companyDomain?: string; + linkedinUrl?: string; + updatedAt?: string; +} + +/** Applies a header mapping to the data rows. */ +export function applyMapping( + rows: readonly string[][], + mapping: Readonly>, +): MappedRow[] { + return rows.map((cells) => { + const mapped: MappedRow = {}; + + for (const [field, index] of Object.entries(mapping)) { + const value = cells[index]; + if (value !== undefined && value.trim() !== '') { + mapped[field as keyof MappedRow] = value; + } + } + + return mapped; + }); +} + +/** One CSV cell: quoted only when it has to be. */ +export function csvCell(value: unknown): string { + const text = value === null || value === undefined ? '' : String(value); + return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; +} + +/** Rows of objects as CSV, in the column order given, with a header line. */ +export function toCsv( + rows: readonly Readonly>[], + columns: readonly string[], +): string { + const lines = rows.map((row) => columns.map((column) => csvCell(row[column])).join(',')); + return `${[columns.join(','), ...lines].join('\n')}\n`; +} diff --git a/packages/domain/src/index.ts b/packages/domain/src/index.ts index 16c92c0..64dac34 100644 --- a/packages/domain/src/index.ts +++ b/packages/domain/src/index.ts @@ -25,6 +25,8 @@ export * from './playbooks'; export * from './plans'; export * from './credits'; export * from './contact-import'; +export * from './lead-screen'; +export * from './csv'; export * from './web-presence'; export * from './rules'; export * from './pipeline'; diff --git a/packages/domain/src/lead-screen.test.ts b/packages/domain/src/lead-screen.test.ts new file mode 100644 index 0000000..31ae0ab --- /dev/null +++ b/packages/domain/src/lead-screen.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, test } from 'bun:test'; +import { + describeFindings, + generatedNamePart, + isConsumerMailDomain, + isRelayDomain, + isRoleLocal, + nameFromAddress, + parseFindings, + screenContext, + screenLead, + type ScreenRow, +} from './lead-screen'; +import { cleanContact } from './contact-import'; + +const flags = (row: ScreenRow, rows: readonly ScreenRow[] = [row]) => + screenLead(row, screenContext(rows)).map((finding) => finding.flag); + +describe('screenLead', () => { + test('a real person at a real company passes', () => { + expect( + flags({ email: 'ada.lovelace@acme.com', firstName: 'Ada', lastName: 'Lovelace' }), + ).toEqual([]); + }); + + test('gmail is not a flag: real buyers use it', () => { + expect(flags({ email: 'dave@gmail.com', name: 'Dave Mackenzie' })).toEqual([]); + }); + + test('a name with digits is generated', () => { + expect(flags({ email: 'x1@acme.com', firstName: 'Jalen', lastName: 'Borer853' })).toContain( + 'generated_name', + ); + expect(generatedNamePart({ name: 'Kuhic12 Lemke' })).toBe('Kuhic12'); + expect(generatedNamePart({ name: 'Mary-Jane Watson' })).toBeUndefined(); + }); + + test('relay hosts are flagged, including every simplelogin TLD', () => { + for (const domain of [ + 'passmail.net', + 'passinbox.com', + 'passfwd.com', + 'aleeas.com', + 'simplelogin.com', + 'simplelogin.fr', + 'slmail.me', + 'mozmail.com', + 'duck.com', + 'addy.io', + 'anonaddy.me', + 'hidingmail.com', + 'agentmail.to', + ]) { + expect(isRelayDomain(domain)).toBe(true); + expect(flags({ email: `someone@${domain}` })).toContain('relay_address'); + } + expect(isRelayDomain('acme.com')).toBe(false); + }); + + test('known temp-mail domains are flagged on their own', () => { + expect(flags({ email: 'abc@maildock.store' })).toContain('temp_mail_domain'); + expect(flags({ email: 'abc@fake.legal' })).toContain('temp_mail_domain'); + }); + + test('three unrelated signups at one obscure domain is a temp-mail domain', () => { + const rows = [ + { email: 'xk2p9@zzmail.biz' }, + { email: 'qq81l@zzmail.biz' }, + { email: 'mw0aa@zzmail.biz' }, + ]; + expect(flags(rows[0]!, rows)).toContain('temp_mail_domain'); + // Two is not enough to say. + expect(flags(rows[0]!, rows.slice(0, 2))).not.toContain('temp_mail_domain'); + }); + + test('colleagues at one company are not a temp-mail domain', () => { + const named = [ + { email: 'ada.lovelace@acme.com' }, + { email: 'bob.ng@acme.com' }, + { email: 'cy.tan@acme.com' }, + ]; + expect(flags(named[0]!, named)).toEqual([]); + + const claimed = [ + { email: 'a1@acme.com', companyDomain: 'acme.com' }, + { email: 'b2@acme.com' }, + { email: 'c3@acme.com' }, + ]; + expect(flags(claimed[1]!, claimed)).not.toContain('temp_mail_domain'); + }); + + test('agents, bots and test accounts are flagged', () => { + expect(flags({ email: 'sales-bot@acme.com' })).toContain('agent_account'); + expect(flags({ email: 'qa.test3@acme.com' })).toContain('agent_account'); + expect(flags({ email: 'dana@acme.com', name: 'Dana Test' })).toContain('agent_account'); + expect(flags({ email: 'agentsmith@acme.com' })).not.toContain('agent_account'); + }); + + test('role inboxes are flagged', () => { + expect(flags({ email: 'info@acme.com' })).toContain('role_address'); + expect(flags({ email: 'sales.team@acme.com' })).toContain('role_address'); + expect(isRoleLocal('hello2')).toBe(true); + expect(isRoleLocal('ada')).toBe(false); + }); + + test('findings explain themselves and survive storage', () => { + const findings = screenLead({ email: 'info@passmail.net' }); + expect(describeFindings(findings)).toContain('relay_address: passmail.net'); + expect(parseFindings(JSON.stringify(findings))).toEqual(findings); + expect(parseFindings('not json')).toEqual([]); + }); +}); + +describe('isConsumerMailDomain', () => { + test('webmail, relays, temp mail and newsletter hosts are never a company', () => { + for (const domain of ['gmail.com', 'substack.com', 'passmail.net', 'mailinator.com', 'qq.com']) + expect(isConsumerMailDomain(domain)).toBe(true); + expect(isConsumerMailDomain('acme.com')).toBe(false); + }); + + test('the importer never stores a webmail host as the company', () => { + const result = cleanContact({ email: 'dave@acme.com', companyDomain: 'gmail.com' }); + expect(result.ok && result.contact.companyDomain).toBeFalsy(); + }); +}); + +describe('nameFromAddress', () => { + test('reads first.last, first_last and first-last', () => { + expect(nameFromAddress('scott.perry@acme.com')).toEqual({ + firstName: 'Scott', + lastName: 'Perry', + }); + expect(nameFromAddress('gabriella_fiore@x.com')).toEqual({ + firstName: 'Gabriella', + lastName: 'Fiore', + }); + expect(nameFromAddress('ann-lee+news@x.com')).toEqual({ firstName: 'Ann', lastName: 'Lee' }); + }); + + test('refuses what is not clearly a name', () => { + for (const email of [ + 'jsmith@x.com', + 'jordan.i@x.com', + 'john.smith42@x.com', + 'info.team@x.com', + 'sales.us@x.com', + 'a.b.c@x.com', + 'test.bot@x.com', + ]) { + expect(nameFromAddress(email)).toBeUndefined(); + } + }); +}); diff --git a/packages/domain/src/lead-screen.ts b/packages/domain/src/lead-screen.ts new file mode 100644 index 0000000..8944cd3 --- /dev/null +++ b/packages/domain/src/lead-screen.ts @@ -0,0 +1,445 @@ +/** + * Screening a lead list before anything is sent to it. + * + * The contact cleaner (`contact-import.ts`) decides whether a row has a + * mailbox at all. This decides whether the mailbox is worth a message, and it + * is deliberately a *flag* rather than a reject: a screened lead is stored, + * shown with its reason, and held back from sending until a human says + * otherwise. Dropping them would repeat the failure this exists to fix, where + * a list shrank on import and nobody could say which rows went or why. + * + * Every rule here came out of a real list. A 6,960-address signup export held + * generated names ("Borer853"), relay mailboxes that hide who is behind them + * (passmail, aleeas, simplelogin), throwaway domains shared by a handful of + * unrelated signups, agents and test accounts, and role inboxes. Mailing those + * costs sender reputation and buys nothing. + * + * Pure functions over rows, so each rule can be argued with in a test. + */ + +import { isDisposableDomain, isFreemailDomain } from './contact-import'; + +/** Why a lead is held back from sending. */ +export const SCREEN_FLAGS = [ + 'generated_name', + 'relay_address', + 'temp_mail_domain', + 'agent_account', + 'role_address', +] as const; +export type ScreenFlag = (typeof SCREEN_FLAGS)[number]; + +export interface ScreenFinding { + readonly flag: ScreenFlag; + /** One sentence a person can check, naming the value that tripped it. */ + readonly detail: string; +} + +/** The columns screening reads. Everything but the address is optional. */ +export interface ScreenRow { + readonly email?: string | undefined; + readonly name?: string | undefined; + readonly firstName?: string | undefined; + readonly lastName?: string | undefined; + readonly companyDomain?: string | undefined; +} + +/** + * Forwarding services whose whole point is that the address does not say who + * is behind it. The person may be real; the mailbox tells you nothing about an + * employer, and a reply-rate on a relay is a reply-rate on a filter. + */ +const RELAY_DOMAINS = new Set([ + 'passmail.net', + 'passmail.com', + 'passinbox.com', + 'passfwd.com', + 'aleeas.com', + 'slmail.me', + 'mozmail.com', + 'duck.com', + 'addy.io', + 'anonaddy.com', + 'anonaddy.me', + 'hidingmail.com', + 'agentmail.to', + '33mail.com', + 'privaterelay.appleid.com', + 'relay.firefox.com', +]); + +/** Relay brands that register under several TLDs: simplelogin.com, .co, .io, .fr... */ +const RELAY_BRANDS = ['simplelogin', 'anonaddy']; + +/** + * Temp-mail domains seen in our own lists that are not on the cleaner's + * disposable list. The shared-domain rule below catches most of these on its + * own; the ones here were seen with fewer than three signups. + */ +const TEMP_MAIL_DOMAINS = new Set(['maildock.store', 'fake.legal']); + +/** + * Mailbox hosts that are not companies, beyond the consumer webmail the + * cleaner already knows: newsletter platforms (a substack.com address is a + * publication, not a person at Substack) and regional webmail. + */ +const NOT_A_COMPANY = new Set([ + 'substack.com', + 'mail.com', + 'yandex.com', + 'rambler.ru', + 'list.ru', + 'bk.ru', + 'inbox.ru', + 'wp.pl', + 'o2.pl', + 'interia.pl', + 'hotmail.fr', + 'hotmail.de', + 'hotmail.it', + 'hotmail.es', + 'outlook.de', + 'outlook.fr', + 'live.fr', + 'live.de', + 'yahoo.fr', + 'yahoo.de', + 'yahoo.es', + 'yahoo.it', + 'yahoo.ca', + 'yahoo.co.in', + 'yahoo.co.jp', + 'yahoo.com.br', + 'uol.com.br', + 'bol.com.br', + 'gmx.net', + 'gmx.at', + 'gmx.ch', + 'protonmail.ch', + 'tuta.io', + 'tutanota.de', + 'skiff.com', + 'zohomail.com', +]); + +/** + * Inboxes a team reads, not a person. Wider than the cleaner's list, which + * only rejects mailboxes nobody reads at all: `sales@` is read, it is just not + * somebody, and a cold message to it lands in a ticket queue. + */ +const ROLE_LOCALS = new Set([ + 'info', + 'admin', + 'administrator', + 'support', + 'help', + 'helpdesk', + 'sales', + 'contact', + 'contacts', + 'hello', + 'hi', + 'hey', + 'team', + 'office', + 'mail', + 'email', + 'enquiries', + 'enquiry', + 'inquiries', + 'inquiry', + 'billing', + 'accounts', + 'accounting', + 'finance', + 'invoices', + 'jobs', + 'careers', + 'hr', + 'recruiting', + 'press', + 'media', + 'pr', + 'marketing', + 'legal', + 'privacy', + 'security', + 'compliance', + 'webmaster', + 'hostmaster', + 'it', + 'dev', + 'developers', + 'engineering', + 'ops', + 'feedback', + 'partners', + 'partnerships', + 'service', + 'customerservice', + 'orders', + 'shop', + 'store', + 'reception', + 'general', +]); + +/** Words that mark an address or a name as a machine or a test, not a buyer. */ +const AGENT_WORDS = new Set([ + 'bot', + 'bots', + 'agent', + 'agents', + 'aiagent', + 'test', + 'tests', + 'testing', + 'tester', + 'qa', + 'demo', + 'dummy', + 'sandbox', + 'staging', + 'automation', + 'automated', + 'crawler', + 'scraper', + 'gpt', + 'chatgpt', + 'claude', + 'openclaw', + 'llm', +]); + +const lower = (value: string | undefined): string => (value ?? '').trim().toLowerCase(); + +function domainOf(email: string): string { + const at = email.lastIndexOf('@'); + return at >= 0 ? email.slice(at + 1) : ''; +} + +function localOf(email: string): string { + const at = email.lastIndexOf('@'); + return (at >= 0 ? email.slice(0, at) : email).split('+')[0] ?? ''; +} + +/** `slmail.me`, `x.simplelogin.fr`, `privaterelay.appleid.com`: a forwarding service. */ +export function isRelayDomain(domain: string): boolean { + const host = lower(domain); + if (!host) return false; + for (const relay of RELAY_DOMAINS) { + if (host === relay || host.endsWith(`.${relay}`)) return true; + } + const labels = host.split('.'); + return labels.some((label) => RELAY_BRANDS.includes(label)); +} + +/** + * True when an address at this domain says nothing about an employer: + * consumer webmail, a relay, a throwaway, a newsletter platform. + * + * The one test every "is this a company?" question goes through, so a gmail + * address is never crawled as if Google employed the lead, and a relay is + * never looked up as a company page. + */ +export function isConsumerMailDomain(domain: string): boolean { + const host = lower(domain).replace(/^www\./, ''); + if (!host) return true; + return ( + isFreemailDomain(host) || + isDisposableDomain(host) || + NOT_A_COMPANY.has(host) || + TEMP_MAIL_DOMAINS.has(host) || + isRelayDomain(host) + ); +} + +/** `info@`, `sales.team@`: an inbox a team reads. */ +export function isRoleLocal(local: string): boolean { + const bare = lower(local).replace(/\d+$/, ''); + if (ROLE_LOCALS.has(bare)) return true; + const parts = bare.split(/[._-]+/).filter(Boolean); + return parts.length > 0 && parts.every((part) => ROLE_LOCALS.has(part)); +} + +/** + * A name a generator produced rather than a person typed. + * + * Seeded fake-data libraries suffix surnames with digits ("Borer853", + * "Kuhic12"), and a person almost never puts a number in their own name. + * Only letters-then-digits inside a name part counts: a handle like `dave2` + * arriving in the email column is the cleaner's business, not this one's. + */ +export function generatedNamePart(row: ScreenRow): string | undefined { + const parts = [row.firstName, row.lastName, ...(row.name ?? '').split(/\s+/)] + .map((part) => (part ?? '').trim()) + .filter(Boolean); + return parts.find((part) => /\p{L}{2,}\d{1,4}$/u.test(part) || /^\d+\p{L}+/u.test(part)); +} + +function agentWord(row: ScreenRow, local: string): string | undefined { + const tokens = [ + ...local.split(/[._\-\d]+/), + ...[row.name, row.firstName, row.lastName].flatMap((value) => + lower(value).split(/[\s._\-\d]+/), + ), + ].filter(Boolean); + return tokens.find((token) => AGENT_WORDS.has(token)); +} + +/** Whether an address's local part reads as somebody's name. */ +function nameLike(row: ScreenRow, local: string): boolean { + const bare = lower(local).replace(/\d+$/, ''); + if (/^[a-z]{2,}[._-][a-z]{2,}$/.test(bare)) return true; + const first = lower(row.firstName ?? row.name?.split(/\s+/)[0]); + const last = lower(row.lastName ?? row.name?.split(/\s+/).slice(1).join(' ')); + return Boolean( + (first.length >= 2 && bare.includes(first)) || (last.length >= 2 && bare.includes(last)), + ); +} + +/** What the whole list says about each domain in it. */ +export interface ScreenContext { + readonly domains: ReadonlyMap; +} + +export interface DomainStats { + /** Leads in the list at this domain. */ + readonly leads: number; + /** Of those, how many name it as their company. */ + readonly claimed: number; + /** Of those, how many have an address that reads as their name. */ + readonly nameLike: number; +} + +/** + * Counts the list by domain, once, so the shared-domain rule can see past the + * row it is judging. Built over the *whole* list: a chunk of five hundred + * would miss a temp-mail domain whose signups were spread across chunks. + */ +export function screenContext(rows: readonly ScreenRow[]): ScreenContext { + const domains = new Map(); + for (const row of rows) { + const email = lower(row.email); + const domain = domainOf(email); + if (!domain) continue; + const stats = domains.get(domain) ?? { leads: 0, claimed: 0, nameLike: 0 }; + stats.leads += 1; + if (lower(row.companyDomain).replace(/^www\./, '') === domain) stats.claimed += 1; + if (nameLike(row, localOf(email))) stats.nameLike += 1; + domains.set(domain, stats); + } + return { domains }; +} + +/** Three unrelated signups at one obscure domain is a throwaway, not a company. */ +export const SHARED_DOMAIN_MIN = 3; + +/** + * Every reason this lead should not be mailed, or none. + * + * Consumer webmail is not a flag: plenty of real buyers use gmail. It is only + * never treated as their company (see `isConsumerMailDomain`). + */ +export function screenLead(row: ScreenRow, context?: ScreenContext): ScreenFinding[] { + const email = lower(row.email); + const domain = domainOf(email); + const local = localOf(email); + const findings: ScreenFinding[] = []; + + const generated = generatedNamePart(row); + if (generated) { + findings.push({ + flag: 'generated_name', + detail: `the name "${generated}" carries digits, which a fake-data generator does and a person does not`, + }); + } + + if (domain && isRelayDomain(domain)) { + findings.push({ + flag: 'relay_address', + detail: `${domain} is a forwarding relay that hides who is behind the address`, + }); + } + + if (domain && TEMP_MAIL_DOMAINS.has(domain)) { + findings.push({ flag: 'temp_mail_domain', detail: `${domain} is a temp-mail domain` }); + } else if (domain && !isConsumerMailDomain(domain) && context) { + const stats = context.domains.get(domain); + // Colleagues share a domain too. They differ from a throwaway in that + // somebody names it as their company, or their addresses spell names. + if ( + stats && + stats.leads >= SHARED_DOMAIN_MIN && + stats.claimed === 0 && + stats.nameLike * 2 < stats.leads + ) { + findings.push({ + flag: 'temp_mail_domain', + detail: + `${domain} is shared by ${stats.leads} leads in this list, none names it as their ` + + 'company and their addresses are not names: it looks like a temp-mail domain', + }); + } + } + + const agent = agentWord(row, local); + if (agent) { + findings.push({ + flag: 'agent_account', + detail: `"${agent}" in the address or name marks a bot, agent or test account`, + }); + } + + if (local && isRoleLocal(local)) { + findings.push({ + flag: 'role_address', + detail: `${local}@ is a team inbox, not a person`, + }); + } + + return findings; +} + +/** One line for a report cell: `relay_address: ...; role_address: ...`. */ +export function describeFindings(findings: readonly ScreenFinding[]): string { + return findings.map((finding) => `${finding.flag}: ${finding.detail}`).join('; '); +} + +/** Parses stored findings, tolerating anything malformed as "none". */ +export function parseFindings(json: string | null | undefined): ScreenFinding[] { + if (!json) return []; + try { + const parsed: unknown = JSON.parse(json); + if (!Array.isArray(parsed)) return []; + return parsed.filter( + (entry): entry is ScreenFinding => + Boolean(entry) && + typeof (entry as ScreenFinding).flag === 'string' && + typeof (entry as ScreenFinding).detail === 'string', + ); + } catch { + return []; + } +} + +/** + * `scott.perry@acme.com` -> Scott / Perry. + * + * Stricter than `nameFromEmail`, which only makes a display label: these go + * into first_name and last_name, where a wrong value is a wrong greeting. Only + * the unambiguous shape counts: exactly two alphabetic parts of two or more + * letters joined by `.`, `_` or `-`, neither a role word. `jsmith`, `john.s` + * and `john.smith42` say too little to be a name. + */ +export function nameFromAddress( + email: string, +): { readonly firstName: string; readonly lastName: string } | undefined { + const local = lower(email.slice(0, email.lastIndexOf('@'))).replace(/\+.*$/, ''); + const match = /^([a-z]{2,})[._-]([a-z]{2,})$/.exec(local); + if (!match?.[1] || !match[2]) return undefined; + if (ROLE_LOCALS.has(match[1]) || ROLE_LOCALS.has(match[2])) return undefined; + if (AGENT_WORDS.has(match[1]) || AGENT_WORDS.has(match[2])) return undefined; + const cap = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + return { firstName: cap(match[1]), lastName: cap(match[2]) }; +} diff --git a/packages/pipeline/src/autopilot.ts b/packages/pipeline/src/autopilot.ts index a05337c..2d05687 100644 --- a/packages/pipeline/src/autopilot.ts +++ b/packages/pipeline/src/autopilot.ts @@ -48,6 +48,7 @@ import { AUTOPILOT_ACTOR, } from './outreach-email'; import { budgetStatus } from './metering'; +import { holdReason } from './lead-screen'; import { ownDomains, refuseRecipient } from './recipient-guard'; export interface AutopilotDeps { @@ -132,6 +133,8 @@ interface Candidate { readonly believed_minor: number; readonly outreach_eligible: number; readonly identity_confidence: number; + /** Lead screening's findings while they hold this person back; null otherwise. */ + readonly screen_findings: string | null; readonly min_outreach_confidence: number; readonly draft_id: string | null; readonly subject: string | null; @@ -360,6 +363,9 @@ export async function runAutopilot( c.approval_mode, c.budget_json, p.display_name, p.status AS person_status, p.believed_minor, p.outreach_eligible, p.identity_confidence, + (SELECT ls.findings FROM lead_screens ls + WHERE ls.workspace_id = r.workspace_id AND ls.person_id = p.id + AND ls.allowed_at IS NULL) AS screen_findings, w.min_outreach_confidence, d.id AS draft_id, d.subject, d.body, d.checks_json, co.name AS company_name, co.contact_email AS company_contact_email, @@ -531,6 +537,9 @@ export async function runAutopilot( approvalMode: row.approval_mode as 'trusted_automation', hasConnectedAccount: sender !== undefined, personSuppressed: row.person_status === 'suppressed' || row.outreach_eligible === 0, + ...(holdReason(row.screen_findings) + ? { personScreenedOut: holdReason(row.screen_findings) } + : {}), personBelievedMinor: row.believed_minor === 1, personDeleted: row.person_status === 'deleted', identityConfidence: row.identity_confidence, diff --git a/packages/pipeline/src/cadence-runner.ts b/packages/pipeline/src/cadence-runner.ts index 9e12884..13038d4 100644 --- a/packages/pipeline/src/cadence-runner.ts +++ b/packages/pipeline/src/cadence-runner.ts @@ -19,6 +19,7 @@ import { matchKeysForPerson } from './suppression-keys'; import type { PolicyDecision, PolicyRequest } from '@outreachgraph/policy'; import { advanceCadences, type AdvanceResult, type DueEnrollment } from './cadence'; import { budgetStatus } from './metering'; +import { screenHold } from './lead-screen'; export interface RunCadencesDeps { readonly db: Client; @@ -102,11 +103,13 @@ async function policyInputsFor( const suppressed = person?.status === 'suppressed' || (await isSuppressed(db, enrollment.workspace_id, enrollment.person_id)); + const screenedOut = await screenHold(db, enrollment.workspace_id, enrollment.person_id); return { approvalMode: (campaign?.approval_mode ?? 'draft_and_approve') as 'draft_and_approve', hasConnectedAccount: connected || (step.network === 'email' && platformEmailEnabled), personSuppressed: suppressed, + ...(screenedOut ? { personScreenedOut: screenedOut } : {}), personBelievedMinor: person?.believed_minor === 1, personDeleted: person?.status === 'deleted', identityConfidence: person?.identity_confidence ?? 0, diff --git a/packages/pipeline/src/contact-import.ts b/packages/pipeline/src/contact-import.ts index 6ee060a..c49365f 100644 --- a/packages/pipeline/src/contact-import.ts +++ b/packages/pipeline/src/contact-import.ts @@ -15,10 +15,15 @@ import { cleanContact, + describeFindings, newId, + screenContext, + screenLead, type CleanContact, type RawContact, type RejectReason, + type ScreenContext, + type ScreenFinding, } from '@outreachgraph/domain'; import { now, queryOne, type Client } from '@outreachgraph/db'; @@ -77,9 +82,45 @@ export interface ChunkResult { /** Of the merged, how many people the row's newer data changed. */ readonly updated: number; readonly rejected: number; + /** Of the imported and merged, how many screening holds back from sending. */ + readonly flagged: number; readonly personIds: readonly string[]; + /** Every row that became (or matched) a person, in file order. */ + readonly stored: readonly StoredRow[]; } +/** One row that became, or matched, a person. */ +export interface StoredRow { + /** 1-based row number in the file (the header is not counted). */ + readonly row: number; + readonly email: string; + readonly personId: string; + /** False when the address was already on file. */ + readonly created: boolean; + /** What screening found; empty for a clean lead. */ + readonly findings: readonly ScreenFinding[]; +} + +export interface ChunkOptions { + readonly startRow?: number; + /** + * The whole list's screening context. Without it the chunk is screened + * against itself, which still catches everything but a temp-mail domain + * whose signups were spread across chunks. + */ + readonly screen?: ScreenContext; + /** The importer chose to send to screened leads anyway: hold nobody back. */ + readonly allowFlagged?: boolean; + readonly userId?: string | undefined; +} + +type Unscreened = Omit & { + readonly raw?: RawContact; + readonly contact?: CleanContact; +}; + +type Unfinished = Omit & { stored: readonly Unscreened[] }; + /** True when an insert lost a race against the unique index. */ function isUniqueViolation(error: unknown): boolean { const message = String((error as { message?: string })?.message ?? error); @@ -98,7 +139,7 @@ export async function importContactChunk( db: Client, importId: string, rows: readonly RawContact[], - options: { readonly startRow?: number } = {}, + options: ChunkOptions = {}, ): Promise { const batch = await queryOne<{ workspace_id: string; @@ -117,7 +158,7 @@ export async function importContactChunk( // ---------------------------------------------------------------- clean // Pure and local: no database, so five hundred rows cost nothing here. const seen = new Set(); - const clean: { row: number; contact: CleanContact }[] = []; + const clean: { row: number; contact: CleanContact; raw: RawContact }[] = []; const rejects: { row: number; email?: string; reason: string; detail: string }[] = []; for (const [offset, raw] of rows.entries()) { @@ -135,7 +176,7 @@ export async function importContactChunk( } seen.add(result.contact.dedupeKey); - clean.push({ row: rowNumber, contact: result.contact }); + clean.push({ row: rowNumber, contact: result.contact, raw }); } // ------------------------------------------------------------- existing @@ -179,7 +220,7 @@ export async function importContactChunk( for (const reject of rejects) { statements.push({ sql: `INSERT INTO contact_import_rejects (id, import_id, row_number, email, reason, detail, - created_at) VALUES (?, ?, ?, ?, ?, ?, ?)`, + outcome, created_at) VALUES (?, ?, ?, ?, ?, ?, 'rejected', ?)`, args: [ newId('contactImportReject'), importId, @@ -210,23 +251,7 @@ export async function importContactChunk( // The slow path stores and patches row by row, so nothing is left for // the batched patch below. const retried = await storeOneAtATime(db, importId, batch, clean, rejects.length); - await db.execute({ - sql: `UPDATE contact_imports - SET total_rows = total_rows + ?, imported = imported + ?, - merged = merged + ?, updated = updated + ?, rejected = rejected + ?, - updated_at = ? - WHERE id = ?`, - args: [ - rows.length, - retried.imported, - retried.merged, - retried.updated, - retried.rejected, - now(), - importId, - ], - }); - return retried; + return finishChunk(db, importId, batch.workspace_id, rows, retried, options); } } @@ -252,15 +277,147 @@ export async function importContactChunk( }), ); - await db.execute({ + const stored: Unscreened[] = [ + ...fresh.map((entry, index) => ({ + row: entry.row, + email: entry.contact.email, + personId: freshIds[index]!, + created: true, + raw: entry.raw, + contact: entry.contact, + })), + ...known.flatMap((entry) => { + const personId = existing.get(entry.contact.dedupeKey); + return personId + ? [ + { + row: entry.row, + email: entry.contact.email, + personId, + created: false, + raw: entry.raw, + contact: entry.contact, + }, + ] + : []; + }), + ]; + + return finishChunk( + db, + importId, + batch.workspace_id, + rows, + { imported, merged, updated, rejected, personIds, stored }, + options, + ); +} + +/** + * Screens what was stored, writes the held leads into the report, and closes + * the chunk's counters. Shared by the batched path and the slow one. + */ +async function finishChunk( + db: Client, + importId: string, + workspaceId: string, + rows: readonly RawContact[], + result: Unfinished, + options: ChunkOptions, +): Promise { + const context = options.screen ?? screenContext(rows); + const stamp = now(); + const statements: { sql: string; args: (string | number | null)[] }[] = []; + const stored: StoredRow[] = []; + let flagged = 0; + + for (const entry of [...result.stored].sort((a, b) => a.row - b.row)) { + const raw = entry.raw ?? { email: entry.email }; + const findings = screenLead( + { + email: entry.email, + name: raw.name, + firstName: raw.firstName, + lastName: raw.lastName, + companyDomain: entry.contact?.companyDomain ?? raw.companyDomain, + }, + context, + ); + stored.push({ + row: entry.row, + email: entry.email, + personId: entry.personId, + created: entry.created, + findings, + }); + if (findings.length === 0) continue; + + flagged += 1; + statements.push( + { + // An earlier "send to them anyway" survives a re-import: a human + // already looked at this person. + sql: `INSERT INTO lead_screens (workspace_id, person_id, findings, screened_at, + allowed_at, allowed_by) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (workspace_id, person_id) DO UPDATE SET + findings = excluded.findings, screened_at = excluded.screened_at, + allowed_at = COALESCE(lead_screens.allowed_at, excluded.allowed_at), + allowed_by = COALESCE(lead_screens.allowed_by, excluded.allowed_by)`, + args: [ + workspaceId, + entry.personId, + JSON.stringify(findings), + stamp, + options.allowFlagged ? stamp : null, + options.allowFlagged ? (options.userId ?? 'import') : null, + ], + }, + { + sql: `INSERT INTO contact_import_rejects (id, import_id, row_number, email, reason, detail, + outcome, created_at) VALUES (?, ?, ?, ?, ?, ?, 'flagged', ?)`, + args: [ + newId('contactImportReject'), + importId, + entry.row, + entry.email, + findings.map((finding) => finding.flag).join('+'), + describeFindings(findings) + + (options.allowFlagged ? ' (allowed at import: will be sent to)' : ''), + stamp, + ], + }, + ); + } + + statements.push({ sql: `UPDATE contact_imports SET total_rows = total_rows + ?, imported = imported + ?, - merged = merged + ?, updated = updated + ?, rejected = rejected + ?, updated_at = ? + merged = merged + ?, updated = updated + ?, rejected = rejected + ?, + flagged = flagged + ?, updated_at = ? WHERE id = ?`, - args: [rows.length, imported, merged, updated, rejected, now(), importId], + args: [ + rows.length, + result.imported, + result.merged, + result.updated, + result.rejected, + flagged, + stamp, + importId, + ], }); - - return { imported, merged, updated, rejected, personIds }; + await db.batch(statements); + + return { + imported: result.imported, + merged: result.merged, + updated: result.updated, + rejected: result.rejected, + flagged, + personIds: result.personIds, + stored, + }; } /** @@ -366,10 +523,11 @@ async function storeOneAtATime( db: Client, importId: string, batch: { workspace_id: string; consent_basis: string; consent_source: string | null }, - clean: readonly { row: number; contact: CleanContact }[], + clean: readonly { row: number; contact: CleanContact; raw: RawContact }[], alreadyRejected: number, -): Promise { +): Promise { const personIds: string[] = []; + const stored: Unscreened[] = []; let imported = 0; let merged = 0; let updated = 0; @@ -393,6 +551,14 @@ async function storeOneAtATime( if (!outcome.created) updated += changed; personIds.push(outcome.personId); + stored.push({ + row: entry.row, + email: entry.contact.email, + personId: outcome.personId, + created: outcome.created, + raw: entry.raw, + contact: entry.contact, + }); } catch (error) { rejected += 1; await recordReject( @@ -406,7 +572,7 @@ async function storeOneAtATime( } } - return { imported, merged, updated, rejected, personIds }; + return { imported, merged, updated, rejected, personIds, stored }; } async function recordReject( diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index 8d3d5b3..76c8242 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -175,9 +175,19 @@ export { finishContactImport, importContactChunk, startContactImport, + type ChunkOptions, type ChunkResult, type StartImportInput, + type StoredRow, } from './contact-import'; +export { + countScreenedHeld, + holdReason, + screenedLeads, + screenHold, + setScreenAllowed, + type ScreenedLead, +} from './lead-screen'; export { enrichContact, sweepContactEnrichment, diff --git a/packages/pipeline/src/lead-screen.ts b/packages/pipeline/src/lead-screen.ts new file mode 100644 index 0000000..e5fd95c --- /dev/null +++ b/packages/pipeline/src/lead-screen.ts @@ -0,0 +1,136 @@ +/** + * Screened leads: who is held back from cold outreach, and the human override. + * + * Screening itself is pure (`@outreachgraph/domain`, `screenLead`) and runs at + * import. This is what reads its verdict back. The hold is enforced where every + * other "do not contact" is, as an input to the policy engine + * (`personScreenedOut`), so the approval queue, autopilot, cadences and card + * generation all refuse the same person for the same reason, and an answer to + * someone who wrote back is never blocked by it. + */ + +import { describeFindings, parseFindings, type ScreenFinding } from '@outreachgraph/domain'; +import { now, queryAll, queryOne, type Client } from '@outreachgraph/db'; + +/** + * Why this person is held back from cold outreach in this workspace, or + * undefined when they are not (never screened, screened clean, or allowed). + */ +export async function screenHold( + db: Client, + workspaceId: string, + personId: string, +): Promise { + const row = await queryOne<{ findings: string }>( + db, + `SELECT findings FROM lead_screens + WHERE workspace_id = ? AND person_id = ? AND allowed_at IS NULL`, + [workspaceId, personId], + ); + return holdReason(row?.findings); +} + +/** The policy engine's reason text from stored findings, or undefined for none. */ +export function holdReason(json: string | null | undefined): string | undefined { + const findings = parseFindings(json); + return findings.length > 0 + ? `${describeFindings(findings)}. Allow the lead to send to them anyway.` + : undefined; +} + +/** + * Sends to a screened lead anyway (`allow: true`), or holds them back again. + * Returns false when screening never held this person. + */ +export async function setScreenAllowed( + db: Client, + input: { + readonly workspaceId: string; + readonly personId: string; + readonly allow: boolean; + readonly userId?: string | undefined; + }, +): Promise { + const result = await db.execute({ + sql: `UPDATE lead_screens SET allowed_at = ?, allowed_by = ? + WHERE workspace_id = ? AND person_id = ?`, + args: [ + input.allow ? now() : null, + input.allow ? (input.userId ?? 'api') : null, + input.workspaceId, + input.personId, + ], + }); + return result.rowsAffected > 0; +} + +export interface ScreenedLead { + readonly personId: string; + readonly name: string; + readonly email: string | null; + readonly findings: readonly ScreenFinding[]; + readonly screenedAt: string; + readonly allowedAt: string | null; +} + +/** + * The campaign's screened leads, held first. With `includeAllowed` false + * (the default) only the ones still held back are listed. + */ +export async function screenedLeads( + db: Client, + input: { + readonly workspaceId: string; + readonly campaignId: string; + readonly includeAllowed?: boolean; + readonly limit?: number; + }, +): Promise { + const rows = await queryAll<{ + person_id: string; + display_name: string; + email: string | null; + findings: string; + screened_at: string; + allowed_at: string | null; + }>( + db, + `SELECT ls.person_id, p.display_name, ls.findings, ls.screened_at, ls.allowed_at, + (SELECT pe.address FROM person_emails pe + WHERE pe.person_id = ls.person_id AND pe.workspace_id = ls.workspace_id + ORDER BY pe.created_at LIMIT 1) AS email + FROM lead_screens ls + JOIN campaign_people cp ON cp.person_id = ls.person_id AND cp.campaign_id = ? + JOIN people p ON p.id = ls.person_id + WHERE ls.workspace_id = ? AND ls.findings != '[]' AND p.status != 'deleted' + ${input.includeAllowed ? '' : 'AND ls.allowed_at IS NULL'} + ORDER BY (ls.allowed_at IS NULL) DESC, ls.screened_at DESC + LIMIT ?`, + [input.campaignId, input.workspaceId, Math.min(input.limit ?? 200, 1000)], + ); + + return rows.map((row) => ({ + personId: row.person_id, + name: row.display_name, + email: row.email, + findings: parseFindings(row.findings), + screenedAt: row.screened_at, + allowedAt: row.allowed_at, + })); +} + +/** How many of a campaign's leads screening is holding back right now. */ +export async function countScreenedHeld( + db: Client, + workspaceId: string, + campaignId: string, +): Promise { + const row = await queryOne<{ n: number }>( + db, + `SELECT COUNT(*) AS n FROM lead_screens ls + JOIN campaign_people cp ON cp.person_id = ls.person_id AND cp.campaign_id = ? + WHERE ls.workspace_id = ? AND ls.findings != '[]' AND ls.allowed_at IS NULL`, + [campaignId, workspaceId], + ); + return Number(row?.n ?? 0); +} diff --git a/packages/pipeline/src/pipeline.ts b/packages/pipeline/src/pipeline.ts index fc3c704..11c00a9 100644 --- a/packages/pipeline/src/pipeline.ts +++ b/packages/pipeline/src/pipeline.ts @@ -38,6 +38,7 @@ import { rescoreProspect } from './jobs'; import { recordDiscovered, recordStatus } from './stages'; import { storeDiscoveredPhoto } from './photos'; import { enqueueFindEmail } from './find-email-queue'; +import { screenHold } from './lead-screen'; export interface PipelineOptions { readonly db: Client; @@ -988,6 +989,9 @@ async function createRecommendation( const counts = await actionCounts(db, workspaceId, personId); const flags = await featureFlags(db, workspaceId); const connected = await connectedNetworks(db, workspaceId); + // A screened lead (relay mailbox, generated name...) gets research but no + // cold card until a human allows them. + const screenedOut = await screenHold(db, workspaceId, personId); const result = generateRecommendation({ personId, @@ -1006,6 +1010,7 @@ async function createRecommendation( // Overridden per network by `connectedNetworks` above. hasConnectedAccount: connected.size > 0 || options.emailSendingEnabled === true, personSuppressed: person.status === 'suppressed', + ...(screenedOut ? { personScreenedOut: screenedOut } : {}), personBelievedMinor: person.believed_minor === 1, personDeleted: person.status === 'deleted', identityConfidence: person.identity_confidence, diff --git a/packages/policy/src/engine.test.ts b/packages/policy/src/engine.test.ts index e5af184..46338e6 100644 --- a/packages/policy/src/engine.test.ts +++ b/packages/policy/src/engine.test.ts @@ -205,6 +205,36 @@ describe('person eligibility', () => { } }); + test('a lead held by screening gets no cold message, but an answer still goes', () => { + const cold = evaluatePolicy( + request({ + network: 'email', + action: 'send_email', + personScreenedOut: 'relay_address: passmail.net is a forwarding relay', + }), + ); + expect(cold.decision).toBe('deny'); + expect(cold.gate).toBe('person_ineligible'); + expect(cold.reason).toContain('Held by lead screening: relay_address'); + + const answer = evaluatePolicy( + request({ + network: 'email', + action: 'send_email', + personScreenedOut: 'role_address: info@ is a team inbox', + isFollowUp: true, + conversationOpen: true, + }), + ); + expect(answer.decision).not.toBe('deny'); + + // Research is not contact: a held lead can still be read about. + const research = evaluatePolicy( + request({ action: 'refresh_research', personScreenedOut: 'generated_name' }), + ); + expect(research.decision).not.toBe('deny'); + }); + test('suppression outranks trusted automation', () => { const result = evaluatePolicy( request({ personSuppressed: true, approvalMode: 'trusted_automation' }), diff --git a/packages/policy/src/engine.ts b/packages/policy/src/engine.ts index 6284e8c..5fd9941 100644 --- a/packages/policy/src/engine.ts +++ b/packages/policy/src/engine.ts @@ -82,6 +82,13 @@ export interface PolicyRequest { readonly personSuppressed: boolean; readonly personBelievedMinor: boolean; readonly personDeleted: boolean; + /** + * Why lead screening holds this person back from cold outreach (a generated + * name, a relay mailbox, a role inbox...), until a human allows them. + * Omitted when screening has nothing against them. Never blocks a follow-up + * on a conversation they opened: someone who wrote back is real enough. + */ + readonly personScreenedOut?: string | undefined; readonly identityConfidence: number; readonly minIdentityConfidence: number; @@ -307,6 +314,13 @@ export function evaluatePolicy(request: PolicyRequest): PolicyResult { if (request.personBelievedMinor) { restrict('person_ineligible', 'deny', 'This person is believed to be a minor.'); } + if ( + request.personScreenedOut && + isOutboundAction(request.action) && + !(request.isFollowUp === true && request.conversationOpen === true) + ) { + restrict('person_ineligible', 'deny', `Held by lead screening: ${request.personScreenedOut}`); + } // 5. Wrong-person outreach destroys trust, so low-confidence identities may // be researched but never contacted (PRD §9.4, §48 Decision 4). From 8f80185c22a1f914417f52cbfa7037f75529ec19 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Tue, 6 Oct 2026 19:14:09 +0000 Subject: [PATCH 2/3] Lead enrichment: names from addresses, LinkedIn profile and title by search, PDL behind a key Fills only blanks, for leads in live campaigns: first/last from unambiguous first.last@ addresses (free); People Data Labs by email when PDL_API_KEY is set (licensed for storage, 3/day by default); then a Google search of LinkedIn through the existing ValueSERP key, taking a profile only when the result title carries every name part and the company (or, with no company, one profile whose slug is the name), the title off "Name - Title - Company", and the company's LinkedIn page as fallback. Ported from cli-tools linkedin-lookup with its tests. Every search is cached in serp_cache (reruns are free), capped per run and per workspace per day (enrichment_usage), run at concurrency 10 with a 45 s timeout, and a 402 pauses the provider until the next UTC day. A worker sweep runs beside the tick; POST /autogtm/campaigns/{id}/enrich starts one on demand and GET .../enrichment reports what is missing. UI on the product page, `og leads enrich|enrichment`, MCP enrich_campaign_leads and get_lead_enrichment. Migration 0052. PDL_API_KEY added to the deploy's secret sync (unset until a key is assigned). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-dev2.yml | 3 +- apps/api/src/app.ts | 8 + apps/api/src/append-leads.test.ts | 55 ++ apps/api/src/autogtm-docs.ts | 45 ++ apps/api/src/autogtm.ts | 83 +++ apps/cli/src/commands.ts | 37 +- apps/mcp/src/mcp.test.ts | 15 + apps/mcp/src/tools.ts | 44 ++ apps/server/src/index.ts | 62 ++ apps/web/app/(app)/products/[id]/page.tsx | 2 + apps/web/components/lead-enrichment.tsx | 124 ++++ migrations-pg/0052_lead_enrichment.sql | 19 + migrations/0052_lead_enrichment.sql | 37 ++ packages/pipeline/src/index.ts | 13 + packages/pipeline/src/lead-enrichment.test.ts | 251 ++++++++ packages/pipeline/src/lead-enrichment.ts | 579 ++++++++++++++++++ packages/providers/src/index.ts | 19 + packages/providers/src/pdl/index.ts | 126 ++++ packages/providers/src/valueserp/index.ts | 13 + .../providers/src/valueserp/linkedin.test.ts | 167 +++++ packages/providers/src/valueserp/linkedin.ts | 150 +++++ 21 files changed, 1849 insertions(+), 3 deletions(-) create mode 100644 apps/web/components/lead-enrichment.tsx create mode 100644 migrations-pg/0052_lead_enrichment.sql create mode 100644 migrations/0052_lead_enrichment.sql create mode 100644 packages/pipeline/src/lead-enrichment.test.ts create mode 100644 packages/pipeline/src/lead-enrichment.ts create mode 100644 packages/providers/src/pdl/index.ts create mode 100644 packages/providers/src/valueserp/linkedin.test.ts create mode 100644 packages/providers/src/valueserp/linkedin.ts diff --git a/.github/workflows/deploy-dev2.yml b/.github/workflows/deploy-dev2.yml index 483a3a2..2017da4 100644 --- a/.github/workflows/deploy-dev2.yml +++ b/.github/workflows/deploy-dev2.yml @@ -52,9 +52,10 @@ jobs: DEV2_HOST: ${{ secrets.DEV2_HOST }} VALUESERP_API_KEY: ${{ secrets.VALUESERP_API_KEY }} CHOVY_CAMPAIGN_SECRET: ${{ secrets.CHOVY_CAMPAIGN_SECRET }} + PDL_API_KEY: ${{ secrets.PDL_API_KEY }} run: | lines="" - for key in VALUESERP_API_KEY CHOVY_CAMPAIGN_SECRET; do + for key in VALUESERP_API_KEY CHOVY_CAMPAIGN_SECRET PDL_API_KEY; do value=$(printenv "$key" || true) [ -n "$value" ] && lines="$lines$key=$value"$'\n' done diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index abac45f..ae1b4a0 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -87,6 +87,7 @@ import { budgetStatus, crawlDedupeKey, screenHold, + type LeadEnrichDeps, startContactImport, importContactChunk, intakeSocialPeople, @@ -336,6 +337,12 @@ export interface AppOptions { * people by company. Absent without a key, which leaves pasting URLs. */ readonly jobSearcher?: WebSearcher | undefined; + /** + * Lead enrichment (names from addresses, ValueSERP LinkedIn search, People + * Data Labs), for the on-demand run a campaign page can start. Absent: the + * route still fills names from addresses. + */ + readonly leadEnrichment?: Omit | undefined; /** Test seam for the job boards and company sites a posting is read from. */ readonly jobReader?: JobReaderOptions | undefined; /** @@ -1375,6 +1382,7 @@ export function createApp(options: AppOptions): Hono { approve: (db, actor, recommendation) => approveRecommendation(db, options, actor, recommendation, {}), requireVerifiedEmail: (db, actor) => requireVerifiedEmail(db, actor), + ...(options.leadEnrichment ? { leadEnrichment: options.leadEnrichment } : {}), }), ); diff --git a/apps/api/src/append-leads.test.ts b/apps/api/src/append-leads.test.ts index e37c8ea..c57a10c 100644 --- a/apps/api/src/append-leads.test.ts +++ b/apps/api/src/append-leads.test.ts @@ -159,6 +159,61 @@ describe('POST /autogtm/campaigns/:id/leads', () => { T, ); + test( + 'enrich runs in the background and the status reports what was filled', + async () => { + const seeded = await seedDatabase('append-enrich'); + active = seeded; + const asked: string[] = []; + const app = createApp({ + db: seeded.db, + authenticate: async () => ACTOR, + leadEnrichment: { + searcher: { + async search(query) { + asked.push(query); + return query.includes('Scott Perry') + ? [ + { + title: 'Scott Perry - VP Sales - Northwind | LinkedIn', + link: 'https://www.linkedin.com/in/scott-perry-9', + }, + ] + : []; + }, + }, + }, + }); + + await send(app, 'POST', `/autogtm/campaigns/${SEED.campaignId}/leads`, { + leads: [{ email: 'scott.perry@northwind.io', company_domain: 'northwind.io' }], + }); + + const started = await send(app, 'POST', `/autogtm/campaigns/${SEED.campaignId}/enrich`, {}); + expect(started.status).toBe(202); + + let status: Record = { running: true }; + for (let i = 0; i < 100 && status.running; i += 1) { + await Bun.sleep(100); + status = (await ( + await app.request(`/api/v1/autogtm/campaigns/${SEED.campaignId}/enrichment`) + ).json()) as Record; + } + expect(status.running).toBe(false); + expect(status.last_run).toMatchObject({ titles: 1, profiles: 1 }); + + const scott = await queryOne<{ current_title: string; first_name: string }>( + seeded.db, + `SELECT p.current_title, p.first_name FROM people p + JOIN person_emails pe ON pe.person_id = p.id + WHERE pe.address = 'scott.perry@northwind.io'`, + ); + expect(scott).toEqual({ current_title: 'VP Sales', first_name: 'Scott' }); + expect(asked.some((q) => q.includes('"Scott Perry" northwind'))).toBe(true); + }, + T, + ); + test( 'takes the CSV itself and names a missing email column', async () => { diff --git a/apps/api/src/autogtm-docs.ts b/apps/api/src/autogtm-docs.ts index 4ac2594..11bd9cc 100644 --- a/apps/api/src/autogtm-docs.ts +++ b/apps/api/src/autogtm-docs.ts @@ -601,6 +601,51 @@ export const OPERATIONS: readonly Operation[] = [ properties: { allow: { type: 'boolean' } }, }, }, + { + method: 'post', + path: '/autogtm/campaigns/{campaign_id}/enrich', + id: 'enrichLeads', + tag: 'Campaigns', + summary: 'Fill in leads’ missing name, job title and LinkedIn', + description: + 'Starts a run and answers 202 at once. Names come from unambiguous addresses ' + + '(first.last@), free. Job title and LinkedIn profile come from a Google search of LinkedIn, ' + + 'taken only when the result carries every part of the name and the company; the company’s ' + + 'LinkedIn page is the fallback. People Data Labs is used first when configured. Only blanks ' + + 'are filled, every search is cached so a rerun costs nothing, and searches are capped per ' + + 'run (`max_searches`) and per day. Read the outcome from GET .../enrichment.', + params: [idParam('campaign_id', 'The campaign.')], + body: { + type: 'object', + properties: { + max_searches: { type: 'integer', minimum: 0, maximum: 1000 }, + max_leads: { type: 'integer', minimum: 1, maximum: 5000 }, + }, + }, + status: 202, + }, + { + method: 'get', + path: '/autogtm/campaigns/{campaign_id}/enrichment', + id: 'getEnrichment', + tag: 'Campaigns', + summary: 'What a campaign’s leads are missing, and the last enrichment run', + params: [idParam('campaign_id', 'The campaign.')], + response: { + type: 'object', + properties: { + leads: { type: 'integer' }, + missing_name: { type: 'integer' }, + missing_title: { type: 'integer' }, + missing_linkedin: { type: 'integer' }, + looked_up: { type: 'integer' }, + providers: { type: 'object' }, + today: { type: 'object' }, + running: { type: 'boolean' }, + last_run: { type: 'object' }, + }, + }, + }, { method: 'get', path: '/autogtm/campaigns/import/{task_id}', diff --git a/apps/api/src/autogtm.ts b/apps/api/src/autogtm.ts index 21af8f3..394fe41 100644 --- a/apps/api/src/autogtm.ts +++ b/apps/api/src/autogtm.ts @@ -48,6 +48,9 @@ import { domainMatchKey, emailMatchKey, enqueue, + enrichLeads, + enrichmentStatus, + type LeadEnrichDeps, finishContactImport, importContactChunk, normaliseDomain, @@ -101,8 +104,13 @@ export interface AutogtmDeps { ) => Promise; /** Refuses an unverified account anything that reaches a stranger. */ readonly requireVerifiedEmail: (db: Client, actor: RequestActor) => Promise; + /** The enrichment providers; absent means names from addresses only. */ + readonly leadEnrichment?: Omit; } +/** Campaigns with an on-demand enrichment run in flight, and the last result of each. */ +const enrichRuns = new Map(); + // ------------------------------------------------------------------ schemas const budgetBody = z.object({ @@ -219,6 +227,12 @@ const appendBody = z const allowBody = z.object({ allow: z.boolean() }); +const enrichBody = z.object({ + /** New (uncached) searches this run may spend; the daily cap still applies. */ + max_searches: z.number().int().min(0).max(1_000).optional(), + max_leads: z.number().int().min(1).max(5_000).optional(), +}); + // ------------------------------------------------------------------ rows interface CampaignRow { @@ -957,6 +971,75 @@ export function autogtmRoutes(deps: AutogtmDeps): Hono { }); }); + r.get('/campaigns/:id/enrichment', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + const campaign = await ownedCampaign(db, actor.workspaceId, c.req.param('id')); + const status = await enrichmentStatus( + { db, ...deps.leadEnrichment }, + { workspaceId: actor.workspaceId, campaignId: campaign.id }, + ); + const run = enrichRuns.get(campaign.id); + return c.json({ + campaign_id: campaign.id, + ...status, + running: run?.running === true, + ...(run?.last ? { last_run: run.last } : {}), + }); + }); + + /** + * Starts a run over this campaign's leads and answers at once: searches take + * 10-45 s each, so the run continues after the response and its outcome is + * read back from GET .../enrichment. + */ + r.post('/campaigns/:id/enrich', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + requireWriter(actor, 'enriching leads'); + const campaign = await ownedCampaign(db, actor.workspaceId, c.req.param('id')); + const body = await parse(c.req.raw, enrichBody); + + const current = enrichRuns.get(campaign.id); + if (current?.running) { + return c.json({ campaign_id: campaign.id, started: false, running: true }, 202); + } + const startedAt = now(); + enrichRuns.set(campaign.id, { + running: true, + startedAt, + ...(current?.last ? { last: current.last } : {}), + }); + void enrichLeads( + { db, ...deps.leadEnrichment }, + { + workspaceId: actor.workspaceId, + campaignId: campaign.id, + limit: body.max_leads ?? 200, + maxSearches: body.max_searches ?? 100, + }, + ) + .then((result) => { + enrichRuns.set(campaign.id, { + running: false, + startedAt, + last: { ...result, started_at: startedAt, finished_at: now() }, + }); + }) + .catch((error: unknown) => { + enrichRuns.set(campaign.id, { + running: false, + startedAt, + last: { + error: error instanceof Error ? error.message : String(error), + started_at: startedAt, + }, + }); + }); + + return c.json({ campaign_id: campaign.id, started: true, running: true }, 202); + }); + r.post('/leads/:person_id/screening', async (c) => { const actor = c.get('actor'); const db = c.get('db'); diff --git a/apps/cli/src/commands.ts b/apps/cli/src/commands.ts index 3c46cbf..58cafcd 100644 --- a/apps/cli/src/commands.ts +++ b/apps/cli/src/commands.ts @@ -273,6 +273,39 @@ async function runLeads({ client, args, flags }: CommandContext): Promise`); + const base = `/autogtm/campaigns/${encodeURIComponent(target)}`; + if (verb === 'enrich') { + const max = flagString(flags, 'max'); + await client.post(`${base}/enrich`, max ? { max_searches: Number(max) } : {}); + } + const status = (await client.get(`${base}/enrichment`)) as Record; + const today = (status.today ?? {}) as Record; + const last = status.last_run as Record | undefined; + return [ + verb === 'enrich' + ? 'Started. It runs in the background; check: og leads enrichment ' + target + : '', + `${text(status, 'leads', '0')} leads: missing ${text(status, 'missing_title', '0')} titles, ` + + `${text(status, 'missing_linkedin', '0')} LinkedIn, ${text(status, 'missing_name', '0')} names`, + `searches today: ${text(today, 'searches', '0')} of ${text(today, 'searches_cap', '0')}` + + `${status.running ? ' (a run is in progress)' : ''}`, + ...(last + ? [ + last.error + ? `last run failed: ${text(last, 'error')}` + : `last run: +${text(last, 'names', '0')} names, +${text(last, 'titles', '0')} titles, ` + + `+${text(last, 'profiles', '0')} profiles, +${text(last, 'companies', '0')} company pages, ` + + `${text(last, 'searches', '0')} searches (${text(last, 'cached', '0')} cached)` + + `${last.stopped ? `; stopped: ${text(last, 'stopped')}` : ''}`, + ] + : []), + ] + .filter(Boolean) + .join('\n'); + } + if (verb === 'allow' || verb === 'hold') { if (!target) throw new Error(`usage: og leads ${verb} `); await client.post(`/autogtm/leads/${encodeURIComponent(target)}/screening`, { @@ -283,7 +316,7 @@ async function runLeads({ client, args, flags }: CommandContext): Promise { @@ -1662,7 +1695,7 @@ export const COMMANDS: readonly Command[] = [ { name: 'leads', usage: - 'og leads add --consent-source "" [--allow-flagged] [--keep-project-duplicates] [--report out.csv] | report [--csv] | screened [--all] | allow | hold ', + 'og leads add --consent-source "" [--allow-flagged] [--keep-project-duplicates] [--report out.csv] | report [--csv] | screened [--all] | enrich [--max ] | enrichment | allow | hold ', summary: 'Add a CSV of leads to a running campaign, with a per-row report and screening.', run: runLeads, }, diff --git a/apps/mcp/src/mcp.test.ts b/apps/mcp/src/mcp.test.ts index 1055fda..85b486d 100644 --- a/apps/mcp/src/mcp.test.ts +++ b/apps/mcp/src/mcp.test.ts @@ -358,4 +358,19 @@ describe('lead tools', () => { expect(calls[1]?.body).toEqual({ allow: true }); expect(calls[2]?.url).toBe('https://api.test/api/v1/autogtm/campaigns/import/cim_1/report'); }); + + test('enrich_campaign_leads starts a run and reads the status back', async () => { + const { fetchImpl, calls } = recorder(ok({ running: true })); + const client = createClient(CONFIG, fetchImpl); + + await runTool(toolByName('enrich_campaign_leads')!, client, { + campaignId: 'cmp_1', + maxSearches: 20, + }); + expect(calls.map((c) => `${c.method} ${c.url}`)).toEqual([ + 'POST https://api.test/api/v1/autogtm/campaigns/cmp_1/enrich', + 'GET https://api.test/api/v1/autogtm/campaigns/cmp_1/enrichment', + ]); + expect(calls[0]?.body).toEqual({ max_searches: 20 }); + }); }); diff --git a/apps/mcp/src/tools.ts b/apps/mcp/src/tools.ts index f5950c2..b1ea2fe 100644 --- a/apps/mcp/src/tools.ts +++ b/apps/mcp/src/tools.ts @@ -1006,6 +1006,50 @@ export const TOOLS: readonly ToolDefinition[] = [ allow: args.allow === true, }), }, + { + name: 'enrich_campaign_leads', + title: 'Fill in leads’ missing name, title and LinkedIn', + description: + 'Starts a background run over a campaign’s leads: names from unambiguous addresses ' + + '(first.last@, free), then People Data Labs when configured, then a Google search of ' + + 'LinkedIn taken only when the result carries the full name and the company. Fills blanks ' + + 'only, caches every search, and is capped per run and per day. Returns the status; call ' + + 'get_lead_enrichment later for the outcome.', + readOnly: false, + inputSchema: { + type: 'object', + properties: { + campaignId: { type: 'string' }, + maxSearches: { type: 'number', description: 'New searches this run may spend.' }, + }, + required: ['campaignId'], + }, + run: async (client, args) => { + const base = `/autogtm/campaigns/${encodeURIComponent(require(args, 'campaignId'))}`; + await client.post( + `${base}/enrich`, + typeof args.maxSearches === 'number' ? { max_searches: args.maxSearches } : {}, + ); + return client.get(`${base}/enrichment`); + }, + }, + { + name: 'get_lead_enrichment', + title: 'What a campaign’s leads are missing', + description: + 'How many of a campaign’s leads lack a name, job title or LinkedIn, which enrichment ' + + 'providers are configured, searches spent today against the cap, and the last run.', + readOnly: true, + inputSchema: { + type: 'object', + properties: { campaignId: { type: 'string' } }, + required: ['campaignId'], + }, + run: (client, args) => + client.get( + `/autogtm/campaigns/${encodeURIComponent(require(args, 'campaignId'))}/enrichment`, + ), + }, { name: 'list_inbox', title: 'List conversations', diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index 4797ce8..efa0273 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -90,12 +90,16 @@ import { checkLinkedInAcceptances, linkedInSessionForWorkspace, workspacesWithLinkedInSession, + enrichLeads, + workspacesAwaitingLeadEnrichment, + type LeadEnrichDeps, } from '@outreachgraph/pipeline'; import { BlueskyFeedSource, BlueskyProvider, createSmtpProber, NostrSource, + PeopleDataLabsClient, RedditSource, RssSource, SiteProvider, @@ -301,6 +305,34 @@ if (!chovy) console.log('no CHOVY_CAMPAIGN_SECRET: ideas are found and ranked, B const photoLookupsPerDay = Number(process.env.PHOTO_LOOKUPS_PER_DAY ?? 300); +/** + * Lead enrichment: a lead's missing name, job title and LinkedIn. Names from + * the address are free and always on; the LinkedIn search is the same + * ValueSERP account (a shorter per-request timeout, since a sweep at + * concurrency ten would rather drop a slow query than wait a minute on it); + * People Data Labs only with PDL_API_KEY. Both are capped per workspace per + * day, and a 402 pauses that provider until the next UTC day. + */ +const leadEnrichment: LeadEnrichDeps = { + db, + ...(process.env.VALUESERP_API_KEY + ? { + searcher: new ValueSerpClient({ + apiKey: process.env.VALUESERP_API_KEY, + searchTimeoutMs: 45_000, + }), + } + : {}), + ...(process.env.PDL_API_KEY + ? { pdl: new PeopleDataLabsClient({ apiKey: process.env.PDL_API_KEY }) } + : {}), + searchesPerDay: Number(process.env.LINKEDIN_SEARCHES_PER_DAY ?? 300), + pdlPerDay: Number(process.env.PDL_LOOKUPS_PER_DAY ?? 3), + concurrency: Number(process.env.LEAD_ENRICH_CONCURRENCY ?? 10), +}; +if (!leadEnrichment.pdl) console.log('no PDL_API_KEY: lead enrichment uses addresses and search'); +let leadEnrichmentRunning = false; + if (!photoFinder) console.log('no VALUESERP_API_KEY: lead photos come from public profiles and crawls'); @@ -472,6 +504,7 @@ const api = createApp({ ...(encryptionKey ? { encryptionKey } : {}), ...(appUrl ? { appUrl } : {}), ...(jobSearcher ? { jobSearcher } : {}), + leadEnrichment, ...(chovy ? { chovy } : {}), ...(process.env.API_TOKEN ? { serviceToken: process.env.API_TOKEN } : {}), // A person editing their own OpenProfile.md carries an OpenAccess bearer @@ -1097,6 +1130,35 @@ async function tick(): Promise { // which is both the cheapest grounded claim the product can make and the // fastest-decaying one. Each watch has its own interval, so a workspace // watching one Bluesky handle costs one query per tick and nothing else. + // A name, a title and a LinkedIn for leads that arrived as a bare address. + // A search takes 10-45 s, so the sweep runs beside the tick rather than + // inside it, one run at a time, each bounded by its own caps. + if (!leadEnrichmentRunning) { + const waiting = await workspacesAwaitingLeadEnrichment(db); + if (waiting.length) { + leadEnrichmentRunning = true; + void (async () => { + for (const workspaceId of waiting) { + try { + const r = await enrichLeads(leadEnrichment, { workspaceId, maxSearches: 40 }); + if (r.looked > 0) { + console.log( + `lead enrichment: ${workspaceId} looked at ${r.looked}, +${r.names} names, ` + + `+${r.titles} titles, +${r.profiles} profiles, +${r.companies} company pages, ` + + `${r.pdlMatches} PDL matches, ${r.searches} searches (${r.cached} cached)` + + `${r.stopped ? `; stopped: ${r.stopped}` : ''}`, + ); + } + } catch (error) { + console.error(`lead enrichment failed for ${workspaceId}`, error); + } + } + })().finally(() => { + leadEnrichmentRunning = false; + }); + } + } + // The Idea Generator: what people keep asking for on Reddit. A scan paces the // public archive at ~2.5s a request, so it runs beside the tick rather than // inside it, one at a time. diff --git a/apps/web/app/(app)/products/[id]/page.tsx b/apps/web/app/(app)/products/[id]/page.tsx index ee3f527..90c4090 100644 --- a/apps/web/app/(app)/products/[id]/page.tsx +++ b/apps/web/app/(app)/products/[id]/page.tsx @@ -3,6 +3,7 @@ import { notFound, redirect } from 'next/navigation'; import { Avatar } from '../../../../components/avatar'; import { AddLeads } from '../../../../components/add-leads'; import { CampaignControls } from '../../../../components/campaign-controls'; +import { LeadEnrichment } from '../../../../components/lead-enrichment'; import { ScreenedLeads } from '../../../../components/screened-leads'; import { ApiUnavailableError, @@ -140,6 +141,7 @@ export default async function ProductPage({ params }: { params: Promise<{ id: st {campaign ? ( <> + ) : null} diff --git a/apps/web/components/lead-enrichment.tsx b/apps/web/components/lead-enrichment.tsx new file mode 100644 index 0000000..ff0a2e0 --- /dev/null +++ b/apps/web/components/lead-enrichment.tsx @@ -0,0 +1,124 @@ +'use client'; + +import { useCallback, useEffect, useState } from 'react'; + +/** + * What a campaign's leads are missing, and a button that fills it in. + * + * A worker sweep already does this a little at a time; the button is for the + * moment after an import, when the owner wants titles before the first drafts + * are written. Searches take 10-45 s each, so the run continues in the + * background and this polls for the outcome. + */ + +interface Status { + readonly leads: number; + readonly missing_name: number; + readonly missing_title: number; + readonly missing_linkedin: number; + readonly providers: { readonly valueserp: boolean; readonly pdl: boolean }; + readonly today: { readonly searches: number; readonly searches_cap: number }; + readonly paused?: { readonly valueserp?: string; readonly pdl?: string }; + readonly running: boolean; + readonly last_run?: { + readonly names?: number; + readonly titles?: number; + readonly profiles?: number; + readonly companies?: number; + readonly searches?: number; + readonly cached?: number; + readonly stopped?: string; + readonly error?: string; + }; +} + +export function LeadEnrichment({ campaignId }: { campaignId: string }) { + const [status, setStatus] = useState(); + const [error, setError] = useState(); + const url = `/api/v1/autogtm/campaigns/${encodeURIComponent(campaignId)}`; + + const load = useCallback(async () => { + const response = await fetch(`${url}/enrichment`, { credentials: 'same-origin' }); + if (response.ok) setStatus((await response.json()) as Status); + }, [url]); + + useEffect(() => { + void load().catch(() => undefined); + }, [load]); + + useEffect(() => { + if (!status?.running) return; + const timer = setInterval(() => void load().catch(() => undefined), 5_000); + return () => clearInterval(timer); + }, [status?.running, load]); + + async function start(): Promise { + setError(undefined); + const response = await fetch(`${url}/enrich`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + credentials: 'same-origin', + body: JSON.stringify({}), + }); + if (!response.ok) { + setError(`Could not start (${response.status}).`); + return; + } + await load(); + } + + if (!status || status.leads === 0) return null; + const missing = status.missing_title + status.missing_linkedin + status.missing_name; + const last = status.last_run; + + return ( +
+
+

+ {missing === 0 ? ( + 'Every lead has a name, a title and a LinkedIn.' + ) : ( + <> + Missing: {status.missing_title.toLocaleString()} titles,{' '} + {status.missing_linkedin.toLocaleString()} LinkedIn,{' '} + {status.missing_name.toLocaleString()} names. + + )} +

+ {missing > 0 ? ( + + ) : null} +
+

+ Names from addresses (free) + {status.providers.pdl ? ', People Data Labs' : ''} + {status.providers.valueserp + ? `, then a Google search of LinkedIn (${status.today.searches} of ${status.today.searches_cap} searches used today)` + : '. LinkedIn search is not configured on this deployment'} + . Only blanks are filled. + {status.paused?.valueserp + ? ` Search is out of credits until ${status.paused.valueserp}.` + : ''} +

+ {last && !status.running ? ( +

+ {last.error + ? `Last run failed: ${last.error}` + : `Last run: +${last.names ?? 0} names, +${last.titles ?? 0} titles, +${last.profiles ?? 0} profiles, +${last.companies ?? 0} company pages from ${last.searches ?? 0} searches (${last.cached ?? 0} cached)${last.stopped ? `; stopped: ${last.stopped}` : ''}.`} +

+ ) : null} + {error ? ( +

+ {error} +

+ ) : null} +
+ ); +} diff --git a/migrations-pg/0052_lead_enrichment.sql b/migrations-pg/0052_lead_enrichment.sql new file mode 100644 index 0000000..6f84be3 --- /dev/null +++ b/migrations-pg/0052_lead_enrichment.sql @@ -0,0 +1,19 @@ +-- 0052_lead_enrichment.sql (Postgres). See migrations/0052_lead_enrichment.sql for the reasoning. +-- Two nullable ADD COLUMNs (no rewrite) and two new, empty tables. No index on an existing table. + +ALTER TABLE people ADD COLUMN IF NOT EXISTS linkedin_looked_up_at text; +ALTER TABLE companies ADD COLUMN IF NOT EXISTS linkedin_url text; + +create table if not exists serp_cache ( + query text PRIMARY KEY, + results text NOT NULL, + fetched_at text NOT NULL +); + +create table if not exists enrichment_usage ( + workspace_id text NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + day text NOT NULL, + provider text NOT NULL, + lookups bigint NOT NULL DEFAULT 0, + PRIMARY KEY (workspace_id, day, provider) +); diff --git a/migrations/0052_lead_enrichment.sql b/migrations/0052_lead_enrichment.sql new file mode 100644 index 0000000..ba808a4 --- /dev/null +++ b/migrations/0052_lead_enrichment.sql @@ -0,0 +1,37 @@ +-- 0052_lead_enrichment.sql +-- +-- Filling in a lead's missing name, job title and LinkedIn from free and +-- cheap sources (the address itself, a Google search of LinkedIn, and People +-- Data Labs when a key is configured). +-- +-- people.linkedin_looked_up_at is the "already tried" stamp, like +-- photo_looked_up_at: most searches find nothing, and a sweep that only +-- recorded hits would pay for the same misses forever. Deliberately no index: +-- the sweep reaches people through campaign_people, never by scanning this. +-- +-- companies.linkedin_url is the company page, the fallback when no person +-- profile is evidenced. +-- +-- serp_cache keeps every search's raw results, keyed by the query, so a rerun +-- (or a tightened matching rule) costs nothing. Public search results, not +-- per workspace. +-- +-- enrichment_usage counts paid lookups per workspace per day, so the daily cap +-- survives a restart. + +ALTER TABLE people ADD COLUMN linkedin_looked_up_at TEXT; +ALTER TABLE companies ADD COLUMN linkedin_url TEXT; + +CREATE TABLE IF NOT EXISTS serp_cache ( + query TEXT PRIMARY KEY, + results TEXT NOT NULL, + fetched_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS enrichment_usage ( + workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + day TEXT NOT NULL, + provider TEXT NOT NULL, + lookups INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (workspace_id, day, provider) +); diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index 76c8242..1d11513 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -180,6 +180,19 @@ export { type StartImportInput, type StoredRow, } from './contact-import'; +export { + DEFAULT_PDL_PER_DAY, + DEFAULT_SEARCHES_PER_DAY, + enrichLeads, + enrichmentPausedUntil, + enrichmentStatus, + lookupsToday, + resetEnrichmentPauses, + workspacesAwaitingLeadEnrichment, + type LeadEnrichDeps, + type LeadEnrichInput, + type LeadEnrichResult, +} from './lead-enrichment'; export { countScreenedHeld, holdReason, diff --git a/packages/pipeline/src/lead-enrichment.test.ts b/packages/pipeline/src/lead-enrichment.test.ts new file mode 100644 index 0000000..c383ed7 --- /dev/null +++ b/packages/pipeline/src/lead-enrichment.test.ts @@ -0,0 +1,251 @@ +/** + * Filling in a lead's name, title and LinkedIn: only blanks, only on + * evidence, and never paying twice for the same search. + */ + +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { now, queryAll, queryOne, type Client } from '@outreachgraph/db'; +import { + SearchOutOfCredits, + type PersonEnricher, + type WebResult, + type WebSearcher, +} from '@outreachgraph/providers'; +import { seedDatabase, SEED, type SeededDatabase } from '../../../apps/api/src/test-seed'; +import { + enrichLeads, + enrichmentStatus, + lookupsToday, + resetEnrichmentPauses, +} from './lead-enrichment'; + +let seeded: SeededDatabase | undefined; + +beforeEach(() => resetEnrichmentPauses()); +afterEach(() => { + seeded?.cleanup(); + seeded = undefined; +}); + +const T = 60_000; + +/** A lead who arrived as a bare address, in the seeded campaign. */ +async function addLead(db: Client, id: string, email: string, companyDomain?: string) { + const stamp = now(); + const companyId = companyDomain ? `co_${id}` : null; + await db.batch([ + ...(companyDomain + ? [ + { + sql: `INSERT INTO companies (id, name, domain, technologies, created_at, updated_at) + VALUES (?, ?, ?, '[]', ?, ?)`, + args: [companyId, companyDomain, companyDomain, stamp, stamp], + }, + ] + : []), + { + sql: `INSERT INTO people (id, display_name, current_company_id, identity_confidence, status, + outreach_eligible, created_at, updated_at) + VALUES (?, ?, ?, 0.9, 'active', 1, ?, ?)`, + args: [id, email.split('@')[0]!, companyId, stamp, stamp], + }, + { + sql: `INSERT INTO person_emails (id, workspace_id, person_id, address, dedupe_key, source, + verified, created_at) VALUES (?, ?, ?, ?, ?, 'import', 1, ?)`, + args: [`pe_${id}`, SEED.workspaceId, id, email, email, stamp], + }, + { + sql: `INSERT INTO campaign_people (campaign_id, person_id, workspace_id, status, + interaction_state, discovered_at, updated_at) + VALUES (?, ?, ?, 'discovered', 'never_contacted', ?, ?)`, + args: [SEED.campaignId, id, SEED.workspaceId, stamp, stamp], + }, + ]); +} + +function searcher(answers: Record): { + asked: string[]; + searcher: WebSearcher; +} { + const asked: string[] = []; + return { + asked, + searcher: { + async search(query) { + asked.push(query); + return answers[query] ?? []; + }, + }, + }; +} + +describe('enrichLeads', () => { + test( + 'names from the address, title and profile from the search, company page as fallback', + async () => { + seeded = await seedDatabase('enrich-leads'); + const { db } = seeded; + await addLead(db, 'per_scott', 'scott.perry@northwind.io', 'northwind.io'); + await addLead(db, 'per_kim', 'kim.ode@globex.dev', 'globex.dev'); + + const { asked, searcher: fake } = searcher({ + 'site:linkedin.com/in "Scott Perry" northwind': [ + { + title: 'Scott Perry - VP Sales - Northwind | LinkedIn', + link: 'https://www.linkedin.com/in/scott-perry-9', + }, + ], + 'site:linkedin.com/company "globex.dev"': [ + { title: 'Globex | LinkedIn', link: 'https://www.linkedin.com/company/globex/' }, + ], + }); + + const result = await enrichLeads( + { db, searcher: fake, concurrency: 1 }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId }, + ); + + expect(result).toMatchObject({ names: 2, titles: 1, profiles: 1, companies: 1 }); + + const scott = await queryOne<{ + first_name: string; + last_name: string; + display_name: string; + current_title: string; + linkedin_looked_up_at: string | null; + }>( + db, + `SELECT first_name, last_name, display_name, current_title, linkedin_looked_up_at + FROM people WHERE id = 'per_scott'`, + ); + expect(scott).toMatchObject({ + first_name: 'Scott', + last_name: 'Perry', + display_name: 'Scott Perry', + current_title: 'VP Sales', + }); + expect(scott?.linkedin_looked_up_at).toBeTruthy(); + + const profile = await queryOne<{ profile_url: string; confidence: number }>( + db, + `SELECT profile_url, confidence FROM social_identities + WHERE person_id = 'per_scott' AND network = 'linkedin'`, + ); + expect(profile?.profile_url).toBe('https://www.linkedin.com/in/scott-perry-9'); + expect(profile!.confidence).toBeLessThan(0.85); + + const globex = await queryOne<{ linkedin_url: string }>( + db, + `SELECT linkedin_url FROM companies WHERE id = 'co_per_kim'`, + ); + expect(globex?.linkedin_url).toBe('https://www.linkedin.com/company/globex'); + + // Rerun after forgetting who was looked up: every answer comes from the cache. + await db.execute({ sql: 'UPDATE people SET linkedin_looked_up_at = NULL', args: [] }); + await db.execute({ sql: 'UPDATE companies SET linkedin_url = NULL', args: [] }); + const before = asked.length; + const again = await enrichLeads( + { db, searcher: fake, concurrency: 1 }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId }, + ); + expect(asked.length).toBe(before); + expect(again.searches).toBe(0); + expect(again.cached).toBeGreaterThan(0); + expect(await lookupsToday(db, SEED.workspaceId, 'valueserp')).toBe(before); + }, + T, + ); + + test( + 'stops at the cap and on a 402, leaving the unanswered for the next run', + async () => { + seeded = await seedDatabase('enrich-caps'); + const { db } = seeded; + for (const name of ['ann.lee', 'bo.chan', 'cy.diaz']) { + await addLead(db, `per_${name.replace('.', '_')}`, `${name}@initech.co`); + } + + const capped = await enrichLeads( + { db, searcher: searcher({}).searcher, concurrency: 1 }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId, maxSearches: 1 }, + ); + expect(capped.searches).toBe(1); + expect(capped.stopped).toContain('cap'); + // At most the one person whose question was answered is stamped; the + // rest wait for the next run rather than being marked as misses. + const stamped = await queryAll<{ id: string }>( + db, + 'SELECT id FROM people WHERE linkedin_looked_up_at IS NOT NULL', + ); + expect(stamped.length).toBeLessThanOrEqual(1); + + const broke: WebSearcher = { + async search() { + throw new SearchOutOfCredits('ValueSERP'); + }, + }; + const stopped = await enrichLeads( + { db, searcher: broke, concurrency: 1 }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId }, + ); + expect(stopped.stopped).toContain('402'); + // The names still came from the addresses for free. + const named = await queryOne<{ n: number }>( + db, + `SELECT COUNT(*) AS n FROM people + WHERE first_name IS NOT NULL AND id IN ('per_ann_lee', 'per_bo_chan', 'per_cy_diaz')`, + ); + expect(Number(named?.n)).toBe(3); + }, + T, + ); + + test( + 'People Data Labs first, within its daily cap, and the status says what is missing', + async () => { + seeded = await seedDatabase('enrich-pdl'); + const { db } = seeded; + await addLead(db, 'per_x', 'xq@hooli.xyz', 'hooli.xyz'); + + const asked: string[] = []; + const pdl: PersonEnricher = { + name: 'pdl', + async enrichByEmail(email) { + asked.push(email); + return { + firstName: 'Xavier', + lastName: 'Quinn', + title: 'Head of Data', + linkedinUrl: 'https://www.linkedin.com/in/xquinn', + }; + }, + }; + const { asked: searched, searcher: fake } = searcher({}); + + const status0 = await enrichmentStatus( + { db, pdl, searcher: fake }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId }, + ); + expect(status0.missing_title).toBe(1); + + const result = await enrichLeads( + { db, pdl, searcher: fake, pdlPerDay: 1, concurrency: 1 }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId }, + ); + expect(asked).toEqual(['xq@hooli.xyz']); + expect(result).toMatchObject({ pdlMatches: 1, titles: 1, profiles: 1 }); + // Title and profile were found, so no search was spent on Xavier (the + // seeded Jane still has no profile, and is searched for). + expect(searched.filter((q) => /Xavier|hooli/.test(q))).toEqual([]); + + const status = await enrichmentStatus( + { db, pdl, searcher: fake, pdlPerDay: 1 }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId }, + ); + expect(status.missing_title).toBe(0); + expect(status.today.pdl_lookups).toBe(1); + expect(status.providers).toEqual({ address: true, valueserp: true, pdl: true }); + }, + T, + ); +}); diff --git a/packages/pipeline/src/lead-enrichment.ts b/packages/pipeline/src/lead-enrichment.ts new file mode 100644 index 0000000..07014d4 --- /dev/null +++ b/packages/pipeline/src/lead-enrichment.ts @@ -0,0 +1,579 @@ +/** + * Filling in what a lead is missing: name, job title, LinkedIn. + * + * An imported row is often just an address. A message written to "there" + * with no idea what the person does reads as exactly what it is, so before a + * lead is written to, this fills the blanks from the cheapest source that can + * answer, in order, and only ever fills blanks: + * + * 1. **The address itself.** `scott.perry@acme.com` is Scott Perry. Free, + * and only the unambiguous shape (`nameFromAddress`). + * 2. **People Data Labs**, when a key is configured. Paid per match, capped + * per day, and licensed for storage (Apollo is not, and is never used). + * 3. **A Google search of LinkedIn** through ValueSERP: the profile, and + * the job title off the result's headline, taken only when the result is + * evidence (see `pickPerson`). The company's LinkedIn page is the + * fallback when no person profile is evidenced. + * + * Every search's raw results are cached in `serp_cache`, so a rerun costs + * nothing, and the searches are bounded three ways: per run, per workspace per + * day (counted in `enrichment_usage`, so a restart cannot reset it), and by + * HTTP 402, which stops the run cleanly. ValueSERP takes 10-45 s a query, so + * the work runs at a concurrency of about ten with a per-request timeout. + * + * This reads search results. It never fetches LinkedIn, never logs in, and the + * profiles it records are research at a confidence no machine acts on. + */ + +import { isConsumerMailDomain, nameFromAddress, newId } from '@outreachgraph/domain'; +import { + companyQuery, + linkedinPath, + personQuery, + pickCompany, + pickPerson, + SearchOutOfCredits, + titleFromResult, + type PersonEnricher, + type SerpResult, + type WebSearcher, +} from '@outreachgraph/providers'; +import { now, queryAll, queryOne, type Client } from '@outreachgraph/db'; + +/** A search result is research, below any outreach threshold (same as photos). */ +const SEARCH_IDENTITY_CONFIDENCE = 0.6; +/** A licensed provider matched the address itself: stronger than a name search. */ +const PDL_IDENTITY_CONFIDENCE = 0.8; + +export const DEFAULT_SEARCHES_PER_DAY = 300; +export const DEFAULT_PDL_PER_DAY = 3; +const DEFAULT_PEOPLE_PER_RUN = 60; +const DEFAULT_SEARCHES_PER_RUN = 100; +const DEFAULT_CONCURRENCY = 10; + +export interface LeadEnrichDeps { + readonly db: Client; + /** Google results (ValueSERP). Absent: names from addresses only. */ + readonly searcher?: WebSearcher | undefined; + /** People Data Labs. Absent: not used. */ + readonly pdl?: PersonEnricher | undefined; + readonly now?: Date; + readonly concurrency?: number; + readonly searchesPerDay?: number; + readonly pdlPerDay?: number; +} + +export interface LeadEnrichInput { + readonly workspaceId: string; + /** Only this campaign's leads. */ + readonly campaignId?: string | undefined; + /** People looked at in this run. */ + readonly limit?: number; + /** New (uncached) searches this run may spend. */ + readonly maxSearches?: number; +} + +export interface LeadEnrichResult { + readonly looked: number; + readonly names: number; + readonly titles: number; + readonly profiles: number; + readonly companies: number; + readonly pdlMatches: number; + readonly searches: number; + readonly cached: number; + /** Why the run stopped spending early, when it did. */ + readonly stopped?: string; +} + +interface Subject { + readonly person_id: string; + readonly display_name: string; + readonly first_name: string | null; + readonly last_name: string | null; + readonly current_title: string | null; + readonly company_id: string | null; + readonly company_domain: string | null; + readonly company_linkedin: string | null; + readonly email: string | null; + readonly linkedin: string | null; +} + +/** Workspace-wide: paused after a 402 until the next UTC day. */ +const pausedUntil = new Map(); + +/** Forgets every 402 pause. For tests, and for an operator who just topped up. */ +export function resetEnrichmentPauses(): void { + pausedUntil.clear(); +} + +export function enrichmentPausedUntil(provider: 'valueserp' | 'pdl'): string | undefined { + const until = pausedUntil.get(provider); + return until && until > new Date().toISOString() ? until : undefined; +} + +function nextUtcDay(at: Date): string { + const next = new Date(at); + next.setUTCHours(24, 0, 0, 0); + return next.toISOString(); +} + +/** Lookups spent today by this workspace with this provider. */ +export async function lookupsToday( + db: Client, + workspaceId: string, + provider: string, + at: Date = new Date(), +): Promise { + const row = await queryOne<{ lookups: number }>( + db, + 'SELECT lookups FROM enrichment_usage WHERE workspace_id = ? AND day = ? AND provider = ?', + [workspaceId, at.toISOString().slice(0, 10), provider], + ); + return Number(row?.lookups ?? 0); +} + +async function countLookup(db: Client, workspaceId: string, provider: string, at: Date) { + await db.execute({ + sql: `INSERT INTO enrichment_usage (workspace_id, day, provider, lookups) VALUES (?, ?, ?, 1) + ON CONFLICT (workspace_id, day, provider) DO UPDATE + SET lookups = enrichment_usage.lookups + 1`, + args: [workspaceId, at.toISOString().slice(0, 10), provider], + }); +} + +/** The people a run would look at: in a live campaign, missing something, never tried. */ +async function subjects(db: Client, input: LeadEnrichInput, limit: number): Promise { + return queryAll( + db, + `SELECT DISTINCT p.id AS person_id, p.display_name, p.first_name, p.last_name, + p.current_title, co.id AS company_id, co.domain AS company_domain, + co.linkedin_url AS company_linkedin, + (SELECT pe.address FROM person_emails pe + WHERE pe.person_id = p.id AND pe.workspace_id = cp.workspace_id + ORDER BY pe.created_at LIMIT 1) AS email, + (SELECT si.profile_url FROM social_identities si + WHERE si.person_id = p.id AND si.network = 'linkedin' LIMIT 1) AS linkedin + FROM campaign_people cp + JOIN campaigns c ON c.id = cp.campaign_id + JOIN people p ON p.id = cp.person_id + LEFT JOIN companies co ON co.id = p.current_company_id + WHERE cp.workspace_id = ? ${input.campaignId ? 'AND cp.campaign_id = ?' : ''} + AND c.status != 'archived' + AND p.status = 'active' AND p.kind = 'person' + AND p.linkedin_looked_up_at IS NULL + AND (p.current_title IS NULL OR p.first_name IS NULL OR p.last_name IS NULL + OR NOT EXISTS (SELECT 1 FROM social_identities si + WHERE si.person_id = p.id AND si.network = 'linkedin')) + LIMIT ?`, + input.campaignId ? [input.workspaceId, input.campaignId, limit] : [input.workspaceId, limit], + ); +} + +/** Workspaces with live-campaign leads still missing a name, title or profile. */ +export async function workspacesAwaitingLeadEnrichment(db: Client): Promise { + const rows = await queryAll<{ workspace_id: string }>( + db, + `SELECT DISTINCT cp.workspace_id + FROM campaign_people cp + JOIN campaigns c ON c.id = cp.campaign_id + JOIN people p ON p.id = cp.person_id + WHERE c.status IN ('active', 'running') AND p.status = 'active' AND p.kind = 'person' + AND p.linkedin_looked_up_at IS NULL + AND (p.current_title IS NULL OR p.first_name IS NULL + OR NOT EXISTS (SELECT 1 FROM social_identities si + WHERE si.person_id = p.id AND si.network = 'linkedin'))`, + ); + return rows.map((row) => row.workspace_id); +} + +/** + * One run over the next batch of leads. + * + * A person is stamped as looked up only when every question asked about them + * was answered (from the cache or a search). One skipped because the run hit + * its cap, ran out of credits or timed out is left for the next run. + */ +export async function enrichLeads( + deps: LeadEnrichDeps, + input: LeadEnrichInput, +): Promise { + const { db } = deps; + const at = deps.now ?? new Date(); + const result = { + looked: 0, + names: 0, + titles: 0, + profiles: 0, + companies: 0, + pdlMatches: 0, + searches: 0, + cached: 0, + stopped: undefined as string | undefined, + }; + + const searchRoom = deps.searcher + ? Math.max( + 0, + Math.min( + input.maxSearches ?? DEFAULT_SEARCHES_PER_RUN, + (deps.searchesPerDay ?? DEFAULT_SEARCHES_PER_DAY) - + (await lookupsToday(db, input.workspaceId, 'valueserp', at)), + ), + ) + : 0; + let pdlRoom = deps.pdl + ? Math.max( + 0, + (deps.pdlPerDay ?? DEFAULT_PDL_PER_DAY) - + (await lookupsToday(db, input.workspaceId, 'pdl', at)), + ) + : 0; + if (enrichmentPausedUntil('pdl')) pdlRoom = 0; + const searchPaused = enrichmentPausedUntil('valueserp'); + if (searchPaused) result.stopped = `ValueSERP is out of credits until ${searchPaused}`; + + const people = await subjects(db, input, input.limit ?? DEFAULT_PEOPLE_PER_RUN); + const inflight = new Map>(); + + /** The query's results, from the cache or one search; undefined when it could not ask. */ + const ask = async (query: string): Promise => { + const cached = await queryOne<{ results: string }>( + db, + 'SELECT results FROM serp_cache WHERE query = ?', + [query], + ); + if (cached) { + result.cached += 1; + try { + return JSON.parse(cached.results) as SerpResult[]; + } catch { + return []; + } + } + const existing = inflight.get(query); + if (existing) return existing; + if (!deps.searcher || searchPaused || result.stopped || result.searches >= searchRoom) { + if (!result.stopped && deps.searcher && result.searches >= searchRoom) { + result.stopped = `reached the search cap (${searchRoom} this run)`; + } + return undefined; + } + result.searches += 1; + const searcher = deps.searcher; + const pending = (async () => { + try { + const results = await searcher.search(query, { num: 10 }); + const kept = results.map(({ title, link, snippet }) => ({ title, link, snippet })); + await db.execute({ + sql: `INSERT INTO serp_cache (query, results, fetched_at) VALUES (?, ?, ?) + ON CONFLICT (query) DO UPDATE SET results = excluded.results, + fetched_at = excluded.fetched_at`, + args: [query, JSON.stringify(kept), now()], + }); + await countLookup(db, input.workspaceId, 'valueserp', at); + return kept; + } catch (error) { + if (error instanceof SearchOutOfCredits) { + pausedUntil.set('valueserp', nextUtcDay(at)); + result.stopped = error.message; + } + // A timeout or a refusal is not an answer: the person is retried later. + return undefined; + } + })(); + inflight.set(query, pending); + return pending; + }; + + const enrichOne = async (subject: Subject): Promise => { + const stamp = now(); + const statements: { sql: string; args: (string | number | null)[] }[] = []; + let first = subject.first_name; + let last = subject.last_name; + let title = subject.current_title; + let linkedin = subject.linkedin; + let complete = true; + + const emailDomain = subject.email?.split('@')[1]?.toLowerCase(); + const companyDomain = + subject.company_domain ?? + (emailDomain && !isConsumerMailDomain(emailDomain) ? emailDomain : undefined); + + // 1. The address itself. + if ((!first || !last) && subject.email) { + const named = nameFromAddress(subject.email); + if (named) { + first = first ?? named.firstName; + last = last ?? named.lastName; + const display = /\s/.test(subject.display_name.trim()) + ? subject.display_name + : `${named.firstName} ${named.lastName}`; + statements.push({ + sql: `UPDATE people SET first_name = COALESCE(first_name, ?), + last_name = COALESCE(last_name, ?), display_name = ?, updated_at = ? + WHERE id = ?`, + args: [named.firstName, named.lastName, display, stamp, subject.person_id], + }); + statements.push(provenance(subject.person_id, 'first_name', named.firstName, 'address')); + result.names += 1; + } + } + + // 2. People Data Labs, by address, for whoever still lacks a title or profile. + if ((!title || !linkedin) && subject.email && pdlRoom > 0 && deps.pdl) { + pdlRoom -= 1; + try { + await countLookup(db, input.workspaceId, 'pdl', at); + const found = await deps.pdl.enrichByEmail(subject.email); + if (found) { + result.pdlMatches += 1; + if ((!first || !last) && found.firstName && found.lastName) { + first = found.firstName; + last = found.lastName; + statements.push({ + sql: `UPDATE people SET first_name = COALESCE(first_name, ?), + last_name = COALESCE(last_name, ?), updated_at = ? WHERE id = ?`, + args: [found.firstName, found.lastName, stamp, subject.person_id], + }); + } + if (!title && found.title) { + title = found.title; + statements.push(...setTitle(subject.person_id, found.title, 'pdl', found.linkedinUrl)); + result.titles += 1; + } + const profile = linkedinPath(found.linkedinUrl, 'in'); + if (!linkedin && profile) { + linkedin = profile; + statements.push(addProfile(subject.person_id, profile, PDL_IDENTITY_CONFIDENCE, 'pdl')); + result.profiles += 1; + } + } + } catch (error) { + if (error instanceof SearchOutOfCredits) { + pausedUntil.set('pdl', nextUtcDay(at)); + pdlRoom = 0; + } + } + } + + // 3. A Google search of LinkedIn: the profile and its headline. + if ((!title || !linkedin) && first && last) { + const subjectForSearch = { firstName: first, lastName: last, companyDomain }; + const query = personQuery(subjectForSearch); + const results = await ask(query); + if (!results) { + complete = false; + } else { + const url = pickPerson(results, subjectForSearch); + if (url) { + if (!linkedin) { + linkedin = url; + statements.push( + addProfile(subject.person_id, url, SEARCH_IDENTITY_CONFIDENCE, 'search'), + ); + result.profiles += 1; + } + const hit = results.find((r) => linkedinPath(r.link, 'in') === url); + const headline = hit ? titleFromResult(hit.title ?? '', companyDomain ?? '') : ''; + if (!title && headline) { + title = headline; + statements.push(...setTitle(subject.person_id, headline, 'valueserp', url)); + result.titles += 1; + } + } + } + } + + // 4. The company's page, when no person profile is evidenced. + if (!linkedin && companyDomain && subject.company_id && !subject.company_linkedin) { + const results = await ask(companyQuery(companyDomain)); + if (!results) { + complete = false; + } else { + const url = pickCompany(results, companyDomain); + if (url) { + statements.push({ + sql: `UPDATE companies SET linkedin_url = ?, updated_at = ? + WHERE id = ? AND linkedin_url IS NULL`, + args: [url, stamp, subject.company_id], + }); + result.companies += 1; + } + } + } + + if (complete) { + statements.push({ + sql: 'UPDATE people SET linkedin_looked_up_at = ? WHERE id = ?', + args: [stamp, subject.person_id], + }); + } + if (statements.length > 0) await db.batch(statements); + }; + + let next = 0; + const worker = async () => { + while (next < people.length) { + const subject = people[next++]!; + result.looked += 1; + try { + await enrichOne(subject); + } catch (error) { + console.error(`lead enrichment failed for ${subject.person_id}`, error); + } + } + }; + await Promise.all( + Array.from( + { length: Math.min(deps.concurrency ?? DEFAULT_CONCURRENCY, people.length) }, + worker, + ), + ); + + return { + looked: result.looked, + names: result.names, + titles: result.titles, + profiles: result.profiles, + companies: result.companies, + pdlMatches: result.pdlMatches, + searches: result.searches, + cached: result.cached, + ...(result.stopped ? { stopped: result.stopped } : {}), + }; +} + +function provenance( + personId: string, + field: string, + value: string, + via: 'address' | 'pdl' | 'valueserp', + url?: string, +): { sql: string; args: (string | number | null)[] } { + const stamp = now(); + const [sourceType, license, confidence] = + via === 'address' + ? ['derived', 'derived_inference', 0.7] + : via === 'pdl' + ? ['provider', 'licensed_enrichment', PDL_IDENTITY_CONFIDENCE] + : ['public_web', 'public_web', SEARCH_IDENTITY_CONFIDENCE]; + return { + sql: `INSERT INTO field_provenance (id, entity_kind, entity_id, field, value, source_type, + provider, source_url, license_class, confidence, observed_at, created_at) + VALUES (?, 'person', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + args: [ + newId('fieldProvenance'), + personId, + field, + value, + sourceType, + via, + url ?? null, + license, + confidence, + stamp, + stamp, + ], + }; +} + +function setTitle( + personId: string, + title: string, + via: 'pdl' | 'valueserp', + url?: string, +): { sql: string; args: (string | number | null)[] }[] { + return [ + { + sql: `UPDATE people SET current_title = ?, updated_at = ? + WHERE id = ? AND current_title IS NULL`, + args: [title, now(), personId], + }, + provenance(personId, 'current_title', title, via, url), + ]; +} + +function addProfile( + personId: string, + url: string, + confidence: number, + via: 'pdl' | 'search', +): { sql: string; args: (string | number | null)[] } { + const stamp = now(); + return { + // Guarded in SQL as well: two runs racing on one person record one profile. + sql: `INSERT INTO social_identities (id, person_id, network, handle, profile_url, confidence, + source_type, verified_by, first_seen_at, last_verified_at) + SELECT ?, ?, 'linkedin', ?, ?, ?, ?, ?, ?, ? + WHERE NOT EXISTS (SELECT 1 FROM social_identities + WHERE person_id = ? AND network = 'linkedin')`, + args: [ + newId('socialIdentity'), + personId, + decodeURIComponent(url.split('/in/')[1] ?? '') || null, + url, + confidence, + via === 'pdl' ? 'provider' : 'public_web', + JSON.stringify([via]), + stamp, + stamp, + personId, + ], + }; +} + +/** What a campaign's leads are missing, and what enrichment can still spend today. */ +export async function enrichmentStatus( + deps: LeadEnrichDeps, + input: { readonly workspaceId: string; readonly campaignId: string }, +) { + const row = await queryOne<{ + leads: number; + no_name: number; + no_title: number; + no_linkedin: number; + looked_up: number; + }>( + deps.db, + `SELECT COUNT(*) AS leads, + SUM(CASE WHEN p.first_name IS NULL OR p.last_name IS NULL THEN 1 ELSE 0 END) AS no_name, + SUM(CASE WHEN p.current_title IS NULL THEN 1 ELSE 0 END) AS no_title, + SUM(CASE WHEN NOT EXISTS (SELECT 1 FROM social_identities si + WHERE si.person_id = p.id AND si.network = 'linkedin') + THEN 1 ELSE 0 END) AS no_linkedin, + SUM(CASE WHEN p.linkedin_looked_up_at IS NOT NULL THEN 1 ELSE 0 END) AS looked_up + FROM campaign_people cp JOIN people p ON p.id = cp.person_id + WHERE cp.campaign_id = ? AND cp.workspace_id = ? AND p.status = 'active'`, + [input.campaignId, input.workspaceId], + ); + const at = deps.now ?? new Date(); + const [searches, pdl] = await Promise.all([ + lookupsToday(deps.db, input.workspaceId, 'valueserp', at), + lookupsToday(deps.db, input.workspaceId, 'pdl', at), + ]); + return { + leads: Number(row?.leads ?? 0), + missing_name: Number(row?.no_name ?? 0), + missing_title: Number(row?.no_title ?? 0), + missing_linkedin: Number(row?.no_linkedin ?? 0), + looked_up: Number(row?.looked_up ?? 0), + providers: { + address: true, + valueserp: Boolean(deps.searcher), + pdl: Boolean(deps.pdl), + }, + today: { + searches, + searches_cap: deps.searcher ? (deps.searchesPerDay ?? DEFAULT_SEARCHES_PER_DAY) : 0, + pdl_lookups: pdl, + pdl_cap: deps.pdl ? (deps.pdlPerDay ?? DEFAULT_PDL_PER_DAY) : 0, + }, + paused: { + ...(enrichmentPausedUntil('valueserp') + ? { valueserp: enrichmentPausedUntil('valueserp') } + : {}), + ...(enrichmentPausedUntil('pdl') ? { pdl: enrichmentPausedUntil('pdl') } : {}), + }, + }; +} diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index 6cfe5d4..9a082dc 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -33,6 +33,7 @@ export { deriveEvidence, type EvidenceContext } from './evidence'; export { ValueSerpClient, + SearchOutOfCredits, carriesName, corroborate, isLinkedInProfile, @@ -43,6 +44,24 @@ export { type WebResult, type WebSearcher, } from './valueserp'; +export { + companyQuery, + companyToken, + linkedinPath, + personQuery, + pickCompany, + pickPerson, + registrable, + titleFromResult, + type LinkedinSubject, + type SerpResult, +} from './valueserp/linkedin'; +export { + PeopleDataLabsClient, + type PersonEnricher, + type PersonEnrichment, + type PeopleDataLabsOptions, +} from './pdl'; export { readJobPosting, diff --git a/packages/providers/src/pdl/index.ts b/packages/providers/src/pdl/index.ts new file mode 100644 index 0000000..c74ec9d --- /dev/null +++ b/packages/providers/src/pdl/index.ts @@ -0,0 +1,126 @@ +/** + * People Data Labs: a person's name, title, LinkedIn and employer from their + * email address. + * + * Optional, behind `PDL_API_KEY`, and the only paid person source here on + * purpose: PDL's terms allow storing what it returns inside a product, which + * Apollo's do not (Apollo data must never be written to this database). A + * free key carries 100 matches a month, and a 404 (no match) is not charged, + * so the caller caps lookups per day and asks only for leads still missing a + * title or a profile after the free sources. + * + * Measured on our own list (2026-10-06): PDL's titles are Apollo's titles + * verbatim (a shared upstream), work addresses match about half the time, + * personal addresses rarely. + */ + +import { SearchOutOfCredits } from '../valueserp'; + +export interface PersonEnrichment { + readonly firstName?: string; + readonly lastName?: string; + readonly title?: string; + /** `https://www.linkedin.com/in/` */ + readonly linkedinUrl?: string; + readonly companyName?: string; + /** The employer's domain, e.g. `acme.com`. */ + readonly companyDomain?: string; + /** PDL's own 1-10 confidence that the match is this person. */ + readonly likelihood?: number; +} + +export interface PersonEnricher { + readonly name: string; + /** One person by address, or undefined for no match. Throws SearchOutOfCredits on 402. */ + enrichByEmail(email: string): Promise; +} + +export interface PeopleDataLabsOptions { + readonly apiKey: string; + readonly baseUrl?: string; + readonly fetchImpl?: typeof fetch; + readonly timeoutMs?: number; + /** Matches below this likelihood are not taken. PDL's scale is 1-10. */ + readonly minLikelihood?: number; +} + +interface PdlPerson { + readonly first_name?: unknown; + readonly last_name?: unknown; + readonly job_title?: unknown; + readonly linkedin_url?: unknown; + readonly job_company_name?: unknown; + readonly job_company_website?: unknown; +} + +const text = (value: unknown): string | undefined => + typeof value === 'string' && value.trim() ? value.trim() : undefined; + +const titleCase = (value: string | undefined): string | undefined => + value?.replace(/\b\p{L}/gu, (letter) => letter.toUpperCase()); + +export class PeopleDataLabsClient implements PersonEnricher { + readonly name = 'pdl'; + private readonly apiKey: string; + private readonly baseUrl: string; + private readonly fetchImpl: typeof fetch; + private readonly timeoutMs: number; + private readonly minLikelihood: number; + + constructor(options: PeopleDataLabsOptions) { + this.apiKey = options.apiKey; + this.baseUrl = (options.baseUrl ?? 'https://api.peopledatalabs.com').replace(/\/$/, ''); + this.fetchImpl = options.fetchImpl ?? fetch; + this.timeoutMs = options.timeoutMs ?? 30_000; + this.minLikelihood = options.minLikelihood ?? 6; + } + + async enrichByEmail(email: string): Promise { + const url = new URL('/v5/person/enrich', this.baseUrl); + url.searchParams.set('email', email); + url.searchParams.set('min_likelihood', String(this.minLikelihood)); + + const response = await this.fetchImpl(url, { + headers: { 'X-Api-Key': this.apiKey, accept: 'application/json' }, + signal: AbortSignal.timeout(this.timeoutMs), + }); + if (response.status === 404) return undefined; + if (response.status === 402) throw new SearchOutOfCredits('People Data Labs'); + if (!response.ok) throw new Error(`People Data Labs answered ${response.status}`); + + const body = (await response.json().catch(() => ({}))) as { + likelihood?: unknown; + data?: PdlPerson; + }; + const person = body.data; + if (!person) return undefined; + + const linkedin = text(person.linkedin_url); + const website = text(person.job_company_website) + ?.toLowerCase() + .replace(/^https?:\/\//, '') + .replace(/^www\./, '') + .split(/[/?#]/)[0]; + + const out: PersonEnrichment = { + ...(titleCase(text(person.first_name)) + ? { firstName: titleCase(text(person.first_name))! } + : {}), + ...(titleCase(text(person.last_name)) + ? { lastName: titleCase(text(person.last_name))! } + : {}), + ...(titleCase(text(person.job_title)) ? { title: titleCase(text(person.job_title))! } : {}), + ...(linkedin + ? { + linkedinUrl: `https://www.${linkedin.replace(/^https?:\/\//, '').replace(/^www\./, '')}`, + } + : {}), + ...(titleCase(text(person.job_company_name)) + ? { companyName: titleCase(text(person.job_company_name))! } + : {}), + ...(website && website.includes('.') ? { companyDomain: website } : {}), + ...(typeof body.likelihood === 'number' ? { likelihood: body.likelihood } : {}), + }; + return out; + } +} diff --git a/packages/providers/src/valueserp/index.ts b/packages/providers/src/valueserp/index.ts index 1fd461b..845f84a 100644 --- a/packages/providers/src/valueserp/index.ts +++ b/packages/providers/src/valueserp/index.ts @@ -67,6 +67,16 @@ export interface ValueSerpOptions { const DEFAULT_BASE = 'https://api.valueserp.com'; +/** A paid provider answered HTTP 402: no credits left, so stop asking. */ +export class SearchOutOfCredits extends Error { + readonly provider: string; + constructor(provider: string) { + super(`${provider} is out of credits (HTTP 402)`); + this.name = 'SearchOutOfCredits'; + this.provider = provider; + } +} + /** One image result, as much of it as this adapter reads. */ interface ImageResult { readonly title?: string; @@ -113,6 +123,9 @@ export class ValueSerpClient implements ProfilePhotoFinder, WebSearcher { organic_results?: { title?: unknown; link?: unknown; snippet?: unknown }[]; request_info?: { success?: boolean; message?: string }; }; + // Out of credits is not a miss: every search after it would fail too, so + // a sweep stops on it rather than stamping people as looked up. + if (response.status === 402) throw new SearchOutOfCredits('ValueSERP'); if (!response.ok || body.request_info?.success === false) { throw new Error( `ValueSERP refused the search (${response.status}): ${body.request_info?.message ?? 'no reason given'}`, diff --git a/packages/providers/src/valueserp/linkedin.test.ts b/packages/providers/src/valueserp/linkedin.test.ts new file mode 100644 index 0000000..db8c042 --- /dev/null +++ b/packages/providers/src/valueserp/linkedin.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, test } from 'bun:test'; +import { + companyToken, + personQuery, + pickCompany, + pickPerson, + titleFromResult, + type LinkedinSubject, + type SerpResult, +} from './linkedin'; +import { SearchOutOfCredits, ValueSerpClient } from './index'; +import { PeopleDataLabsClient } from '../pdl'; + +const ada: LinkedinSubject = { firstName: 'Ada', lastName: 'Lovelace', companyDomain: 'acme.com' }; + +describe('LinkedIn matching', () => { + test('names the company by its first label', () => { + expect(companyToken('acme.com')).toBe('acme'); + expect(companyToken('shop.acme.co.uk')).toBe('acme'); + expect(companyToken('ai.nookplot.com')).toBe('nookplot'); + }); + + test('searches profiles by quoted name and company', () => { + expect(personQuery(ada)).toBe('site:linkedin.com/in "Ada Lovelace" acme'); + expect(personQuery({ firstName: 'Bob', lastName: 'Ng' })).toBe('site:linkedin.com/in "Bob Ng"'); + }); + + test('takes a profile that carries the name and the company', () => { + const results: SerpResult[] = [ + { + title: 'Ada Lovelace - Engineer - Initech | LinkedIn', + link: 'https://www.linkedin.com/in/ada-other', + snippet: 'Initech', + }, + { + title: 'Adá LOVELACE - CTO - Acme | LinkedIn', + link: 'https://uk.linkedin.com/in/ada-l?trk=x', + snippet: '', + }, + ]; + expect(pickPerson(results, ada)).toBe('https://www.linkedin.com/in/ada-l'); + }); + + test('without a company, takes only a single matching profile whose slug is the name', () => { + const bob = { firstName: 'Bob', lastName: 'Ng' }; + const one = [ + { title: 'Bob Ng | LinkedIn', link: 'https://www.linkedin.com/in/bob-ng' }, + { title: 'Posts', link: 'https://www.linkedin.com/posts/x' }, + ]; + expect(pickPerson(one, bob)).toBe('https://www.linkedin.com/in/bob-ng'); + expect( + pickPerson( + [...one, { title: 'Bob Ng - Chef', link: 'https://www.linkedin.com/in/bob-ng-2' }], + bob, + ), + ).toBe(''); + expect( + pickPerson([{ title: 'Bob Ng | LinkedIn', link: 'https://www.linkedin.com/in/b8812' }], bob), + ).toBe(''); + }); + + test('rejects other people and non-profile pages', () => { + expect( + pickPerson([{ title: 'Ada Byron - Acme', link: 'https://www.linkedin.com/in/ab' }], ada), + ).toBe(''); + expect( + pickPerson( + [{ title: 'Ada Lovelace - Acme', link: 'https://www.linkedin.com/company/acme' }], + ada, + ), + ).toBe(''); + }); + + test('takes a company page that names the domain or the company, not a passing mention', () => { + expect( + pickCompany( + [{ title: 'Acme Corp | LinkedIn', link: 'https://www.linkedin.com/company/acme-corp/' }], + 'acme.com', + ), + ).toBe('https://www.linkedin.com/company/acme-corp'); + expect( + pickCompany( + [{ title: 'Other | LinkedIn', link: 'https://www.linkedin.com/company/other' }], + 'acme.com', + ), + ).toBe(''); + expect( + pickCompany( + [ + { + title: 'National City Adult School', + link: 'https://www.linkedin.com/company/ncas', + snippet: 'part of Sweetwater schools', + }, + ], + 'sweetwaterschools.net', + ), + ).toBe(''); + expect( + pickCompany( + [ + { + title: 'Adult School', + link: 'https://www.linkedin.com/company/ncas', + snippet: 'sweetwaterschools.net', + }, + ], + 'sweetwaterschools.net', + ), + ).toBe('https://www.linkedin.com/company/ncas'); + }); + + test('reads the headline out of a profile title', () => { + expect(titleFromResult('Ada Lovelace - CTO - Acme | LinkedIn', 'acme.com')).toBe('CTO'); + expect(titleFromResult('Ada Lovelace – Head of Growth | LinkedIn', 'acme.com')).toBe( + 'Head of Growth', + ); + expect(titleFromResult('Ada Lovelace - Acme | LinkedIn', 'acme.com')).toBe(''); + expect(titleFromResult('Ada Lovelace | LinkedIn')).toBe(''); + }); +}); + +describe('out of credits', () => { + const answer = (status: number, body: unknown = {}) => + (async () => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + })) as unknown as typeof fetch; + + test('ValueSERP turns a 402 into SearchOutOfCredits', async () => { + const client = new ValueSerpClient({ apiKey: 'k', fetchImpl: answer(402) }); + await expect(client.search('q')).rejects.toBeInstanceOf(SearchOutOfCredits); + }); + + test('People Data Labs: a match, no match, and a 402', async () => { + const hit = new PeopleDataLabsClient({ + apiKey: 'k', + fetchImpl: answer(200, { + likelihood: 8, + data: { + first_name: 'ada', + last_name: 'lovelace', + job_title: 'chief technology officer', + linkedin_url: 'linkedin.com/in/ada-l', + job_company_name: 'acme', + job_company_website: 'acme.com', + }, + }), + }); + expect(await hit.enrichByEmail('ada@acme.com')).toEqual({ + firstName: 'Ada', + lastName: 'Lovelace', + title: 'Chief Technology Officer', + linkedinUrl: 'https://www.linkedin.com/in/ada-l', + companyName: 'Acme', + companyDomain: 'acme.com', + likelihood: 8, + }); + + const miss = new PeopleDataLabsClient({ apiKey: 'k', fetchImpl: answer(404) }); + expect(await miss.enrichByEmail('nobody@acme.com')).toBeUndefined(); + + const broke = new PeopleDataLabsClient({ apiKey: 'k', fetchImpl: answer(402) }); + await expect(broke.enrichByEmail('ada@acme.com')).rejects.toBeInstanceOf(SearchOutOfCredits); + }); +}); diff --git a/packages/providers/src/valueserp/linkedin.ts b/packages/providers/src/valueserp/linkedin.ts new file mode 100644 index 0000000..25c3dce --- /dev/null +++ b/packages/providers/src/valueserp/linkedin.ts @@ -0,0 +1,150 @@ +/** + * Finding a lead's LinkedIn profile and job title through Google results, + * because LinkedIn's own search needs a login and this product never logs in + * there (see CLAUDE.md: LinkedIn acts only through the member's session). + * + * Ported from cli-tools `linkedin-lookup`, where it ran over ~2,000 queries + * of a real user list. A result is taken only when it is evidence, by the + * same standard the photo lookup uses (`carriesName`): + * + * - the link is a `linkedin.com/in/` profile and its title carries + * every part of the name; + * - with a company, the company's name (the domain's first label) is in the + * title or the snippet too; + * - with no company, exactly one profile in the results carries the name, + * and its slug is the name: "John Smith" alone is everybody. + * + * The title is read off the result title, which Google shows as + * `Name - Title - Company | LinkedIn`. A lead still without a profile but + * with a company domain gets the company's `linkedin.com/company/` + * page instead, taken only when the result names the domain or the company. + */ + +import { carriesName } from './index'; + +export interface SerpResult { + readonly title?: string | undefined; + readonly link?: string | undefined; + readonly snippet?: string | undefined; +} + +export interface LinkedinSubject { + readonly firstName: string; + readonly lastName: string; + /** The employer's own domain, when known. Never a webmail host. */ + readonly companyDomain?: string | undefined; +} + +function fold(value: string): string { + return value.normalize('NFD').replace(/[̀-ͯ]/g, '').toLowerCase(); +} + +/** `shop.acme.co.uk` -> `acme.co.uk`: the domain a company registered. */ +export function registrable(domain: string): string { + const labels = domain + .toLowerCase() + .replace(/^www\./, '') + .split('.'); + const secondLast = labels[labels.length - 2] ?? ''; + const last = labels[labels.length - 1] ?? ''; + const keep = + labels.length >= 3 && /^(co|com|org|net|ac|gov|edu)$/.test(secondLast) && last.length === 2 + ? 3 + : 2; + return labels.slice(-keep).join('.'); +} + +/** `shop.acme.co.uk` -> `acme`: the label a company is usually called by. */ +export function companyToken(domain: string): string { + return registrable(domain).split('.')[0] ?? ''; +} + +/** The profile or company slug as words: `/in/ada-lovelace-42` -> `ada lovelace 42`. */ +function slugWords(url: string): string { + return (url.split('/').pop() ?? '').replace(/[-_.]+/g, ' '); +} + +/** A LinkedIn profile (`in`) or company page URL, canonical, or '' for anything else. */ +export function linkedinPath(link: string | undefined, kind: 'in' | 'company'): string { + if (!link) return ''; + try { + const url = new URL(link); + const host = url.hostname.toLowerCase(); + if (host !== 'linkedin.com' && !host.endsWith('.linkedin.com')) return ''; + const match = (kind === 'in' ? /^\/in\/([^/]+)/ : /^\/company\/([^/]+)/).exec(url.pathname); + if (!match?.[1]) return ''; + return `https://www.linkedin.com/${kind}/${decodeURIComponent(match[1])}`; + } catch { + return ''; + } +} + +export function personQuery(subject: LinkedinSubject): string { + const name = `${subject.firstName} ${subject.lastName}`.trim(); + const company = subject.companyDomain ? companyToken(subject.companyDomain) : ''; + return `site:linkedin.com/in "${name}"${company ? ` ${company}` : ''}`; +} + +/** The one profile the results evidence for this person, or ''. */ +export function pickPerson(results: readonly SerpResult[], subject: LinkedinSubject): string { + const name = `${subject.firstName} ${subject.lastName}`; + const company = subject.companyDomain ? fold(companyToken(subject.companyDomain)) : ''; + const profiles = new Set(); + for (const result of results) { + const url = linkedinPath(result.link, 'in'); + if (!url || !carriesName(result.title ?? '', name)) continue; + if (company) { + if (company.length >= 3 && fold(`${result.title} ${result.snippet}`).includes(company)) { + return url; + } + } else if (carriesName(slugWords(url), name)) { + profiles.add(url); + } + } + return profiles.size === 1 ? ([...profiles][0] ?? '') : ''; +} + +/** + * The headline from a profile result's title. With two parts the second is + * as often the company as the title, so it only counts when it is not the + * company's name. + */ +export function titleFromResult(title: string, companyDomain = ''): string { + const parts = title + .replace(/\s*[|·]\s*LinkedIn.*$/i, '') + .split(/\s+[-–—]\s+/) + .map((part) => part.trim()); + const candidate = parts.length >= 2 ? (parts[1] ?? '') : ''; + if (candidate.length < 2 || candidate.length > 100 || candidate.endsWith('...')) return ''; + const company = companyDomain ? fold(companyToken(companyDomain)) : ''; + if (parts.length === 2 && company && fold(candidate).replace(/\s+/g, '').includes(company)) { + return ''; + } + return candidate; +} + +export function companyQuery(domain: string): string { + return `site:linkedin.com/company "${registrable(domain)}"`; +} + +/** + * The company page whose title or slug names the company, or whose text + * carries the domain itself. A snippet that merely mentions the name is not + * enough: a school district's name turns up on every one of its schools' pages. + */ +export function pickCompany(results: readonly SerpResult[], domain: string): string { + const site = registrable(domain); + const token = fold(companyToken(domain)); + for (const result of results) { + const url = linkedinPath(result.link, 'company'); + if (!url) continue; + const named = fold(`${result.title} ${slugWords(url)}`).replace(/\s+/g, ''); + if ( + fold(`${result.title} ${result.snippet}`).includes(site) || + (token.length >= 3 && named.includes(token)) + ) { + return url; + } + } + return ''; +} From 4c6a5a5e4975432adc041722e5d5bfa52c567edf Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Tue, 6 Oct 2026 19:15:28 +0000 Subject: [PATCH 3/3] Import report: a flagged role inbox reads as screening, not as a dead mailbox Seen live on the first prod append: info@ came back as 'nobody reads that mailbox', the cleaner's text for noreply@. Co-Authored-By: Claude Opus 5.5 --- apps/api/src/autogtm.ts | 4 ++-- apps/api/src/import-report.ts | 20 +++++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/apps/api/src/autogtm.ts b/apps/api/src/autogtm.ts index 394fe41..1877661 100644 --- a/apps/api/src/autogtm.ts +++ b/apps/api/src/autogtm.ts @@ -891,7 +891,7 @@ export function autogtmRoutes(deps: AutogtmDeps): Hono { if (c.req.query('format') === 'csv') return importReportCsv(c, report); return c.json({ ...report, - rows: report.rows.map((row) => ({ ...row, why: reasonText(row.reason) })), + rows: report.rows.map((row) => ({ ...row, why: reasonText(row.reason, row.outcome) })), }); }); @@ -1911,7 +1911,7 @@ async function importIntoCampaign(db: Client, actor: RequestActor, input: Import crawls_queued: crawls, report: reportRows .slice(0, INLINE_REPORT_ROWS) - .map((row) => ({ ...row, why: reasonText(row.reason) })), + .map((row) => ({ ...row, why: reasonText(row.reason, row.outcome) })), report_truncated: reportRows.length > INLINE_REPORT_ROWS, report_url: `/api/v1/autogtm/campaigns/import/${importId}/report?format=csv`, }; diff --git a/apps/api/src/import-report.ts b/apps/api/src/import-report.ts index 901a854..e8650d1 100644 --- a/apps/api/src/import-report.ts +++ b/apps/api/src/import-report.ts @@ -55,11 +55,25 @@ const REASON_TEXT: Readonly> = { suppressed: 'on a suppress list', }; +/** + * Screening's words for its flags. `role_address` means something different + * there: the cleaner rejects a mailbox nobody reads (noreply@), screening + * holds back one a team reads (info@). + */ +const FLAG_TEXT: Readonly> = { + generated_name: 'a generated name, held back from sending', + relay_address: 'a relay address that hides the person, held back from sending', + temp_mail_domain: 'a temp-mail domain, held back from sending', + agent_account: 'a bot, agent or test account, held back from sending', + role_address: 'a team inbox, not a person, held back from sending', +}; + /** A reason as words, for the UI and the CSV `why` column. */ -export function reasonText(reason: string): string { +export function reasonText(reason: string, outcome?: string): string { + const words = outcome === 'flagged' ? { ...REASON_TEXT, ...FLAG_TEXT } : REASON_TEXT; return reason .split('+') - .map((part) => REASON_TEXT[part] ?? part.replace(/_/g, ' ')) + .map((part) => words[part] ?? part.replace(/_/g, ' ')) .join('; '); } @@ -130,7 +144,7 @@ export function importReportCsv(c: Context, report: ImportReport): Response { email: row.email, outcome: row.outcome, reason: row.reason, - why: reasonText(row.reason), + why: reasonText(row.reason, row.outcome), detail: row.detail, })), ['row', 'email', 'outcome', 'reason', 'why', 'detail'],