diff --git a/.github/workflows/deploy-dev2.yml b/.github/workflows/deploy-dev2.yml index 483a3a2..2017da4 100644 --- a/.github/workflows/deploy-dev2.yml +++ b/.github/workflows/deploy-dev2.yml @@ -52,9 +52,10 @@ jobs: DEV2_HOST: ${{ secrets.DEV2_HOST }} VALUESERP_API_KEY: ${{ secrets.VALUESERP_API_KEY }} CHOVY_CAMPAIGN_SECRET: ${{ secrets.CHOVY_CAMPAIGN_SECRET }} + PDL_API_KEY: ${{ secrets.PDL_API_KEY }} run: | lines="" - for key in VALUESERP_API_KEY CHOVY_CAMPAIGN_SECRET; do + for key in VALUESERP_API_KEY CHOVY_CAMPAIGN_SECRET PDL_API_KEY; do value=$(printenv "$key" || true) [ -n "$value" ] && lines="$lines$key=$value"$'\n' done diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index abac45f..ae1b4a0 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -87,6 +87,7 @@ import { budgetStatus, crawlDedupeKey, screenHold, + type LeadEnrichDeps, startContactImport, importContactChunk, intakeSocialPeople, @@ -336,6 +337,12 @@ export interface AppOptions { * people by company. Absent without a key, which leaves pasting URLs. */ readonly jobSearcher?: WebSearcher | undefined; + /** + * Lead enrichment (names from addresses, ValueSERP LinkedIn search, People + * Data Labs), for the on-demand run a campaign page can start. Absent: the + * route still fills names from addresses. + */ + readonly leadEnrichment?: Omit | undefined; /** Test seam for the job boards and company sites a posting is read from. */ readonly jobReader?: JobReaderOptions | undefined; /** @@ -1375,6 +1382,7 @@ export function createApp(options: AppOptions): Hono { approve: (db, actor, recommendation) => approveRecommendation(db, options, actor, recommendation, {}), requireVerifiedEmail: (db, actor) => requireVerifiedEmail(db, actor), + ...(options.leadEnrichment ? { leadEnrichment: options.leadEnrichment } : {}), }), ); diff --git a/apps/api/src/append-leads.test.ts b/apps/api/src/append-leads.test.ts index e37c8ea..c57a10c 100644 --- a/apps/api/src/append-leads.test.ts +++ b/apps/api/src/append-leads.test.ts @@ -159,6 +159,61 @@ describe('POST /autogtm/campaigns/:id/leads', () => { T, ); + test( + 'enrich runs in the background and the status reports what was filled', + async () => { + const seeded = await seedDatabase('append-enrich'); + active = seeded; + const asked: string[] = []; + const app = createApp({ + db: seeded.db, + authenticate: async () => ACTOR, + leadEnrichment: { + searcher: { + async search(query) { + asked.push(query); + return query.includes('Scott Perry') + ? [ + { + title: 'Scott Perry - VP Sales - Northwind | LinkedIn', + link: 'https://www.linkedin.com/in/scott-perry-9', + }, + ] + : []; + }, + }, + }, + }); + + await send(app, 'POST', `/autogtm/campaigns/${SEED.campaignId}/leads`, { + leads: [{ email: 'scott.perry@northwind.io', company_domain: 'northwind.io' }], + }); + + const started = await send(app, 'POST', `/autogtm/campaigns/${SEED.campaignId}/enrich`, {}); + expect(started.status).toBe(202); + + let status: Record = { running: true }; + for (let i = 0; i < 100 && status.running; i += 1) { + await Bun.sleep(100); + status = (await ( + await app.request(`/api/v1/autogtm/campaigns/${SEED.campaignId}/enrichment`) + ).json()) as Record; + } + expect(status.running).toBe(false); + expect(status.last_run).toMatchObject({ titles: 1, profiles: 1 }); + + const scott = await queryOne<{ current_title: string; first_name: string }>( + seeded.db, + `SELECT p.current_title, p.first_name FROM people p + JOIN person_emails pe ON pe.person_id = p.id + WHERE pe.address = 'scott.perry@northwind.io'`, + ); + expect(scott).toEqual({ current_title: 'VP Sales', first_name: 'Scott' }); + expect(asked.some((q) => q.includes('"Scott Perry" northwind'))).toBe(true); + }, + T, + ); + test( 'takes the CSV itself and names a missing email column', async () => { diff --git a/apps/api/src/autogtm-docs.ts b/apps/api/src/autogtm-docs.ts index 4ac2594..11bd9cc 100644 --- a/apps/api/src/autogtm-docs.ts +++ b/apps/api/src/autogtm-docs.ts @@ -601,6 +601,51 @@ export const OPERATIONS: readonly Operation[] = [ properties: { allow: { type: 'boolean' } }, }, }, + { + method: 'post', + path: '/autogtm/campaigns/{campaign_id}/enrich', + id: 'enrichLeads', + tag: 'Campaigns', + summary: 'Fill in leads’ missing name, job title and LinkedIn', + description: + 'Starts a run and answers 202 at once. Names come from unambiguous addresses ' + + '(first.last@), free. Job title and LinkedIn profile come from a Google search of LinkedIn, ' + + 'taken only when the result carries every part of the name and the company; the company’s ' + + 'LinkedIn page is the fallback. People Data Labs is used first when configured. Only blanks ' + + 'are filled, every search is cached so a rerun costs nothing, and searches are capped per ' + + 'run (`max_searches`) and per day. Read the outcome from GET .../enrichment.', + params: [idParam('campaign_id', 'The campaign.')], + body: { + type: 'object', + properties: { + max_searches: { type: 'integer', minimum: 0, maximum: 1000 }, + max_leads: { type: 'integer', minimum: 1, maximum: 5000 }, + }, + }, + status: 202, + }, + { + method: 'get', + path: '/autogtm/campaigns/{campaign_id}/enrichment', + id: 'getEnrichment', + tag: 'Campaigns', + summary: 'What a campaign’s leads are missing, and the last enrichment run', + params: [idParam('campaign_id', 'The campaign.')], + response: { + type: 'object', + properties: { + leads: { type: 'integer' }, + missing_name: { type: 'integer' }, + missing_title: { type: 'integer' }, + missing_linkedin: { type: 'integer' }, + looked_up: { type: 'integer' }, + providers: { type: 'object' }, + today: { type: 'object' }, + running: { type: 'boolean' }, + last_run: { type: 'object' }, + }, + }, + }, { method: 'get', path: '/autogtm/campaigns/import/{task_id}', diff --git a/apps/api/src/autogtm.ts b/apps/api/src/autogtm.ts index 21af8f3..1877661 100644 --- a/apps/api/src/autogtm.ts +++ b/apps/api/src/autogtm.ts @@ -48,6 +48,9 @@ import { domainMatchKey, emailMatchKey, enqueue, + enrichLeads, + enrichmentStatus, + type LeadEnrichDeps, finishContactImport, importContactChunk, normaliseDomain, @@ -101,8 +104,13 @@ export interface AutogtmDeps { ) => Promise; /** Refuses an unverified account anything that reaches a stranger. */ readonly requireVerifiedEmail: (db: Client, actor: RequestActor) => Promise; + /** The enrichment providers; absent means names from addresses only. */ + readonly leadEnrichment?: Omit; } +/** Campaigns with an on-demand enrichment run in flight, and the last result of each. */ +const enrichRuns = new Map(); + // ------------------------------------------------------------------ schemas const budgetBody = z.object({ @@ -219,6 +227,12 @@ const appendBody = z const allowBody = z.object({ allow: z.boolean() }); +const enrichBody = z.object({ + /** New (uncached) searches this run may spend; the daily cap still applies. */ + max_searches: z.number().int().min(0).max(1_000).optional(), + max_leads: z.number().int().min(1).max(5_000).optional(), +}); + // ------------------------------------------------------------------ rows interface CampaignRow { @@ -877,7 +891,7 @@ export function autogtmRoutes(deps: AutogtmDeps): Hono { if (c.req.query('format') === 'csv') return importReportCsv(c, report); return c.json({ ...report, - rows: report.rows.map((row) => ({ ...row, why: reasonText(row.reason) })), + rows: report.rows.map((row) => ({ ...row, why: reasonText(row.reason, row.outcome) })), }); }); @@ -957,6 +971,75 @@ export function autogtmRoutes(deps: AutogtmDeps): Hono { }); }); + r.get('/campaigns/:id/enrichment', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + const campaign = await ownedCampaign(db, actor.workspaceId, c.req.param('id')); + const status = await enrichmentStatus( + { db, ...deps.leadEnrichment }, + { workspaceId: actor.workspaceId, campaignId: campaign.id }, + ); + const run = enrichRuns.get(campaign.id); + return c.json({ + campaign_id: campaign.id, + ...status, + running: run?.running === true, + ...(run?.last ? { last_run: run.last } : {}), + }); + }); + + /** + * Starts a run over this campaign's leads and answers at once: searches take + * 10-45 s each, so the run continues after the response and its outcome is + * read back from GET .../enrichment. + */ + r.post('/campaigns/:id/enrich', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + requireWriter(actor, 'enriching leads'); + const campaign = await ownedCampaign(db, actor.workspaceId, c.req.param('id')); + const body = await parse(c.req.raw, enrichBody); + + const current = enrichRuns.get(campaign.id); + if (current?.running) { + return c.json({ campaign_id: campaign.id, started: false, running: true }, 202); + } + const startedAt = now(); + enrichRuns.set(campaign.id, { + running: true, + startedAt, + ...(current?.last ? { last: current.last } : {}), + }); + void enrichLeads( + { db, ...deps.leadEnrichment }, + { + workspaceId: actor.workspaceId, + campaignId: campaign.id, + limit: body.max_leads ?? 200, + maxSearches: body.max_searches ?? 100, + }, + ) + .then((result) => { + enrichRuns.set(campaign.id, { + running: false, + startedAt, + last: { ...result, started_at: startedAt, finished_at: now() }, + }); + }) + .catch((error: unknown) => { + enrichRuns.set(campaign.id, { + running: false, + startedAt, + last: { + error: error instanceof Error ? error.message : String(error), + started_at: startedAt, + }, + }); + }); + + return c.json({ campaign_id: campaign.id, started: true, running: true }, 202); + }); + r.post('/leads/:person_id/screening', async (c) => { const actor = c.get('actor'); const db = c.get('db'); @@ -1828,7 +1911,7 @@ async function importIntoCampaign(db: Client, actor: RequestActor, input: Import crawls_queued: crawls, report: reportRows .slice(0, INLINE_REPORT_ROWS) - .map((row) => ({ ...row, why: reasonText(row.reason) })), + .map((row) => ({ ...row, why: reasonText(row.reason, row.outcome) })), report_truncated: reportRows.length > INLINE_REPORT_ROWS, report_url: `/api/v1/autogtm/campaigns/import/${importId}/report?format=csv`, }; diff --git a/apps/api/src/import-report.ts b/apps/api/src/import-report.ts index 901a854..e8650d1 100644 --- a/apps/api/src/import-report.ts +++ b/apps/api/src/import-report.ts @@ -55,11 +55,25 @@ const REASON_TEXT: Readonly> = { suppressed: 'on a suppress list', }; +/** + * Screening's words for its flags. `role_address` means something different + * there: the cleaner rejects a mailbox nobody reads (noreply@), screening + * holds back one a team reads (info@). + */ +const FLAG_TEXT: Readonly> = { + generated_name: 'a generated name, held back from sending', + relay_address: 'a relay address that hides the person, held back from sending', + temp_mail_domain: 'a temp-mail domain, held back from sending', + agent_account: 'a bot, agent or test account, held back from sending', + role_address: 'a team inbox, not a person, held back from sending', +}; + /** A reason as words, for the UI and the CSV `why` column. */ -export function reasonText(reason: string): string { +export function reasonText(reason: string, outcome?: string): string { + const words = outcome === 'flagged' ? { ...REASON_TEXT, ...FLAG_TEXT } : REASON_TEXT; return reason .split('+') - .map((part) => REASON_TEXT[part] ?? part.replace(/_/g, ' ')) + .map((part) => words[part] ?? part.replace(/_/g, ' ')) .join('; '); } @@ -130,7 +144,7 @@ export function importReportCsv(c: Context, report: ImportReport): Response { email: row.email, outcome: row.outcome, reason: row.reason, - why: reasonText(row.reason), + why: reasonText(row.reason, row.outcome), detail: row.detail, })), ['row', 'email', 'outcome', 'reason', 'why', 'detail'], diff --git a/apps/cli/src/commands.ts b/apps/cli/src/commands.ts index 3c46cbf..58cafcd 100644 --- a/apps/cli/src/commands.ts +++ b/apps/cli/src/commands.ts @@ -273,6 +273,39 @@ async function runLeads({ client, args, flags }: CommandContext): Promise`); + const base = `/autogtm/campaigns/${encodeURIComponent(target)}`; + if (verb === 'enrich') { + const max = flagString(flags, 'max'); + await client.post(`${base}/enrich`, max ? { max_searches: Number(max) } : {}); + } + const status = (await client.get(`${base}/enrichment`)) as Record; + const today = (status.today ?? {}) as Record; + const last = status.last_run as Record | undefined; + return [ + verb === 'enrich' + ? 'Started. It runs in the background; check: og leads enrichment ' + target + : '', + `${text(status, 'leads', '0')} leads: missing ${text(status, 'missing_title', '0')} titles, ` + + `${text(status, 'missing_linkedin', '0')} LinkedIn, ${text(status, 'missing_name', '0')} names`, + `searches today: ${text(today, 'searches', '0')} of ${text(today, 'searches_cap', '0')}` + + `${status.running ? ' (a run is in progress)' : ''}`, + ...(last + ? [ + last.error + ? `last run failed: ${text(last, 'error')}` + : `last run: +${text(last, 'names', '0')} names, +${text(last, 'titles', '0')} titles, ` + + `+${text(last, 'profiles', '0')} profiles, +${text(last, 'companies', '0')} company pages, ` + + `${text(last, 'searches', '0')} searches (${text(last, 'cached', '0')} cached)` + + `${last.stopped ? `; stopped: ${text(last, 'stopped')}` : ''}`, + ] + : []), + ] + .filter(Boolean) + .join('\n'); + } + if (verb === 'allow' || verb === 'hold') { if (!target) throw new Error(`usage: og leads ${verb} `); await client.post(`/autogtm/leads/${encodeURIComponent(target)}/screening`, { @@ -283,7 +316,7 @@ async function runLeads({ client, args, flags }: CommandContext): Promise { @@ -1662,7 +1695,7 @@ export const COMMANDS: readonly Command[] = [ { name: 'leads', usage: - 'og leads add --consent-source "" [--allow-flagged] [--keep-project-duplicates] [--report out.csv] | report [--csv] | screened [--all] | allow | hold ', + 'og leads add --consent-source "" [--allow-flagged] [--keep-project-duplicates] [--report out.csv] | report [--csv] | screened [--all] | enrich [--max ] | enrichment | allow | hold ', summary: 'Add a CSV of leads to a running campaign, with a per-row report and screening.', run: runLeads, }, diff --git a/apps/mcp/src/mcp.test.ts b/apps/mcp/src/mcp.test.ts index 1055fda..85b486d 100644 --- a/apps/mcp/src/mcp.test.ts +++ b/apps/mcp/src/mcp.test.ts @@ -358,4 +358,19 @@ describe('lead tools', () => { expect(calls[1]?.body).toEqual({ allow: true }); expect(calls[2]?.url).toBe('https://api.test/api/v1/autogtm/campaigns/import/cim_1/report'); }); + + test('enrich_campaign_leads starts a run and reads the status back', async () => { + const { fetchImpl, calls } = recorder(ok({ running: true })); + const client = createClient(CONFIG, fetchImpl); + + await runTool(toolByName('enrich_campaign_leads')!, client, { + campaignId: 'cmp_1', + maxSearches: 20, + }); + expect(calls.map((c) => `${c.method} ${c.url}`)).toEqual([ + 'POST https://api.test/api/v1/autogtm/campaigns/cmp_1/enrich', + 'GET https://api.test/api/v1/autogtm/campaigns/cmp_1/enrichment', + ]); + expect(calls[0]?.body).toEqual({ max_searches: 20 }); + }); }); diff --git a/apps/mcp/src/tools.ts b/apps/mcp/src/tools.ts index f5950c2..b1ea2fe 100644 --- a/apps/mcp/src/tools.ts +++ b/apps/mcp/src/tools.ts @@ -1006,6 +1006,50 @@ export const TOOLS: readonly ToolDefinition[] = [ allow: args.allow === true, }), }, + { + name: 'enrich_campaign_leads', + title: 'Fill in leads’ missing name, title and LinkedIn', + description: + 'Starts a background run over a campaign’s leads: names from unambiguous addresses ' + + '(first.last@, free), then People Data Labs when configured, then a Google search of ' + + 'LinkedIn taken only when the result carries the full name and the company. Fills blanks ' + + 'only, caches every search, and is capped per run and per day. Returns the status; call ' + + 'get_lead_enrichment later for the outcome.', + readOnly: false, + inputSchema: { + type: 'object', + properties: { + campaignId: { type: 'string' }, + maxSearches: { type: 'number', description: 'New searches this run may spend.' }, + }, + required: ['campaignId'], + }, + run: async (client, args) => { + const base = `/autogtm/campaigns/${encodeURIComponent(require(args, 'campaignId'))}`; + await client.post( + `${base}/enrich`, + typeof args.maxSearches === 'number' ? { max_searches: args.maxSearches } : {}, + ); + return client.get(`${base}/enrichment`); + }, + }, + { + name: 'get_lead_enrichment', + title: 'What a campaign’s leads are missing', + description: + 'How many of a campaign’s leads lack a name, job title or LinkedIn, which enrichment ' + + 'providers are configured, searches spent today against the cap, and the last run.', + readOnly: true, + inputSchema: { + type: 'object', + properties: { campaignId: { type: 'string' } }, + required: ['campaignId'], + }, + run: (client, args) => + client.get( + `/autogtm/campaigns/${encodeURIComponent(require(args, 'campaignId'))}/enrichment`, + ), + }, { name: 'list_inbox', title: 'List conversations', diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index 4797ce8..efa0273 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -90,12 +90,16 @@ import { checkLinkedInAcceptances, linkedInSessionForWorkspace, workspacesWithLinkedInSession, + enrichLeads, + workspacesAwaitingLeadEnrichment, + type LeadEnrichDeps, } from '@outreachgraph/pipeline'; import { BlueskyFeedSource, BlueskyProvider, createSmtpProber, NostrSource, + PeopleDataLabsClient, RedditSource, RssSource, SiteProvider, @@ -301,6 +305,34 @@ if (!chovy) console.log('no CHOVY_CAMPAIGN_SECRET: ideas are found and ranked, B const photoLookupsPerDay = Number(process.env.PHOTO_LOOKUPS_PER_DAY ?? 300); +/** + * Lead enrichment: a lead's missing name, job title and LinkedIn. Names from + * the address are free and always on; the LinkedIn search is the same + * ValueSERP account (a shorter per-request timeout, since a sweep at + * concurrency ten would rather drop a slow query than wait a minute on it); + * People Data Labs only with PDL_API_KEY. Both are capped per workspace per + * day, and a 402 pauses that provider until the next UTC day. + */ +const leadEnrichment: LeadEnrichDeps = { + db, + ...(process.env.VALUESERP_API_KEY + ? { + searcher: new ValueSerpClient({ + apiKey: process.env.VALUESERP_API_KEY, + searchTimeoutMs: 45_000, + }), + } + : {}), + ...(process.env.PDL_API_KEY + ? { pdl: new PeopleDataLabsClient({ apiKey: process.env.PDL_API_KEY }) } + : {}), + searchesPerDay: Number(process.env.LINKEDIN_SEARCHES_PER_DAY ?? 300), + pdlPerDay: Number(process.env.PDL_LOOKUPS_PER_DAY ?? 3), + concurrency: Number(process.env.LEAD_ENRICH_CONCURRENCY ?? 10), +}; +if (!leadEnrichment.pdl) console.log('no PDL_API_KEY: lead enrichment uses addresses and search'); +let leadEnrichmentRunning = false; + if (!photoFinder) console.log('no VALUESERP_API_KEY: lead photos come from public profiles and crawls'); @@ -472,6 +504,7 @@ const api = createApp({ ...(encryptionKey ? { encryptionKey } : {}), ...(appUrl ? { appUrl } : {}), ...(jobSearcher ? { jobSearcher } : {}), + leadEnrichment, ...(chovy ? { chovy } : {}), ...(process.env.API_TOKEN ? { serviceToken: process.env.API_TOKEN } : {}), // A person editing their own OpenProfile.md carries an OpenAccess bearer @@ -1097,6 +1130,35 @@ async function tick(): Promise { // which is both the cheapest grounded claim the product can make and the // fastest-decaying one. Each watch has its own interval, so a workspace // watching one Bluesky handle costs one query per tick and nothing else. + // A name, a title and a LinkedIn for leads that arrived as a bare address. + // A search takes 10-45 s, so the sweep runs beside the tick rather than + // inside it, one run at a time, each bounded by its own caps. + if (!leadEnrichmentRunning) { + const waiting = await workspacesAwaitingLeadEnrichment(db); + if (waiting.length) { + leadEnrichmentRunning = true; + void (async () => { + for (const workspaceId of waiting) { + try { + const r = await enrichLeads(leadEnrichment, { workspaceId, maxSearches: 40 }); + if (r.looked > 0) { + console.log( + `lead enrichment: ${workspaceId} looked at ${r.looked}, +${r.names} names, ` + + `+${r.titles} titles, +${r.profiles} profiles, +${r.companies} company pages, ` + + `${r.pdlMatches} PDL matches, ${r.searches} searches (${r.cached} cached)` + + `${r.stopped ? `; stopped: ${r.stopped}` : ''}`, + ); + } + } catch (error) { + console.error(`lead enrichment failed for ${workspaceId}`, error); + } + } + })().finally(() => { + leadEnrichmentRunning = false; + }); + } + } + // The Idea Generator: what people keep asking for on Reddit. A scan paces the // public archive at ~2.5s a request, so it runs beside the tick rather than // inside it, one at a time. diff --git a/apps/web/app/(app)/products/[id]/page.tsx b/apps/web/app/(app)/products/[id]/page.tsx index ee3f527..90c4090 100644 --- a/apps/web/app/(app)/products/[id]/page.tsx +++ b/apps/web/app/(app)/products/[id]/page.tsx @@ -3,6 +3,7 @@ import { notFound, redirect } from 'next/navigation'; import { Avatar } from '../../../../components/avatar'; import { AddLeads } from '../../../../components/add-leads'; import { CampaignControls } from '../../../../components/campaign-controls'; +import { LeadEnrichment } from '../../../../components/lead-enrichment'; import { ScreenedLeads } from '../../../../components/screened-leads'; import { ApiUnavailableError, @@ -140,6 +141,7 @@ export default async function ProductPage({ params }: { params: Promise<{ id: st {campaign ? ( <> + ) : null} diff --git a/apps/web/components/lead-enrichment.tsx b/apps/web/components/lead-enrichment.tsx new file mode 100644 index 0000000..ff0a2e0 --- /dev/null +++ b/apps/web/components/lead-enrichment.tsx @@ -0,0 +1,124 @@ +'use client'; + +import { useCallback, useEffect, useState } from 'react'; + +/** + * What a campaign's leads are missing, and a button that fills it in. + * + * A worker sweep already does this a little at a time; the button is for the + * moment after an import, when the owner wants titles before the first drafts + * are written. Searches take 10-45 s each, so the run continues in the + * background and this polls for the outcome. + */ + +interface Status { + readonly leads: number; + readonly missing_name: number; + readonly missing_title: number; + readonly missing_linkedin: number; + readonly providers: { readonly valueserp: boolean; readonly pdl: boolean }; + readonly today: { readonly searches: number; readonly searches_cap: number }; + readonly paused?: { readonly valueserp?: string; readonly pdl?: string }; + readonly running: boolean; + readonly last_run?: { + readonly names?: number; + readonly titles?: number; + readonly profiles?: number; + readonly companies?: number; + readonly searches?: number; + readonly cached?: number; + readonly stopped?: string; + readonly error?: string; + }; +} + +export function LeadEnrichment({ campaignId }: { campaignId: string }) { + const [status, setStatus] = useState(); + const [error, setError] = useState(); + const url = `/api/v1/autogtm/campaigns/${encodeURIComponent(campaignId)}`; + + const load = useCallback(async () => { + const response = await fetch(`${url}/enrichment`, { credentials: 'same-origin' }); + if (response.ok) setStatus((await response.json()) as Status); + }, [url]); + + useEffect(() => { + void load().catch(() => undefined); + }, [load]); + + useEffect(() => { + if (!status?.running) return; + const timer = setInterval(() => void load().catch(() => undefined), 5_000); + return () => clearInterval(timer); + }, [status?.running, load]); + + async function start(): Promise { + setError(undefined); + const response = await fetch(`${url}/enrich`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + credentials: 'same-origin', + body: JSON.stringify({}), + }); + if (!response.ok) { + setError(`Could not start (${response.status}).`); + return; + } + await load(); + } + + if (!status || status.leads === 0) return null; + const missing = status.missing_title + status.missing_linkedin + status.missing_name; + const last = status.last_run; + + return ( +
+
+

+ {missing === 0 ? ( + 'Every lead has a name, a title and a LinkedIn.' + ) : ( + <> + Missing: {status.missing_title.toLocaleString()} titles,{' '} + {status.missing_linkedin.toLocaleString()} LinkedIn,{' '} + {status.missing_name.toLocaleString()} names. + + )} +

+ {missing > 0 ? ( + + ) : null} +
+

+ Names from addresses (free) + {status.providers.pdl ? ', People Data Labs' : ''} + {status.providers.valueserp + ? `, then a Google search of LinkedIn (${status.today.searches} of ${status.today.searches_cap} searches used today)` + : '. LinkedIn search is not configured on this deployment'} + . Only blanks are filled. + {status.paused?.valueserp + ? ` Search is out of credits until ${status.paused.valueserp}.` + : ''} +

+ {last && !status.running ? ( +

+ {last.error + ? `Last run failed: ${last.error}` + : `Last run: +${last.names ?? 0} names, +${last.titles ?? 0} titles, +${last.profiles ?? 0} profiles, +${last.companies ?? 0} company pages from ${last.searches ?? 0} searches (${last.cached ?? 0} cached)${last.stopped ? `; stopped: ${last.stopped}` : ''}.`} +

+ ) : null} + {error ? ( +

+ {error} +

+ ) : null} +
+ ); +} diff --git a/migrations-pg/0052_lead_enrichment.sql b/migrations-pg/0052_lead_enrichment.sql new file mode 100644 index 0000000..6f84be3 --- /dev/null +++ b/migrations-pg/0052_lead_enrichment.sql @@ -0,0 +1,19 @@ +-- 0052_lead_enrichment.sql (Postgres). See migrations/0052_lead_enrichment.sql for the reasoning. +-- Two nullable ADD COLUMNs (no rewrite) and two new, empty tables. No index on an existing table. + +ALTER TABLE people ADD COLUMN IF NOT EXISTS linkedin_looked_up_at text; +ALTER TABLE companies ADD COLUMN IF NOT EXISTS linkedin_url text; + +create table if not exists serp_cache ( + query text PRIMARY KEY, + results text NOT NULL, + fetched_at text NOT NULL +); + +create table if not exists enrichment_usage ( + workspace_id text NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + day text NOT NULL, + provider text NOT NULL, + lookups bigint NOT NULL DEFAULT 0, + PRIMARY KEY (workspace_id, day, provider) +); diff --git a/migrations/0052_lead_enrichment.sql b/migrations/0052_lead_enrichment.sql new file mode 100644 index 0000000..ba808a4 --- /dev/null +++ b/migrations/0052_lead_enrichment.sql @@ -0,0 +1,37 @@ +-- 0052_lead_enrichment.sql +-- +-- Filling in a lead's missing name, job title and LinkedIn from free and +-- cheap sources (the address itself, a Google search of LinkedIn, and People +-- Data Labs when a key is configured). +-- +-- people.linkedin_looked_up_at is the "already tried" stamp, like +-- photo_looked_up_at: most searches find nothing, and a sweep that only +-- recorded hits would pay for the same misses forever. Deliberately no index: +-- the sweep reaches people through campaign_people, never by scanning this. +-- +-- companies.linkedin_url is the company page, the fallback when no person +-- profile is evidenced. +-- +-- serp_cache keeps every search's raw results, keyed by the query, so a rerun +-- (or a tightened matching rule) costs nothing. Public search results, not +-- per workspace. +-- +-- enrichment_usage counts paid lookups per workspace per day, so the daily cap +-- survives a restart. + +ALTER TABLE people ADD COLUMN linkedin_looked_up_at TEXT; +ALTER TABLE companies ADD COLUMN linkedin_url TEXT; + +CREATE TABLE IF NOT EXISTS serp_cache ( + query TEXT PRIMARY KEY, + results TEXT NOT NULL, + fetched_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS enrichment_usage ( + workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + day TEXT NOT NULL, + provider TEXT NOT NULL, + lookups INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (workspace_id, day, provider) +); diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index 76c8242..1d11513 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -180,6 +180,19 @@ export { type StartImportInput, type StoredRow, } from './contact-import'; +export { + DEFAULT_PDL_PER_DAY, + DEFAULT_SEARCHES_PER_DAY, + enrichLeads, + enrichmentPausedUntil, + enrichmentStatus, + lookupsToday, + resetEnrichmentPauses, + workspacesAwaitingLeadEnrichment, + type LeadEnrichDeps, + type LeadEnrichInput, + type LeadEnrichResult, +} from './lead-enrichment'; export { countScreenedHeld, holdReason, diff --git a/packages/pipeline/src/lead-enrichment.test.ts b/packages/pipeline/src/lead-enrichment.test.ts new file mode 100644 index 0000000..c383ed7 --- /dev/null +++ b/packages/pipeline/src/lead-enrichment.test.ts @@ -0,0 +1,251 @@ +/** + * Filling in a lead's name, title and LinkedIn: only blanks, only on + * evidence, and never paying twice for the same search. + */ + +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { now, queryAll, queryOne, type Client } from '@outreachgraph/db'; +import { + SearchOutOfCredits, + type PersonEnricher, + type WebResult, + type WebSearcher, +} from '@outreachgraph/providers'; +import { seedDatabase, SEED, type SeededDatabase } from '../../../apps/api/src/test-seed'; +import { + enrichLeads, + enrichmentStatus, + lookupsToday, + resetEnrichmentPauses, +} from './lead-enrichment'; + +let seeded: SeededDatabase | undefined; + +beforeEach(() => resetEnrichmentPauses()); +afterEach(() => { + seeded?.cleanup(); + seeded = undefined; +}); + +const T = 60_000; + +/** A lead who arrived as a bare address, in the seeded campaign. */ +async function addLead(db: Client, id: string, email: string, companyDomain?: string) { + const stamp = now(); + const companyId = companyDomain ? `co_${id}` : null; + await db.batch([ + ...(companyDomain + ? [ + { + sql: `INSERT INTO companies (id, name, domain, technologies, created_at, updated_at) + VALUES (?, ?, ?, '[]', ?, ?)`, + args: [companyId, companyDomain, companyDomain, stamp, stamp], + }, + ] + : []), + { + sql: `INSERT INTO people (id, display_name, current_company_id, identity_confidence, status, + outreach_eligible, created_at, updated_at) + VALUES (?, ?, ?, 0.9, 'active', 1, ?, ?)`, + args: [id, email.split('@')[0]!, companyId, stamp, stamp], + }, + { + sql: `INSERT INTO person_emails (id, workspace_id, person_id, address, dedupe_key, source, + verified, created_at) VALUES (?, ?, ?, ?, ?, 'import', 1, ?)`, + args: [`pe_${id}`, SEED.workspaceId, id, email, email, stamp], + }, + { + sql: `INSERT INTO campaign_people (campaign_id, person_id, workspace_id, status, + interaction_state, discovered_at, updated_at) + VALUES (?, ?, ?, 'discovered', 'never_contacted', ?, ?)`, + args: [SEED.campaignId, id, SEED.workspaceId, stamp, stamp], + }, + ]); +} + +function searcher(answers: Record): { + asked: string[]; + searcher: WebSearcher; +} { + const asked: string[] = []; + return { + asked, + searcher: { + async search(query) { + asked.push(query); + return answers[query] ?? []; + }, + }, + }; +} + +describe('enrichLeads', () => { + test( + 'names from the address, title and profile from the search, company page as fallback', + async () => { + seeded = await seedDatabase('enrich-leads'); + const { db } = seeded; + await addLead(db, 'per_scott', 'scott.perry@northwind.io', 'northwind.io'); + await addLead(db, 'per_kim', 'kim.ode@globex.dev', 'globex.dev'); + + const { asked, searcher: fake } = searcher({ + 'site:linkedin.com/in "Scott Perry" northwind': [ + { + title: 'Scott Perry - VP Sales - Northwind | LinkedIn', + link: 'https://www.linkedin.com/in/scott-perry-9', + }, + ], + 'site:linkedin.com/company "globex.dev"': [ + { title: 'Globex | LinkedIn', link: 'https://www.linkedin.com/company/globex/' }, + ], + }); + + const result = await enrichLeads( + { db, searcher: fake, concurrency: 1 }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId }, + ); + + expect(result).toMatchObject({ names: 2, titles: 1, profiles: 1, companies: 1 }); + + const scott = await queryOne<{ + first_name: string; + last_name: string; + display_name: string; + current_title: string; + linkedin_looked_up_at: string | null; + }>( + db, + `SELECT first_name, last_name, display_name, current_title, linkedin_looked_up_at + FROM people WHERE id = 'per_scott'`, + ); + expect(scott).toMatchObject({ + first_name: 'Scott', + last_name: 'Perry', + display_name: 'Scott Perry', + current_title: 'VP Sales', + }); + expect(scott?.linkedin_looked_up_at).toBeTruthy(); + + const profile = await queryOne<{ profile_url: string; confidence: number }>( + db, + `SELECT profile_url, confidence FROM social_identities + WHERE person_id = 'per_scott' AND network = 'linkedin'`, + ); + expect(profile?.profile_url).toBe('https://www.linkedin.com/in/scott-perry-9'); + expect(profile!.confidence).toBeLessThan(0.85); + + const globex = await queryOne<{ linkedin_url: string }>( + db, + `SELECT linkedin_url FROM companies WHERE id = 'co_per_kim'`, + ); + expect(globex?.linkedin_url).toBe('https://www.linkedin.com/company/globex'); + + // Rerun after forgetting who was looked up: every answer comes from the cache. + await db.execute({ sql: 'UPDATE people SET linkedin_looked_up_at = NULL', args: [] }); + await db.execute({ sql: 'UPDATE companies SET linkedin_url = NULL', args: [] }); + const before = asked.length; + const again = await enrichLeads( + { db, searcher: fake, concurrency: 1 }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId }, + ); + expect(asked.length).toBe(before); + expect(again.searches).toBe(0); + expect(again.cached).toBeGreaterThan(0); + expect(await lookupsToday(db, SEED.workspaceId, 'valueserp')).toBe(before); + }, + T, + ); + + test( + 'stops at the cap and on a 402, leaving the unanswered for the next run', + async () => { + seeded = await seedDatabase('enrich-caps'); + const { db } = seeded; + for (const name of ['ann.lee', 'bo.chan', 'cy.diaz']) { + await addLead(db, `per_${name.replace('.', '_')}`, `${name}@initech.co`); + } + + const capped = await enrichLeads( + { db, searcher: searcher({}).searcher, concurrency: 1 }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId, maxSearches: 1 }, + ); + expect(capped.searches).toBe(1); + expect(capped.stopped).toContain('cap'); + // At most the one person whose question was answered is stamped; the + // rest wait for the next run rather than being marked as misses. + const stamped = await queryAll<{ id: string }>( + db, + 'SELECT id FROM people WHERE linkedin_looked_up_at IS NOT NULL', + ); + expect(stamped.length).toBeLessThanOrEqual(1); + + const broke: WebSearcher = { + async search() { + throw new SearchOutOfCredits('ValueSERP'); + }, + }; + const stopped = await enrichLeads( + { db, searcher: broke, concurrency: 1 }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId }, + ); + expect(stopped.stopped).toContain('402'); + // The names still came from the addresses for free. + const named = await queryOne<{ n: number }>( + db, + `SELECT COUNT(*) AS n FROM people + WHERE first_name IS NOT NULL AND id IN ('per_ann_lee', 'per_bo_chan', 'per_cy_diaz')`, + ); + expect(Number(named?.n)).toBe(3); + }, + T, + ); + + test( + 'People Data Labs first, within its daily cap, and the status says what is missing', + async () => { + seeded = await seedDatabase('enrich-pdl'); + const { db } = seeded; + await addLead(db, 'per_x', 'xq@hooli.xyz', 'hooli.xyz'); + + const asked: string[] = []; + const pdl: PersonEnricher = { + name: 'pdl', + async enrichByEmail(email) { + asked.push(email); + return { + firstName: 'Xavier', + lastName: 'Quinn', + title: 'Head of Data', + linkedinUrl: 'https://www.linkedin.com/in/xquinn', + }; + }, + }; + const { asked: searched, searcher: fake } = searcher({}); + + const status0 = await enrichmentStatus( + { db, pdl, searcher: fake }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId }, + ); + expect(status0.missing_title).toBe(1); + + const result = await enrichLeads( + { db, pdl, searcher: fake, pdlPerDay: 1, concurrency: 1 }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId }, + ); + expect(asked).toEqual(['xq@hooli.xyz']); + expect(result).toMatchObject({ pdlMatches: 1, titles: 1, profiles: 1 }); + // Title and profile were found, so no search was spent on Xavier (the + // seeded Jane still has no profile, and is searched for). + expect(searched.filter((q) => /Xavier|hooli/.test(q))).toEqual([]); + + const status = await enrichmentStatus( + { db, pdl, searcher: fake, pdlPerDay: 1 }, + { workspaceId: SEED.workspaceId, campaignId: SEED.campaignId }, + ); + expect(status.missing_title).toBe(0); + expect(status.today.pdl_lookups).toBe(1); + expect(status.providers).toEqual({ address: true, valueserp: true, pdl: true }); + }, + T, + ); +}); diff --git a/packages/pipeline/src/lead-enrichment.ts b/packages/pipeline/src/lead-enrichment.ts new file mode 100644 index 0000000..07014d4 --- /dev/null +++ b/packages/pipeline/src/lead-enrichment.ts @@ -0,0 +1,579 @@ +/** + * Filling in what a lead is missing: name, job title, LinkedIn. + * + * An imported row is often just an address. A message written to "there" + * with no idea what the person does reads as exactly what it is, so before a + * lead is written to, this fills the blanks from the cheapest source that can + * answer, in order, and only ever fills blanks: + * + * 1. **The address itself.** `scott.perry@acme.com` is Scott Perry. Free, + * and only the unambiguous shape (`nameFromAddress`). + * 2. **People Data Labs**, when a key is configured. Paid per match, capped + * per day, and licensed for storage (Apollo is not, and is never used). + * 3. **A Google search of LinkedIn** through ValueSERP: the profile, and + * the job title off the result's headline, taken only when the result is + * evidence (see `pickPerson`). The company's LinkedIn page is the + * fallback when no person profile is evidenced. + * + * Every search's raw results are cached in `serp_cache`, so a rerun costs + * nothing, and the searches are bounded three ways: per run, per workspace per + * day (counted in `enrichment_usage`, so a restart cannot reset it), and by + * HTTP 402, which stops the run cleanly. ValueSERP takes 10-45 s a query, so + * the work runs at a concurrency of about ten with a per-request timeout. + * + * This reads search results. It never fetches LinkedIn, never logs in, and the + * profiles it records are research at a confidence no machine acts on. + */ + +import { isConsumerMailDomain, nameFromAddress, newId } from '@outreachgraph/domain'; +import { + companyQuery, + linkedinPath, + personQuery, + pickCompany, + pickPerson, + SearchOutOfCredits, + titleFromResult, + type PersonEnricher, + type SerpResult, + type WebSearcher, +} from '@outreachgraph/providers'; +import { now, queryAll, queryOne, type Client } from '@outreachgraph/db'; + +/** A search result is research, below any outreach threshold (same as photos). */ +const SEARCH_IDENTITY_CONFIDENCE = 0.6; +/** A licensed provider matched the address itself: stronger than a name search. */ +const PDL_IDENTITY_CONFIDENCE = 0.8; + +export const DEFAULT_SEARCHES_PER_DAY = 300; +export const DEFAULT_PDL_PER_DAY = 3; +const DEFAULT_PEOPLE_PER_RUN = 60; +const DEFAULT_SEARCHES_PER_RUN = 100; +const DEFAULT_CONCURRENCY = 10; + +export interface LeadEnrichDeps { + readonly db: Client; + /** Google results (ValueSERP). Absent: names from addresses only. */ + readonly searcher?: WebSearcher | undefined; + /** People Data Labs. Absent: not used. */ + readonly pdl?: PersonEnricher | undefined; + readonly now?: Date; + readonly concurrency?: number; + readonly searchesPerDay?: number; + readonly pdlPerDay?: number; +} + +export interface LeadEnrichInput { + readonly workspaceId: string; + /** Only this campaign's leads. */ + readonly campaignId?: string | undefined; + /** People looked at in this run. */ + readonly limit?: number; + /** New (uncached) searches this run may spend. */ + readonly maxSearches?: number; +} + +export interface LeadEnrichResult { + readonly looked: number; + readonly names: number; + readonly titles: number; + readonly profiles: number; + readonly companies: number; + readonly pdlMatches: number; + readonly searches: number; + readonly cached: number; + /** Why the run stopped spending early, when it did. */ + readonly stopped?: string; +} + +interface Subject { + readonly person_id: string; + readonly display_name: string; + readonly first_name: string | null; + readonly last_name: string | null; + readonly current_title: string | null; + readonly company_id: string | null; + readonly company_domain: string | null; + readonly company_linkedin: string | null; + readonly email: string | null; + readonly linkedin: string | null; +} + +/** Workspace-wide: paused after a 402 until the next UTC day. */ +const pausedUntil = new Map(); + +/** Forgets every 402 pause. For tests, and for an operator who just topped up. */ +export function resetEnrichmentPauses(): void { + pausedUntil.clear(); +} + +export function enrichmentPausedUntil(provider: 'valueserp' | 'pdl'): string | undefined { + const until = pausedUntil.get(provider); + return until && until > new Date().toISOString() ? until : undefined; +} + +function nextUtcDay(at: Date): string { + const next = new Date(at); + next.setUTCHours(24, 0, 0, 0); + return next.toISOString(); +} + +/** Lookups spent today by this workspace with this provider. */ +export async function lookupsToday( + db: Client, + workspaceId: string, + provider: string, + at: Date = new Date(), +): Promise { + const row = await queryOne<{ lookups: number }>( + db, + 'SELECT lookups FROM enrichment_usage WHERE workspace_id = ? AND day = ? AND provider = ?', + [workspaceId, at.toISOString().slice(0, 10), provider], + ); + return Number(row?.lookups ?? 0); +} + +async function countLookup(db: Client, workspaceId: string, provider: string, at: Date) { + await db.execute({ + sql: `INSERT INTO enrichment_usage (workspace_id, day, provider, lookups) VALUES (?, ?, ?, 1) + ON CONFLICT (workspace_id, day, provider) DO UPDATE + SET lookups = enrichment_usage.lookups + 1`, + args: [workspaceId, at.toISOString().slice(0, 10), provider], + }); +} + +/** The people a run would look at: in a live campaign, missing something, never tried. */ +async function subjects(db: Client, input: LeadEnrichInput, limit: number): Promise { + return queryAll( + db, + `SELECT DISTINCT p.id AS person_id, p.display_name, p.first_name, p.last_name, + p.current_title, co.id AS company_id, co.domain AS company_domain, + co.linkedin_url AS company_linkedin, + (SELECT pe.address FROM person_emails pe + WHERE pe.person_id = p.id AND pe.workspace_id = cp.workspace_id + ORDER BY pe.created_at LIMIT 1) AS email, + (SELECT si.profile_url FROM social_identities si + WHERE si.person_id = p.id AND si.network = 'linkedin' LIMIT 1) AS linkedin + FROM campaign_people cp + JOIN campaigns c ON c.id = cp.campaign_id + JOIN people p ON p.id = cp.person_id + LEFT JOIN companies co ON co.id = p.current_company_id + WHERE cp.workspace_id = ? ${input.campaignId ? 'AND cp.campaign_id = ?' : ''} + AND c.status != 'archived' + AND p.status = 'active' AND p.kind = 'person' + AND p.linkedin_looked_up_at IS NULL + AND (p.current_title IS NULL OR p.first_name IS NULL OR p.last_name IS NULL + OR NOT EXISTS (SELECT 1 FROM social_identities si + WHERE si.person_id = p.id AND si.network = 'linkedin')) + LIMIT ?`, + input.campaignId ? [input.workspaceId, input.campaignId, limit] : [input.workspaceId, limit], + ); +} + +/** Workspaces with live-campaign leads still missing a name, title or profile. */ +export async function workspacesAwaitingLeadEnrichment(db: Client): Promise { + const rows = await queryAll<{ workspace_id: string }>( + db, + `SELECT DISTINCT cp.workspace_id + FROM campaign_people cp + JOIN campaigns c ON c.id = cp.campaign_id + JOIN people p ON p.id = cp.person_id + WHERE c.status IN ('active', 'running') AND p.status = 'active' AND p.kind = 'person' + AND p.linkedin_looked_up_at IS NULL + AND (p.current_title IS NULL OR p.first_name IS NULL + OR NOT EXISTS (SELECT 1 FROM social_identities si + WHERE si.person_id = p.id AND si.network = 'linkedin'))`, + ); + return rows.map((row) => row.workspace_id); +} + +/** + * One run over the next batch of leads. + * + * A person is stamped as looked up only when every question asked about them + * was answered (from the cache or a search). One skipped because the run hit + * its cap, ran out of credits or timed out is left for the next run. + */ +export async function enrichLeads( + deps: LeadEnrichDeps, + input: LeadEnrichInput, +): Promise { + const { db } = deps; + const at = deps.now ?? new Date(); + const result = { + looked: 0, + names: 0, + titles: 0, + profiles: 0, + companies: 0, + pdlMatches: 0, + searches: 0, + cached: 0, + stopped: undefined as string | undefined, + }; + + const searchRoom = deps.searcher + ? Math.max( + 0, + Math.min( + input.maxSearches ?? DEFAULT_SEARCHES_PER_RUN, + (deps.searchesPerDay ?? DEFAULT_SEARCHES_PER_DAY) - + (await lookupsToday(db, input.workspaceId, 'valueserp', at)), + ), + ) + : 0; + let pdlRoom = deps.pdl + ? Math.max( + 0, + (deps.pdlPerDay ?? DEFAULT_PDL_PER_DAY) - + (await lookupsToday(db, input.workspaceId, 'pdl', at)), + ) + : 0; + if (enrichmentPausedUntil('pdl')) pdlRoom = 0; + const searchPaused = enrichmentPausedUntil('valueserp'); + if (searchPaused) result.stopped = `ValueSERP is out of credits until ${searchPaused}`; + + const people = await subjects(db, input, input.limit ?? DEFAULT_PEOPLE_PER_RUN); + const inflight = new Map>(); + + /** The query's results, from the cache or one search; undefined when it could not ask. */ + const ask = async (query: string): Promise => { + const cached = await queryOne<{ results: string }>( + db, + 'SELECT results FROM serp_cache WHERE query = ?', + [query], + ); + if (cached) { + result.cached += 1; + try { + return JSON.parse(cached.results) as SerpResult[]; + } catch { + return []; + } + } + const existing = inflight.get(query); + if (existing) return existing; + if (!deps.searcher || searchPaused || result.stopped || result.searches >= searchRoom) { + if (!result.stopped && deps.searcher && result.searches >= searchRoom) { + result.stopped = `reached the search cap (${searchRoom} this run)`; + } + return undefined; + } + result.searches += 1; + const searcher = deps.searcher; + const pending = (async () => { + try { + const results = await searcher.search(query, { num: 10 }); + const kept = results.map(({ title, link, snippet }) => ({ title, link, snippet })); + await db.execute({ + sql: `INSERT INTO serp_cache (query, results, fetched_at) VALUES (?, ?, ?) + ON CONFLICT (query) DO UPDATE SET results = excluded.results, + fetched_at = excluded.fetched_at`, + args: [query, JSON.stringify(kept), now()], + }); + await countLookup(db, input.workspaceId, 'valueserp', at); + return kept; + } catch (error) { + if (error instanceof SearchOutOfCredits) { + pausedUntil.set('valueserp', nextUtcDay(at)); + result.stopped = error.message; + } + // A timeout or a refusal is not an answer: the person is retried later. + return undefined; + } + })(); + inflight.set(query, pending); + return pending; + }; + + const enrichOne = async (subject: Subject): Promise => { + const stamp = now(); + const statements: { sql: string; args: (string | number | null)[] }[] = []; + let first = subject.first_name; + let last = subject.last_name; + let title = subject.current_title; + let linkedin = subject.linkedin; + let complete = true; + + const emailDomain = subject.email?.split('@')[1]?.toLowerCase(); + const companyDomain = + subject.company_domain ?? + (emailDomain && !isConsumerMailDomain(emailDomain) ? emailDomain : undefined); + + // 1. The address itself. + if ((!first || !last) && subject.email) { + const named = nameFromAddress(subject.email); + if (named) { + first = first ?? named.firstName; + last = last ?? named.lastName; + const display = /\s/.test(subject.display_name.trim()) + ? subject.display_name + : `${named.firstName} ${named.lastName}`; + statements.push({ + sql: `UPDATE people SET first_name = COALESCE(first_name, ?), + last_name = COALESCE(last_name, ?), display_name = ?, updated_at = ? + WHERE id = ?`, + args: [named.firstName, named.lastName, display, stamp, subject.person_id], + }); + statements.push(provenance(subject.person_id, 'first_name', named.firstName, 'address')); + result.names += 1; + } + } + + // 2. People Data Labs, by address, for whoever still lacks a title or profile. + if ((!title || !linkedin) && subject.email && pdlRoom > 0 && deps.pdl) { + pdlRoom -= 1; + try { + await countLookup(db, input.workspaceId, 'pdl', at); + const found = await deps.pdl.enrichByEmail(subject.email); + if (found) { + result.pdlMatches += 1; + if ((!first || !last) && found.firstName && found.lastName) { + first = found.firstName; + last = found.lastName; + statements.push({ + sql: `UPDATE people SET first_name = COALESCE(first_name, ?), + last_name = COALESCE(last_name, ?), updated_at = ? WHERE id = ?`, + args: [found.firstName, found.lastName, stamp, subject.person_id], + }); + } + if (!title && found.title) { + title = found.title; + statements.push(...setTitle(subject.person_id, found.title, 'pdl', found.linkedinUrl)); + result.titles += 1; + } + const profile = linkedinPath(found.linkedinUrl, 'in'); + if (!linkedin && profile) { + linkedin = profile; + statements.push(addProfile(subject.person_id, profile, PDL_IDENTITY_CONFIDENCE, 'pdl')); + result.profiles += 1; + } + } + } catch (error) { + if (error instanceof SearchOutOfCredits) { + pausedUntil.set('pdl', nextUtcDay(at)); + pdlRoom = 0; + } + } + } + + // 3. A Google search of LinkedIn: the profile and its headline. + if ((!title || !linkedin) && first && last) { + const subjectForSearch = { firstName: first, lastName: last, companyDomain }; + const query = personQuery(subjectForSearch); + const results = await ask(query); + if (!results) { + complete = false; + } else { + const url = pickPerson(results, subjectForSearch); + if (url) { + if (!linkedin) { + linkedin = url; + statements.push( + addProfile(subject.person_id, url, SEARCH_IDENTITY_CONFIDENCE, 'search'), + ); + result.profiles += 1; + } + const hit = results.find((r) => linkedinPath(r.link, 'in') === url); + const headline = hit ? titleFromResult(hit.title ?? '', companyDomain ?? '') : ''; + if (!title && headline) { + title = headline; + statements.push(...setTitle(subject.person_id, headline, 'valueserp', url)); + result.titles += 1; + } + } + } + } + + // 4. The company's page, when no person profile is evidenced. + if (!linkedin && companyDomain && subject.company_id && !subject.company_linkedin) { + const results = await ask(companyQuery(companyDomain)); + if (!results) { + complete = false; + } else { + const url = pickCompany(results, companyDomain); + if (url) { + statements.push({ + sql: `UPDATE companies SET linkedin_url = ?, updated_at = ? + WHERE id = ? AND linkedin_url IS NULL`, + args: [url, stamp, subject.company_id], + }); + result.companies += 1; + } + } + } + + if (complete) { + statements.push({ + sql: 'UPDATE people SET linkedin_looked_up_at = ? WHERE id = ?', + args: [stamp, subject.person_id], + }); + } + if (statements.length > 0) await db.batch(statements); + }; + + let next = 0; + const worker = async () => { + while (next < people.length) { + const subject = people[next++]!; + result.looked += 1; + try { + await enrichOne(subject); + } catch (error) { + console.error(`lead enrichment failed for ${subject.person_id}`, error); + } + } + }; + await Promise.all( + Array.from( + { length: Math.min(deps.concurrency ?? DEFAULT_CONCURRENCY, people.length) }, + worker, + ), + ); + + return { + looked: result.looked, + names: result.names, + titles: result.titles, + profiles: result.profiles, + companies: result.companies, + pdlMatches: result.pdlMatches, + searches: result.searches, + cached: result.cached, + ...(result.stopped ? { stopped: result.stopped } : {}), + }; +} + +function provenance( + personId: string, + field: string, + value: string, + via: 'address' | 'pdl' | 'valueserp', + url?: string, +): { sql: string; args: (string | number | null)[] } { + const stamp = now(); + const [sourceType, license, confidence] = + via === 'address' + ? ['derived', 'derived_inference', 0.7] + : via === 'pdl' + ? ['provider', 'licensed_enrichment', PDL_IDENTITY_CONFIDENCE] + : ['public_web', 'public_web', SEARCH_IDENTITY_CONFIDENCE]; + return { + sql: `INSERT INTO field_provenance (id, entity_kind, entity_id, field, value, source_type, + provider, source_url, license_class, confidence, observed_at, created_at) + VALUES (?, 'person', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + args: [ + newId('fieldProvenance'), + personId, + field, + value, + sourceType, + via, + url ?? null, + license, + confidence, + stamp, + stamp, + ], + }; +} + +function setTitle( + personId: string, + title: string, + via: 'pdl' | 'valueserp', + url?: string, +): { sql: string; args: (string | number | null)[] }[] { + return [ + { + sql: `UPDATE people SET current_title = ?, updated_at = ? + WHERE id = ? AND current_title IS NULL`, + args: [title, now(), personId], + }, + provenance(personId, 'current_title', title, via, url), + ]; +} + +function addProfile( + personId: string, + url: string, + confidence: number, + via: 'pdl' | 'search', +): { sql: string; args: (string | number | null)[] } { + const stamp = now(); + return { + // Guarded in SQL as well: two runs racing on one person record one profile. + sql: `INSERT INTO social_identities (id, person_id, network, handle, profile_url, confidence, + source_type, verified_by, first_seen_at, last_verified_at) + SELECT ?, ?, 'linkedin', ?, ?, ?, ?, ?, ?, ? + WHERE NOT EXISTS (SELECT 1 FROM social_identities + WHERE person_id = ? AND network = 'linkedin')`, + args: [ + newId('socialIdentity'), + personId, + decodeURIComponent(url.split('/in/')[1] ?? '') || null, + url, + confidence, + via === 'pdl' ? 'provider' : 'public_web', + JSON.stringify([via]), + stamp, + stamp, + personId, + ], + }; +} + +/** What a campaign's leads are missing, and what enrichment can still spend today. */ +export async function enrichmentStatus( + deps: LeadEnrichDeps, + input: { readonly workspaceId: string; readonly campaignId: string }, +) { + const row = await queryOne<{ + leads: number; + no_name: number; + no_title: number; + no_linkedin: number; + looked_up: number; + }>( + deps.db, + `SELECT COUNT(*) AS leads, + SUM(CASE WHEN p.first_name IS NULL OR p.last_name IS NULL THEN 1 ELSE 0 END) AS no_name, + SUM(CASE WHEN p.current_title IS NULL THEN 1 ELSE 0 END) AS no_title, + SUM(CASE WHEN NOT EXISTS (SELECT 1 FROM social_identities si + WHERE si.person_id = p.id AND si.network = 'linkedin') + THEN 1 ELSE 0 END) AS no_linkedin, + SUM(CASE WHEN p.linkedin_looked_up_at IS NOT NULL THEN 1 ELSE 0 END) AS looked_up + FROM campaign_people cp JOIN people p ON p.id = cp.person_id + WHERE cp.campaign_id = ? AND cp.workspace_id = ? AND p.status = 'active'`, + [input.campaignId, input.workspaceId], + ); + const at = deps.now ?? new Date(); + const [searches, pdl] = await Promise.all([ + lookupsToday(deps.db, input.workspaceId, 'valueserp', at), + lookupsToday(deps.db, input.workspaceId, 'pdl', at), + ]); + return { + leads: Number(row?.leads ?? 0), + missing_name: Number(row?.no_name ?? 0), + missing_title: Number(row?.no_title ?? 0), + missing_linkedin: Number(row?.no_linkedin ?? 0), + looked_up: Number(row?.looked_up ?? 0), + providers: { + address: true, + valueserp: Boolean(deps.searcher), + pdl: Boolean(deps.pdl), + }, + today: { + searches, + searches_cap: deps.searcher ? (deps.searchesPerDay ?? DEFAULT_SEARCHES_PER_DAY) : 0, + pdl_lookups: pdl, + pdl_cap: deps.pdl ? (deps.pdlPerDay ?? DEFAULT_PDL_PER_DAY) : 0, + }, + paused: { + ...(enrichmentPausedUntil('valueserp') + ? { valueserp: enrichmentPausedUntil('valueserp') } + : {}), + ...(enrichmentPausedUntil('pdl') ? { pdl: enrichmentPausedUntil('pdl') } : {}), + }, + }; +} diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index 6cfe5d4..9a082dc 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -33,6 +33,7 @@ export { deriveEvidence, type EvidenceContext } from './evidence'; export { ValueSerpClient, + SearchOutOfCredits, carriesName, corroborate, isLinkedInProfile, @@ -43,6 +44,24 @@ export { type WebResult, type WebSearcher, } from './valueserp'; +export { + companyQuery, + companyToken, + linkedinPath, + personQuery, + pickCompany, + pickPerson, + registrable, + titleFromResult, + type LinkedinSubject, + type SerpResult, +} from './valueserp/linkedin'; +export { + PeopleDataLabsClient, + type PersonEnricher, + type PersonEnrichment, + type PeopleDataLabsOptions, +} from './pdl'; export { readJobPosting, diff --git a/packages/providers/src/pdl/index.ts b/packages/providers/src/pdl/index.ts new file mode 100644 index 0000000..c74ec9d --- /dev/null +++ b/packages/providers/src/pdl/index.ts @@ -0,0 +1,126 @@ +/** + * People Data Labs: a person's name, title, LinkedIn and employer from their + * email address. + * + * Optional, behind `PDL_API_KEY`, and the only paid person source here on + * purpose: PDL's terms allow storing what it returns inside a product, which + * Apollo's do not (Apollo data must never be written to this database). A + * free key carries 100 matches a month, and a 404 (no match) is not charged, + * so the caller caps lookups per day and asks only for leads still missing a + * title or a profile after the free sources. + * + * Measured on our own list (2026-10-06): PDL's titles are Apollo's titles + * verbatim (a shared upstream), work addresses match about half the time, + * personal addresses rarely. + */ + +import { SearchOutOfCredits } from '../valueserp'; + +export interface PersonEnrichment { + readonly firstName?: string; + readonly lastName?: string; + readonly title?: string; + /** `https://www.linkedin.com/in/` */ + readonly linkedinUrl?: string; + readonly companyName?: string; + /** The employer's domain, e.g. `acme.com`. */ + readonly companyDomain?: string; + /** PDL's own 1-10 confidence that the match is this person. */ + readonly likelihood?: number; +} + +export interface PersonEnricher { + readonly name: string; + /** One person by address, or undefined for no match. Throws SearchOutOfCredits on 402. */ + enrichByEmail(email: string): Promise; +} + +export interface PeopleDataLabsOptions { + readonly apiKey: string; + readonly baseUrl?: string; + readonly fetchImpl?: typeof fetch; + readonly timeoutMs?: number; + /** Matches below this likelihood are not taken. PDL's scale is 1-10. */ + readonly minLikelihood?: number; +} + +interface PdlPerson { + readonly first_name?: unknown; + readonly last_name?: unknown; + readonly job_title?: unknown; + readonly linkedin_url?: unknown; + readonly job_company_name?: unknown; + readonly job_company_website?: unknown; +} + +const text = (value: unknown): string | undefined => + typeof value === 'string' && value.trim() ? value.trim() : undefined; + +const titleCase = (value: string | undefined): string | undefined => + value?.replace(/\b\p{L}/gu, (letter) => letter.toUpperCase()); + +export class PeopleDataLabsClient implements PersonEnricher { + readonly name = 'pdl'; + private readonly apiKey: string; + private readonly baseUrl: string; + private readonly fetchImpl: typeof fetch; + private readonly timeoutMs: number; + private readonly minLikelihood: number; + + constructor(options: PeopleDataLabsOptions) { + this.apiKey = options.apiKey; + this.baseUrl = (options.baseUrl ?? 'https://api.peopledatalabs.com').replace(/\/$/, ''); + this.fetchImpl = options.fetchImpl ?? fetch; + this.timeoutMs = options.timeoutMs ?? 30_000; + this.minLikelihood = options.minLikelihood ?? 6; + } + + async enrichByEmail(email: string): Promise { + const url = new URL('/v5/person/enrich', this.baseUrl); + url.searchParams.set('email', email); + url.searchParams.set('min_likelihood', String(this.minLikelihood)); + + const response = await this.fetchImpl(url, { + headers: { 'X-Api-Key': this.apiKey, accept: 'application/json' }, + signal: AbortSignal.timeout(this.timeoutMs), + }); + if (response.status === 404) return undefined; + if (response.status === 402) throw new SearchOutOfCredits('People Data Labs'); + if (!response.ok) throw new Error(`People Data Labs answered ${response.status}`); + + const body = (await response.json().catch(() => ({}))) as { + likelihood?: unknown; + data?: PdlPerson; + }; + const person = body.data; + if (!person) return undefined; + + const linkedin = text(person.linkedin_url); + const website = text(person.job_company_website) + ?.toLowerCase() + .replace(/^https?:\/\//, '') + .replace(/^www\./, '') + .split(/[/?#]/)[0]; + + const out: PersonEnrichment = { + ...(titleCase(text(person.first_name)) + ? { firstName: titleCase(text(person.first_name))! } + : {}), + ...(titleCase(text(person.last_name)) + ? { lastName: titleCase(text(person.last_name))! } + : {}), + ...(titleCase(text(person.job_title)) ? { title: titleCase(text(person.job_title))! } : {}), + ...(linkedin + ? { + linkedinUrl: `https://www.${linkedin.replace(/^https?:\/\//, '').replace(/^www\./, '')}`, + } + : {}), + ...(titleCase(text(person.job_company_name)) + ? { companyName: titleCase(text(person.job_company_name))! } + : {}), + ...(website && website.includes('.') ? { companyDomain: website } : {}), + ...(typeof body.likelihood === 'number' ? { likelihood: body.likelihood } : {}), + }; + return out; + } +} diff --git a/packages/providers/src/valueserp/index.ts b/packages/providers/src/valueserp/index.ts index 1fd461b..845f84a 100644 --- a/packages/providers/src/valueserp/index.ts +++ b/packages/providers/src/valueserp/index.ts @@ -67,6 +67,16 @@ export interface ValueSerpOptions { const DEFAULT_BASE = 'https://api.valueserp.com'; +/** A paid provider answered HTTP 402: no credits left, so stop asking. */ +export class SearchOutOfCredits extends Error { + readonly provider: string; + constructor(provider: string) { + super(`${provider} is out of credits (HTTP 402)`); + this.name = 'SearchOutOfCredits'; + this.provider = provider; + } +} + /** One image result, as much of it as this adapter reads. */ interface ImageResult { readonly title?: string; @@ -113,6 +123,9 @@ export class ValueSerpClient implements ProfilePhotoFinder, WebSearcher { organic_results?: { title?: unknown; link?: unknown; snippet?: unknown }[]; request_info?: { success?: boolean; message?: string }; }; + // Out of credits is not a miss: every search after it would fail too, so + // a sweep stops on it rather than stamping people as looked up. + if (response.status === 402) throw new SearchOutOfCredits('ValueSERP'); if (!response.ok || body.request_info?.success === false) { throw new Error( `ValueSERP refused the search (${response.status}): ${body.request_info?.message ?? 'no reason given'}`, diff --git a/packages/providers/src/valueserp/linkedin.test.ts b/packages/providers/src/valueserp/linkedin.test.ts new file mode 100644 index 0000000..db8c042 --- /dev/null +++ b/packages/providers/src/valueserp/linkedin.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, test } from 'bun:test'; +import { + companyToken, + personQuery, + pickCompany, + pickPerson, + titleFromResult, + type LinkedinSubject, + type SerpResult, +} from './linkedin'; +import { SearchOutOfCredits, ValueSerpClient } from './index'; +import { PeopleDataLabsClient } from '../pdl'; + +const ada: LinkedinSubject = { firstName: 'Ada', lastName: 'Lovelace', companyDomain: 'acme.com' }; + +describe('LinkedIn matching', () => { + test('names the company by its first label', () => { + expect(companyToken('acme.com')).toBe('acme'); + expect(companyToken('shop.acme.co.uk')).toBe('acme'); + expect(companyToken('ai.nookplot.com')).toBe('nookplot'); + }); + + test('searches profiles by quoted name and company', () => { + expect(personQuery(ada)).toBe('site:linkedin.com/in "Ada Lovelace" acme'); + expect(personQuery({ firstName: 'Bob', lastName: 'Ng' })).toBe('site:linkedin.com/in "Bob Ng"'); + }); + + test('takes a profile that carries the name and the company', () => { + const results: SerpResult[] = [ + { + title: 'Ada Lovelace - Engineer - Initech | LinkedIn', + link: 'https://www.linkedin.com/in/ada-other', + snippet: 'Initech', + }, + { + title: 'Adá LOVELACE - CTO - Acme | LinkedIn', + link: 'https://uk.linkedin.com/in/ada-l?trk=x', + snippet: '', + }, + ]; + expect(pickPerson(results, ada)).toBe('https://www.linkedin.com/in/ada-l'); + }); + + test('without a company, takes only a single matching profile whose slug is the name', () => { + const bob = { firstName: 'Bob', lastName: 'Ng' }; + const one = [ + { title: 'Bob Ng | LinkedIn', link: 'https://www.linkedin.com/in/bob-ng' }, + { title: 'Posts', link: 'https://www.linkedin.com/posts/x' }, + ]; + expect(pickPerson(one, bob)).toBe('https://www.linkedin.com/in/bob-ng'); + expect( + pickPerson( + [...one, { title: 'Bob Ng - Chef', link: 'https://www.linkedin.com/in/bob-ng-2' }], + bob, + ), + ).toBe(''); + expect( + pickPerson([{ title: 'Bob Ng | LinkedIn', link: 'https://www.linkedin.com/in/b8812' }], bob), + ).toBe(''); + }); + + test('rejects other people and non-profile pages', () => { + expect( + pickPerson([{ title: 'Ada Byron - Acme', link: 'https://www.linkedin.com/in/ab' }], ada), + ).toBe(''); + expect( + pickPerson( + [{ title: 'Ada Lovelace - Acme', link: 'https://www.linkedin.com/company/acme' }], + ada, + ), + ).toBe(''); + }); + + test('takes a company page that names the domain or the company, not a passing mention', () => { + expect( + pickCompany( + [{ title: 'Acme Corp | LinkedIn', link: 'https://www.linkedin.com/company/acme-corp/' }], + 'acme.com', + ), + ).toBe('https://www.linkedin.com/company/acme-corp'); + expect( + pickCompany( + [{ title: 'Other | LinkedIn', link: 'https://www.linkedin.com/company/other' }], + 'acme.com', + ), + ).toBe(''); + expect( + pickCompany( + [ + { + title: 'National City Adult School', + link: 'https://www.linkedin.com/company/ncas', + snippet: 'part of Sweetwater schools', + }, + ], + 'sweetwaterschools.net', + ), + ).toBe(''); + expect( + pickCompany( + [ + { + title: 'Adult School', + link: 'https://www.linkedin.com/company/ncas', + snippet: 'sweetwaterschools.net', + }, + ], + 'sweetwaterschools.net', + ), + ).toBe('https://www.linkedin.com/company/ncas'); + }); + + test('reads the headline out of a profile title', () => { + expect(titleFromResult('Ada Lovelace - CTO - Acme | LinkedIn', 'acme.com')).toBe('CTO'); + expect(titleFromResult('Ada Lovelace – Head of Growth | LinkedIn', 'acme.com')).toBe( + 'Head of Growth', + ); + expect(titleFromResult('Ada Lovelace - Acme | LinkedIn', 'acme.com')).toBe(''); + expect(titleFromResult('Ada Lovelace | LinkedIn')).toBe(''); + }); +}); + +describe('out of credits', () => { + const answer = (status: number, body: unknown = {}) => + (async () => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + })) as unknown as typeof fetch; + + test('ValueSERP turns a 402 into SearchOutOfCredits', async () => { + const client = new ValueSerpClient({ apiKey: 'k', fetchImpl: answer(402) }); + await expect(client.search('q')).rejects.toBeInstanceOf(SearchOutOfCredits); + }); + + test('People Data Labs: a match, no match, and a 402', async () => { + const hit = new PeopleDataLabsClient({ + apiKey: 'k', + fetchImpl: answer(200, { + likelihood: 8, + data: { + first_name: 'ada', + last_name: 'lovelace', + job_title: 'chief technology officer', + linkedin_url: 'linkedin.com/in/ada-l', + job_company_name: 'acme', + job_company_website: 'acme.com', + }, + }), + }); + expect(await hit.enrichByEmail('ada@acme.com')).toEqual({ + firstName: 'Ada', + lastName: 'Lovelace', + title: 'Chief Technology Officer', + linkedinUrl: 'https://www.linkedin.com/in/ada-l', + companyName: 'Acme', + companyDomain: 'acme.com', + likelihood: 8, + }); + + const miss = new PeopleDataLabsClient({ apiKey: 'k', fetchImpl: answer(404) }); + expect(await miss.enrichByEmail('nobody@acme.com')).toBeUndefined(); + + const broke = new PeopleDataLabsClient({ apiKey: 'k', fetchImpl: answer(402) }); + await expect(broke.enrichByEmail('ada@acme.com')).rejects.toBeInstanceOf(SearchOutOfCredits); + }); +}); diff --git a/packages/providers/src/valueserp/linkedin.ts b/packages/providers/src/valueserp/linkedin.ts new file mode 100644 index 0000000..25c3dce --- /dev/null +++ b/packages/providers/src/valueserp/linkedin.ts @@ -0,0 +1,150 @@ +/** + * Finding a lead's LinkedIn profile and job title through Google results, + * because LinkedIn's own search needs a login and this product never logs in + * there (see CLAUDE.md: LinkedIn acts only through the member's session). + * + * Ported from cli-tools `linkedin-lookup`, where it ran over ~2,000 queries + * of a real user list. A result is taken only when it is evidence, by the + * same standard the photo lookup uses (`carriesName`): + * + * - the link is a `linkedin.com/in/` profile and its title carries + * every part of the name; + * - with a company, the company's name (the domain's first label) is in the + * title or the snippet too; + * - with no company, exactly one profile in the results carries the name, + * and its slug is the name: "John Smith" alone is everybody. + * + * The title is read off the result title, which Google shows as + * `Name - Title - Company | LinkedIn`. A lead still without a profile but + * with a company domain gets the company's `linkedin.com/company/` + * page instead, taken only when the result names the domain or the company. + */ + +import { carriesName } from './index'; + +export interface SerpResult { + readonly title?: string | undefined; + readonly link?: string | undefined; + readonly snippet?: string | undefined; +} + +export interface LinkedinSubject { + readonly firstName: string; + readonly lastName: string; + /** The employer's own domain, when known. Never a webmail host. */ + readonly companyDomain?: string | undefined; +} + +function fold(value: string): string { + return value.normalize('NFD').replace(/[̀-ͯ]/g, '').toLowerCase(); +} + +/** `shop.acme.co.uk` -> `acme.co.uk`: the domain a company registered. */ +export function registrable(domain: string): string { + const labels = domain + .toLowerCase() + .replace(/^www\./, '') + .split('.'); + const secondLast = labels[labels.length - 2] ?? ''; + const last = labels[labels.length - 1] ?? ''; + const keep = + labels.length >= 3 && /^(co|com|org|net|ac|gov|edu)$/.test(secondLast) && last.length === 2 + ? 3 + : 2; + return labels.slice(-keep).join('.'); +} + +/** `shop.acme.co.uk` -> `acme`: the label a company is usually called by. */ +export function companyToken(domain: string): string { + return registrable(domain).split('.')[0] ?? ''; +} + +/** The profile or company slug as words: `/in/ada-lovelace-42` -> `ada lovelace 42`. */ +function slugWords(url: string): string { + return (url.split('/').pop() ?? '').replace(/[-_.]+/g, ' '); +} + +/** A LinkedIn profile (`in`) or company page URL, canonical, or '' for anything else. */ +export function linkedinPath(link: string | undefined, kind: 'in' | 'company'): string { + if (!link) return ''; + try { + const url = new URL(link); + const host = url.hostname.toLowerCase(); + if (host !== 'linkedin.com' && !host.endsWith('.linkedin.com')) return ''; + const match = (kind === 'in' ? /^\/in\/([^/]+)/ : /^\/company\/([^/]+)/).exec(url.pathname); + if (!match?.[1]) return ''; + return `https://www.linkedin.com/${kind}/${decodeURIComponent(match[1])}`; + } catch { + return ''; + } +} + +export function personQuery(subject: LinkedinSubject): string { + const name = `${subject.firstName} ${subject.lastName}`.trim(); + const company = subject.companyDomain ? companyToken(subject.companyDomain) : ''; + return `site:linkedin.com/in "${name}"${company ? ` ${company}` : ''}`; +} + +/** The one profile the results evidence for this person, or ''. */ +export function pickPerson(results: readonly SerpResult[], subject: LinkedinSubject): string { + const name = `${subject.firstName} ${subject.lastName}`; + const company = subject.companyDomain ? fold(companyToken(subject.companyDomain)) : ''; + const profiles = new Set(); + for (const result of results) { + const url = linkedinPath(result.link, 'in'); + if (!url || !carriesName(result.title ?? '', name)) continue; + if (company) { + if (company.length >= 3 && fold(`${result.title} ${result.snippet}`).includes(company)) { + return url; + } + } else if (carriesName(slugWords(url), name)) { + profiles.add(url); + } + } + return profiles.size === 1 ? ([...profiles][0] ?? '') : ''; +} + +/** + * The headline from a profile result's title. With two parts the second is + * as often the company as the title, so it only counts when it is not the + * company's name. + */ +export function titleFromResult(title: string, companyDomain = ''): string { + const parts = title + .replace(/\s*[|·]\s*LinkedIn.*$/i, '') + .split(/\s+[-–—]\s+/) + .map((part) => part.trim()); + const candidate = parts.length >= 2 ? (parts[1] ?? '') : ''; + if (candidate.length < 2 || candidate.length > 100 || candidate.endsWith('...')) return ''; + const company = companyDomain ? fold(companyToken(companyDomain)) : ''; + if (parts.length === 2 && company && fold(candidate).replace(/\s+/g, '').includes(company)) { + return ''; + } + return candidate; +} + +export function companyQuery(domain: string): string { + return `site:linkedin.com/company "${registrable(domain)}"`; +} + +/** + * The company page whose title or slug names the company, or whose text + * carries the domain itself. A snippet that merely mentions the name is not + * enough: a school district's name turns up on every one of its schools' pages. + */ +export function pickCompany(results: readonly SerpResult[], domain: string): string { + const site = registrable(domain); + const token = fold(companyToken(domain)); + for (const result of results) { + const url = linkedinPath(result.link, 'company'); + if (!url) continue; + const named = fold(`${result.title} ${slugWords(url)}`).replace(/\s+/g, ''); + if ( + fold(`${result.title} ${result.snippet}`).includes(site) || + (token.length >= 3 && named.includes(token)) + ) { + return url; + } + } + return ''; +}