diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 5a15866..23f34ca 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -38,12 +38,15 @@ import { } from './auth'; import { evaluatePolicy, isExecutable, POLICY_VERSION } from '@outreachgraph/policy'; import { draftForRecommendation, type TextModel } from '@outreachgraph/ai'; -import { runPipeline } from '@outreachgraph/pipeline'; -import { GitHubProvider } from '@outreachgraph/providers'; +import { batchStatus, enqueue, runPipeline } from '@outreachgraph/pipeline'; +import { GitHubProvider, normaliseUrl } from '@outreachgraph/providers'; import { ConsoleMailer, verificationEmail, type Mailer } from '@outreachgraph/email'; import { ApiError, canApprove, type AppEnv, type RequestActor } from './context'; import * as repo from './repository'; +/** One paste, one reviewable unit of work. */ +const MAX_BULK_URLS = 100; + export interface AppOptions { readonly db: Client; /** @@ -393,6 +396,135 @@ export function createApp(options: AppOptions): Hono { }); }); + // ------------------------------------------------------------ by url + /** + * Adds prospects from company URLs (PRD §8, URL-first intake). + * + * Accepts one URL or up to a hundred, and answers immediately with a batch + * id. Nothing is fetched inside the request: a single URL fans out to a + * crawl, an extraction, an identity resolution per person named and a draft + * per recommendation, and doing that inline for a hundred URLs is not a slow + * request but a request that never returns. + * + * A URL already queued is reported as a duplicate rather than rejected. The + * same address appearing twice in one paste is a normal thing for a human to + * do and must not fail the other ninety-nine. + */ + api.post('/prospects/by-url', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + + if (!canApprove(actor)) throw ApiError.forbidden('adding prospects'); + + const raw = safeJson(await c.req.raw.text()); + const submitted = Array.isArray(raw.urls) ? raw.urls : [raw.url]; + + const cleaned: string[] = []; + const rejected: { url: string; reason: string }[] = []; + const seen = new Set(); + + for (const entry of submitted) { + if (typeof entry !== 'string' || !entry.trim()) continue; + + let parsed: URL; + try { + parsed = new URL(normaliseUrl(entry)); + } catch { + rejected.push({ url: String(entry), reason: 'not a url' }); + continue; + } + + if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { + rejected.push({ url: entry, reason: 'unsupported scheme' }); + continue; + } + + // `new URL` is far more permissive than a domain is: prefixing a scheme + // onto free text such as "not a url at all" produces a URL object with a + // one-word host, which would otherwise become a crawl job that can only + // ever fail. Require something that looks like a registrable domain. + if ( + !/^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/i.test(parsed.hostname) + ) { + rejected.push({ url: entry, reason: 'not a domain' }); + continue; + } + + // Deduped by host, not by the full string: `example.com`, + // `https://example.com/` and `https://www.example.com` are one company, + // and crawling a homepage three times helps nobody. + const key = parsed.hostname.replace(/^www\./, '').toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + + cleaned.push(parsed.toString()); + } + + if (cleaned.length === 0) { + throw ApiError.badRequest('enter at least one company URL', { urls: ['none were usable'] }); + } + + if (cleaned.length > MAX_BULK_URLS) { + throw ApiError.badRequest(`at most ${MAX_BULK_URLS} URLs at a time`, { + urls: [`received ${cleaned.length}`], + }); + } + + const batchId = newId('job'); + const queued: string[] = []; + const duplicates: string[] = []; + + for (const url of cleaned) { + const result = await enqueue(db, { + workspaceId: actor.workspaceId, + kind: 'crawl_site', + payload: { url }, + batchId, + // One outstanding crawl per host per workspace. Releases itself when + // the job finishes, so the site can be re-crawled later. + dedupeKey: `crawl:${new URL(url).hostname.replace(/^www\./, '')}`, + }); + + if (result.queued) queued.push(url); + else duplicates.push(url); + } + + await repo.audit(db, { + workspaceId: actor.workspaceId, + actorKind: 'user', + actorId: actor.userId, + eventType: 'prospects.bulk_queued', + entityKind: 'job', + entityId: batchId, + detail: { queued: queued.length, duplicates: duplicates.length, rejected: rejected.length }, + }); + + return c.json( + { + batchId, + queued: queued.length, + urls: queued, + ...(duplicates.length ? { duplicates } : {}), + ...(rejected.length ? { rejected } : {}), + }, + 202, + ); + }); + + /** + * Progress for one bulk submission. + * + * Reports the individual rows, not just counts: after a hundred URLs the + * useful question is which ones failed and why, and that answer is on the row. + */ + api.get('/batches/:id', async (c) => { + const actor = c.get('actor'); + const status = await batchStatus(c.get('db'), actor.workspaceId, c.req.param('id')); + + if (!status) throw ApiError.notFound('batch'); + return c.json(status); + }); + // -------------------------------------------------------------- people /** * The prospect list, ranked the way the UI ranks everything else — by diff --git a/apps/api/src/by-url.test.ts b/apps/api/src/by-url.test.ts new file mode 100644 index 0000000..205bc46 --- /dev/null +++ b/apps/api/src/by-url.test.ts @@ -0,0 +1,201 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import type { Hono } from 'hono'; +import { queryAll } from '@outreachgraph/db'; +import { createApp } from './app'; +import type { AppEnv, RequestActor } from './context'; +import { seedDatabase, SEED, type SeededDatabase } from './test-seed'; + +const ACTOR: RequestActor = { + userId: SEED.userId, + workspaceId: SEED.workspaceId, + organizationId: SEED.organizationId, + role: 'owner', +}; + +let active: SeededDatabase | undefined; + +afterEach(() => { + active?.cleanup(); + active = undefined; +}); + +async function harness( + label: string, + actor: RequestActor | null = ACTOR, +): Promise<{ app: Hono; seeded: SeededDatabase }> { + const seeded = await seedDatabase(label); + active = seeded; + + const app = createApp({ + db: seeded.db, + authenticate: async () => actor ?? undefined, + }); + + return { app, seeded }; +} + +async function post(app: Hono, path: string, body: unknown): Promise { + return app.request(`/api/v1${path}`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }); +} + +describe('POST /prospects/by-url', () => { + test('a single URL is accepted and queued, not crawled inline', async () => { + const { app, seeded } = await harness('byurl-single'); + + const response = await post(app, '/prospects/by-url', { url: 'https://loopwright.io' }); + const body = await response.json(); + + // 202, not 200: the work has been accepted, not done. + expect(response.status).toBe(202); + expect(body.queued).toBe(1); + expect(body.batchId).toStartWith('job_'); + + const jobs = await queryAll<{ kind: string; status: string; payload_json: string }>( + seeded.db, + 'SELECT kind, status, payload_json FROM jobs', + ); + + expect(jobs).toHaveLength(1); + expect(jobs[0]!.kind).toBe('crawl_site'); + expect(jobs[0]!.status).toBe('pending'); + expect(JSON.parse(jobs[0]!.payload_json).url).toContain('loopwright.io'); + }); + + test('a bare domain is accepted', async () => { + const { app } = await harness('byurl-bare'); + + const response = await post(app, '/prospects/by-url', { url: 'loopwright.io' }); + const body = await response.json(); + + expect(response.status).toBe(202); + expect(body.urls[0]).toBe('https://loopwright.io/'); + }); + + test('the same company twice in one paste is queued once', async () => { + const { app } = await harness('byurl-dupe-in-batch'); + + const response = await post(app, '/prospects/by-url', { + urls: ['https://loopwright.io', 'https://www.loopwright.io/', 'loopwright.io'], + }); + const body = await response.json(); + + // Deduped by host: www and the bare domain are the same company, and + // crawling a homepage three times helps nobody. + expect(body.queued).toBe(1); + }); + + test('a URL already queued is reported, not rejected', async () => { + const { app } = await harness('byurl-dupe-across'); + + await post(app, '/prospects/by-url', { url: 'https://loopwright.io' }); + const second = await post(app, '/prospects/by-url', { + urls: ['https://loopwright.io', 'https://other.example'], + }); + const body = await second.json(); + + expect(body.queued).toBe(1); + expect(body.duplicates).toHaveLength(1); + // The point: one repeat must not take the other URL down with it. + expect(body.urls[0]).toContain('other.example'); + }); + + test('unusable entries are named rather than silently dropped', async () => { + const { app } = await harness('byurl-rejected'); + + const response = await post(app, '/prospects/by-url', { + urls: ['https://good.example', 'ftp://files.example', 'not a url at all'], + }); + const body = await response.json(); + + expect(body.queued).toBe(1); + expect(body.rejected.map((r: { url: string }) => r.url)).toContain('ftp://files.example'); + }); + + test('a submission with nothing usable is a 400', async () => { + const { app } = await harness('byurl-empty'); + + const response = await post(app, '/prospects/by-url', { urls: ['', ' '] }); + expect(response.status).toBe(400); + }); + + test('more than a hundred URLs is refused', async () => { + const { app } = await harness('byurl-too-many'); + + const urls = Array.from({ length: 101 }, (_, i) => `https://site-${i}.example`); + const response = await post(app, '/prospects/by-url', { urls }); + + expect(response.status).toBe(400); + expect((await response.json()).error.message).toContain('100'); + }); + + test('a hundred URLs is accepted and answers immediately', async () => { + const { app, seeded } = await harness('byurl-hundred'); + + const urls = Array.from({ length: 100 }, (_, i) => `https://site-${i}.example`); + const response = await post(app, '/prospects/by-url', { urls }); + + expect(response.status).toBe(202); + expect((await response.json()).queued).toBe(100); + + const rows = await queryAll<{ n: number }>(seeded.db, 'SELECT COUNT(*) AS n FROM jobs'); + expect(Number(rows[0]!.n)).toBe(100); + }); + + test('a caller who cannot approve cannot add prospects', async () => { + const { app } = await harness('byurl-forbidden', { ...ACTOR, role: 'viewer' }); + + const response = await post(app, '/prospects/by-url', { url: 'https://loopwright.io' }); + expect(response.status).toBe(403); + }); +}); + +describe('GET /batches/:id', () => { + test('progress reports each URL, not just a count', async () => { + const { app } = await harness('batch-progress'); + + const created = await post(app, '/prospects/by-url', { + urls: ['https://one.example', 'https://two.example'], + }); + const { batchId } = await created.json(); + + const response = await app.request(`/api/v1/batches/${batchId}`); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.total).toBe(2); + expect(body.pending).toBe(2); + // "Which ones, and why" is the question after a hundred URLs, and it can + // only be answered from the individual rows. + expect(body.items.map((i: { url: string }) => i.url).sort()).toEqual([ + 'https://one.example/', + 'https://two.example/', + ]); + }); + + test('an unknown batch is a 404', async () => { + const { app } = await harness('batch-missing'); + + const response = await app.request('/api/v1/batches/job_nope'); + expect(response.status).toBe(404); + }); + + test('another workspace cannot read this batch', async () => { + const { app, seeded } = await harness('batch-scoped'); + + const created = await post(app, '/prospects/by-url', { url: 'https://one.example' }); + const { batchId } = await created.json(); + + // A second app over the *same* database, not a second harness: calling the + // helper twice would replace the tracked handle and leak the first file. + const other = createApp({ + db: seeded.db, + authenticate: async () => ({ ...ACTOR, workspaceId: 'wsp_someone_else' }), + }); + + expect((await other.request(`/api/v1/batches/${batchId}`)).status).toBe(404); + }); +}); diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index d76265b..68e22b2 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -29,8 +29,11 @@ import { expireSignals, processDeletion, rescoreProspect, + runPipelineForCandidate, type QueuedJob, } from '@outreachgraph/pipeline'; +import { BlueskyProvider, SiteProvider } from '@outreachgraph/providers'; +import { queryOne } from '@outreachgraph/db'; const PORT = Number(process.env.PORT ?? 8080); const WEB_PORT = Number(process.env.WEB_PORT ?? 3001); @@ -225,8 +228,79 @@ let inFlight: Promise = Promise.resolve(); * success. A throw here is how the queue is told to retry or, once attempts * run out, to keep the row as `failed` with the reason attached. */ +/** The site crawler, sharing the composer's key for its extraction fallback. */ +const site = new SiteProvider(model ? { model } : {}); + +/** + * Providers the fan-out may consult once a candidate exists. + * + * Bluesky only, for now, and deliberately: its AppView needs no key and no + * contract, so it can run here without a commercial decision attached. X and + * the enrichment vendors both do, and shipping an adapter that cannot be + * exercised would be code pretending to be a feature. + */ +const fanOutProviders = [new BlueskyProvider()]; + +/** + * Fetches one company URL and runs everyone it names. + * + * A page that names nobody is a completed job, not a failed one. Homepages + * frequently describe a company without naming a single person, and retrying + * that four more times would spend the crawl budget re-reading a page whose + * answer will not change. + */ +async function crawlSite(job: QueuedJob): Promise { + const { url } = job.payload as { url?: string }; + if (!url) throw new Error('crawl_site needs a url'); + + const result = await site.crawl(url); + + if (result.outcome !== 'ok') { + // A refusal is the site's answer and re-asking will not change it, so this + // resolves the job rather than throwing it back into the retry loop. + console.log(`crawl ${url}: ${result.outcome}${result.detail ? ` (${result.detail})` : ''}`); + return; + } + + const campaign = await queryOne<{ id: string }>( + db, + `SELECT id FROM campaigns WHERE workspace_id = ? ORDER BY created_at LIMIT 1`, + [job.workspaceId], + ); + + if (!campaign) throw new Error(`workspace ${job.workspaceId} has no campaign`); + + let queued = 0; + for (const candidate of result.people) { + await runPipelineForCandidate( + { + db, + workspaceId: job.workspaceId, + campaignId: campaign.id, + providers: fanOutProviders, + ...(model ? { model } : {}), + }, + candidate, + { + capabilities: site.capabilities(), + // No anchor: nobody named on a company page is proven to be that + // person. Every identity found there is a claim for the resolver. + }, + ); + queued += 1; + } + + console.log( + `crawl ${url}: ${result.company.name ?? 'unnamed company'}, ${queued} people` + + ` (${result.usedSignals.join(', ') || 'no signals'})`, + ); +} + async function runJob(job: QueuedJob): Promise { switch (job.kind) { + case 'crawl_site': + await crawlSite(job); + return; case 'rescore_prospect': { const { campaignId, personId } = job.payload as { campaignId?: string; personId?: string }; if (!campaignId || !personId) diff --git a/migrations/0008_job_batches.sql b/migrations/0008_job_batches.sql new file mode 100644 index 0000000..48e31d4 --- /dev/null +++ b/migrations/0008_job_batches.sql @@ -0,0 +1,16 @@ +-- Batches (PRD §8, URL-first intake). +-- +-- Pasting a hundred URLs produces a hundred independent jobs, and the person +-- who pasted them wants one answer: how far along is it, and which ones failed. +-- Without a grouping key the API can only report on jobs it already holds the +-- ids of, which means the client has to remember a hundred of them. +-- +-- A column rather than a `batches` table. A batch has no attributes of its own +-- beyond the jobs in it — no name, no owner beyond the workspace already on +-- every job, no lifecycle that is not just the aggregate of its members. A +-- table would add a join and a second thing to keep consistent for nothing. + +ALTER TABLE jobs ADD COLUMN batch_id TEXT; + +-- The progress query: every job in one batch, oldest first. +CREATE INDEX idx_jobs_batch ON jobs(batch_id, created_at) WHERE batch_id IS NOT NULL; diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index 8beb504..8bba285 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -7,7 +7,13 @@ * dependency direction a lie. */ -export { runPipeline, type PipelineOptions, type PipelineResult } from './pipeline'; +export { + runPipeline, + runPipelineForCandidate, + type CandidateOrigin, + type PipelineOptions, + type PipelineResult, +} from './pipeline'; export { expireSignals, markSourceUnavailable, @@ -18,6 +24,7 @@ export { type JobResult, } from './jobs'; export { + batchStatus, claimNext, completeJob, drainQueue, @@ -26,6 +33,8 @@ export { queueDepth, reclaimStalled, JOB_STATUSES, + type BatchItem, + type BatchStatus, type DrainSummary, type EnqueueInput, type EnqueueResult, diff --git a/packages/pipeline/src/jobs.ts b/packages/pipeline/src/jobs.ts index fd1e60d..c8a19aa 100644 --- a/packages/pipeline/src/jobs.ts +++ b/packages/pipeline/src/jobs.ts @@ -24,6 +24,8 @@ export const JOB_KINDS = [ 'expire_signals', 'process_deletion', 'poll_drop', + /** Fetch one company URL, read it, and run everyone it names. */ + 'crawl_site', ] as const; export type JobKind = (typeof JOB_KINDS)[number]; diff --git a/packages/pipeline/src/pipeline.ts b/packages/pipeline/src/pipeline.ts index 3f8c9bc..6774bc2 100644 --- a/packages/pipeline/src/pipeline.ts +++ b/packages/pipeline/src/pipeline.ts @@ -16,11 +16,13 @@ import { resolveIdentity, type EvidenceInput } from '@outreachgraph/identity'; import { deriveEvidence, extractSignals, + findIdentities, GitHubProvider, GitHubRateLimitError, type ExtractedSignal, type PersonCandidate, type PersonEnrichmentProvider, + type ProviderCapabilities, } from '@outreachgraph/providers'; import { generateRecommendation, type CandidateSignal } from '@outreachgraph/recommend'; import { draftForRecommendation, type TextModel } from '@outreachgraph/ai'; @@ -54,17 +56,14 @@ export interface PipelineResult { /** * Runs the full chain for one GitHub handle. * - * GitHub-first is deliberate: it is free, its profiles carry self-declared - * cross-links, and the launch wedge is developer tooling (PRD §45). + * GitHub-first was the launch wedge: it is free and its profiles carry + * self-declared cross-links (PRD §45). It is now one way in among others — + * this resolves a handle to a candidate and hands over to the shared chain. */ export async function runPipeline( options: PipelineOptions, handle: string, ): Promise { - const { db, workspaceId, campaignId } = options; - const stamp = now(); - - // ---------------------------------------------------------------- enrich const github = options.github ?? new GitHubProvider(); const enrichment = await github.enrich({ handles: { github: handle } }); @@ -77,10 +76,44 @@ export async function runPipeline( }; } - const candidate = enrichment.candidate; + return runPipelineForCandidate(options, enrichment.candidate, { + capabilities: github.capabilities(), + // The GitHub account this run started from is the anchor: it is the person, + // not an inference about them, so it links without a confidence test. + anchorNetwork: 'github', + githubHandle: handle, + }); +} + +export interface CandidateOrigin { + /** Whose data this is, for provenance and evidence weighting. */ + readonly capabilities: ProviderCapabilities; + /** A network whose identity *is* the person, rather than a match for them. */ + readonly anchorNetwork?: Network; + /** Enables the GitHub research stage when the candidate has a handle. */ + readonly githubHandle?: string; +} + +/** + * The chain, from a candidate onwards. + * + * This is the part that does not care where the person came from. A GitHub + * handle, a name lifted off a company homepage and an enrichment vendor's + * record all arrive here as the same normalised candidate, and everything after + * this point — suppression, resolution, research, scoring, recommendation, + * drafting — is identical for all three. + */ +export async function runPipelineForCandidate( + options: PipelineOptions, + candidate: PersonCandidate, + origin: CandidateOrigin, +): Promise { + const { db, workspaceId, campaignId } = options; + const stamp = now(); + const personId = await upsertPerson(db, candidate, stamp); - await recordProvenance(db, personId, candidate, stamp); + await recordProvenance(db, personId, candidate, origin, stamp); await ensureCampaignMembership(db, campaignId, personId, workspaceId, stamp); // Suppression outranks everything: a suppressed person is never researched @@ -97,21 +130,42 @@ export async function runPipeline( } // -------------------------------------------------------------- resolve - const linked = await linkIdentities(db, personId, candidate, github, stamp); + // + // Ask the other configured providers what else they can vouch for before + // resolving. Nothing is merged here — the fan-out gathers claims and the + // resolver below decides which of them are the same person. + const fanned = + options.providers.length > 0 ? await findIdentities(candidate, options.providers) : undefined; + + const enriched = fanned?.candidate ?? candidate; + + const linked = await linkIdentities(db, personId, enriched, origin, stamp); await setStatus(db, campaignId, personId, 'resolved'); // ------------------------------------------------------------- research + // + // Only GitHub produces signals today, so a candidate with no GitHub identity + // reaches the queue with evidence but no activity. That is a real limitation + // rather than a bug: the card still carries the prospect and its source, and + // the composer refuses to invent a reason it cannot ground. + const researchHandle = + origin.githubHandle ?? + candidate.identities.find((identity) => identity.network === 'github')?.handle; + let stored = 0; - try { - const activity = await github.activity(handle); - const context = await extractionContext(db, campaignId); - const extracted = extractSignals(activity.events, activity.repos, context); - stored = await storeSignals(db, workspaceId, personId, extracted, stamp); - await setStatus(db, campaignId, personId, 'researching'); - } catch (error) { - // A quota wall is not a prospect failure — keep what we have and let the - // next tick resume research. - if (!(error instanceof GitHubRateLimitError)) throw error; + if (researchHandle) { + const github = options.github ?? new GitHubProvider(); + try { + const activity = await github.activity(researchHandle); + const context = await extractionContext(db, campaignId); + const extracted = extractSignals(activity.events, activity.repos, context); + stored = await storeSignals(db, workspaceId, personId, extracted, stamp); + await setStatus(db, campaignId, personId, 'researching'); + } catch (error) { + // A quota wall is not a prospect failure — keep what we have and let the + // next tick resume research. + if (!(error instanceof GitHubRateLimitError)) throw error; + } } // ---------------------------------------------------------------- score @@ -237,7 +291,7 @@ async function linkIdentities( db: Client, personId: string, candidate: PersonCandidate, - provider: GitHubProvider, + origin: CandidateOrigin, stamp: string, ): Promise { const workspace = await queryOne<{ auto_merge_threshold: number; candidate_threshold: number }>( @@ -262,14 +316,16 @@ async function linkIdentities( const evidence: EvidenceInput[] = deriveEvidence({ identity, candidate, - capabilities: provider.capabilities(), + capabilities: origin.capabilities, crossLinkedHandles: declared, ...(candidate.companyName ? { platformEmployer: candidate.companyName } : {}), }); - // The GitHub account itself is the anchor, not an inference about it. + // The anchor identity *is* the person the run started from, not an + // inference about them, so it links without a confidence test. A crawl has + // no anchor: every identity on a company page is a claim to be resolved. const resolution = - identity.network === 'github' + origin.anchorNetwork && identity.network === origin.anchorNetwork ? { decision: 'merge' as const, score: 1, verifiedBy: ['provider_asserted_link'] } : resolveIdentity(evidence, { thresholds }); @@ -277,7 +333,7 @@ async function linkIdentities( await db.execute({ sql: `INSERT INTO social_identities (id, person_id, network, handle, platform_user_id, profile_url, confidence, source_type, verified_by, first_seen_at, last_verified_at) - VALUES (?, ?, ?, ?, ?, ?, ?, 'official_api', ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -- The uniqueness index is partial, so the conflict target has to -- repeat its WHERE clause or SQLite refuses to match it. ON CONFLICT(network, platform_user_id) WHERE platform_user_id IS NOT NULL @@ -290,6 +346,7 @@ async function linkIdentities( identity.platformUserId ?? null, identity.profileUrl ?? null, resolution.score, + origin.capabilities.sourceType, JSON.stringify(resolution.verifiedBy), stamp, stamp, @@ -519,6 +576,7 @@ async function recordProvenance( db: Client, personId: string, candidate: PersonCandidate, + origin: CandidateOrigin, stamp: string, ): Promise { const fields: [string, string | undefined][] = [ @@ -531,15 +589,22 @@ async function recordProvenance( for (const [field, value] of fields) { if (!value) continue; await db.execute({ + // Attribution follows the provider that actually supplied the value. + // Hardcoding GitHub's here was harmless while GitHub was the only source; + // with a crawl in the mix it would label a scraped name as an API fact + // and mis-classify what may be retained and exported (PRD §35). sql: `INSERT INTO field_provenance (id, entity_kind, entity_id, field, value, source_type, provider, source_record_id, license_class, confidence, observed_at, created_at) - VALUES (?, 'person', ?, ?, ?, 'official_api', 'github', ?, 'public_api', 1.0, ?, ?)`, + VALUES (?, 'person', ?, ?, ?, ?, ?, ?, ?, 1.0, ?, ?)`, args: [ newId('fieldProvenance'), personId, field, value, + origin.capabilities.sourceType, + origin.capabilities.slug, candidate.sourceRecordId ?? null, + origin.capabilities.licenseClass, candidate.observedAt, stamp, ], diff --git a/packages/pipeline/src/queue.ts b/packages/pipeline/src/queue.ts index 721a70d..fd68cd0 100644 --- a/packages/pipeline/src/queue.ts +++ b/packages/pipeline/src/queue.ts @@ -50,6 +50,8 @@ export interface EnqueueInput { readonly maxAttempts?: number; /** Suppresses a duplicate while an identical job is still outstanding. */ readonly dedupeKey?: string; + /** Groups this job with the rest of one bulk submission. */ + readonly batchId?: string; } export interface EnqueueResult { @@ -77,8 +79,8 @@ export async function enqueue(db: Client, input: EnqueueInput): Promise { + const rows = await queryAll<{ + id: string; + status: string; + attempts: number; + payload_json: string; + last_error: string | null; + }>( + db, + `SELECT id, status, attempts, payload_json, last_error + FROM jobs WHERE batch_id = ? AND workspace_id = ? ORDER BY created_at`, + [batchId, workspaceId], + ); + + if (rows.length === 0) return undefined; + + const counts: Record = { pending: 0, running: 0, done: 0, failed: 0 }; + const items: BatchItem[] = []; + + for (const row of rows) { + const status = ( + (JOB_STATUSES as readonly string[]).includes(row.status) ? row.status : 'pending' + ) as JobStatus; + counts[status] += 1; + + const url = safeParse(row.payload_json).url; + + items.push({ + id: row.id, + status, + attempts: Number(row.attempts), + ...(typeof url === 'string' ? { url } : {}), + ...(row.last_error ? { lastError: row.last_error } : {}), + }); + } + + return { batchId, total: rows.length, ...counts, items }; +} + /** Queue depth by status, for the tick log and, later, an operator view. */ export async function queueDepth(db: Client): Promise> { const rows = await queryAll<{ status: string; n: number }>( diff --git a/packages/providers/src/bluesky/bluesky.test.ts b/packages/providers/src/bluesky/bluesky.test.ts new file mode 100644 index 0000000..c956ec0 --- /dev/null +++ b/packages/providers/src/bluesky/bluesky.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, test } from 'bun:test'; +import { BlueskyProvider, BlueskyRateLimitError } from './provider'; +import { findIdentities } from '../fan-out'; +import type { PersonCandidate, PersonEnrichmentProvider } from '../provider'; + +function jsonFetch(status: number, body: unknown) { + return async (): Promise => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); +} + +const PROFILE = { + did: 'did:plc:abc123', + handle: 'alex.bsky.social', + displayName: 'Alex Chen', + description: 'agent reliability', +}; + +describe('BlueskyProvider', () => { + test('a known handle resolves to an identity carrying the stable did', async () => { + const provider = new BlueskyProvider({ fetchImpl: jsonFetch(200, PROFILE) }); + + const result = await provider.enrich({ handles: { bluesky: 'alex.bsky.social' } }); + + expect(result.matchConfidence).toBe(1); + expect(result.candidate?.fullName).toBe('Alex Chen'); + // The did, not the handle: handles on this network are renameable. + expect(result.candidate?.identities[0]!.platformUserId).toBe('did:plc:abc123'); + expect(result.candidate?.identities[0]!.profileUrl).toContain('alex.bsky.social'); + }); + + test('a name alone resolves to nothing', async () => { + let called = false; + const provider = new BlueskyProvider({ + fetchImpl: async () => { + called = true; + return new Response('{}', { status: 200 }); + }, + }); + + const result = await provider.enrich({ fullName: 'Alex Chen' }); + + // Taking the first search hit for a display name is how the wrong human + // reaches an approval queue. Finding nobody is the correct answer. + expect(result.matchConfidence).toBe(0); + expect(called).toBe(false); + }); + + test('an unknown actor is an answer, not an error', async () => { + const provider = new BlueskyProvider({ + fetchImpl: jsonFetch(400, { error: 'InvalidRequest' }), + }); + + const result = await provider.enrich({ handles: { bluesky: 'nobody.bsky.social' } }); + expect(result.candidate).toBeUndefined(); + }); + + test('a quota wall is typed so callers can keep partial work', async () => { + const provider = new BlueskyProvider({ fetchImpl: jsonFetch(429, {}) }); + + await expect( + provider.enrich({ handles: { bluesky: 'alex.bsky.social' } }), + ).rejects.toBeInstanceOf(BlueskyRateLimitError); + }); + + test('search returns candidates for keywords', async () => { + const provider = new BlueskyProvider({ fetchImpl: jsonFetch(200, { actors: [PROFILE] }) }); + + const result = await provider.search({ keywords: ['agent reliability'] }); + expect(result.candidates).toHaveLength(1); + expect(result.candidates[0]!.identities[0]!.network).toBe('bluesky'); + }); + + test('it reports itself as free, so the waterfall reaches it early', () => { + expect(new BlueskyProvider().capabilities().costPerEnrichmentUsd).toBe(0); + }); +}); + +describe('findIdentities', () => { + const BASE: PersonCandidate = { + fullName: 'Alex Chen', + identities: [{ network: 'github', handle: 'alexchen' }], + observedAt: '2026-08-13T00:00:00.000Z', + }; + + function stub( + slug: string, + networks: readonly ('github' | 'bluesky')[], + result: () => Promise<{ + candidate?: PersonCandidate; + matchConfidence: number; + costUsd: number; + }>, + ): PersonEnrichmentProvider { + return { + capabilities: () => ({ + slug, + displayName: slug, + networks, + canSearch: false, + canEnrich: true, + licenseClass: 'public_api', + sourceType: 'official_api', + costPerEnrichmentUsd: 0, + }), + search: async () => ({ candidates: [], costUsd: 0 }), + enrich: result, + costEstimate: async () => 0, + }; + } + + test('a provider is only asked about a network we already hold a handle for', async () => { + let asked = false; + const bluesky = stub('bluesky', ['bluesky'], async () => { + asked = true; + return { matchConfidence: 0, costUsd: 0 }; + }); + + await findIdentities(BASE, [bluesky]); + + // The candidate has a GitHub handle and nothing else. Asking Bluesky to + // find "Alex Chen" would be a guess, so it is not asked at all. + expect(asked).toBe(false); + }); + + test('discovered identities are folded in without duplicating what we had', async () => { + const candidate: PersonCandidate = { + ...BASE, + identities: [ + { network: 'github', handle: 'alexchen' }, + { network: 'bluesky', handle: 'alex.bsky.social' }, + ], + }; + + const provider = stub('bluesky', ['bluesky'], async () => ({ + candidate: { + ...candidate, + identities: [ + { network: 'bluesky', handle: 'alex.bsky.social', platformUserId: 'did:plc:abc' }, + ], + }, + matchConfidence: 1, + costUsd: 0, + })); + + const result = await findIdentities(candidate, [provider]); + + // The returned identity keys on the did while ours keyed on the handle, so + // both survive — the resolver, not the fan-out, decides they are one person. + expect(result.attempts[0]!.found).toBe(1); + expect(result.candidate.identities.length).toBeGreaterThanOrEqual(2); + }); + + test('one provider failing does not lose the others', async () => { + const angry = stub('angry', ['github'], async () => { + throw new Error('rate limited'); + }); + const calm = stub('calm', ['github'], async () => ({ + candidate: { ...BASE, identities: [{ network: 'bluesky', handle: 'found.bsky.social' }] }, + matchConfidence: 1, + costUsd: 0, + })); + + const result = await findIdentities(BASE, [angry, calm]); + + expect(result.attempts.find((a) => a.provider === 'angry')?.ok).toBe(false); + expect(result.attempts.find((a) => a.provider === 'calm')?.found).toBe(1); + expect(result.candidate.identities.map((i) => i.network)).toContain('bluesky'); + }); + + test('with no providers the candidate is returned untouched', async () => { + const result = await findIdentities(BASE, []); + expect(result.candidate).toEqual(BASE); + expect(result.attempts).toHaveLength(0); + }); +}); diff --git a/packages/providers/src/bluesky/provider.ts b/packages/providers/src/bluesky/provider.ts new file mode 100644 index 0000000..45f998b --- /dev/null +++ b/packages/providers/src/bluesky/provider.ts @@ -0,0 +1,150 @@ +/** + * Bluesky as an identity source (capability matrix: `bluesky/observe` is + * `official_api`, "public AppView feed and profile data"). + * + * First after GitHub for one reason: the AppView is genuinely public. No key, + * no contract, no per-seat cost — so it can ship and be tested without a + * commercial decision attached, which is not true of X or an enrichment vendor. + * + * Read-only. Every write action on this network is `official_api` in the + * matrix and belongs to the execution path, not to research. + */ + +import type { FetchLike } from '../site/fetch'; +import type { + CandidateIdentity, + PersonCandidate, + PersonEnrichmentInput, + PersonEnrichmentProvider, + PersonEnrichmentResult, + PersonSearchInput, + PersonSearchResult, + ProviderCapabilities, +} from '../provider'; + +export const BLUESKY_API = 'https://public.api.bsky.app'; + +export interface BlueskyProviderOptions { + readonly baseUrl?: string; + readonly fetchImpl?: FetchLike; + readonly timeoutMs?: number; +} + +interface ActorProfile { + did?: string; + handle?: string; + displayName?: string; + description?: string; +} + +/** Raised for a quota wall, so callers can keep partial work as GitHub does. */ +export class BlueskyRateLimitError extends Error { + constructor() { + super('bluesky rate limit reached'); + this.name = 'BlueskyRateLimitError'; + } +} + +export class BlueskyProvider implements PersonEnrichmentProvider { + readonly #baseUrl: string; + readonly #fetch: FetchLike; + readonly #timeoutMs: number; + + constructor(options: BlueskyProviderOptions = {}) { + this.#baseUrl = options.baseUrl ?? BLUESKY_API; + this.#fetch = options.fetchImpl ?? fetch; + this.#timeoutMs = options.timeoutMs ?? 10_000; + } + + capabilities(): ProviderCapabilities { + return { + slug: 'bluesky', + displayName: 'Bluesky', + networks: ['bluesky'], + canSearch: true, + canEnrich: true, + licenseClass: 'public_api', + sourceType: 'official_api', + costPerEnrichmentUsd: 0, + }; + } + + async #get(path: string, params: Record): Promise { + const url = new URL(`/xrpc/${path}`, this.#baseUrl); + for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value); + + const response = await this.#fetch(url.toString(), { + headers: { accept: 'application/json' }, + signal: AbortSignal.timeout(this.#timeoutMs), + }); + + if (response.status === 429) throw new BlueskyRateLimitError(); + // A missing actor is an answer, not a fault: most people are not here. + if (response.status === 400 || response.status === 404) return undefined; + if (!response.ok) throw new Error(`bluesky ${path} failed: ${response.status}`); + + return (await response.json()) as T; + } + + async search(input: PersonSearchInput): Promise { + const query = input.keywords?.join(' ').trim(); + if (!query) return { candidates: [], costUsd: 0 }; + + const result = await this.#get<{ actors?: ActorProfile[] }>('app.bsky.actor.searchActors', { + q: query, + limit: String(Math.min(input.limit ?? 20, 50)), + }); + + const observedAt = new Date().toISOString(); + const candidates = (result?.actors ?? []) + .filter((actor) => actor.handle) + .map((actor) => toCandidate(actor, observedAt)); + + return { candidates, costUsd: 0 }; + } + + /** + * Resolves one Bluesky identity. + * + * Only a known handle is looked up. Searching by display name and taking the + * first hit is how a product confidently attaches the wrong person: names are + * not unique and this network has no verification to break the tie. Finding + * nobody is the correct answer to a name. + */ + async enrich(input: PersonEnrichmentInput): Promise { + const handle = input.handles?.bluesky; + if (!handle) return { matchConfidence: 0, costUsd: 0 }; + + const profile = await this.#get('app.bsky.actor.getProfile', { actor: handle }); + if (!profile?.handle) return { matchConfidence: 0, costUsd: 0 }; + + return { + candidate: toCandidate(profile, new Date().toISOString()), + // The handle was supplied and the profile exists: this is a resolution, + // not a guess. + matchConfidence: 1, + costUsd: 0, + }; + } + + async costEstimate(_input: PersonSearchInput | PersonEnrichmentInput): Promise { + return 0; + } +} + +function toCandidate(actor: ActorProfile, observedAt: string): PersonCandidate { + const identity: CandidateIdentity = { + network: 'bluesky', + handle: actor.handle!, + ...(actor.did ? { platformUserId: actor.did } : {}), + profileUrl: `https://bsky.app/profile/${actor.handle}`, + providerConfidence: 1, + }; + + return { + fullName: actor.displayName?.trim() || actor.handle!, + identities: [identity], + ...(actor.did ? { sourceRecordId: actor.did } : {}), + observedAt, + }; +} diff --git a/packages/providers/src/fan-out.ts b/packages/providers/src/fan-out.ts new file mode 100644 index 0000000..b3a26aa --- /dev/null +++ b/packages/providers/src/fan-out.ts @@ -0,0 +1,105 @@ +/** + * Asking every configured provider what else it knows about one person. + * + * The waterfall picks *one* provider to answer a question cheaply. This is the + * other shape: a candidate already exists, and each provider is asked to add + * the identities it can vouch for. A crawl that produced a name and an X handle + * gains a Bluesky profile here, and only then does the resolver decide whether + * any of it is the same human. + * + * Nothing here merges anything. It gathers claims; `@outreachgraph/identity` + * weighs them. Keeping those apart is what stops a provider's confidence from + * quietly becoming the product's confidence. + */ + +import type { Network } from '@outreachgraph/domain'; +import type { CandidateIdentity, PersonCandidate, PersonEnrichmentProvider } from './provider'; + +export interface FanOutAttempt { + readonly provider: string; + readonly ok: boolean; + readonly found: number; + readonly error?: string; +} + +export interface FanOutResult { + /** The candidate with everything discovered folded in. */ + readonly candidate: PersonCandidate; + readonly attempts: readonly FanOutAttempt[]; + readonly costUsd: number; +} + +function key(identity: CandidateIdentity): string { + return `${identity.network}:${(identity.platformUserId ?? identity.handle ?? '').toLowerCase()}`; +} + +/** + * Runs every provider that can add something, and folds the results together. + * + * A provider is only asked when the candidate already carries a handle it can + * resolve. Handing a display name to a network with no verification and taking + * the first match is how the wrong person ends up in an approval queue, so + * discovery here is strictly "resolve what we were told", never "find someone + * who looks like this". + * + * One provider failing never fails the fan-out. A quota wall on one network is + * not a reason to discard what the others returned. + */ +export async function findIdentities( + candidate: PersonCandidate, + providers: readonly PersonEnrichmentProvider[], +): Promise { + const merged = new Map(); + for (const identity of candidate.identities) merged.set(key(identity), identity); + + const attempts: FanOutAttempt[] = []; + let costUsd = 0; + + const known: Partial> = {}; + for (const identity of candidate.identities) { + if (identity.handle) known[identity.network] = identity.handle; + } + + for (const provider of providers) { + const capabilities = provider.capabilities(); + if (!capabilities.canEnrich) continue; + + // Only ask a provider about a network we already hold a handle for. + const usable = capabilities.networks.some((network) => known[network]); + if (!usable) continue; + + try { + const result = await provider.enrich({ + fullName: candidate.fullName, + ...(candidate.companyDomain ? { companyDomain: candidate.companyDomain } : {}), + ...(candidate.companyName ? { companyName: candidate.companyName } : {}), + handles: known, + }); + + costUsd += result.costUsd; + + let found = 0; + for (const identity of result.candidate?.identities ?? []) { + const id = key(identity); + if (merged.has(id)) continue; + merged.set(id, identity); + found += 1; + } + + attempts.push({ provider: capabilities.slug, ok: true, found }); + } catch (error) { + attempts.push({ + provider: capabilities.slug, + ok: false, + found: 0, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + return { + candidate: { ...candidate, identities: [...merged.values()] }, + attempts, + costUsd, + }; +} diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index b0c0514..59d7e97 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -54,3 +54,47 @@ export { } from './github/provider'; export { extractSignals, type ExtractedSignal, type ExtractionContext } from './github/signals'; + +export { + SiteProvider, + normaliseUrl, + type CrawlResult, + type SiteProviderOptions, +} from './site/provider'; + +export { + fetchPage, + loadRobots, + USER_AGENT, + type FetchLike, + type FetchOptions, + type FetchOutcome, + type FetchedPage, +} from './site/fetch'; + +export { + extractCompany, + extractPeople, + extractSite, + networkForUrl, + type ExtractedCompany, + type SiteExtraction, +} from './site/extract'; + +export { + extractWithModel, + visibleText, + type ExtractionModel, + type ModelExtraction, +} from './site/model-extract'; + +export { isAllowed, parseRobots, type RobotsRules } from './site/robots'; + +export { + BlueskyProvider, + BlueskyRateLimitError, + BLUESKY_API, + type BlueskyProviderOptions, +} from './bluesky/provider'; + +export { findIdentities, type FanOutAttempt, type FanOutResult } from './fan-out'; diff --git a/packages/providers/src/site/extract.ts b/packages/providers/src/site/extract.ts new file mode 100644 index 0000000..8323331 --- /dev/null +++ b/packages/providers/src/site/extract.ts @@ -0,0 +1,302 @@ +/** + * Deterministic extraction — the first half of the hybrid. + * + * Everything here reads markup the site published *about itself*: JSON-LD, + * OpenGraph, `rel="me"`, outbound links to networks we know. That matters + * beyond convenience. The grounding rule says no claim without stored + * evidence, and a value lifted from a site's own structured data has evidence + * by construction — we can point at the element it came from. + * + * The model only runs when this finds nothing, so a site with decent markup + * never costs a token. + */ + +import type { Network } from '@outreachgraph/domain'; +import type { CandidateIdentity, PersonCandidate } from '../provider'; + +export interface ExtractedCompany { + readonly name?: string; + readonly description?: string; + readonly domain?: string; + readonly location?: string; + /** Company-level profiles found on the page, e.g. its own X account. */ + readonly identities: readonly CandidateIdentity[]; +} + +export interface SiteExtraction { + readonly company: ExtractedCompany; + readonly people: readonly PersonCandidate[]; + /** Which routes produced something, for the log and for tests. */ + readonly usedSignals: readonly string[]; +} + +/** Hosts that identify a network, longest-first so `x.com` beats a bare match. */ +const NETWORK_HOSTS: readonly (readonly [RegExp, Network])[] = [ + [/(^|\.)github\.com$/i, 'github'], + [/(^|\.)twitter\.com$/i, 'x'], + [/(^|\.)x\.com$/i, 'x'], + [/(^|\.)bsky\.app$/i, 'bluesky'], + [/(^|\.)linkedin\.com$/i, 'linkedin'], + [/(^|\.)reddit\.com$/i, 'reddit'], + [/(^|\.)youtube\.com$/i, 'youtube'], + [/(^|\.)instagram\.com$/i, 'instagram'], +]; + +export function networkForUrl(url: string): Network | undefined { + try { + const host = new URL(url).hostname; + return NETWORK_HOSTS.find(([pattern]) => pattern.test(host))?.[1]; + } catch { + return undefined; + } +} + +/** `https://github.com/octocat/repo` → `octocat`. */ +function handleFromUrl(url: string): string | undefined { + try { + const segments = new URL(url).pathname.split('/').filter(Boolean); + const first = segments[0]; + if (!first) return undefined; + // Company pages and static sections are not handles. + if (/^(company|school|showcase|in|pub|c|channel|user|r)$/i.test(first)) return segments[1]; + return first; + } catch { + return undefined; + } +} + +function decodeEntities(text: string): string { + return text + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/�?39;/g, "'") + .replace(/'/g, "'") + .replace(/ /g, ' ') + .replace(/&/g, '&'); +} + +function collapse(text: string): string { + return decodeEntities(text).replace(/\s+/g, ' ').trim(); +} + +/** Every ` + + + bluesky + `; + + test('the organisation is read from JSON-LD, not the page title', () => { + const company = extractCompany(HTML, 'https://loopwright.io/'); + + // says "Loopwright — agent reliability"; the structured statement + // is the company's actual name and has to win. + expect(company.name).toBe('Loopwright'); + expect(company.domain).toBe('loopwright.io'); + expect(company.description).toContain('Reliability tooling'); + }); + + test('sameAs and rel=me both become identities', () => { + const company = extractCompany(HTML, 'https://loopwright.io/'); + const networks = company.identities.map((i) => i.network).sort(); + + expect(networks).toContain('github'); + expect(networks).toContain('x'); + expect(networks).toContain('bluesky'); + }); + + test('a JSON-LD person carries their title and company', () => { + const company = extractCompany(HTML, 'https://loopwright.io/'); + const people = extractPeople(HTML, 'https://loopwright.io/', company); + + expect(people).toHaveLength(1); + expect(people[0]!.fullName).toBe('Alex Chen'); + expect(people[0]!.title).toBe('Staff Engineer'); + expect(people[0]!.companyName).toBe('Loopwright'); + expect(people[0]!.identities[0]!.handle).toBe('alexchen'); + }); + + test('an @graph wrapper is unwrapped', () => { + const html = `<script type="application/ld+json"> + {"@graph":[{"@type":"Organization","name":"Graphed Co"}]}</script>`; + + expect(extractCompany(html, 'https://graphed.co/').name).toBe('Graphed Co'); + }); + + test('malformed JSON-LD does not stop the other routes', () => { + const html = `<html><head> + <script type="application/ld+json">{ not json at all </script> + <meta property="og:site_name" content="Still Here" /> + </head></html>`; + + expect(extractCompany(html, 'https://x.example/').name).toBe('Still Here'); + }); + + test('a linkedin company path yields the company slug, not the word company', () => { + expect(networkForUrl('https://www.linkedin.com/company/loopwright')).toBe('linkedin'); + const company = extractCompany( + '<a href="https://www.linkedin.com/company/loopwright">li</a>', + 'https://loopwright.io/', + ); + expect(company.identities[0]!.handle).toBe('loopwright'); + }); + + test('a page with no structured data names nobody', () => { + const result = extractSite('<html><body><p>We are a team.</p></body></html>', 'https://a.io/'); + expect(result.people).toHaveLength(0); + }); +}); + +describe('model extraction', () => { + function modelReturning(text: string, refused = false): ExtractionModel { + return { generate: async () => ({ text, refused }) }; + } + + test('visible text drops script, style and markup', () => { + const text = visibleText( + '<html><style>a{color:red}</style><script>var x=1</script><p>Hello world</p></html>', + ); + + expect(text).toBe('Hello world'); + }); + + test('a fenced JSON reply is parsed', async () => { + const model = modelReturning( + '```json\n{"company":{"name":"Bespoke Co"},"people":[{"fullName":"Dana Reed","title":"CTO"}]}\n```', + ); + + const result = await extractWithModel(model, '<p>text</p>', 'https://b.co/', { + identities: [], + domain: 'b.co', + }); + + expect(result.company.name).toBe('Bespoke Co'); + expect(result.people[0]!.fullName).toBe('Dana Reed'); + expect(result.people[0]!.companyDomain).toBe('b.co'); + }); + + test('a single-word name is discarded', async () => { + const model = modelReturning( + '{"people":[{"fullName":"Engineering"},{"fullName":"Ada Byron"}]}', + ); + + const result = await extractWithModel(model, '<p>x</p>', 'https://b.co/', { identities: [] }); + + // "Engineering" is a department. Turning it into a prospect would put a + // fabricated human into the approval queue. + expect(result.people.map((p) => p.fullName)).toEqual(['Ada Byron']); + }); + + test('a refusal yields nothing rather than throwing', async () => { + const result = await extractWithModel(modelReturning('no', true), '<p>x</p>', 'https://b.co/', { + identities: [], + }); + + expect(result.people).toHaveLength(0); + expect(result.company.name).toBeUndefined(); + }); + + test('an unparseable reply yields nothing rather than throwing', async () => { + const result = await extractWithModel( + modelReturning('I am unable to find any JSON here'), + '<p>x</p>', + 'https://b.co/', + { identities: [] }, + ); + + expect(result.people).toHaveLength(0); + }); + + test('a thrown model error does not fail the crawl', async () => { + const model: ExtractionModel = { + generate: async () => { + throw new Error('api down'); + }, + }; + + const result = await extractWithModel(model, '<p>x</p>', 'https://b.co/', { identities: [] }); + expect(result.people).toHaveLength(0); + }); +}); + +describe('SiteProvider', () => { + const ROBOTS = { body: '', type: 'text/plain' }; + + test('a well-marked-up page never reaches the model', async () => { + let modelCalls = 0; + const model: ExtractionModel = { + generate: async () => { + modelCalls += 1; + return { text: '{}', refused: false }; + }, + }; + + const provider = new SiteProvider({ + model, + fetchImpl: stubFetch({ + 'https://loop.io/robots.txt': ROBOTS, + 'https://loop.io/': { + body: `<script type="application/ld+json"> + {"@type":"Organization","name":"Loop"}</script> + <script type="application/ld+json"> + {"@type":"Person","name":"Alex Chen"}</script>`, + }, + }), + }); + + const result = await provider.crawl('https://loop.io/'); + + expect(result.company.name).toBe('Loop'); + expect(result.people).toHaveLength(1); + // The whole economic argument for the hybrid: structured data is free. + expect(modelCalls).toBe(0); + expect(result.usedSignals).not.toContain('model'); + }); + + test('a bespoke page falls through to the model', async () => { + const model: ExtractionModel = { + generate: async () => ({ + text: '{"company":{"name":"Handmade Ltd"},"people":[{"fullName":"Sam Rivers","title":"Founder"}]}', + refused: false, + }), + }; + + const provider = new SiteProvider({ + model, + fetchImpl: stubFetch({ + 'https://handmade.example/robots.txt': ROBOTS, + 'https://handmade.example/': { + body: '<html><body><h1>We build things</h1><p>Sam Rivers, Founder</p></body></html>', + }, + }), + }); + + const result = await provider.crawl('https://handmade.example/'); + + expect(result.company.name).toBe('Handmade Ltd'); + expect(result.people[0]!.fullName).toBe('Sam Rivers'); + expect(result.usedSignals).toContain('model'); + }); + + test('without a model it degrades to deterministic-only', async () => { + const provider = new SiteProvider({ + fetchImpl: stubFetch({ + 'https://plain.example/robots.txt': ROBOTS, + 'https://plain.example/': { body: '<html><body><p>Nothing structured</p></body></html>' }, + }), + }); + + const result = await provider.crawl('https://plain.example/'); + + expect(result.outcome).toBe('ok'); + expect(result.people).toHaveLength(0); + }); + + test('a blocked page is a result, not a throw', async () => { + const provider = new SiteProvider({ + fetchImpl: stubFetch({ + 'https://blocked.example/robots.txt': { + body: 'User-agent: *\nDisallow: /', + type: 'text/plain', + }, + }), + }); + + const result = await provider.crawl('https://blocked.example/'); + + expect(result.outcome).toBe('robots_denied'); + expect(result.people).toHaveLength(0); + }); + + test('a bare domain is normalised to https', () => { + expect(normaliseUrl('example.com')).toBe('https://example.com'); + expect(normaliseUrl('http://example.com')).toBe('http://example.com'); + }); + + test('it does not claim it can search', () => { + expect(new SiteProvider().capabilities().canSearch).toBe(false); + }); +});