diff --git a/apps/web/components/add-leads.tsx b/apps/web/components/add-leads.tsx index 5b43249..90d77c9 100644 --- a/apps/web/components/add-leads.tsx +++ b/apps/web/components/add-leads.tsx @@ -16,7 +16,11 @@ import { applyMapping, parseCsv, type MappedRow } from '../lib/csv'; * named as such rather than quietly sent to or quietly dropped. */ -const MAX_ROWS = 5_000; +/** The most one file may hold. Sent in chunks, so this is not a request size. */ +const MAX_ROWS = 100_000; + +/** Rows per request: the API takes at most 5,000 leads per call. */ +const CHUNK_ROWS = 5_000; interface ReportRow { readonly row: number | null; @@ -39,7 +43,44 @@ interface AppendResult { readonly flagged_held: number; readonly report: readonly ReportRow[]; readonly report_truncated: boolean; - readonly report_url: string; + /** Absent when the file went up in several chunks; the report is built here instead. */ + readonly report_url?: string; +} + +/** Adds one chunk's result to the running total, row numbers shifted to the file's. */ +function mergeResults(total: AppendResult | undefined, next: AppendResult, offset: number) { + const shifted = next.report.map((row) => ({ + ...row, + row: row.row === null ? null : row.row + offset, + })); + if (!total) return { ...next, report: shifted }; + return { + received: total.received + next.received, + added: total.added + next.added, + imported: total.imported + next.imported, + merged: total.merged + next.merged, + updated: total.updated + next.updated, + rejected: total.rejected + next.rejected, + skipped: total.skipped + next.skipped, + flagged: total.flagged + next.flagged, + flagged_held: total.flagged_held + next.flagged_held, + report: [...total.report, ...shifted], + report_truncated: total.report_truncated || next.report_truncated, + } satisfies AppendResult; +} + +/** The combined report as a CSV, for a file sent in several chunks. */ +function reportCsv(report: readonly ReportRow[]): string { + const cell = (value: string | number | null) => { + const text = value === null ? '' : String(value); + return /[",\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; + }; + return [ + 'row,email,outcome,reason,why,detail', + ...report.map((r) => + [r.row, r.email, r.outcome, r.reason, r.why, r.detail].map(cell).join(','), + ), + ].join('\n'); } const OUTCOME_LABEL: Readonly> = { @@ -60,6 +101,7 @@ export function AddLeads({ campaignId }: { campaignId: string }) { const [busy, setBusy] = useState(false); const [result, setResult] = useState(); const [error, setError] = useState(); + const [progress, setProgress] = useState(); function onFile(event: ChangeEvent): void { const file = event.target.files?.[0]; @@ -84,7 +126,9 @@ export function AddLeads({ campaignId }: { campaignId: string }) { } if (body.length > MAX_ROWS) { setRows([]); - setError(`${body.length.toLocaleString()} rows: split the file into ${MAX_ROWS} or fewer.`); + setError( + `${body.length.toLocaleString()} rows: the most one file can hold is ${MAX_ROWS.toLocaleString()}.`, + ); return; } setMapping(found); @@ -96,45 +140,71 @@ export function AddLeads({ campaignId }: { campaignId: string }) { async function run(): Promise { setBusy(true); setError(undefined); + setResult(undefined); + let total: AppendResult | undefined; + const chunks = Math.ceil(rows.length / CHUNK_ROWS); try { - const response = await fetch( - `/api/v1/autogtm/campaigns/${encodeURIComponent(campaignId)}/leads`, - { - method: 'POST', - headers: { 'content-type': 'application/json' }, - credentials: 'same-origin', - body: JSON.stringify({ - filename, - consent_source: consentSource, - allow_flagged: allowFlagged, - leads: rows.map((row) => ({ - email: row.email, - name: row.name, - first_name: row.firstName, - last_name: row.lastName, - company: row.company, - company_domain: row.companyDomain, - job_title: row.title, - location: row.location, - linkedin_url: row.linkedinUrl, - updated_at: row.updatedAt, - })), - }), - }, - ); - const payload = (await response.json().catch(() => ({}))) as AppendResult & { - error?: { message?: string }; - }; - if (!response.ok) { - setError(payload.error?.message ?? `That failed (${response.status}).`); - return; + // One request per 5,000 rows, in order. Re-running after a failure is + // safe: leads already added are skipped. + for (let i = 0; i < chunks; i += 1) { + const offset = i * CHUNK_ROWS; + const chunk = rows.slice(offset, offset + CHUNK_ROWS); + if (chunks > 1) { + setProgress( + `Sending rows ${(offset + 1).toLocaleString()}-${(offset + chunk.length).toLocaleString()} of ${rows.length.toLocaleString()}…`, + ); + } + const response = await fetch( + `/api/v1/autogtm/campaigns/${encodeURIComponent(campaignId)}/leads`, + { + method: 'POST', + headers: { 'content-type': 'application/json' }, + credentials: 'same-origin', + body: JSON.stringify({ + filename, + consent_source: consentSource, + allow_flagged: allowFlagged, + leads: chunk.map((row) => ({ + email: row.email, + name: row.name, + first_name: row.firstName, + last_name: row.lastName, + company: row.company, + company_domain: row.companyDomain, + job_title: row.title, + location: row.location, + linkedin_url: row.linkedinUrl, + updated_at: row.updatedAt, + })), + }), + }, + ); + const payload = (await response.json().catch(() => ({}))) as AppendResult & { + error?: { message?: string }; + }; + if (!response.ok) { + if (total) setResult(total); + setError( + `${payload.error?.message ?? `That failed (${response.status}).`}` + + (i > 0 ? ` Stopped at row ${(offset + 1).toLocaleString()}; re-run to finish.` : ''), + ); + return; + } + total = mergeResults(total, payload, offset); + } + if (total && chunks > 1) { + const { report_url: _single, ...combined } = total; + void _single; + total = combined; } - setResult(payload); + setResult(total); setRows([]); router.refresh(); } catch { + if (total) setResult(total); setError('Lost the connection. Re-running is safe: leads already added are skipped.'); } finally { + setProgress(undefined); setBusy(false); } } @@ -205,6 +275,8 @@ export function AddLeads({ campaignId }: { campaignId: string }) { ) : null} + {progress ?

{progress}

: null} + {result ? : null} {error ? ( @@ -262,7 +334,14 @@ function ImportReportView({ result }: { result: AppendResult }) { ) : null} - + Download the full report (CSV)