Merge main into feat/tech-stack

# Conflicts:
#	packages/providers/src/index.ts

Author: ralyodio

apps/api/src/mailboxes.test.ts

@@ -5,8 +5,8 @@
 
 import { afterEach, describe, expect, test } from 'bun:test';
 import type { Hono } from 'hono';
-import { now, type Client } from '@outreachgraph/db';
-import { recordReplyCheck } from '@outreachgraph/pipeline';
+import { now, queryAll, type Client } from '@outreachgraph/db';
+import { recordReplyCheck, sweepBlacklists } from '@outreachgraph/pipeline';
 import { generateSecretKey, parseSecretKey } from '@outreachgraph/secrets';
 import { createApp } from './app';
 import type { AppEnv, RequestActor } from './context';
@@ -137,6 +137,51 @@ describe('GET /mailboxes', () => {
     expect(body.mailboxes.find((m) => m.id === ana.id)!.repliesError).toBeNull();
   });
 
+  test('the daily blocklist sweep records a listing, alerts once, and drops health', async () => {
+    const { app, db } = await harness('mailboxes-blocklist');
+    await call(app, 'PUT', '/integrations/email', {
+      host: 'mail.acme.test',
+      port: 587,
+      secure: false,
+      username: 'cy@acme.test',
+      password: 'pw',
+      fromEmail: 'cy@acme.test',
+      imapHost: 'mail.acme.test',
+      skipVerification: true,
+    });
+
+    const resolve4 = async (name: string): Promise<string[]> => {
+      if (name === 'acme.test.dbl.spamhaus.org') return ['127.0.1.2'];
+      throw Object.assign(new Error('not found'), { code: 'ENOTFOUND' });
+    };
+    const before = (
+      (await (await call(app, 'GET', '/mailboxes')).json()) as { mailboxes: MailboxJson[] }
+    ).mailboxes[0]!;
+
+    const swept = await sweepBlacklists(db, SEED.workspaceId, { deps: { resolve4 } });
+    expect(swept).toEqual({ checked: 1, listed: 1 });
+
+    const after = (
+      (await (await call(app, 'GET', '/mailboxes')).json()) as {
+        mailboxes: Array<MailboxJson & { blacklistedOn: string[] }>;
+      }
+    ).mailboxes[0]!;
+    expect(after.blacklistedOn).toEqual(['Spamhaus DBL']);
+    expect(after.healthIssues[0]).toContain('Spamhaus DBL');
+    expect(after.healthScore).toBeLessThan(before.healthScore);
+
+    // Checked today: the next sweep leaves it alone, so no second alert.
+    expect(await sweepBlacklists(db, SEED.workspaceId, { deps: { resolve4 } })).toEqual({
+      checked: 0,
+      listed: 0,
+    });
+    const alerts = await queryAll<{ message: string }>(
+      db,
+      `SELECT message FROM workflow_events WHERE message LIKE '%is listed on%'`,
+    );
+    expect(alerts).toHaveLength(1);
+  });
+
   test('404s the DNS report for a mailbox in another workspace', async () => {
     const { app } = await harness('mailboxes-dns-404');
     expect((await call(app, 'GET', '/mailboxes/ita_nope/dns')).status).toBe(404);

apps/cli/src/commands.ts

@@ -1729,10 +1729,13 @@ export const COMMANDS: readonly Command[] = [
         };
         return [
           `${text(dns, 'domain')}: ${text(dns, 'status')}`,
-          ...['spf', 'dkim', 'dmarc', 'mx'].map((key) => {
-            const check = (dns[key] ?? {}) as Record<string, unknown>;
-            return `  ${pad(key.toUpperCase(), 6)} ${pad(text(check, 'status'), 5)} ${text(check, 'detail')}`;
-          }),
+          ...['spf', 'dkim', 'dmarc', 'mx', 'blacklist']
+            .filter((key) => dns[key] !== undefined)
+            .map((key) => {
+              const check = (dns[key] ?? {}) as Record<string, unknown>;
+              const label = key === 'blacklist' ? 'BL' : key.toUpperCase();
+              return `  ${pad(label, 6)} ${pad(text(check, 'status'), 5)} ${text(check, 'detail')}`;
+            }),
         ].join('\n');
       }
 

apps/server/src/index.ts

@@ -70,6 +70,7 @@ import {
   workspacesAwaitingPhotos,
   runCadences,
   promoteAbWinners,
+  sweepBlacklists,
   runPlanner,
   bumpQuietThreads,
   runCrawlJob,
@@ -128,6 +129,7 @@ const AB_SWEEP_MS = 3_600_000;
 const lastAbSweepAt = new Map<string, number>();
 const lastBumpSweepAt = new Map<string, number>();
 const lastPlannerSweepAt = new Map<string, number>();
+const lastBlacklistSweepAt = new Map<string, number>();
 
 /** Last successful poll per workspace, so the slower clock survives a tick. */
 const lastPolledAt = new Map<string, number>();
@@ -1390,6 +1392,23 @@ async function tick(): Promise<void> {
       }
     }
 
+    // Deliverability: each active mailbox's domain (and self-hosted server)
+    // against the public blocklists, once a day, a few per run. A new
+    // listing is an error in the live feed and drops the mailbox's health.
+    if (Date.now() - (lastBlacklistSweepAt.get(workspace.id) ?? 0) >= AB_SWEEP_MS) {
+      lastBlacklistSweepAt.set(workspace.id, Date.now());
+      try {
+        const swept = await sweepBlacklists(db, workspace.id);
+        if (swept.listed > 0) {
+          console.warn(
+            `blocklists ${workspace.id}: ${swept.listed} of ${swept.checked} mailbox(es) listed`,
+          );
+        }
+      } catch (error) {
+        console.error(`blocklist sweep failed for ${workspace.id}`, error);
+      }
+    }
+
     // The Outreach Planner: this month's plays, launched once per product per
     // month from engagement data. Checked hourly so a play whose segment was
     // empty on the 1st still launches once the data arrives.

apps/web/app/(app)/planner/page.tsx

@@ -87,6 +87,12 @@ export default async function PlannerPage() {
             This month builds fresh lists: every product’s seeds are re-read.
           </p>
         )}
+        {!view.tracksEngagement ? (
+          <p className="text-ink-muted pt-1 text-xs">
+            Opens and clicks are not tracked (plain-text email, as the planner recommends), so plays
+            pick people by delivery and replies: written to, delivered, no answer.
+          </p>
+        ) : null}
         <p className="text-ink-muted pt-1 text-xs">Next month: {view.next.label}</p>
       </section>
 

apps/web/components/mailboxes-view.tsx

@@ -536,6 +536,7 @@ interface DnsReportView {
   spf: DnsCheckView;
   dkim: DnsCheckView;
   dmarc: DnsCheckView;
+  blacklist?: DnsCheckView;
   status: 'pass' | 'warn' | 'fail';
 }
 
@@ -571,6 +572,9 @@ function DnsTable({ report, error }: { report?: DnsReportView; error?: string })
     ['DKIM', report.dkim],
     ['DMARC', report.dmarc],
     ['MX', report.mx],
+    ...(report.blacklist
+      ? ([['Blocklists', report.blacklist]] as Array<[string, DnsCheckView]>)
+      : []),
   ];
 
   return (

apps/web/lib/api.ts

@@ -858,6 +858,7 @@ export interface PlannerView {
     readonly sequence: string;
   }>;
   readonly next: { readonly period: string; readonly label: string };
+  readonly tracksEngagement: boolean;
   readonly offerings: ReadonlyArray<{
     readonly offeringId: string;
     readonly name: string;

migrations-pg/0056_mailbox_blacklists.sql

@@ -0,0 +1,10 @@
+-- 0056_mailbox_blacklists.sql (Postgres). See migrations/0056_mailbox_blacklists.sql for the reasoning.
+-- One new, empty table. Nothing existing is altered.
+
+create table if not exists mailbox_blacklist_checks (
+  account_id text PRIMARY KEY REFERENCES integration_accounts(id) ON DELETE CASCADE,
+  workspace_id text NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
+  listed_on text NOT NULL DEFAULT '[]',
+  results_json text NOT NULL DEFAULT '[]',
+  checked_at text NOT NULL
+);

migrations/0056_mailbox_blacklists.sql

@@ -0,0 +1,15 @@
+-- 0056_mailbox_blacklists.sql
+--
+-- The deliverability check, daily, by the worker: is a mailbox's sending
+-- domain (or its self-hosted SMTP server's IP) on a public blocklist?
+-- One row per mailbox, the latest check. listed_on is a JSON array of list
+-- names, empty when clean; results_json keeps every answer, including the
+-- lists that refused to answer, so "clean" and "could not check" differ.
+
+CREATE TABLE IF NOT EXISTS mailbox_blacklist_checks (
+  account_id    TEXT PRIMARY KEY REFERENCES integration_accounts(id) ON DELETE CASCADE,
+  workspace_id  TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
+  listed_on     TEXT NOT NULL DEFAULT '[]',
+  results_json  TEXT NOT NULL DEFAULT '[]',
+  checked_at    TEXT NOT NULL
+);

packages/domain/src/mailbox-health.ts

@@ -119,6 +119,8 @@ export interface MailboxHealthInput {
   readonly replyCheckFailed: boolean;
   /** Warm-up progress, 0 to 1; 1 when off or finished. */
   readonly warmupProgress: number;
+  /** Public blocklists that name the sending domain or server, from the daily check. */
+  readonly blacklistedOn?: readonly string[];
 }
 
 export interface MailboxHealth {
@@ -164,6 +166,13 @@ export function mailboxHealth(input: MailboxHealthInput): MailboxHealth {
     issues.push('Could not read the inbox on the last check');
   }
 
+  if (input.blacklistedOn && input.blacklistedOn.length > 0) {
+    score -= 40;
+    issues.unshift(
+      `Listed on ${input.blacklistedOn.join(', ')}: request delisting before sending more`,
+    );
+  }
+
   if (progress < 1) issues.push('Warming up');
 
   return { score: Math.max(0, Math.min(100, score)), bounceRisk: risk, issues };

packages/pipeline/src/index.ts

@@ -135,8 +135,11 @@ export {
 } from './warmup-network';
 export {
   detectMailbox,
+  blacklistCheck,
   listMailboxes,
   mailboxDns,
+  recordBlacklistCheck,
+  sweepBlacklists,
   recordReplyCheck,
   MailboxDetectError,
   type DetectedMailbox,