From ab3019603401c863007bb0e7e9200519442a151a Mon Sep 17 00:00:00 2001
From: Anthony Ettinger <anthony@profullstack.com>
Date: Wed, 7 Oct 2026 05:56:33 +0000
Subject: [PATCH] List sources: retry a timed-out search instead of closing the
 week

In production most ValueSERP news searches took over 60 s, timed out, and
were recorded as done for the week, so funding and appointment scans for
most products never ran. A scan that errored without results is now left
unrecorded and tried on the next hourly sweep (still counted against the
daily cap), and news searches get a 150 s timeout.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 packages/pipeline/src/list-sources.test.ts | 28 ++++++++++++++++++++++
 packages/pipeline/src/list-sources.ts      | 24 ++++++++++++-------
 packages/providers/src/valueserp/index.ts  |  7 ++++--
 3 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/packages/pipeline/src/list-sources.test.ts b/packages/pipeline/src/list-sources.test.ts
index d0d8d96..3d3f0cb 100644
--- a/packages/pipeline/src/list-sources.test.ts
+++ b/packages/pipeline/src/list-sources.test.ts
@@ -102,6 +102,34 @@ describe('scanListSources', () => {
     expect(calls.length).toBe(before);
   });
 
+  test('a timed-out search is tried again on the next sweep, not closed for the week', async () => {
+    seeded = await seedDatabase('lists-retry');
+    const { db } = seeded;
+    await productIsPlanned(db);
+    const { searcher } = fakeSearcher();
+    let fail = true;
+    const flaky = {
+      search: searcher.search,
+      async searchNews(query: string) {
+        if (fail) throw new Error('ValueSERP news did not answer within 150 s');
+        return searcher.searchNews(query);
+      },
+    };
+
+    const first = await scanListSources({ db, searcher: flaky, now: AT }, SEED.workspaceId);
+    expect(first.filter((scan) => scan.error).map((scan) => scan.kind)).toEqual([
+      'funding',
+      'leadership',
+    ]);
+
+    fail = false;
+    const second = await scanListSources({ db, searcher: flaky, now: AT }, SEED.workspaceId);
+    expect(second.map((scan) => [scan.kind, scan.items])).toEqual([
+      ['funding', 1],
+      ['leadership', 0],
+    ]);
+  });
+
   test('stops at the daily search cap', async () => {
     seeded = await seedDatabase('lists-cap');
     const { db } = seeded;
diff --git a/packages/pipeline/src/list-sources.ts b/packages/pipeline/src/list-sources.ts
index 1401c5d..fffb354 100644
--- a/packages/pipeline/src/list-sources.ts
+++ b/packages/pipeline/src/list-sources.ts
@@ -132,15 +132,21 @@ export async function scanListSources(
           break outer;
         }
       }
-      await recordRun(db, {
-        workspaceId,
-        offeringId: offering.id,
-        kind,
-        key,
-        items,
-        ...(error ? { error } : {}),
-        at,
-      });
+      // A failed search (ValueSERP news routinely takes over a minute) is not
+      // a finished scan: left unrecorded, it is tried again on the next hourly
+      // sweep, each try still counted against the daily cap. Only a scan that
+      // completed — even one that found nothing — closes the week.
+      if (!error || items > 0) {
+        await recordRun(db, {
+          workspaceId,
+          offeringId: offering.id,
+          kind,
+          key,
+          items,
+          ...(error ? { error } : {}),
+          at,
+        });
+      }
       scans.push({ offeringId: offering.id, kind, items, ...(error ? { error } : {}) });
     }
   }
diff --git a/packages/providers/src/valueserp/index.ts b/packages/providers/src/valueserp/index.ts
index 9753e7f..7fe66df 100644
--- a/packages/providers/src/valueserp/index.ts
+++ b/packages/providers/src/valueserp/index.ts
@@ -177,8 +177,11 @@ export class ValueSerpClient implements ProfilePhotoFinder, WebSearcher, NewsSea
     url.searchParams.set('gl', 'us');
     url.searchParams.set('hl', 'en');
 
+    // Google News through ValueSERP is slower than web search: answers of
+    // 60-120 s are normal, and a minute's timeout lost most of them.
+    const newsTimeoutMs = Math.max(this.searchTimeoutMs, 150_000);
     const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), this.searchTimeoutMs);
+    const timer = setTimeout(() => controller.abort(), newsTimeoutMs);
     try {
       const response = await this.fetchImpl(url, {
         signal: controller.signal,
@@ -216,7 +219,7 @@ export class ValueSerpClient implements ProfilePhotoFinder, WebSearcher, NewsSea
       );
     } catch (error) {
       if (controller.signal.aborted) {
-        throw new Error(`ValueSERP did not answer within ${this.searchTimeoutMs / 1000} s`);
+        throw new Error(`ValueSERP news did not answer within ${newsTimeoutMs / 1000} s`);
       }
       throw error;
     } finally {