List sources: retry a timed-out search instead of closing the week

packages/pipeline/src/list-sources.test.ts

@@ -102,6 +102,34 @@ describe('scanListSources', () => {
     expect(calls.length).toBe(before);
   });
 
+  test('a timed-out search is tried again on the next sweep, not closed for the week', async () => {
+    seeded = await seedDatabase('lists-retry');
+    const { db } = seeded;
+    await productIsPlanned(db);
+    const { searcher } = fakeSearcher();
+    let fail = true;
+    const flaky = {
+      search: searcher.search,
+      async searchNews(query: string) {
+        if (fail) throw new Error('ValueSERP news did not answer within 150 s');
+        return searcher.searchNews(query);
+      },
+    };
+
+    const first = await scanListSources({ db, searcher: flaky, now: AT }, SEED.workspaceId);
+    expect(first.filter((scan) => scan.error).map((scan) => scan.kind)).toEqual([
+      'funding',
+      'leadership',
+    ]);
+
+    fail = false;
+    const second = await scanListSources({ db, searcher: flaky, now: AT }, SEED.workspaceId);
+    expect(second.map((scan) => [scan.kind, scan.items])).toEqual([
+      ['funding', 1],
+      ['leadership', 0],
+    ]);
+  });
+
   test('stops at the daily search cap', async () => {
     seeded = await seedDatabase('lists-cap');
     const { db } = seeded;

packages/pipeline/src/list-sources.ts

@@ -132,15 +132,21 @@ export async function scanListSources(
           break outer;
         }
       }
-      await recordRun(db, {
-        workspaceId,
-        offeringId: offering.id,
-        kind,
-        key,
-        items,
-        ...(error ? { error } : {}),
-        at,
-      });
+      // A failed search (ValueSERP news routinely takes over a minute) is not
+      // a finished scan: left unrecorded, it is tried again on the next hourly
+      // sweep, each try still counted against the daily cap. Only a scan that
+      // completed — even one that found nothing — closes the week.
+      if (!error || items > 0) {
+        await recordRun(db, {
+          workspaceId,
+          offeringId: offering.id,
+          kind,
+          key,
+          items,
+          ...(error ? { error } : {}),
+          at,
+        });
+      }
       scans.push({ offeringId: offering.id, kind, items, ...(error ? { error } : {}) });
     }
   }

packages/providers/src/valueserp/index.ts

@@ -177,8 +177,11 @@ export class ValueSerpClient implements ProfilePhotoFinder, WebSearcher, NewsSea
     url.searchParams.set('gl', 'us');
     url.searchParams.set('hl', 'en');
 
+    // Google News through ValueSERP is slower than web search: answers of
+    // 60-120 s are normal, and a minute's timeout lost most of them.
+    const newsTimeoutMs = Math.max(this.searchTimeoutMs, 150_000);
     const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), this.searchTimeoutMs);
+    const timer = setTimeout(() => controller.abort(), newsTimeoutMs);
     try {
       const response = await this.fetchImpl(url, {
         signal: controller.signal,
@@ -216,7 +219,7 @@ export class ValueSerpClient implements ProfilePhotoFinder, WebSearcher, NewsSea
       );
     } catch (error) {
       if (controller.signal.aborted) {
-        throw new Error(`ValueSERP did not answer within ${this.searchTimeoutMs / 1000} s`);
+        throw new Error(`ValueSERP news did not answer within ${newsTimeoutMs / 1000} s`);
       }
       throw error;
     } finally {