Charge the Lightpanda headless browser like a training crawler

src/lib/crawl-gateway.ts

@@ -18,7 +18,14 @@ function env(key: string): string | undefined {
 // Meta-ExternalFetcher, Applebot-Extended / Applebot). The extras below are the
 // agents the old hand-written robots.txt named that the library does not, kept
 // on the record so an auditor grepping for them still finds a decision.
-export const TRAINING = [...TRAINING_AGENTS, "cohere-ai"];
+//
+// Lightpanda is a headless browser, not a crawler with an operator page, but on
+// 2026-10-06 it was 86% of the site's traffic: 999 residential addresses in one
+// hour, each loading one page plus its prefetches (~81 requests in 27 s) and
+// leaving. That stays under the per-address allowance in lib/throttle.ts, and
+// because it runs JavaScript every visit counted as a live reader in the
+// analytics (322 "active users"). It names itself, so it is charged like GPTBot.
+export const TRAINING = [...TRAINING_AGENTS, "cohere-ai", "Lightpanda"];
 export const RETRIEVAL = [
   ...RETRIEVAL_AGENTS,
   "Applebot",

test/crawl-gateway.test.mjs

@@ -0,0 +1,35 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { x402Proxy } from '@profullstack/x402-gateway/next';
+import { gateway } from '../src/lib/crawl-gateway.ts';
+
+const gate = x402Proxy(gateway);
+
+const CHROME =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36';
+
+function req(path, headers) {
+  return new Request(`https://p0dcasters.com${path}`, {
+    headers: { 'x-forwarded-for': '37.231.157.22', ...headers },
+  });
+}
+
+test('Lightpanda is charged like a training crawler', async () => {
+  const res = await gate(req('/podcast/broken-heart', { 'user-agent': 'Lightpanda/1.0' }));
+  assert.equal(res?.status, 402);
+});
+
+test('Lightpanda is charged on the prefetch requests too', async () => {
+  const res = await gate(req('/browse?_rsc=wnvFOH2u9NsTKups', { 'user-agent': 'Lightpanda/1.0', rsc: '1' }));
+  assert.equal(res?.status, 402);
+});
+
+test('a real browser is untouched', async () => {
+  const res = await gate(req('/podcast/broken-heart', { 'user-agent': CHROME, 'sec-fetch-mode': 'navigate' }));
+  assert.equal(res ?? undefined, undefined);
+});
+
+test('Lightpanda may still read robots.txt', async () => {
+  const res = await gate(req('/robots.txt', { 'user-agent': 'Lightpanda/1.0' }));
+  assert.notEqual(res?.status, 402);
+});