From 0c56741cce64104320d9d03eb8a70035e1290b58 Mon Sep 17 00:00:00 2001
From: Anthony Ettinger <anthony@chovy.com>
Date: Tue, 6 Oct 2026 01:23:55 +0000
Subject: [PATCH] fix(gateway): charge the Lightpanda headless browser like a
 training crawler

Lightpanda/1.0 was 86% of p0dcasters traffic on 2026-10-06 (21,263 of
24,787 requests in an hour) from 999 residential addresses. Each one loads
a page plus its prefetches (~81 requests in 27 s) and leaves, so it never
crosses the 100/min per-address allowance. It runs JavaScript, so every
visit showed up as a live reader in analytics (322 "active users").

It names itself, so it goes on the TRAINING list: 402 with the crawl-pass
offer, and robots.txt now names it too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 src/lib/crawl-gateway.ts    |  9 ++++++++-
 test/crawl-gateway.test.mjs | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)
 create mode 100644 test/crawl-gateway.test.mjs

diff --git a/src/lib/crawl-gateway.ts b/src/lib/crawl-gateway.ts
index 8223656..f4ffd8e 100644
--- a/src/lib/crawl-gateway.ts
+++ b/src/lib/crawl-gateway.ts
@@ -18,7 +18,14 @@ function env(key: string): string | undefined {
 // Meta-ExternalFetcher, Applebot-Extended / Applebot). The extras below are the
 // agents the old hand-written robots.txt named that the library does not, kept
 // on the record so an auditor grepping for them still finds a decision.
-export const TRAINING = [...TRAINING_AGENTS, "cohere-ai"];
+//
+// Lightpanda is a headless browser, not a crawler with an operator page, but on
+// 2026-10-06 it was 86% of the site's traffic: 999 residential addresses in one
+// hour, each loading one page plus its prefetches (~81 requests in 27 s) and
+// leaving. That stays under the per-address allowance in lib/throttle.ts, and
+// because it runs JavaScript every visit counted as a live reader in the
+// analytics (322 "active users"). It names itself, so it is charged like GPTBot.
+export const TRAINING = [...TRAINING_AGENTS, "cohere-ai", "Lightpanda"];
 export const RETRIEVAL = [
   ...RETRIEVAL_AGENTS,
   "Applebot",
diff --git a/test/crawl-gateway.test.mjs b/test/crawl-gateway.test.mjs
new file mode 100644
index 0000000..58d435f
--- /dev/null
+++ b/test/crawl-gateway.test.mjs
@@ -0,0 +1,35 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { x402Proxy } from '@profullstack/x402-gateway/next';
+import { gateway } from '../src/lib/crawl-gateway.ts';
+
+const gate = x402Proxy(gateway);
+
+const CHROME =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36';
+
+function req(path, headers) {
+  return new Request(`https://p0dcasters.com${path}`, {
+    headers: { 'x-forwarded-for': '37.231.157.22', ...headers },
+  });
+}
+
+test('Lightpanda is charged like a training crawler', async () => {
+  const res = await gate(req('/podcast/broken-heart', { 'user-agent': 'Lightpanda/1.0' }));
+  assert.equal(res?.status, 402);
+});
+
+test('Lightpanda is charged on the prefetch requests too', async () => {
+  const res = await gate(req('/browse?_rsc=wnvFOH2u9NsTKups', { 'user-agent': 'Lightpanda/1.0', rsc: '1' }));
+  assert.equal(res?.status, 402);
+});
+
+test('a real browser is untouched', async () => {
+  const res = await gate(req('/podcast/broken-heart', { 'user-agent': CHROME, 'sec-fetch-mode': 'navigate' }));
+  assert.equal(res ?? undefined, undefined);
+});
+
+test('Lightpanda may still read robots.txt', async () => {
+  const res = await gate(req('/robots.txt', { 'user-agent': 'Lightpanda/1.0' }));
+  assert.notEqual(res?.status, 402);
+});