Skip to content

Commit e7e40a7

Browse files
ralyodioclaude
ralyodio
and
claude
authored
Declared visitors card + crawlproof actors extension (CLI 0.5.0) (#342)
* feat(tracker): declared visitors card + `crawlproof actors extension` (CLI 0.5.0) Dashboard: the project stats page gets a "Declared visitors" card after the headline tiles: declared humans / agents, contradictions in red, and named actors (only the viewer's own or public ones, via declaredSummary, the same filter the API uses). With nothing declared it is one muted line linking to Settings -> Declared actors. CLI: `crawlproof actors extension <email|id> [--out] [--label]` mints a token and writes an unpacked MV3 extension (lib/tracker/declareExtension) whose single declarativeNetRequest rule sets Crawlproof-Actor on <base>/api/track only, so an agent's browser declares itself without a code change and the sites it visits never see the token. Verified end to end: generated for riotcoder, loaded in Chrome for Testing, the visit counted on the actor. Fix: `actors revoke --token=<id>` collided with the CLI-wide --token (API key override) and was sent as the bearer: 401 "Malformed token". Now --token-id. Broken since 0.4.0. Docs: the extension, how to load it, and the branded-Chrome caveat. @profullstack/crawlproof 0.4.0 -> 0.5.0. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: build the fake API key instead of a credential-shaped literal ThreatCrush flagged the regression test's fixture as a hardcoded credential. It never was one; constructing it the way the rest of the suite does keeps the scanner signal clean without a dismissal. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent d4e617c commit e7e40a7

10 files changed

Lines changed: 375 additions & 9 deletions

File tree

‎app/(app)/dashboard/projects/[id]/stats/declared-card.tsx‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import Link from "next/link";
2+
import type { DeclaredSummary } from "@/lib/tracker/actorStore";
3+
4+
// Declared actors on this site (lib/tracker/actors.ts): what visitors SAID
5+
// they are, kept apart from the measured tiles above it. Names appear only
6+
// for the viewer's own actors or ones their owners made public; everyone else
7+
// is in the per-kind totals and nowhere else (declaredSummary does the
8+
// filtering, the same function the API and CLI read).
9+
export function DeclaredCard({ summary, rangeLabel }: { summary: DeclaredSummary | null; rangeLabel: string }) {
10+
// Null means the actor tables are unreadable (or not migrated): say nothing
11+
// rather than print zeros that read as "nobody declared".
12+
if (!summary) return null;
13+
const { human, agent } = summary.totals;
14+
const contradictions = human.contradictions + agent.contradictions;
15+
16+
if (human.events + agent.events === 0) {
17+
return (
18+
<p className="text-xs text-[var(--color-muted)]">
19+
No declared visitors in this window.{" "}
20+
<Link href="/dashboard/settings/actors" className="underline hover:text-[var(--color-foreground)]">
21+
Declare yourself or your agents →
22+
</Link>
23+
</p>
24+
);
25+
}
26+
27+
return (
28+
<section className="card p-4 space-y-3">
29+
<div className="flex flex-wrap items-baseline justify-between gap-2">
30+
<div>
31+
<h2 className="text-lg font-semibold">Declared visitors</h2>
32+
<p className="text-sm text-[var(--color-muted)]">
33+
{rangeLabel}. Self-reported, opt-in: an agent is believed and counted as a bot; a human
34+
never overrides bot detection.
35+
</p>
36+
</div>
37+
<Link href="/dashboard/settings/actors" className="text-sm underline hover:text-[var(--color-foreground)]">
38+
Manage actors →
39+
</Link>
40+
</div>
41+
42+
<div className="grid gap-3 sm:grid-cols-3">
43+
<Tile label="Declared humans" kind={human} />
44+
<Tile label="Declared agents" kind={agent} />
45+
<div className="rounded-md border border-[var(--color-border)] p-3">
46+
<div className="text-xs text-[var(--color-muted)]">Contradictions</div>
47+
<div className={`mt-1 text-2xl font-extrabold ${contradictions ? "text-[var(--color-fail)]" : ""}`}>
48+
{contradictions.toLocaleString()}
49+
</div>
50+
<div className="text-xs text-[var(--color-muted)]">
51+
Hits declared human that detection called a bot
52+
</div>
53+
</div>
54+
</div>
55+
56+
{summary.actors.length > 0 && (
57+
<div className="overflow-x-auto">
58+
<table className="w-full text-sm">
59+
<thead>
60+
<tr className="text-left text-xs text-[var(--color-muted)]">
61+
<th className="py-1 pr-3 font-medium">Actor</th>
62+
<th className="py-1 pr-3 font-medium">Kind</th>
63+
<th className="py-1 pr-3 font-medium text-right">Pageviews</th>
64+
<th className="py-1 pr-3 font-medium text-right">Events</th>
65+
<th className="py-1 font-medium text-right">Contradicted</th>
66+
</tr>
67+
</thead>
68+
<tbody className="divide-y divide-[var(--color-border)]">
69+
{summary.actors.map((a) => (
70+
<tr key={`${a.email}-${a.kind}`}>
71+
<td className="py-1.5 pr-3">
72+
{a.name || a.email}
73+
{a.name && <span className="ml-1 text-xs text-[var(--color-muted)]">{a.email}</span>}
74+
{!a.mine && <span className="ml-1 text-xs text-[var(--color-muted)]">(public)</span>}
75+
</td>
76+
<td className="py-1.5 pr-3">{a.kind}</td>
77+
<td className="py-1.5 pr-3 text-right tabular-nums">{a.pageviews.toLocaleString()}</td>
78+
<td className="py-1.5 pr-3 text-right tabular-nums">{a.events.toLocaleString()}</td>
79+
<td className={`py-1.5 text-right tabular-nums ${a.contradictions ? "text-[var(--color-fail)]" : ""}`}>
80+
{a.contradictions.toLocaleString()}
81+
</td>
82+
</tr>
83+
))}
84+
</tbody>
85+
</table>
86+
</div>
87+
)}
88+
{summary.actors.length === 0 && (
89+
<p className="text-xs text-[var(--color-muted)]">
90+
None of these actors are yours or public, so only the totals are shown.
91+
</p>
92+
)}
93+
</section>
94+
);
95+
}
96+
97+
function Tile({ label, kind }: { label: string; kind: { actors: number; events: number; pageviews: number } }) {
98+
return (
99+
<div className="rounded-md border border-[var(--color-border)] p-3">
100+
<div className="text-xs text-[var(--color-muted)]">{label}</div>
101+
<div className="mt-1 text-2xl font-extrabold">{kind.actors.toLocaleString()}</div>
102+
<div className="text-xs text-[var(--color-muted)]">
103+
{kind.pageviews.toLocaleString()} pageviews, {kind.events.toLocaleString()} events
104+
</div>
105+
</div>
106+
);
107+
}

‎app/(app)/dashboard/projects/[id]/stats/page.tsx‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ import { AutoInstall } from "./auto-install";
2323
import { LiveVisitors } from "./live-visitors";
2424
import { StatsSubnav } from "./stats-subnav";
2525
import { WhoToggle } from "./who-toggle";
26+
import { DeclaredCard } from "./declared-card";
27+
import { serviceClient } from "@/lib/supabase/service";
28+
import { declaredSummary } from "@/lib/tracker/actorStore";
29+
import { DECLARED_DEFINITION } from "@/lib/tracker/actors";
2630
import { getOrMintInstallationToken } from "@/lib/github/installations";
2731
import { listInstallationRepos } from "@/lib/github/app";
2832

@@ -122,6 +126,14 @@ export default async function ProjectStatsPage({
122126
// no connected installations, we just hide the button.
123127
const ghConfigured = !!(env.githubAppId && env.githubAppPrivateKey);
124128
const { data: { user } } = await supabase.auth.getUser();
129+
130+
// Declared actors (opt-in, self-reported). Read with the service client
131+
// because naming an actor needs a join the viewer's RLS cannot see;
132+
// declaredSummary itself filters names to the viewer's own or public ones.
133+
// Best-effort: a failure hides the card instead of breaking the page.
134+
const declared = user
135+
? await declaredSummary(serviceClient(), user.id, id, range, DECLARED_DEFINITION).catch(() => null)
136+
: null;
125137
const installations: Array<{ installation_id: number; account_login: string }> = [];
126138
const ghRepos: Array<{
127139
full_name: string;
@@ -295,6 +307,8 @@ export default async function ProjectStatsPage({
295307
<p className="-mt-1 text-xs text-[var(--color-muted)]">{visitorsCaption}</p>
296308
)}
297309

310+
<DeclaredCard summary={declared} rangeLabel={range.description} />
311+
298312
{grandTotal === 0 && eventTotal === 0 ? (
299313
<section className="card p-4">
300314
<p className="text-sm text-[var(--color-muted)]">

‎app/(marketing)/docs/statistics/page.tsx‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,22 @@ https://example.com/?crp_actor=cpa_…
147147
148148
# or from page code
149149
window.crawlproof?.("actor", "cpa_…"); // null forgets it`}</pre>
150+
<p className="text-sm leading-relaxed">
151+
For an agent&apos;s browser, generate an extension instead of changing
152+
its code. It sends the token on{" "}
153+
<code className="font-mono">/api/track</code> requests and nothing
154+
else, so the sites the agent visits never see it (a blanket extra
155+
header on every request would hand them the token):
156+
</p>
157+
<pre className="overflow-x-auto rounded border border-[var(--color-border)] bg-[#0b0d10] p-3 font-mono text-xs leading-relaxed">{`crawlproof actors extension mybot@example.com --out=./crawlproof-declare
158+
159+
chromium --load-extension=./crawlproof-declare --disable-extensions-except=./crawlproof-declare
160+
# chrome-devtools-mcp: --chromeArg=--load-extension=<dir> --chromeArg=--disable-extensions-except=<dir>
161+
# Playwright: launchPersistentContext(profile, { args: [the same two flags] })`}</pre>
162+
<p className="text-sm leading-relaxed">
163+
Use Chromium or Chrome for Testing: branded Google Chrome 137 and
164+
later ignores <code className="font-mono">--load-extension</code>.
165+
</p>
150166
<p className="text-sm leading-relaxed">
151167
It is self-reported, so the rule is one-way: a declared agent is
152168
believed and counted as a bot; a declared human is recorded but never

‎cli/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1057,7 +1057,7 @@ async function main() {
10571057
return await runActors(args.positional, args.flags as Record<string, string | boolean>, (method, path, body) => apiCall(args, method, path, body), {
10581058
write: (line: string) => process.stdout.write(`${line}\n`),
10591059
error: (line: string) => console.error(line),
1060-
});
1060+
}, { base: apiBase(args) });
10611061
case "dashboard":
10621062
case "roi":
10631063
case "tui":

‎lib/tracker/actorsCli.ts‎

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@
88
// Model and trust rule: lib/tracker/actors.ts. Opt-in and self-reported; an
99
// agent is believed, a human never overrides bot detection.
1010

11+
import { chmodSync, mkdirSync, writeFileSync } from "node:fs";
12+
import { join, resolve } from "node:path";
13+
import { declareExtensionFiles } from "./declareExtension";
14+
1115
type Method = "GET" | "POST" | "PATCH" | "DELETE";
1216
type ApiCall = (method: Method, path: string, body?: Record<string, unknown>) => Promise<{ status: number; json: Record<string, unknown> }>;
1317
type Out = { write: (line: string) => void; error: (line: string) => void };
@@ -27,7 +31,8 @@ export const ACTORS_USAGE = ` actors [list] [--json]
2731
actors add <email> --kind=human|agent [--name=…] [--operator=<human email>]
2832
[--public] [--token-label=…] [--no-token] [--json]
2933
actors token <email|id> [--label=…]
30-
actors revoke <email|id> [--token=<token id>]
34+
actors revoke <email|id> [--token-id=<id>]
35+
actors extension <email|id> [--out=./crawlproof-declare] [--label=…]
3136
Declared actors: say who you are, and whether you are a person, on every
3237
site with the CrawlProof tracker. Opt-in and self-reported. A token
3338
(cpa_…) is the credential, never the email; send it as the
@@ -36,6 +41,13 @@ export const ACTORS_USAGE = ` actors [list] [--json]
3641
is counted as a contradiction. Names are visible to you only unless
3742
--public. Your login address is verified on creation; any other gets a
3843
verification email. Needs an API token.
44+
45+
\`actors extension\` mints a token and writes an unpacked Chrome
46+
extension that sends it on <site>/api/track requests ONLY, for an agent's
47+
browser: chrome --load-extension=<dir> --disable-extensions-except=<dir>.
48+
(A blanket extra header on every request would hand the token to every
49+
site the agent visits.) Chromium or Chrome for Testing; branded Chrome
50+
137+ ignores --load-extension.
3951
`;
4052

4153
/** How to send a fresh actor token. Pure, for tests. */
@@ -57,6 +69,7 @@ export async function runActors(
5769
flags: Record<string, string | boolean>,
5870
call: ApiCall,
5971
out: Out,
72+
opts: { base?: string } = {},
6073
): Promise<number> {
6174
const sub = positional[0] ?? "list";
6275
const json = Boolean(flags.json);
@@ -141,10 +154,12 @@ export async function runActors(
141154
if (r.status >= 400) return fail("revoke", r);
142155
const actor = find(actors, positional[1]);
143156
if (!actor) {
144-
out.error("usage: crawlproof actors revoke <email|id> [--token=<token id>] (no --token revokes the actor and every token)");
157+
out.error("usage: crawlproof actors revoke <email|id> [--token-id=<id>] (no --token-id revokes the actor and every token)");
145158
return 2;
146159
}
147-
const tokenId = typeof flags.token === "string" ? flags.token : undefined;
160+
// Not --token: both CLIs read --token as the API key override, so a token id
161+
// there was sent as the bearer and came back 401 "Malformed token".
162+
const tokenId = typeof flags["token-id"] === "string" ? flags["token-id"] : undefined;
148163
const d = tokenId
149164
? await call("DELETE", `/api/tracker/v1/actors/${actor.id}/tokens?token=${encodeURIComponent(tokenId)}`)
150165
: await call("DELETE", `/api/tracker/v1/actors/${actor.id}`);
@@ -153,6 +168,39 @@ export async function runActors(
153168
return 0;
154169
}
155170

156-
out.error(`unknown: crawlproof actors ${sub} (expected: list | add | token | revoke)`);
171+
if (sub === "extension") {
172+
const { r, actors } = await list();
173+
if (r.status >= 400) return fail("extension", r);
174+
const actor = find(actors, positional[1]);
175+
if (!actor) {
176+
out.error("usage: crawlproof actors extension <email|id> [--out=./crawlproof-declare] [--label=…] (crawlproof actors list shows yours)");
177+
return 2;
178+
}
179+
const dir = resolve(typeof flags.out === "string" ? flags.out : "crawlproof-declare");
180+
const label = typeof flags.label === "string" ? flags.label : "browser extension";
181+
const m = await call("POST", `/api/tracker/v1/actors/${actor.id}/tokens`, { label });
182+
if (m.status >= 400) return fail("extension", m);
183+
const files = declareExtensionFiles({
184+
token: String(m.json.token),
185+
base: opts.base ?? "https://crawlproof.com",
186+
who: `${actor.name || actor.email} (${actor.kind})`,
187+
});
188+
mkdirSync(dir, { recursive: true, mode: 0o700 });
189+
chmodSync(dir, 0o700);
190+
for (const [name, body] of Object.entries(files)) writeFileSync(join(dir, name), body, { mode: 0o600 });
191+
if (json) {
192+
out.write(JSON.stringify({ dir, token_id: m.json.id, prefix: m.json.prefix }, null, 2));
193+
return 0;
194+
}
195+
out.write(`wrote ${dir} for ${actor.email} (${actor.kind}), token ${String(m.json.prefix)}… id ${String(m.json.id)}`);
196+
out.write("");
197+
out.write(` chrome --load-extension=${dir} --disable-extensions-except=${dir} …`);
198+
out.write(` chrome-devtools-mcp --chromeArg=--load-extension=${dir} --chromeArg=--disable-extensions-except=${dir}`);
199+
out.write("");
200+
out.write(`Revoke: crawlproof actors revoke ${actor.email} --token-id=${String(m.json.id)}`);
201+
return 0;
202+
}
203+
204+
out.error(`unknown: crawlproof actors ${sub} (expected: list | add | token | revoke | extension)`);
157205
return 2;
158206
}

‎lib/tracker/declareExtension.ts‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
// The "declare me" Chrome extension for an agent's (or a person's) browser.
2+
//
3+
// An unpacked Manifest V3 extension with one declarativeNetRequest rule: set
4+
// `Crawlproof-Actor: <cpa_ token>` on requests to <base>/api/track and nothing
5+
// else. Scoping the header to the beacon endpoint is the point. A blanket
6+
// "extra header on every request" (Playwright extraHTTPHeaders, Puppeteer
7+
// setExtraHTTPHeaders) hands the token to every site the agent visits, and any
8+
// of them could replay it to pose as that agent.
9+
//
10+
// No dependencies, so both CLIs can bundle it. `crawlproof actors extension`
11+
// writes these files; Chrome loads them with --load-extension=<dir>.
12+
13+
export type ExtensionFiles = Record<"manifest.json" | "rules.json" | "README.txt", string>;
14+
15+
/** "https://crawlproof.com/" -> "https://crawlproof.com". Throws on a non-http(s) base. */
16+
export function trackOrigin(base: string): string {
17+
const u = new URL(base);
18+
if (u.protocol !== "https:" && u.protocol !== "http:") throw new Error(`not an http(s) URL: ${base}`);
19+
return u.origin;
20+
}
21+
22+
export function declareExtensionFiles(input: { token: string; base: string; who: string }): ExtensionFiles {
23+
if (!/^cpa_[A-Za-z0-9_-]{32,124}$/.test(input.token)) throw new Error("not a cpa_ token");
24+
const origin = trackOrigin(input.base);
25+
const manifest = {
26+
manifest_version: 3,
27+
name: "CrawlProof declared actor",
28+
version: "1.0.0",
29+
description: `Declares this browser's visits as ${input.who} to the CrawlProof tracker. Adds one header to ${origin}/api/track requests only.`,
30+
// WithHostAccess: modifyHeaders needs host access to the request URL and
31+
// to the page that sends it, which can be any tracked site.
32+
permissions: ["declarativeNetRequestWithHostAccess"],
33+
host_permissions: ["<all_urls>"],
34+
declarative_net_request: {
35+
rule_resources: [{ id: "declare", enabled: true, path: "rules.json" }],
36+
},
37+
};
38+
const rules = [
39+
{
40+
id: 1,
41+
priority: 1,
42+
action: {
43+
type: "modifyHeaders",
44+
requestHeaders: [{ header: "Crawlproof-Actor", operation: "set", value: input.token }],
45+
},
46+
condition: {
47+
// Left-anchored on the full origin + path: a lookalike host or a page
48+
// whose own URL merely contains this string does not match.
49+
urlFilter: `|${origin}/api/track`,
50+
resourceTypes: ["xmlhttprequest", "ping", "other"],
51+
},
52+
},
53+
];
54+
const readme = [
55+
`CrawlProof declared actor: ${input.who}`,
56+
"",
57+
`Every page this browser loads that runs the CrawlProof tracker is counted as`,
58+
`${input.who}. The token goes only to ${origin}/api/track.`,
59+
"",
60+
"Load it:",
61+
" chrome --load-extension=$PWD --disable-extensions-except=$PWD ...",
62+
" Puppeteer: args: [`--load-extension=${dir}`, `--disable-extensions-except=${dir}`]",
63+
" Playwright: chromium.launchPersistentContext(profile, { args: [same two flags] })",
64+
" chrome-devtools-mcp: --chromeArg=--load-extension=<dir> --chromeArg=--disable-extensions-except=<dir>",
65+
"",
66+
"Branded Google Chrome 137+ ignores --load-extension; use Chromium or Chrome for Testing.",
67+
"rules.json holds the token: keep this folder private (chmod 700).",
68+
"Revoke: crawlproof actors revoke <email> --token-id=<token id>",
69+
"",
70+
].join("\n");
71+
return {
72+
"manifest.json": `${JSON.stringify(manifest, null, 2)}\n`,
73+
"rules.json": `${JSON.stringify(rules, null, 2)}\n`,
74+
"README.txt": readme,
75+
};
76+
}

‎packages/cli/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@profullstack/crawlproof",
3-
"version": "0.4.0",
3+
"version": "0.5.0",
44
"description": "What the fleet costs and what it returns: a live terminal dashboard over CrawlProof traffic, ad delivery and CoinPay banking.",
55
"license": "MIT",
66
"type": "module",

‎packages/cli/src/cli.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import { FINANCE_DAYS, runDashboard } from "../../../cli/dashboard";
1717
import { EMAIL_TRACKING_USAGE, runEmailTracking } from "../../../lib/emailTracking/cli";
1818
import { ACTORS_USAGE, runActors } from "../../../lib/tracker/actorsCli";
1919

20-
export const VERSION = "0.4.0";
20+
export const VERSION = "0.5.0";
2121

2222
type Args = {
2323
command: string;
@@ -420,7 +420,7 @@ export async function main(argv: string[]): Promise<number> {
420420
return await runActors(args.positional, args.flags, (method, path, body) => apiCall(args, method, path, body), {
421421
write: (line: string) => process.stdout.write(`${line}\n`),
422422
error: (line: string) => console.error(line),
423-
});
423+
}, { base: apiBase(args) });
424424
case "email-tracking":
425425
return await runEmailTracking(args.positional, args.flags, (method, path) => apiCall(args, method, path), {
426426
write: (line: string) => process.stdout.write(`${line}\n`),

0 commit comments

Comments
 (0)