Publish the read side as @profullstack/crawlproof, and make crawlproof a command (#241)

* Make `crawlproof` a command, not a cd and an npm incantation

The CLI had no bin and was not on PATH, so every command in its own help and
in this PR's description was a lie unless you first cd'd into the checkout and
typed `npm run cli --`. `crawlproof dashboard` now works from anywhere.

bin/crawlproof.mjs resolves the repo from its own location and runs tsx against
cli/index.ts, preferring the checkout's tsx and falling back to npx so a fresh
clone still starts. It relays the child's signal rather than inventing an exit
code, so a TUI killed with ctrl+c does not look like a clean exit.

The token now also falls back to the `token` field of ~/.crawlproof.json, the
same shape and the same reasoning as ~/.coinpay.json: using the CLI should not
be conditional on remembering to export a secret first.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HvWJ4336pxTFRdRbvsTQeD

* Publish the read side as @profullstack/crawlproof

A dashboard you can only run by cd'ing into a Next.js checkout is not installed
anywhere. packages/cli publishes the two commands a box actually wants —
`dashboard` and `stats` — so cli-tools can vendor it the way it vendors hqtui,
and any machine can `npm i -g @profullstack/crawlproof`.

Only those two, deliberately. Both are token-authed and pure HTTP, which is
what makes them publishable: `audit` needs the audit engines and their model
SDKs, and `sweep` needs a cron secret. Those stay in the repo.

The package is a **build artifact**, not a second copy. lib/dashboard/* and
cli/dashboard.ts stay where the suite already covers them and where the in-repo
CLI already imports them; build.mjs bundles the same files with esbuild and
leaves hqtui and the CoinPay SDK external, since both have their own release
cadence. There is one implementation and it cannot drift.

The stats printer moves to lib/dashboard/stats-text.ts and returns a string
rather than writing to stdout, because two CLIs print it now and a printer that
owns the process can be neither shared nor tested.

Verified from the bundle against production: `crawlproof stats crawlproof.com`
reads the token out of ~/.crawlproof.json with nothing exported, and the
dashboard collects all 52 sites with zero source errors.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HvWJ4336pxTFRdRbvsTQeD

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: ralyodio

bin/crawlproof.mjs

@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+// The `crawlproof` entry point.
+//
+// The CLI is TypeScript inside a Next.js app rather than a built package, so
+// running it means running tsx against cli/index.ts. Everything here is about
+// finding the repo and the right tsx no matter where the caller stood, because
+// the alternative — telling people to cd into a checkout and type
+// `npm run cli --` — is a setup step, and those do not survive contact.
+
+import { spawn } from "node:child_process";
+import { existsSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const repo = resolve(here, "..");
+const entry = join(repo, "cli", "index.ts");
+
+if (!existsSync(entry)) {
+  console.error(`crawlproof: cannot find ${entry}`);
+  console.error("The launcher must sit in bin/ inside the crawlproof.com checkout.");
+  process.exit(1);
+}
+
+// Prefer the checkout's own tsx; fall back to npx so a fresh clone still runs.
+const localTsx = join(repo, "node_modules", ".bin", "tsx");
+const [cmd, prefix] = existsSync(localTsx) ? [localTsx, []] : ["npx", ["--yes", "tsx"]];
+
+const child = spawn(cmd, [...prefix, entry, ...process.argv.slice(2)], {
+  cwd: repo,
+  stdio: "inherit",
+  env: process.env,
+});
+
+child.on("error", (err) => {
+  console.error(`crawlproof: could not start tsx (${err.message})`);
+  console.error(`Run \`npm install\` in ${repo}.`);
+  process.exit(1);
+});
+
+// Relay the child's fate rather than inventing one: a TUI killed by ctrl+c
+// should not look like a clean exit to whatever called this.
+child.on("exit", (code, signal) => {
+  if (signal) {
+    process.kill(process.pid, signal);
+    return;
+  }
+  process.exit(code ?? 0);
+});

cli/index.ts

@@ -185,9 +185,25 @@ async function cmdSweep(args: Args): Promise<number> {
 // Both talk to /api/ads/v1/* with a CrawlProof API token (Social → API
 // tokens), the same token the MCP server and the myna plugin use.
 
+/**
+ * The API token, from the flag, the environment, or ~/.crawlproof.json.
+ *
+ * The config file is last and exists so that using the CLI is not conditional
+ * on remembering to export a secret first. Same shape and same reasoning as
+ * ~/.coinpay.json, which `coinpayAuth` below reads.
+ */
 function apiToken(args: Args): string | null {
-  const token = (args.flags.token as string | undefined) ?? process.env.CRAWLPROOF_TOKEN ?? null;
-  return token && token.trim() ? token.trim() : null;
+  const direct = (args.flags.token as string | undefined) ?? process.env.CRAWLPROOF_TOKEN;
+  if (direct && direct.trim()) return direct.trim();
+
+  try {
+    const home = process.env.HOME ?? process.env.USERPROFILE ?? "";
+    const file = process.env.CRAWLPROOF_CONFIG ?? `${home}/.crawlproof.json`;
+    const token = (JSON.parse(readFileSync(file, "utf8")) as { token?: string }).token;
+    return token && token.trim() ? token.trim() : null;
+  } catch {
+    return null;
+  }
 }
 
 function apiBase(args: Args): string {
@@ -364,29 +380,8 @@ async function cmdStats(args: Args): Promise<number> {
     return 0;
   }
 
-  const project = json.project as { name?: string; url?: string } | undefined;
-  const totals = json.totals as { visitors?: number; pageviews?: number } | undefined;
-  const list = (key: string) => (Array.isArray(json[key]) ? (json[key] as { label: string; value: number }[]) : []);
-
-  process.stdout.write(`${project?.name ?? "project"}  ${range}  ${who}\n`);
-  process.stdout.write(`${totals?.visitors ?? 0} visitors, ${totals?.pageviews ?? 0} pageviews\n`);
-
-  const section = (title: string, items: { label: string; value: number }[]) => {
-    if (!items.length) return;
-    process.stdout.write(`\n${title}\n`);
-    const width = Math.min(46, Math.max(...items.map((i) => i.label.length)));
-    for (const item of items.slice(0, 10)) {
-      process.stdout.write(`  ${item.label.slice(0, width).padEnd(width)}  ${item.value}\n`);
-    }
-  };
-  section("Sources", list("sources"));
-  section("Referrers", list("referrers"));
-  section("Pages", list("pages"));
-
-  // Nothing at all is a real answer, and the likeliest cause is worth naming.
-  if (!(totals?.pageviews ?? 0) && !list("sources").length) {
-    process.stdout.write(`\nNothing in this window. Check the tag is on the page, or widen --range.\n`);
-  }
+  const { renderStats } = await import("../lib/dashboard/stats-text");
+  process.stdout.write(renderStats(json as Parameters<typeof renderStats>[0], { range, who }));
   return 0;
 }
 
@@ -593,7 +588,8 @@ ENV
   CRAWLPROOF_SITE_URL    Override the API base URL for 'report', 'sweep', 'track', 'ads' and 'slots'.
   CRAWLPROOF_PROJECT     Default project UUID for 'track'.
   CRAWLPROOF_TOKEN       API token (crp_…) for 'ads', 'slots', 'stats' and
-                         'dashboard'; --token overrides.
+                         'dashboard'; --token overrides. Falls back to the
+                         'token' field of ~/.crawlproof.json.
   COINPAY_SESSION_TOKEN  CoinPay merchant JWT for the money half of
                          'dashboard'. Defaults to jwtToken in ~/.coinpay.json,
                          which 'coinpay auth login' writes.

lib/dashboard/stats-text.ts

@@ -0,0 +1,47 @@
+// Rendering a /api/tracker/v1/stats answer as plain text.
+//
+// Pure and string-returning rather than writing to stdout, because two CLIs
+// print it now — the in-repo one and the published @profullstack/crawlproof —
+// and a printer that owns the process is a printer that cannot be shared or
+// tested.
+
+export type StatsItem = { label: string; value: number };
+
+export type StatsAnswerish = {
+  project?: { name?: string; url?: string } | null;
+  totals?: { visitors?: number; pageviews?: number } | null;
+  sources?: StatsItem[] | null;
+  referrers?: StatsItem[] | null;
+  pages?: StatsItem[] | null;
+};
+
+const list = (v: StatsItem[] | null | undefined): StatsItem[] => (Array.isArray(v) ? v : []);
+
+export function renderStats(
+  answer: StatsAnswerish,
+  { range, who, rows = 10 }: { range: string; who: string; rows?: number },
+): string {
+  const totals = answer.totals ?? {};
+  const out: string[] = [
+    `${answer.project?.name ?? "project"}  ${range}  ${who}`,
+    `${totals.visitors ?? 0} visitors, ${totals.pageviews ?? 0} pageviews`,
+  ];
+
+  const section = (title: string, items: StatsItem[]) => {
+    if (!items.length) return;
+    out.push("", title);
+    const width = Math.min(46, Math.max(...items.map((i) => i.label.length)));
+    for (const item of items.slice(0, rows)) {
+      out.push(`  ${item.label.slice(0, width).padEnd(width)}  ${item.value}`);
+    }
+  };
+  section("Sources", list(answer.sources));
+  section("Referrers", list(answer.referrers));
+  section("Pages", list(answer.pages));
+
+  // Nothing at all is a real answer, and the likeliest cause is worth naming.
+  if (!(totals.pageviews ?? 0) && !list(answer.sources).length) {
+    out.push("", "Nothing in this window. Check the tag is on the page, or widen --range.");
+  }
+  return `${out.join("\n")}\n`;
+}

package.json

@@ -1,6 +1,9 @@
 {
   "name": "crawlproof",
   "version": "0.3.0",
+  "bin": {
+    "crawlproof": "./bin/crawlproof.mjs"
+  },
   "private": true,
   "license": "AGPL-3.0-only",
   "type": "module",

packages/cli/.gitignore

@@ -0,0 +1,2 @@
+dist/
+node_modules/

packages/cli/README.md

@@ -0,0 +1,74 @@
+# @profullstack/crawlproof
+
+What the fleet costs and what it returns, in a terminal.
+
+```
+npm install -g @profullstack/crawlproof
+crawlproof dashboard
+```
+
+Five live screens over three feeds that are not otherwise in the same place:
+CrawlProof's tracker for who arrived, its ad network for what was delivered,
+and CoinPay for what the bank actually did.
+
+| Screen | Answers |
+| --- | --- |
+| ROI | Monthly burn against revenue, cost per reader, break-even |
+| Traffic | Every site on the account, ranked, with its share of the cost |
+| Ads | Delivery as advertiser and as publisher, and ad-driven arrivals |
+| Money | Earnings, bank position, invoices, income vs spending by month |
+| Spend | Who we pay, largest first, and burn by category |
+
+`1`–`5` or Tab switches screens, `w` cycles the window, `b` cycles humans /
+all / bots, `r` refreshes, `?` explains the arithmetic, `q` quits.
+
+## Two rules the numbers keep
+
+**Self-deal is not revenue.** Where an account advertises on its own slots, ad
+spend and ad earnings are one dollar moving between two pockets. They are shown
+under *Internal* and counted as neither cost nor revenue.
+
+**Personal money is not business cost.** A bank feed carries groceries next to
+servers, so cost is the business scope only — joined from each transaction's
+account to that account's scope.
+
+Everything is normalised to a monthly rate and then prorated onto the traffic
+window, because burn is a rate: the traffic side can be asked for an hour while
+the bank side only answers in weeks.
+
+The dashboard also reports what it cannot know. A site that did not answer is
+missing rather than zero. A vendor list built from one page of a longer ledger
+says so. A fleet whose visits run far above its pageviews — a "visitor" is any
+non-crawler hit, which on a site with a machine-readable endpoint runs orders of
+magnitude above pages anyone read — says that next to the number, and offers the
+per-pageview figure instead.
+
+## Commands
+
+```
+crawlproof dashboard [--range=1h|4h|1d|1w|1m] [--who=humans|bots|all]
+                     [--interval=60] [--sites=a.com,b.com] [--concurrency=8]
+                     [--no-coinpay] [--json]
+crawlproof stats [site] [--range=1d] [--who=humans] [--json]
+```
+
+`--json` prints the same snapshot the screens render, for a script or a box
+with no terminal. Aliases for `dashboard`: `roi`, `tui`.
+
+## Auth
+
+| | |
+| --- | --- |
+| `CRAWLPROOF_TOKEN` | API token (`crp_…`) from Social → API tokens. Also read from the `token` field of `~/.crawlproof.json`. `--token` wins. |
+| `CRAWLPROOF_SITE_URL` | API base, default `https://crawlproof.com`. |
+| `COINPAY_SESSION_TOKEN` | CoinPay merchant JWT for the money screens. Defaults to `jwtToken` in `~/.coinpay.json`, which `coinpay auth login` writes. |
+
+Without a CoinPay session the traffic and ads screens still work and the money
+panels say what is missing, rather than showing zero.
+
+Needs Node 22.6 or newer, and a terminal for the dashboard. `--json` needs
+neither.
+
+## License
+
+MIT

packages/cli/build.mjs

@@ -0,0 +1,36 @@
+// Bundle the publishable CLI out of the app's own source.
+//
+// The point of bundling rather than moving files is that lib/dashboard/* and
+// cli/dashboard.ts stay where the test suite already covers them and where the
+// in-repo CLI already imports them. The package is a build artifact of the
+// same code, so there is one implementation and it cannot drift.
+//
+// hqtui and the CoinPay SDK stay external: they are real dependencies with
+// their own release cadence, declared in package.json and installed alongside.
+
+import { build } from "esbuild";
+import { chmod, readFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const out = join(here, "dist", "cli.mjs");
+
+const pkg = JSON.parse(await readFile(join(here, "package.json"), "utf8"));
+
+await build({
+  entryPoints: [join(here, "src", "bin.ts")],
+  outfile: out,
+  bundle: true,
+  platform: "node",
+  format: "esm",
+  target: "node22",
+  // Anything with its own version belongs in package.json, not inlined here.
+  external: [...Object.keys(pkg.dependencies ?? {}), "node:*"],
+  banner: { js: "#!/usr/bin/env node" },
+  legalComments: "none",
+  logLevel: "info",
+});
+
+await chmod(out, 0o755);
+console.log(`built ${out}`);

packages/cli/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@profullstack/crawlproof",
+  "version": "0.1.0",
+  "description": "What the fleet costs and what it returns: a live terminal dashboard over CrawlProof traffic, ad delivery and CoinPay banking.",
+  "license": "MIT",
+  "type": "module",
+  "homepage": "https://crawlproof.com",
+  "bin": {
+    "crawlproof": "./dist/cli.mjs"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=22.6"
+  },
+  "scripts": {
+    "build": "node build.mjs",
+    "prepublishOnly": "node build.mjs"
+  },
+  "dependencies": {
+    "@profullstack/coinpay": "^0.9.0",
+    "@profullstack/hqtui": "^0.1.11"
+  },
+  "devDependencies": {
+    "esbuild": "^0.25.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "crawlproof",
+    "tui",
+    "dashboard",
+    "analytics",
+    "roi"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/profullstack/crawlproof.com.git",
+    "directory": "packages/cli"
+  }
+}

packages/cli/src/bin.ts

@@ -0,0 +1,7 @@
+// The executable entry. No shebang here: build.mjs adds one as a banner, and
+// two of them is a syntax error rather than a harmless duplicate.
+import { main } from "./cli";
+
+main(process.argv.slice(2)).then((code) => {
+  process.exitCode = code ?? 0;
+});