Skip to content

Commit 3edaa04

Browse files
ralyodioclaude
andauthored
Lead analytics with human visits; show bot crawls apart (#230)
Every headline number on /dashboard, /dashboard/analytics and the per-project stats page was bot-inclusive. On one property 99% of ~257k weekly hits were a single AI training crawler, and the card read as "80k pageviews a day" for a site with a few hundred real readers. Definition, used in SQL and in lib/tracker/humans.ts: a hit is a human when its bucket does NOT start with `bot:` (AI referrals, search, social, referral, direct), a bot when it does. humans + bots = events, exactly. Migration 20260905120000_tracker_human_split.sql (NOT applied yet): - new dashboard_project_traffic(p_project_ids, p_since) -> (project_id, day, humans, bots) from tracker_daily_stats - tracker_project_totals gains humans / prev_humans - tracker_daily_series and tracker_daily_series_multi gain humans - tracker_project_daily_series gains humans and bots - tracker_recent_series gains humans All are drop + create (return types change), same argument signatures, columns only appended, security invoker, grants re-issued. The three *_multi functions carrying `set work_mem` are deliberately untouched. UI: - /dashboard cards: "{n} human visits" headline, "{n} bot hits · Past 7 days" beneath, sparkline of humans, read from dashboard_project_traffic - /dashboard/analytics: tiles are Human visits, AI referrals, Bot crawls, All events; verdict, ranking, per-property trend and sparklines are all human-based; the By property table has Humans / Previous / Change / Trend / AI / Bots; the portfolio chart stacks human visits per property with bot crawls as one dashed unstacked overlay - project stats: tiles are Human visits, AI referrals, Bot crawls; Traffic pulse stacks humans + bots (the bot-inclusive pageview band is gone), with AI referrals and interactions as overlays - every labelled figure carries the definition as hover text and a caption Before the migration is applied the missing `humans` column is derived as events - bots, which is exact for every bucket-based leg, so the pages are correct either way. `humans` is never backfilled from the bot-inclusive event table on pre-bucket days. Tests: lib/tracker/totals.ts and humans.ts are covered in tests/tracker-human-split.test.ts; portfolio-analytics.test.ts covers the new series field and the fallback. Claude-Session: https://claude.ai/code/session_01WJaXiqE9BDoNfoJBhfXroC Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
1 parent bc0d804 commit 3edaa04

12 files changed

Lines changed: 975 additions & 142 deletions

File tree

‎app/(app)/dashboard/analytics/page.tsx‎

Lines changed: 115 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,29 @@ import {
1515
type Trend,
1616
type TrendDirection,
1717
} from "@/lib/tracker/trend";
18+
import {
19+
emptyTotals,
20+
sumTotals,
21+
toProjectTotals,
22+
totalsTrends,
23+
type ProjectTotals,
24+
type TotalsRow,
25+
} from "@/lib/tracker/totals";
26+
import {
27+
AI_REFERRALS_DEFINITION,
28+
ALL_EVENTS_DEFINITION,
29+
BOTS_DEFINITION,
30+
BOTS_LABEL,
31+
HUMANS_DEFINITION,
32+
HUMANS_LABEL,
33+
humansFrom,
34+
} from "@/lib/tracker/humans";
1835
import {
1936
TrackerAnalytics,
2037
type TrackerListItem,
2138
} from "@/components/charts/tracker-analytics";
2239
import {
40+
BOTS_KEY,
2341
OTHER_KEY,
2442
PortfolioTrend,
2543
type PortfolioPoint,
@@ -53,29 +71,14 @@ type PortfolioProject = {
5371
organization_id?: string | null;
5472
};
5573

56-
type TotalsRow = {
57-
project_id: string;
58-
events: number | string;
59-
ai: number | string;
60-
bots: number | string;
61-
prev_events: number | string;
62-
prev_ai: number | string;
63-
prev_bots: number | string;
64-
};
65-
66-
type ProjectTotals = {
67-
events: number;
68-
ai: number;
69-
bots: number;
70-
prevEvents: number;
71-
prevAi: number;
72-
prevBots: number;
73-
};
74-
74+
// Every figure on this page leads with humans (bucket not `bot:`, AI referrals
75+
// included) and shows bot crawls apart. See lib/tracker/humans.ts.
7576
type ProjectRow = {
7677
project: PortfolioProject;
7778
totals: ProjectTotals;
79+
/** Human visits, this window vs the one before. */
7880
trend: Trend;
81+
/** Daily human visits for the sparkline; null when over the row budget. */
7982
samples: number[] | null;
8083
};
8184

@@ -208,33 +211,21 @@ export default async function PortfolioAnalyticsPage({
208211

209212
const totalsByProject = new Map<string, ProjectTotals>();
210213
for (const row of (totalsRes.data ?? []) as TotalsRow[]) {
211-
totalsByProject.set(row.project_id, {
212-
events: Number(row.events),
213-
ai: Number(row.ai),
214-
bots: Number(row.bots),
215-
prevEvents: Number(row.prev_events),
216-
prevAi: Number(row.prev_ai),
217-
prevBots: Number(row.prev_bots),
218-
});
214+
totalsByProject.set(row.project_id, toProjectTotals(row));
219215
}
220216

221217
const portfolio = sumTotals(totalsByProject.values());
222-
const eventsTrend = computeTrend(portfolio.events, portfolio.prevEvents);
223-
const aiTrend = computeTrend(portfolio.ai, portfolio.prevAi);
224-
const botsTrend = computeTrend(portfolio.bots, portfolio.prevBots);
225-
const otherTrend = computeTrend(
226-
otherVisits(portfolio.events, portfolio.ai, portfolio.bots),
227-
otherVisits(portfolio.prevEvents, portfolio.prevAi, portfolio.prevBots),
228-
);
218+
const trends = totalsTrends(portfolio);
229219

230-
// Ranked by current-window volume: the biggest properties get the chart
231-
// bands, and the daily-detail budget is spent on them first.
220+
// Ranked by current-window HUMAN volume: the biggest properties get the
221+
// chart bands, and the daily-detail budget is spent on them first. Ranking
222+
// by events would hand the top band to whichever site a crawler is hitting.
232223
const ranked = projects
233224
.map((project) => ({
234225
project,
235226
totals: totalsByProject.get(project.id) ?? emptyTotals(),
236227
}))
237-
.sort((a, b) => b.totals.events - a.totals.events);
228+
.sort((a, b) => b.totals.humans - a.totals.humans);
238229

239230
const detailCount = Math.max(
240231
CHART_BANDS,
@@ -251,45 +242,54 @@ export default async function PortfolioAnalyticsPage({
251242
perProjectError,
252243
);
253244

245+
// Per-project daily HUMAN counts. The RPC also returns events and bots per
246+
// row; only humans feed the bands and sparklines.
254247
const axis = utcDayAxis(days);
255248
const dailyByProject = new Map<string, Map<string, number>>();
256249
for (const id of detailIds) dailyByProject.set(id, new Map());
257250
for (const row of (perProjectRaw ?? []) as Array<{
258251
project_id: string;
259252
day: string;
260253
events: number | string;
254+
humans?: number | string | null;
255+
bots?: number | string | null;
261256
}>) {
262-
dailyByProject.get(row.project_id)?.set(row.day, Number(row.events));
257+
dailyByProject
258+
.get(row.project_id)
259+
?.set(row.day, humansFrom({ humans: row.humans, events: row.events, bots: row.bots ?? 0 }));
263260
}
264261

265262
const rows: ProjectRow[] = ranked.map(({ project, totals }) => {
266263
const byDay = dailyByProject.get(project.id);
267264
return {
268265
project,
269266
totals,
270-
trend: computeTrend(totals.events, totals.prevEvents),
267+
trend: computeTrend(totals.humans, totals.prevHumans),
271268
samples: byDay ? axis.map((day) => byDay.get(day) ?? 0) : null,
272269
};
273270
});
274271

275272
const withTraffic = rows.filter(
276-
(r) => r.totals.events > 0 || r.totals.prevEvents > 0,
273+
(r) => r.totals.humans > 0 || r.totals.prevHumans > 0,
277274
);
278275
const verdict = portfolioVerdict(
279-
eventsTrend,
276+
trends.humans,
280277
withTraffic.map((r) => r.trend.direction),
281278
);
282279

283-
// Stacked bands for the top properties; "Other" is the exact remainder of
284-
// the full-portfolio series, so the stack always sums to the real total.
285-
const banded = ranked.slice(0, CHART_BANDS).filter((r) => r.totals.events > 0);
280+
// Stacked bands of human visits for the top properties; "Other" is the
281+
// exact remainder of the full-portfolio human series, so the stack always
282+
// sums to the real human total. Bot crawls ride on top as one unstacked
283+
// dashed line so a crawler storm stays visible without swamping the bands.
284+
const banded = ranked.slice(0, CHART_BANDS).filter((r) => r.totals.humans > 0);
286285
const bandIds = banded.map((r) => r.project.id);
287286
const chartSeries: PortfolioSeries[] = banded.map((r) => ({
288287
key: r.project.id,
289288
name: r.project.name,
290289
}));
291290
const hasOther = ranked.length > banded.length;
292291
if (hasOther) chartSeries.push({ key: OTHER_KEY, name: "Other properties" });
292+
chartSeries.push({ key: BOTS_KEY, name: "Bot crawls (all properties)", kind: "overlay" });
293293

294294
const chartData: PortfolioPoint[] = daily.map((point) => {
295295
const row: PortfolioPoint = { date: point.date };
@@ -299,7 +299,8 @@ export default async function PortfolioAnalyticsPage({
299299
row[id] = value;
300300
bandedTotal += value;
301301
}
302-
if (hasOther) row[OTHER_KEY] = Math.max(0, point.events - bandedTotal);
302+
if (hasOther) row[OTHER_KEY] = Math.max(0, point.humans - bandedTotal);
303+
row[BOTS_KEY] = point.bots;
303304
return row;
304305
});
305306

@@ -421,7 +422,7 @@ export default async function PortfolioAnalyticsPage({
421422
const topOperatingSystems = topDeviceItems(deviceRows, (row) => row.os);
422423

423424
const missingSparklines = rows.filter(
424-
(r) => r.samples === null && r.totals.events > 0,
425+
(r) => r.samples === null && r.totals.humans > 0,
425426
).length;
426427

427428
return (
@@ -444,10 +445,30 @@ export default async function PortfolioAnalyticsPage({
444445
</section>
445446

446447
<div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-4">
447-
<TrendMetric label="All events" trend={eventsTrend} tone="accent" />
448-
<TrendMetric label="AI referrals" trend={aiTrend} tone="pass" />
449-
<TrendMetric label="AI / bot crawls" trend={botsTrend} tone="warn" />
450-
<TrendMetric label="Other visits" trend={otherTrend} tone="muted" />
448+
<TrendMetric
449+
label={HUMANS_LABEL}
450+
trend={trends.humans}
451+
tone="accent"
452+
hint={HUMANS_DEFINITION}
453+
/>
454+
<TrendMetric
455+
label="AI referrals"
456+
trend={trends.ai}
457+
tone="pass"
458+
hint={AI_REFERRALS_DEFINITION}
459+
/>
460+
<TrendMetric
461+
label={BOTS_LABEL}
462+
trend={trends.bots}
463+
tone="warn"
464+
hint={BOTS_DEFINITION}
465+
/>
466+
<TrendMetric
467+
label="All events"
468+
trend={trends.events}
469+
tone="muted"
470+
hint={ALL_EVENTS_DEFINITION}
471+
/>
451472
</div>
452473

453474
{portfolio.events === 0 && portfolio.prevEvents === 0 ? (
@@ -464,11 +485,18 @@ export default async function PortfolioAnalyticsPage({
464485
<div>
465486
<h2 className="text-lg font-semibold">Portfolio trend</h2>
466487
<p className="text-sm text-[var(--color-muted)]">
467-
Daily events, stacked by property.
488+
Daily human visits, stacked by property. Bot crawls across
489+
every property are the dashed line, kept out of the stack.
468490
</p>
469491
</div>
470492
<span className="text-xs text-[var(--color-muted)]">
471-
{portfolio.events.toLocaleString()} events
493+
<span title={HUMANS_DEFINITION}>
494+
{portfolio.humans.toLocaleString()} human visits
495+
</span>
496+
{" · "}
497+
<span title={BOTS_DEFINITION}>
498+
{portfolio.bots.toLocaleString()} bot hits
499+
</span>
472500
</span>
473501
</div>
474502
<PortfolioTrend data={chartData} series={chartSeries} />
@@ -479,7 +507,9 @@ export default async function PortfolioAnalyticsPage({
479507
<div>
480508
<h2 className="text-lg font-semibold">By property</h2>
481509
<p className="text-sm text-[var(--color-muted)]">
482-
Each site against its own previous {days} days.
510+
Human visits per site against its own previous {days} days.
511+
AI referrals are part of the human figure; bot crawls are
512+
not.
483513
</p>
484514
</div>
485515
<Link
@@ -603,11 +633,20 @@ function ProjectTrendTable({ rows }: { rows: ProjectRow[] }) {
603633
<thead>
604634
<tr className="border-b border-[var(--color-border)] text-left text-xs text-[var(--color-muted)]">
605635
<th className="py-2 pr-3 font-medium">Property</th>
606-
<th className="py-2 pr-3 text-right font-medium">Events</th>
607-
<th className="py-2 pr-3 text-right font-medium">Previous</th>
636+
<th className="py-2 pr-3 text-right font-medium" title={HUMANS_DEFINITION}>
637+
Humans
638+
</th>
639+
<th className="py-2 pr-3 text-right font-medium" title={HUMANS_DEFINITION}>
640+
Previous
641+
</th>
608642
<th className="py-2 pr-3 text-right font-medium">Change</th>
609643
<th className="py-2 pr-3 font-medium">Trend</th>
610-
<th className="py-2 text-right font-medium">AI / bots</th>
644+
<th className="py-2 pr-3 text-right font-medium" title={AI_REFERRALS_DEFINITION}>
645+
AI
646+
</th>
647+
<th className="py-2 text-right font-medium" title={BOTS_DEFINITION}>
648+
Bots
649+
</th>
611650
</tr>
612651
</thead>
613652
<tbody>
@@ -637,11 +676,11 @@ function ProjectTrendTable({ rows }: { rows: ProjectRow[] }) {
637676
</span>
638677
</Link>
639678
</td>
640-
<td className="py-2 pr-3 text-right tabular-nums">
641-
{totals.events.toLocaleString()}
679+
<td className="py-2 pr-3 text-right tabular-nums" title={HUMANS_DEFINITION}>
680+
{totals.humans.toLocaleString()}
642681
</td>
643682
<td className="py-2 pr-3 text-right tabular-nums text-[var(--color-muted)]">
644-
{totals.prevEvents.toLocaleString()}
683+
{totals.prevHumans.toLocaleString()}
645684
</td>
646685
<td className="py-2 pr-3 text-right">
647686
<TrendChip trend={trend} />
@@ -653,8 +692,11 @@ function ProjectTrendTable({ rows }: { rows: ProjectRow[] }) {
653692
<span className="text-xs text-[var(--color-muted)]">—</span>
654693
)}
655694
</td>
656-
<td className="py-2 text-right tabular-nums text-[var(--color-muted)]">
657-
{totals.ai.toLocaleString()} / {totals.bots.toLocaleString()}
695+
<td className="py-2 pr-3 text-right tabular-nums text-[var(--color-muted)]" title={AI_REFERRALS_DEFINITION}>
696+
{totals.ai.toLocaleString()}
697+
</td>
698+
<td className="py-2 text-right tabular-nums text-[var(--color-muted)]" title={BOTS_DEFINITION}>
699+
{totals.bots.toLocaleString()}
658700
</td>
659701
</tr>
660702
))}
@@ -685,19 +727,24 @@ function TrendMetric({
685727
label,
686728
trend,
687729
tone,
730+
hint,
688731
}: {
689732
label: string;
690733
trend: Trend;
691734
tone: "accent" | "pass" | "warn" | "muted";
735+
/** What this number counts; shown on hover and under the figure. */
736+
hint?: string;
692737
}) {
693738
const color =
694739
tone === "pass"
695740
? "text-green-600"
696741
: tone === "warn"
697742
? "text-yellow-600"
698-
: "text-[var(--color-foreground)]";
743+
: tone === "muted"
744+
? "text-[var(--color-muted)]"
745+
: "text-[var(--color-foreground)]";
699746
return (
700-
<div className="card p-4">
747+
<div className="card p-4" title={hint}>
701748
<p className="text-xs text-[var(--color-muted)]">{label}</p>
702749
<p className={`mt-1 text-2xl font-bold ${color}`}>
703750
{trend.current.toLocaleString()}
@@ -706,6 +753,11 @@ function TrendMetric({
706753
<TrendChip trend={trend} />
707754
<span>from {trend.previous.toLocaleString()}</span>
708755
</p>
756+
{hint && (
757+
<p className="mt-1 text-[11px] leading-snug text-[var(--color-muted)]">
758+
{hint}
759+
</p>
760+
)}
709761
</div>
710762
);
711763
}
@@ -731,29 +783,6 @@ function parseRange(value: string | undefined): Range {
731783
: DEFAULT_RANGE;
732784
}
733785

734-
function emptyTotals(): ProjectTotals {
735-
return { events: 0, ai: 0, bots: 0, prevEvents: 0, prevAi: 0, prevBots: 0 };
736-
}
737-
738-
function sumTotals(all: Iterable<ProjectTotals>): ProjectTotals {
739-
const out = emptyTotals();
740-
for (const t of all) {
741-
out.events += t.events;
742-
out.ai += t.ai;
743-
out.bots += t.bots;
744-
out.prevEvents += t.prevEvents;
745-
out.prevAi += t.prevAi;
746-
out.prevBots += t.prevBots;
747-
}
748-
return out;
749-
}
750-
751-
// Everything that isn't an AI referral or a bot crawl, matching the split the
752-
// per-project stats page uses.
753-
function otherVisits(events: number, ai: number, bots: number) {
754-
return Math.max(0, events - ai - bots);
755-
}
756-
757786
function buildProjectAccessFilter(
758787
userId: string,
759788
projectMemberIds: string[],

0 commit comments

Comments
 (0)