feat(plugin): demand ladder — visit-driven render cadence, dry-run by default (v0.36.0)

packages/plugin/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@harperfast/prerender",
-	"version": "0.35.0",
+	"version": "0.36.0",
 	"type": "module",
 	"description": "Configurable Harper plugin for prerendering pages for bots and crawlers",
 	"license": "Apache-2.0",

packages/plugin/src/configSchema.js

@@ -375,6 +375,18 @@ export const configSchema = group('Prerender plugin configuration.', {
 					'timer; the console’s Recompute button still triggers a one-off pass.',
 				{ unit: 'ms', min: 0 }
 			),
+			snapshotTableCounts: option(
+				true,
+				'Include the four table counts (targets, pages, sitemaps, suppressed) in each backlog ' +
+					'snapshot. The counts go through Harper’s getRecordCount, which on RocksDB tables past ' +
+					'the sampling budget issues ONE synchronous native full-key iteration — measured 2.47s ' +
+					'on a ~2.2M-key table, during which every request routed to that worker waits ' +
+					'(harper-pro#664). False keeps the snapshot itself (the capped backlog/histogram walk and ' +
+					'the queue_health gauges, which never take that walk) while the console shows the counts ' +
+					'as unavailable — the setting for a deployment that disabled the whole snapshot to dodge ' +
+					'#664 and thereby lost its below-floor detector.',
+				{}
+			),
 			pageSize: option(
 				50,
 				'Rows per page for the console’s sitemap-entry and page-cache tables. Also bounds the ' +
@@ -519,6 +531,105 @@ export const configSchema = group('Prerender plugin configuration.', {
 				'target’s stored interval (sitemap `changefreq` / explicit API write) > this default.',
 			{ unit: 'ms', min: 1 }
 		),
+		demand: group(
+			'Demand-driven cadence: move a target UP or DOWN a fixed ladder of render intervals based ' +
+				'on whether bots actually visit it, inside the same total render budget. Hot pages get a ' +
+				'tighter freshness bound; pages nothing crawls get a looser one.\n\n' +
+				'A render interval only bounds staleness for content that drifts with TIME. On this corpus ' +
+				'that is AVAILABILITY (~0.04%/hour, continuous, and directionally in-stock -> out-of-stock, ' +
+				'i.e. the cache claims stock for sold-through items), so each rung is really an ' +
+				'availability-error budget: 6h ~ 0.24%, 12h ~ 0.5%, 24h ~ 1%, 48h ~ 2%. Price does NOT ' +
+				'drift that way — it steps at promotional events, most of the catalog at once — so no ' +
+				'affordable interval bounds it and this does not try.\n\n' +
+				'COST IS NOT SELF-LIMITING. It scales with the fraction of the corpus bots touch, which ' +
+				'grows as search-engine traffic ramps. `maxFastFraction` is the backstop and the level ' +
+				'histogram logged every `statsInterval` is the early warning — watch it before trusting it.',
+			{
+				enabled: option(false, 'Master switch. Off = `resolveRenderInterval` is used unchanged.'),
+				dryRun: option(
+					true,
+					'Compute and LOG every ladder decision but schedule with the unchanged base interval. ' +
+						'A week of this reports the steady-state level distribution — and therefore the render ' +
+						'budget — before you pay for it. Default ON: enabling `enabled` alone changes no ' +
+						'SCHEDULE until this is turned off.\n\n' +
+						'One write does happen in dry-run, deliberately: a rung move persists to ' +
+						'`Target.demandInterval` (only on an actual move, never on hold). That persistence is ' +
+						'what makes the dry-run histogram converge to the steady-state distribution instead ' +
+						'of reporting first-step decisions forever — and it means the measured week is not ' +
+						'free of replicated Target writes (~one per target that moves, per rung walked, plus ' +
+						'boundary pages that flap). Turning the ladder fully off leaves `demandInterval` in ' +
+						'place, ignored; a later re-enable resumes from the stored rung rather than from base.'
+				),
+				ladder: option(
+					[6 * HOUR, 12 * HOUR, 24 * HOUR, 48 * HOUR],
+					'Render intervals a target may occupy, ascending. The route/stored interval is the ' +
+						'CEILING — the ladder reallocates within the cadence the route already grants and never ' +
+						'schedules slower than it. An interval that is not itself a rung participates as its ' +
+						'own top rung: it rests at its granted cadence and may only move through the rungs ' +
+						'FASTER than it — never snapped to a rung in either direction (a 1h route parked at ' +
+						'6h, or a weekly sitemap route pulled to 48h at 3.5x its granted render budget). ' +
+						'Bottoming out at 6h rather than 1h is deliberate: 1h buys ' +
+						'~0.04% availability error against 6h\u2019s ~0.24% for six times the render cost, and the ' +
+						'fast rungs are where a runaway hot set becomes unaffordable.',
+					{ unit: 'ms' }
+				),
+				promoteWindows: option(
+					2,
+					'How many consecutive windows of the CANDIDATE (faster) interval must each contain a ' +
+						'visit before a target is promoted. 1 promotes on "visited at all this interval", which ' +
+						'settles at rendering twice per visit; 2 asks whether a render at the faster rung would ' +
+						'actually have been seen, and settles near once per visit.',
+					{ min: 1 }
+				),
+				maxFastInterval: option(
+					12 * HOUR,
+					'Rungs strictly below this count as "fast" for `maxFastFraction` and the logged ' + '`fastFraction`.',
+					{ unit: 'ms', min: 1 }
+				),
+				maxFastFraction: option(
+					0.05,
+					'Budget backstop: the share of decisions allowed to land on a fast rung. Exceeding it is ' +
+						'logged as a warning — the hot set has grown past what the ladder was sized for.',
+					{ min: 0, max: 1 }
+				),
+				sliceMs: option(
+					6 * HOUR,
+					'Time resolution of the visit ring. Cannot be coarser than the fastest rung or that rung ' +
+						'can never be evaluated.',
+					{ unit: 'ms', min: 1 }
+				),
+				slices: option(
+					16,
+					'Ring length. Must cover promoteWindows x the slowest rung, so the promotion test for the ' +
+						'top rung can see far enough back.',
+					{ min: 2 }
+				),
+				bitsPerSlice: option(
+					1 << 20,
+					'Bloom filter bits per ring slice, rounded UP to a power of two at use (byte sizing and ' +
+						'probe spread both require it). ~1M bits holds ~100k distinct URLs per slice at ~1% ' +
+						'false positives. False positives promote a page nobody asked for — wasted renders, ' +
+						'never staleness — and there are no false negatives.',
+					{ min: 1024 }
+				),
+				hashes: option(7, 'Bloom hash count (k).', { min: 1, max: 32 }),
+				flushInterval: option(
+					5 * MINUTE,
+					'How often a worker merges its in-memory ring slices into this node\u2019s replicated row.',
+					{ unit: 'ms', min: SECOND }
+				),
+				mergeInterval: option(
+					5 * MINUTE,
+					'How often the read side re-unions every node\u2019s rows. The reschedule path runs ~20x/s ' +
+						'and cannot pay a multi-row read per job result, so it reads a cached union this stale.',
+					{ unit: 'ms', min: SECOND }
+				),
+				statsInterval: option(15 * MINUTE, 'How often the level histogram + promote/demote counters are logged.', {
+					unit: 'ms',
+					min: SECOND,
+				}),
+			}
+		),
 		suppression: group(
 			'What happens when a render proves a URL non-indexable (noindex, canonical mismatch, redirect ' +
 				'loop, HTTP error page). The target is not deleted — it is marked `state: suppressed` and ' +

packages/plugin/src/http_handlers/bot_request.js

@@ -18,6 +18,7 @@ import { maybeAccelerateHeal } from '../util/invalidationReenqueue.js';
 import { currentMinuteMs } from '../util/time.js';
 import { writeSchedule } from '../util/renderSchedule.js';
 import { recordCrawl } from '../util/crawlStats.js';
+import { recordVisit } from '../util/visitFilter.js';
 import { deliverResource } from './response.js';
 
 export async function handleBotRequest(request) {
@@ -39,6 +40,17 @@ export async function handleBotRequest(request) {
 			recordCrawl(request.botName, cacheUrl);
 		}
 
+		// Demand signal for the render ladder. Deliberately OUTSIDE the `recordBots` gate: that
+		// gate is about analytics volume, whereas this feeds scheduling — a deployment that turns
+		// analytics down must not silently demote its whole corpus for lack of observed traffic.
+		// Keyed on the device-free URL, since cadence resolves per URL and dropping the device
+		// split halves the distinct count the filter carries. No-op unless render.demand.enabled.
+		// Prerender-class only: those are the only keys the ladder ever probes (proxied and
+		// unclassified paths own no Target), and recording the plentiful junk URLs the CDN
+		// over-forwards would only raise the filter's fill factor — at high fill the
+		// false-positive rate explodes and the ladder degenerates into promoting everything.
+		if (routeClass === PRERENDER) recordVisit(cacheUrl);
+
 		// Debug/observability info surfaced as x-harper-* response headers (only when the
 		// debug header is present). `route` is the matched route entry, if any; `routeClass`
 		// decides whether this request is cached and scheduled at all.

packages/plugin/src/resources/RenderQueue.js

@@ -5,6 +5,7 @@ import { QueueState } from './QueueState.js';
 import { CacheKey } from '../util/cacheKey.js';
 import { canonicalizeUrl } from '../util/url.js';
 import { classifyPath, queryAllowlistFor, resolveRenderInterval, PRERENDER } from '../util/routeClass.js';
+import { decideInterval } from '../util/demandLadder.js';
 import { recordUnroutedPath } from '../util/unrouted.js';
 import { Target, countedStrikes } from './Target.js';
 import { getDesiredPause, setDesiredPause } from '../util/queueControl.js';
@@ -326,7 +327,7 @@ export class RenderQueue extends Resource {
 			const url = CacheKey.extractUrl(cacheKey);
 			const renderTarget = await Target.get({
 				id: url,
-				select: ['renderInterval', 'sitemapUrl', 'state', 'strikes'],
+				select: ['renderInterval', 'sitemapUrl', 'state', 'strikes', 'demandInterval'],
 			});
 			const renderInterval = renderTarget?.renderInterval;
 
@@ -338,7 +339,12 @@ export class RenderQueue extends Resource {
 			// NaN from an arbitrary API PUT — are rejected), else the default. Resolved here
 			// on every cycle, so a route-cadence config change applies on each URL's next
 			// render without touching stored rows.
-			const interval = resolveRenderInterval(url, renderInterval);
+			const base = resolveRenderInterval(url, renderInterval);
+			// The demand ladder reallocates cadence WITHIN `base` (which stays the ceiling) by
+			// whether bots actually visit this URL. Off / dry-run / cold filter all return `base`
+			// unchanged, so this is a no-op until deliberately switched on.
+			const demand = decideInterval(url, base, renderTarget?.demandInterval);
+			const interval = demand.interval;
 			// The cached page expires when the next render is due; the swrTtl window then keeps
 			// it served while the re-render lands, so render latency up to swrTtl never causes
 			// a cache miss.
@@ -370,6 +376,15 @@ export class RenderQueue extends Resource {
 				// continuously and the whole 14× seek win would evaporate.
 				await writeSchedule(cacheKey, { nextRenderTime, fromSitemap: !!renderTarget.sitemapUrl });
 
+				// Persist the rung ONLY on an actual move. 'held' must not write even when the
+				// stored field is absent — absence already resolves to the base ceiling, so writing
+				// it would be redundant, and on first evaluation it would be a corpus-wide storm of
+				// replicated Target patches (~one per render for a full cycle), in dry-run too.
+				// A converged corpus therefore pays nothing here, on the system's hottest path.
+				if (demand.action === 'promoted' || demand.action === 'demoted') {
+					await Target.patch(url, { demandInterval: demand.level });
+				}
+
 				// A suppressed URL that rendered indexable again has healed — put it back in
 				// normal rotation, so the recheck cadence stops and discovery may see it again.
 				if (renderTarget.state === 'suppressed' && result.isIndexable === true) {

packages/plugin/src/schemas/schema.graphql

@@ -99,6 +99,10 @@ type Target @table(database: "render_service") @export {
 	suppressedAt: Date
 	# Consecutive non-indexable verdicts; render.suppression.maxStrikes deletes the target.
 	strikes: Int
+	# Current demand-ladder rung (ms), when render.demand is enabled. Absent = the target has
+	# never been evaluated and starts from its route/stored interval. Written ONLY when the rung
+	# actually changes, so a converged corpus pays no write — same discipline as `strikes`.
+	demandInterval: Long
 }
 
 type RenderSchedule @table(database: "render_schedule") @export {
@@ -220,3 +224,20 @@ type CrawlSketch @table(database: "crawl_stats") {
 	estimate: Float
 	updatedAt: Date
 }
+
+# One ring slice of the bot-visit Bloom filter, per node (`slot|node`) — see util/visitFilter.js.
+# Same shape and rationale as CrawlSketch above: per-thread observations merge into one row per
+# node, rows replicate so any node can answer, and reads union the node rows for the slots they
+# need. Bloom rather than a per-URL `lastVisitedAt` because at search-engine traffic volumes a
+# per-URL write lands at roughly the distinct-URL rate against a replicated table; this is one
+# merged row per node per flush and constant memory regardless of request volume.
+#
+# Deliberately NOT @export: raw filter bits are meaningless outside a merge, and membership is
+# only ever consumed internally by the demand ladder.
+type VisitFilter @table(database: "crawl_stats") {
+	id: String @primaryKey # `${slot}|${node}`
+	slot: Int @indexed
+	node: String
+	bits: Bytes
+	updatedAt: Date
+}

packages/plugin/src/util/backlogSnapshot.js

@@ -228,15 +228,22 @@ export const runBacklogSnapshotOnce = async () => {
 			page_cache: { PrerenderedPage },
 			sitemaps: { Sitemap },
 		} = databases;
-		const counts = {
-			targets: await countTable(Target),
-			pages: await countTable(PrerenderedPage),
-			sitemaps: await countTable(Sitemap),
-			// Suppressed targets replaced the NonIndexable table: an indexed-equality walk,
-			// capped like every other management scan, so a runaway suppression count can't
-			// turn the snapshot into a full table scan.
-			suppressed: await countSuppressed(Target),
-		};
+		// `snapshotTableCounts: false` is the #664 dodge: getRecordCount's native full-key walk is
+		// the ONLY part of this pass that can stall a traffic-serving worker, so a deployment can
+		// drop the counts while keeping the capped backlog walk and the queue_health gauges. The
+		// shape matches countTable's own failure value, which the console already renders.
+		const skipped = { recordCount: null, error: 'disabled' };
+		const counts = !config.management.snapshotTableCounts
+			? { targets: skipped, pages: skipped, sitemaps: skipped, suppressed: skipped }
+			: {
+					targets: await countTable(Target),
+					pages: await countTable(PrerenderedPage),
+					sitemaps: await countTable(Sitemap),
+					// Suppressed targets replaced the NonIndexable table: an indexed-equality walk,
+					// capped like every other management scan, so a runaway suppression count can't
+					// turn the snapshot into a full table scan.
+					suppressed: await countSuppressed(Target),
+				};
 
 		lastRun = { ...stats, counts, node: server.hostname, startedAt, finishedAt: Date.now(), error: null };