Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/cli/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@profullstack/nichedb",
"version": "0.36.0",
"version": "0.36.1",
"description": "CLI and MCP bridge for NicheDB: browse collections, manage sources and feeds, search items, from any deployment",
"type": "module",
"main": "src/index.js",
Expand Down
2 changes: 1 addition & 1 deletion apps/web/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@nichedb/web",
"version": "0.36.0",
"version": "0.36.1",
"private": true,
"type": "module",
"scripts": {
Expand Down
2 changes: 1 addition & 1 deletion apps/worker/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@nichedb/worker",
"version": "0.36.0",
"version": "0.36.1",
"private": true,
"type": "module",
"scripts": {
Expand Down
28 changes: 14 additions & 14 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "niche-db",
"version": "0.36.0",
"version": "0.36.1",
"private": true,
"type": "module",
"description": "An open platform for large, ever-growing databases of real-time data: sources in, feeds out. Web, API, CLI and MCP on one Postgres.",
Expand Down
2 changes: 1 addition & 1 deletion packages/adapters/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@nichedb/adapters",
"version": "0.36.0",
"version": "0.36.1",
"private": true,
"type": "module",
"exports": {
Expand Down
90 changes: 83 additions & 7 deletions packages/adapters/src/espn.js
Original file line number Diff line number Diff line change
Expand Up @@ -906,6 +906,10 @@ export function scheduleQueries(from, to) {
* fails outright, is answered by the undated scoreboard, which carries the NEXT
* fixtures, so an out-of-season league shows its season opener. Without it a
* failure is thrown, so the caller counts the league as failed rather than quiet.
*
* `complete` says whether the answer is the whole window: true when every day
* was answered (a 404 is an answer: nothing on), false when it is the undated
* board standing in. Only a complete answer can say a fixture is gone.
*/
export async function fetchSchedule(client, { league, from, to, fallback = true }) {
const base = `${SITE}/${league.key}/scoreboard`;
Expand All @@ -927,7 +931,7 @@ export async function fetchSchedule(client, { league, from, to, fallback = true
}
if (err?.status === 404) continue;
if (!fallback) throw err;
return parseScoreboard(await client.get(base), league);
return { ...parseScoreboard(await client.get(base), league), complete: false };
}
if (q.days && (data?.events ?? []).length >= QUERY_LIMIT) {
queue.push(...q.days.map((dates) => ({ dates })));
Expand All @@ -944,9 +948,9 @@ export async function fetchSchedule(client, { league, from, to, fallback = true
}
}
if (byKey.size === 0 && fallback) {
return parseScoreboard(await client.get(base), league);
return { ...parseScoreboard(await client.get(base), league), complete: false };
}
return { league: meta, events: [...byKey.values()] };
return { league: meta, events: [...byKey.values()], complete: true };
}

/* ----------------------------------------------------------------- cursor -- */
Expand Down Expand Up @@ -1136,6 +1140,53 @@ export function earliestUpcoming(events, now) {
return best === null ? null : new Date(best).toISOString();
}

/**
* How far back the schedule pass looks for a fixture still held as upcoming.
*
* ESPN does not cancel an "if necessary" playoff game, it deletes it: the NLDS
* Game 5 between the Padres and the Brewers (401908006) left every scoreboard
* the moment Milwaukee won Game 4, so nothing ever wrote over the `pre` it had
* last been given, and it sat on the calendar as scheduled through the time it
* would have started. Three days covers a pass that missed it by a day or two
* without re-asking a week of boards for every league.
*/
export const VANISHED_REACH_MS = 3 * DAY_MS;

/** The held rows a complete board no longer lists. */
export function vanishedFixtures(held, events) {
const listed = new Set(events.map((e) => `${PROVIDER}:fixture:${e.key}`));
return (held ?? []).filter((r) => !listed.has(r.external_id));
}

/**
* A held fixture, written as cancelled: ESPN's own word for a game called off
* (`STATUS_CANCELED` is `post` with "Canceled"), so a consumer that already
* knows how to show a finished game with no score shows this one right, and
* one that only counts `pre` as upcoming stops listing it.
*/
export function cancelledItem(row) {
return {
externalId: row.external_id,
kind: row.kind ?? 'fixture',
title: row.title,
summary: row.summary ?? null,
url: row.url ?? null,
imageUrl: row.image_url ?? null,
publishedAt: row.published_at,
timeKnown: row.time_known ?? true,
precision: row.precision ?? 'minute',
tags: [...(row.tags ?? []).filter((t) => !t.startsWith('state:')), 'state:post', 'canceled'],
data: {
...(row.data ?? {}),
state: 'post',
statusDetail: 'Canceled',
canceled: true,
period: null,
displayClock: null,
},
};
}

export const espnSchedule = defineAdapter({
name: 'espn-schedule',
title: 'ESPN: fixtures',
Expand Down Expand Up @@ -1171,7 +1222,7 @@ export const espnSchedule = defineAdapter({
},
],
async pull(ctx) {
const { config, cursor: prev, env, http, log, deadline } = ctx;
const { config, cursor: prev, env, http, log, deadline, held: heldOf } = ctx;
const client = makeEspnClient({ env, http, log });
const cursor = { ...prev };
const now = Date.now();
Expand Down Expand Up @@ -1200,6 +1251,7 @@ export const espnSchedule = defineAdapter({
const items = [];
let failed = 0;
let done = 0;
let cancelled = 0;
const stopAt = deadline - DEADLINE_MARGIN_MS;
const leftover = await pool(queue, 4, stopAt, async (key) => {
const league = byKey.get(key);
Expand All @@ -1212,14 +1264,38 @@ export const espnSchedule = defineAdapter({
.then((j) => parseLeagueDetail(j)?.region ?? null)
.catch(() => null);
}
const { league: meta, events } = await fetchSchedule(client, {
// What we hold as still to come, back to VANISHED_REACH_MS: the board is
// asked from the earliest of them, so a fixture ESPN has since deleted
// is noticed even once its start has passed out of the window.
const held = heldOf
? await heldOf({
tags: [`league:${league.slug}`, 'state:pre'],
from: new Date(now - VANISHED_REACH_MS),
to: window.to,
})
: [];
const earliest = Math.min(
window.from.getTime(),
...held.map((r) => new Date(r.published_at).getTime()).filter(Number.isFinite),
);
const {
league: meta,
events,
complete,
} = await fetchSchedule(client, {
league,
from: window.from,
from: new Date(earliest),
to: window.to,
fallback: mode === 'full',
});
const metaWithRegion = { ...(meta ?? {}), region: regions[key] ?? null };
for (const f of events) items.push(fixtureItem(f, league, metaWithRegion));
if (complete) {
for (const row of vanishedFixtures(held, events)) {
items.push(cancelledItem(row));
cancelled += 1;
}
}
upcoming[key] = earliestUpcoming(events, now);
done += 1;
} catch (err) {
Expand All @@ -1242,7 +1318,7 @@ export const espnSchedule = defineAdapter({
items,
cursor: next,
nextInMinutes: finished ? undefined : 1,
note: `${mode}: ${items.length} fixtures from ${done} leagues, ${failed} failed${finished ? '' : `, ${leftover.length} left for the next run`}`,
note: `${mode}: ${items.length} fixtures from ${done} leagues, ${failed} failed${cancelled ? `, ${cancelled} gone upstream (cancelled)` : ''}${finished ? '' : `, ${leftover.length} left for the next run`}`,
};
},
});
Expand Down
2 changes: 1 addition & 1 deletion packages/auth/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@nichedb/auth",
"version": "0.36.0",
"version": "0.36.1",
"private": true,
"type": "module",
"exports": {
Expand Down
2 changes: 1 addition & 1 deletion packages/config/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@nichedb/config",
"version": "0.36.0",
"version": "0.36.1",
"private": true,
"type": "module",
"exports": {
Expand Down
2 changes: 1 addition & 1 deletion packages/core/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@nichedb/core",
"version": "0.36.0",
"version": "0.36.1",
"private": true,
"type": "module",
"exports": {
Expand Down
3 changes: 3 additions & 0 deletions packages/core/src/ingest.js
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ export async function runSource(
// What this source wrote last time for these ids, so an adapter that
// keeps a rolling window per item can extend it rather than restate it.
previous: (externalIds) => q.previousItemData({ sourceId: source.id, externalIds }),
// What this source holds under some tags in a time window, as stored. How
// an adapter finds the rows its provider has since deleted.
held: (opts) => q.heldItems({ sourceId: source.id, ...opts }),
});

/*
Expand Down
2 changes: 1 addition & 1 deletion packages/db/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@nichedb/db",
"version": "0.36.0",
"version": "0.36.1",
"private": true,
"type": "module",
"exports": {
Expand Down
20 changes: 20 additions & 0 deletions packages/db/src/queries.js
Original file line number Diff line number Diff line change
Expand Up @@ -1043,6 +1043,26 @@ export async function previousItemData({ sourceId, externalIds }) {
return out;
}

/**
* What this source holds under all of `tags` with a start inside [from, to], as
* the stored rows. For an adapter that has to notice what its provider STOPPED
* saying -- a fixture deleted upstream is never fetched again, so nothing would
* ever overwrite the last thing it said about it.
*/
export async function heldItems({ sourceId, tags = [], from, to, limit = 500 }) {
return sql`
select external_id, kind, title, summary, url, image_url, published_at, time_known,
precision, tags, data
from items
where source_id = ${sourceId}
and tags @> ${pgArray(tags)}::text[]
and published_at >= ${from}
and published_at <= ${to}
order by published_at
limit ${limit}
`;
}

const itemSelection = (db) => db`
i.*, s.slug as source_slug, s.name as source_name, s.adapter,
c.slug as collection_slug, c.name as collection_name, c.early_access
Expand Down
2 changes: 1 addition & 1 deletion packages/enrichers/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@nichedb/enrichers",
"version": "0.36.0",
"version": "0.36.1",
"private": true,
"type": "module",
"exports": {
Expand Down
2 changes: 1 addition & 1 deletion packages/knowledge/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@nichedb/knowledge",
"version": "0.36.0",
"version": "0.36.1",
"private": true,
"type": "module",
"exports": {
Expand Down
2 changes: 1 addition & 1 deletion packages/notify/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@nichedb/notify",
"version": "0.36.0",
"version": "0.36.1",
"private": true,
"type": "module",
"exports": {
Expand Down
Loading
Loading