From 564ed42d4ac59c2555edb463f28f2ccc16632037 Mon Sep 17 00:00:00 2001 From: Baris Can Vural Date: Tue, 3 Nov 2026 17:06:24 +0300 Subject: [PATCH 1/2] fix(api): support TimeSeries time ranges tables (schema version 8) in label lookups ClickHouse 26.10 creates TimeSeries tables of schema version 8 by default, which keep each series' time range in a separate time ranges inner table instead of min_time/max_time in the tags table. The label lookups only looked at the tags table, so match[] was rejected and time bounds were dropped, reading the whole tags table. Read the time ranges table when the tags table has no time bounds. The test fixture pins version 7 for the tags layout where the server knows the setting, and the new tests run where version 8 tables can be created. --- .changeset/time-ranges-label-lookup.md | 6 + .../__tests__/timeseriesEngine.int.test.ts | 142 ++++++++++++++++++ .../api/src/controllers/timeseriesEngine.ts | 91 +++++++---- packages/api/src/fixtures.ts | 59 ++++++-- packages/common-utils/src/core/metadata.ts | 2 +- 5 files changed, 260 insertions(+), 40 deletions(-) create mode 100644 .changeset/time-ranges-label-lookup.md diff --git a/.changeset/time-ranges-label-lookup.md b/.changeset/time-ranges-label-lookup.md new file mode 100644 index 0000000000..e6b85ba444 --- /dev/null +++ b/.changeset/time-ranges-label-lookup.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/api': patch +'@hyperdx/common-utils': patch +--- + +fix: PromQL label lookups (dashboard variables, label autocomplete) now work on TimeSeries tables of schema version 8, the default from ClickHouse 26.10. These keep each series' time range in a separate time ranges table instead of the tags table's min_time/max_time, so match[] failed with "requires store_min_time_and_max_time = 1" and time-bounded lookups read the whole tags table. diff --git a/packages/api/src/controllers/__tests__/timeseriesEngine.int.test.ts b/packages/api/src/controllers/__tests__/timeseriesEngine.int.test.ts index 0ea9586610..e03af77eb3 100644 --- a/packages/api/src/controllers/__tests__/timeseriesEngine.int.test.ts +++ b/packages/api/src/controllers/__tests__/timeseriesEngine.int.test.ts @@ -475,4 +475,146 @@ describe('timeseriesEngine controller', () => { }); }); }); + + // Schema version 8 (the default from ClickHouse 26.10) moved min_time and + // max_time out of the tags table into a separate time ranges table. Older + // servers don't know the `version` setting, so these run only where the + // table can be created. + describe('table of schema version 8 (time ranges table)', () => { + const V8_TABLE = 'test_ts_label_values_v8'; + const V8_NO_RANGES_TABLE = 'test_ts_label_values_v8_no_ranges'; + let supported = true; + + const v8LabelValues = ( + args: Partial[0]> & { + labelName: string; + }, + ) => labelValues({ tableName: V8_TABLE, ...args }); + + beforeAll(async () => { + try { + await seedTimeSeriesTagsTable({ + table: V8_TABLE, + series: SERIES, + withSamples: true, + timeRanges: true, + }); + await seedTimeSeriesTagsTable({ + table: V8_NO_RANGES_TABLE, + series: SERIES, + storeTimeBounds: false, + timeRanges: true, + }); + } catch (e) { + if (!/Unknown setting 'version'/.test(String(e))) throw e; + supported = false; + } + }); + + afterAll(async () => { + for (const table of [V8_TABLE, V8_NO_RANGES_TABLE]) { + await dropTimeSeriesTable({ table }); + } + }); + + it('scopes label values to the window through the time ranges table', async () => { + if (!supported) return; + expect( + await v8LabelValues({ + labelName: 'job', + startMs: sec(RECENT_START_SEC), + endMs: sec(RECENT_START_SEC + HOUR_SEC), + }), + ).toEqual(['api', 'web']); + }); + + it('applies lone bounds against the time ranges', async () => { + if (!supported) return; + expect( + await v8LabelValues({ + labelName: '__name__', + endMs: sec(OLD_START_SEC + HOUR_SEC), + }), + ).toEqual(['old_metric']); + expect( + await v8LabelValues({ + labelName: '__name__', + startMs: sec(RECENT_START_SEC), + }), + ).toEqual(['recent_metric']); + }); + + it('returns nothing for a window no series covers', async () => { + if (!supported) return; + expect( + await v8LabelValues({ + labelName: '__name__', + startMs: sec(OLD_START_SEC + 2 * HOUR_SEC), + endMs: sec(OLD_START_SEC + 3 * HOUR_SEC), + }), + ).toEqual([]); + }); + + it('evaluates match[] and narrows it by the window', async () => { + if (!supported) return; + expect( + await v8LabelValues({ + labelName: 'job', + match: ['recent_metric{job="api"}', 'old_metric'], + }), + ).toEqual(['api', 'batch']); + expect( + await v8LabelValues({ + labelName: 'job', + match: ['{job=~".+"}'], + startMs: sec(RECENT_START_SEC), + endMs: sec(RECENT_START_SEC + HOUR_SEC), + }), + ).toEqual(['api', 'web']); + }); + + it('scopes label names to the window and the selector', async () => { + if (!supported) return; + const names = ( + args: Partial[0]> = {}, + ) => + queryLabelNames({ + client, + connectionId: CONNECTION_ID, + databaseName: DEFAULT_DATABASE, + tableName: V8_TABLE, + ...args, + }); + expect( + await names({ + startMs: sec(RECENT_START_SEC), + endMs: sec(RECENT_START_SEC + HOUR_SEC), + }), + ).toEqual(['__name__', 'job']); + expect(await names({ match: ['old_metric'] })).toEqual([ + '__name__', + 'job', + 'region', + ]); + }); + + it('rejects match[] without a time ranges table, and ignores the bounds', async () => { + if (!supported) return; + await expect( + labelValues({ + tableName: V8_NO_RANGES_TABLE, + labelName: 'job', + match: ['recent_metric'], + }), + ).rejects.toThrow(/store_time_ranges = 1/); + expect( + await labelValues({ + tableName: V8_NO_RANGES_TABLE, + labelName: '__name__', + startMs: sec(RECENT_START_SEC), + endMs: sec(RECENT_START_SEC + HOUR_SEC), + }), + ).toEqual(['old_metric', 'recent_metric']); + }); + }); }); diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts index 64fc8c4b77..80c65f955c 100644 --- a/packages/api/src/controllers/timeseriesEngine.ts +++ b/packages/api/src/controllers/timeseriesEngine.ts @@ -33,10 +33,13 @@ export type TimeSeriesTagsQueryArgs = { }; /** - * Indicates whether the tags inner table associated with the given TimeSeries table - * includes the optional `min_time` and `max_time` columns. + * Where the given TimeSeries table keeps each series' time range, if anywhere: + * - `tags`: the `min_time`/`max_time` columns of the tags inner table (schema + * version 7 or earlier, with `store_min_time_and_max_time` on); + * - `timeRanges`: the time ranges inner table (version 8 and later, with + * `store_time_ranges` on, the default from ClickHouse 26.10). */ -async function timeSeriesTagsTableHasTimeBounds({ +async function timeSeriesTimeBounds({ metadata, connectionId, database, @@ -46,24 +49,32 @@ async function timeSeriesTagsTableHasTimeBounds({ connectionId: string; database: string; table: string; -}): Promise { - try { - const columns = await metadata.getTimeSeriesTableColumns({ - connectionId, - databaseName: database, - tableName: table, - innerTableType: 'Tags', - }); - - const columnNames = new Set(columns.map(column => column.name)); - return columnNames.has('min_time') && columnNames.has('max_time'); - } catch (e) { - logger.warn( - { err: e, database, table }, - 'Failed to check if TimeSeries tags table has time bounds columns', - ); - return false; +}): Promise<'tags' | 'timeRanges' | undefined> { + for (const [innerTableType, bounds] of [ + ['Tags', 'tags'], + ['TimeRanges', 'timeRanges'], + ] as const) { + try { + const columns = await metadata.getTimeSeriesTableColumns({ + connectionId, + databaseName: database, + tableName: table, + innerTableType, + }); + const columnNames = new Set(columns.map(column => column.name)); + if (columnNames.has('min_time') && columnNames.has('max_time')) + return bounds; + } catch (e) { + // A table without a time ranges target (version 7 or earlier, or + // store_time_ranges off) rejects timeSeriesTimeRanges(): expected. + if (innerTableType === 'Tags') + logger.warn( + { err: e, database, table }, + 'Failed to check if TimeSeries tags table has time bounds columns', + ); + } } + return undefined; } // `timeSeriesSelector()` requires both bounds, so we define the minimum and maximum possible times @@ -122,10 +133,11 @@ async function getSeriesFilterConditions({ const selectors = match?.length ? match : undefined; if (startMs == null && endMs == null && selectors == null) return []; - // Both filters lean on min_time/max_time, which the tags inner table carries - // only when the table was created with store_min_time_and_max_time on. + // Both filters lean on each series' time range: min_time/max_time in the + // tags inner table up to schema version 7, a separate time ranges inner + // table from version 8. const metadata = new Metadata(client, new MetadataCache()); - const tableHasTimeBounds = await timeSeriesTagsTableHasTimeBounds({ + const timeBounds = await timeSeriesTimeBounds({ metadata, connectionId, database: databaseName, @@ -135,12 +147,12 @@ async function getSeriesFilterConditions({ const conditions: ChSql[] = []; if (selectors != null) { - // timeSeriesSelector() prunes on min_time/max_time unconditionally, so - // without those columns it cannot run. Unlike the bounds it also cannot - // degrade to "no filter" — that would answer a different question. - if (!tableHasTimeBounds) { + // timeSeriesSelector() prunes on the series' time range, so without one it + // cannot run. Unlike the bounds it also cannot degrade to "no filter" — + // that would answer a different question. + if (timeBounds == null) { throw new Error( - 'match[] requires a TimeSeries table created with store_min_time_and_max_time = 1', + 'match[] requires a TimeSeries table that stores time ranges (store_time_ranges = 1, or store_min_time_and_max_time = 1 before version 8)', ); } conditions.push( @@ -154,7 +166,7 @@ async function getSeriesFilterConditions({ ); } - if (tableHasTimeBounds) { + if (timeBounds === 'tags') { if (endMs != null) conditions.push( chSql`(min_time IS NULL OR min_time <= fromUnixTimestamp64Milli(${{ Int64: endMs }}))`, @@ -163,6 +175,27 @@ async function getSeriesFilterConditions({ conditions.push( chSql`(max_time IS NULL OR max_time >= fromUnixTimestamp64Milli(${{ Int64: startMs }}))`, ); + } else if (timeBounds === 'timeRanges' && selectors == null) { + // The selector above already keeps only series with samples in the + // window. Without one, the time ranges table answers the window; its rows + // are partial aggregates until merged, hence the GROUP BY. + const having: ChSql[] = []; + if (endMs != null) + having.push( + chSql`min(min_time) <= fromUnixTimestamp64Milli(${{ Int64: endMs }})`, + ); + if (startMs != null) + having.push( + chSql`max(max_time) >= fromUnixTimestamp64Milli(${{ Int64: startMs }})`, + ); + conditions.push( + chSql`id IN ( + SELECT id + FROM timeSeriesTimeRanges(${{ Identifier: databaseName }}, ${{ Identifier: tableName }}) + GROUP BY id + HAVING ${concatChSql(' AND ', ...having)} + )`, + ); } return conditions; diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index 26df966c5a..eeb71459f3 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -302,6 +302,10 @@ export type TimeSeriesFixtureSeries = { * `withSamples: true` also writes a sample at each series' `startSec` and * `endSec`. A `match[]` lookup matches only series that have a sample in the * window, so on a tags-only table every selector answers nothing. + * + * `timeRanges: true` creates a schema version 8 table (ClickHouse 26.10+), + * which keeps each series' time range in a separate time ranges inner table + * instead of the tags table's min_time/max_time. */ export const seedTimeSeriesTagsTable = async ({ table, @@ -309,12 +313,14 @@ export const seedTimeSeriesTagsTable = async ({ database = DEFAULT_DATABASE, storeTimeBounds = true, withSamples = false, + timeRanges = false, }: { table: string; series: TimeSeriesFixtureSeries[]; database?: string; storeTimeBounds?: boolean; withSamples?: boolean; + timeRanges?: boolean; }) => { if (withSamples && !storeTimeBounds) { throw new Error( @@ -323,11 +329,29 @@ export const seedTimeSeriesTagsTable = async ({ } await dropTimeSeriesTable({ table, database }); - await executeTimeSeriesSqlCommand( - `CREATE TABLE ${database}.${table} ENGINE = TimeSeries${ - storeTimeBounds ? '' : ' SETTINGS store_min_time_and_max_time = 0' - }`, - ); + const create = (settings: string[]) => + executeTimeSeriesSqlCommand( + `CREATE TABLE ${database}.${table} ENGINE = TimeSeries${ + settings.length ? ` SETTINGS ${settings.join(', ')}` : '' + }`, + ); + if (timeRanges) { + await create([ + 'version = 8', + `store_time_ranges = ${storeTimeBounds ? 1 : 0}`, + ]); + } else { + const settings = storeTimeBounds ? [] : ['store_min_time_and_max_time = 0']; + // ClickHouse 26.10 creates version 8 tables by default, without + // min_time/max_time in the tags table: pin version 7 where the server + // knows the setting (older servers don't, and use that layout anyway). + try { + await create([...settings, 'version = 7']); + } catch (e) { + if (!/Unknown setting 'version'/.test(String(e))) throw e; + await create(settings); + } + } const quoted = (v: string) => `'${v.replace(/'/g, "\\'")}'`; const mapLiteral = (tags: Record) => @@ -335,13 +359,14 @@ export const seedTimeSeriesTagsTable = async ({ .flatMap(([k, v]) => [quoted(k), quoted(v)]) .join(', ')})`; - const columns = storeTimeBounds + const boundsInTags = storeTimeBounds && !timeRanges; + const columns = boundsInTags ? '(metric_name, tags, min_time, max_time)' : '(metric_name, tags)'; const values = series .map(s => { const row = [quoted(s.metricName), mapLiteral(s.tags)]; - if (storeTimeBounds) { + if (boundsInTags) { row.push( `toDateTime64(${s.startSec}, 3)`, `toDateTime64(${s.endSec}, 3)`, @@ -355,13 +380,27 @@ export const seedTimeSeriesTagsTable = async ({ `INSERT INTO TABLE FUNCTION timeSeriesTags('${database}', '${table}') ${columns} VALUES ${values}`, ); - // The engine derives `id` from the tags, so the samples are read back out of - // the tags table rather than recomputed here. + // The engine derives `id` from the tags, so ids are read back out of the + // tags table rather than recomputed here. + if (timeRanges && storeTimeBounds) { + for (const s of series) { + await executeTimeSeriesSqlCommand( + `INSERT INTO TABLE FUNCTION timeSeriesTimeRanges('${database}', '${table}') + SELECT id, toDateTime64(${s.startSec}, 3), toDateTime64(${s.endSec}, 3) + FROM timeSeriesTags('${database}', '${table}') + WHERE metric_name = ${quoted(s.metricName)} AND tags = ${mapLiteral(s.tags)}`, + ); + } + } if (withSamples) { + const bounds = timeRanges + ? `(SELECT id, min(min_time) AS min_time, max(max_time) AS max_time + FROM timeSeriesTimeRanges('${database}', '${table}') GROUP BY id)` + : `timeSeriesTags('${database}', '${table}')`; await executeTimeSeriesSqlCommand( `INSERT INTO TABLE FUNCTION timeSeriesData('${database}', '${table}') SELECT id, ts AS timestamp, 1 AS value - FROM timeSeriesTags('${database}', '${table}') + FROM ${bounds} ARRAY JOIN [min_time, max_time] AS ts`, ); } diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 9953e01a88..dd5a5d7856 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -688,7 +688,7 @@ export class Metadata { connectionId: string; databaseName: string; tableName: string; - innerTableType: 'Tags' | 'Metrics' | 'Data'; + innerTableType: 'Tags' | 'Metrics' | 'Data' | 'TimeRanges'; }) { return this.cache.getOrFetch( `${connectionId}.${databaseName}.${tableName}.${innerTableType}.getTimeSeriesTableColumns`, From 445043da2ab7734320007ec577ea5d2026c7ed33 Mon Sep 17 00:00:00 2001 From: Baris Can Vural Date: Fri, 6 Nov 2026 15:57:30 +0300 Subject: [PATCH 2/2] fix(api): log unexpected failures of the time ranges check --- .../__tests__/timeseriesEngine.int.test.ts | 30 ++++++++++++------- .../api/src/controllers/timeseriesEngine.ts | 22 +++++++++----- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/packages/api/src/controllers/__tests__/timeseriesEngine.int.test.ts b/packages/api/src/controllers/__tests__/timeseriesEngine.int.test.ts index e03af77eb3..32653e22fc 100644 --- a/packages/api/src/controllers/__tests__/timeseriesEngine.int.test.ts +++ b/packages/api/src/controllers/__tests__/timeseriesEngine.int.test.ts @@ -12,6 +12,7 @@ import { seedTimeSeriesTagsTable, TimeSeriesFixtureSeries, } from '@/fixtures'; +import logger from '@/utils/logger'; const CONNECTION_ID = 'timeseries-engine-int-test'; const BOUNDED_TABLE = 'test_ts_label_values'; @@ -319,17 +320,24 @@ describe('timeseriesEngine controller', () => { // Time bounds are best-effort narrowing, so the request widens instead. describe('table without min_time/max_time', () => { it('ignores the bounds instead of failing', async () => { - expect( - await queryLabelValues({ - client, - connectionId: CONNECTION_ID, - databaseName: DEFAULT_DATABASE, - tableName: UNBOUNDED_TABLE, - labelName: '__name__', - startMs: sec(RECENT_START_SEC), - endMs: sec(RECENT_START_SEC + HOUR_SEC), - }), - ).toEqual(['old_metric', 'recent_metric']); + const warn = jest.spyOn(logger, 'warn'); + try { + expect( + await queryLabelValues({ + client, + connectionId: CONNECTION_ID, + databaseName: DEFAULT_DATABASE, + tableName: UNBOUNDED_TABLE, + labelName: '__name__', + startMs: sec(RECENT_START_SEC), + endMs: sec(RECENT_START_SEC + HOUR_SEC), + }), + ).toEqual(['old_metric', 'recent_metric']); + // No time ranges table either: expected, so not logged. + expect(warn).not.toHaveBeenCalled(); + } finally { + warn.mockRestore(); + } }); // A selector needs those columns to evaluate at all. Unlike the bounds it diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts index 80c65f955c..f99172826e 100644 --- a/packages/api/src/controllers/timeseriesEngine.ts +++ b/packages/api/src/controllers/timeseriesEngine.ts @@ -32,6 +32,15 @@ export type TimeSeriesTagsQueryArgs = { signal?: AbortSignal; }; +// What a table without a time ranges table answers (version 7 or earlier, or +// store_time_ranges off): servers before 26.10 have no timeSeriesTimeRanges(), +// newer ones reject the table. Any other error is unexpected. +const isNoTimeRangesTableError = (e: unknown): boolean => + e instanceof Error && + /Unknown table function timeSeriesTimeRanges|has no TimeRanges target table/.test( + e.message, + ); + /** * Where the given TimeSeries table keeps each series' time range, if anywhere: * - `tags`: the `min_time`/`max_time` columns of the tags inner table (schema @@ -65,13 +74,12 @@ async function timeSeriesTimeBounds({ if (columnNames.has('min_time') && columnNames.has('max_time')) return bounds; } catch (e) { - // A table without a time ranges target (version 7 or earlier, or - // store_time_ranges off) rejects timeSeriesTimeRanges(): expected. - if (innerTableType === 'Tags') - logger.warn( - { err: e, database, table }, - 'Failed to check if TimeSeries tags table has time bounds columns', - ); + if (innerTableType === 'TimeRanges' && isNoTimeRangesTableError(e)) + continue; + logger.warn( + { err: e, database, table, innerTableType }, + 'Failed to check if TimeSeries table has time bounds', + ); } } return undefined;