diff --git a/CHANGELOG.md b/CHANGELOG.md index fa8d70da..99928707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,21 @@ All notable changes to this project will be documented in this file. It uses the ## Unreleased +### ⚡ Improvements + +* Preserve declared fractional-second precision when mapping `time`, + `timestamp`, and `timestamptz` columns to chDB types ([#101]). + ### 🐞 Bug Fixes +* Use PostgreSQL session `TimeZone` in chDB so `COPY FROM` preserves + `timestamp` wall-clock values when input omits time zone ([#101]). * Preserve Azure SAS tokens between schema inference and data loading in `CREATE TABLE ... COPY FROM` ([#103]). * Fix GCC 15 build failures caused by discarded `const` qualifiers ([#103]). + [#101]: https://github.com/ClickHouse/pg_chdb/pull/101 + "ClickHouse/pg_chdb#101 Have chDB use postgres session's TimeZone" [#103]: https://github.com/ClickHouse/pg_chdb/pull/103 "ClickHouse/pg_chdb#103 Use const more" diff --git a/doc/chdb_hook.md b/doc/chdb_hook.md index 6f70bfd3..ecc4f856 100644 --- a/doc/chdb_hook.md +++ b/doc/chdb_hook.md @@ -483,7 +483,7 @@ those you need. | bytea | String | | | date | Date32 | | | time | Time64(6) | Override with `String` for formats that don't support times. | -| timestamp | DateTime64(6, 'UTC') | Converted from session time zone. | +| timestamp | DateTime64(6) | Converted from session time zone. | | timestamptz | DateTime64(6, 'UTC') | | | interval | String | Override with an `Interval` unit such as `IntervalDay`. | | uuid | UUID | | diff --git a/src/helper.c b/src/helper.c index f7d96f63..00f46427 100644 --- a/src/helper.c +++ b/src/helper.c @@ -26,6 +26,8 @@ #include "utils/palloc.h" #include "utils/wait_event.h" +#include "pg-clickhouse.h" + #include "helper.h" #include "setup.h" @@ -367,6 +369,8 @@ build_setup( appendBinaryStringInfo(buf, (char*)&ctx->max_threads, sizeof(ctx->max_threads)); appendBinaryStringInfo(buf, (char*)&ctx->max_parsers, sizeof(ctx->max_parsers)); + /* timestamp maps to DateTime64 without time zone, read in this zone */ + append_string(buf, pgch_session_timezone()); append_string(buf, query); uint16_t count = (uint16_t)nparams; diff --git a/src/helper/chdb_helper.c b/src/helper/chdb_helper.c index 10fb9acd..bf1adf01 100644 --- a/src/helper/chdb_helper.c +++ b/src/helper/chdb_helper.c @@ -309,7 +309,8 @@ setup_session( chdb_connection conn, uint16_t max_mem, uint16_t max_threads, - uint16_t max_parsers + uint16_t max_parsers, + str timezone ) { char settings[1024]; snprintf( @@ -321,10 +322,12 @@ setup_session( "output_format_native_encode_types_in_binary_format=0," "date_time_output_format='iso'," "max_threads=%" PRIu16 ",max_parsing_threads=%" PRIu16 - ",max_memory_usage=%" PRIu64, + ",max_memory_usage=%" PRIu64 ",session_timezone='%.*s'", max_threads, max_parsers, - (uint64_t)max_mem * 1024 * 1024 + (uint64_t)max_mem * 1024 * 1024, + (int)timezone.len, + timezone.data ); chdb_result* res = chdb_query(conn, settings, native_format); const char* err = chdb_result_error(res); @@ -346,6 +349,7 @@ main(void) { uint16_t max_mem = take2(&cur); uint16_t max_threads = take2(&cur); uint16_t max_parsers = take2(&cur); + str timezone = take_str(&cur); str query = take_str(&cur); uint16_t npar = take2(&cur); @@ -380,7 +384,7 @@ main(void) { * Unfortunately, setting via argv doesn't work, so we have to set them a * an initial query. https://github.com/chdb-io/chdb-core/issues/191 */ - int status = setup_session(*conn, max_mem, max_threads, max_parsers); + int status = setup_session(*conn, max_mem, max_threads, max_parsers, timezone); if (!status) { switch (cmd_type) { diff --git a/src/hook/copy.c b/src/hook/copy.c index 2cbeb2d5..c5b4769b 100644 --- a/src/hook/copy.c +++ b/src/hook/copy.c @@ -89,8 +89,8 @@ structure_for_attnums(TupleDesc desc, List* attnums) { } /* - * Return a copy of `structure` with every bare `type` clause replaced with - * `String`. + * Return a copy of `structure` with every bare `type` clause, along with any + * parameters it takes, replaced with `String`. */ static char* structure_as_string(const char* structure, const char* type) { @@ -104,20 +104,25 @@ structure_as_string(const char* structure, const char* type) { quoted = !quoted; } if (!quoted && strncmp(pos, type, len) == 0 && pos > structure && - (pos[-1] == ' ' || pos[-1] == '(') && - (pos[len] == '\0' || pos[len] == ',' || pos[len] == ')')) { - appendStringInfoString(&buf, "String"); - pos += len; - } else { - appendStringInfoChar(&buf, *pos++); + (pos[-1] == ' ' || pos[-1] == '(')) { + const char* end = pos + len; + if (*end == '(' && strchr(end, ')')) { + end = strchr(end, ')') + 1; + } + if (*end == '\0' || *end == ',' || *end == ')') { + appendStringInfoString(&buf, "String"); + pos = end; + continue; + } } + appendStringInfoChar(&buf, *pos++); } return buf.data; } /* * Returns true if `format` is one of the formats lacking Time64 support. - * In such cases, `Time64(6)` should be replaced with `String`. + * In such cases, `Time64` should be replaced with `String`. */ static bool format_lacks_time64(const char* format) { @@ -152,7 +157,7 @@ chdb_copy(chdbCopyContext* ctx) { ctx->structure = structure_as_string(ctx->structure, "UUID"); } if (format_lacks_time64(ctx->format)) { - ctx->structure = structure_as_string(ctx->structure, "Time64(6)"); + ctx->structure = structure_as_string(ctx->structure, "Time64"); } } diff --git a/src/setup.h b/src/setup.h index 9376fb1f..e4d7d1fb 100644 --- a/src/setup.h +++ b/src/setup.h @@ -24,6 +24,7 @@ typedef struct chdbHelperContext { * Fields are native endian. * * chdbHelperContext command type, settings + * string session_timezone * string query * uint16 parameter count * string parameter name and value, repeated diff --git a/t/structure.pl b/t/structure.pl index 18e38a96..878249e3 100644 --- a/t/structure.pl +++ b/t/structure.pl @@ -302,7 +302,7 @@ qq{COPY datetime FROM 'file://$dir/nonesuch.csv'}, qr/nonesuch.csv doesn't exist/, qr[\QSELECT * FROM file], - qr[\Q structure: "ts Nullable(DateTime64(6, 'UTC')), tsn Nullable(DateTime64(6, 'UTC')), tstz DateTime64(6, 'UTC'), tstzn DateTime64(6, 'UTC'), date Date32, time Time64(6), timen Time64(6), ttz String, ttzn String, ival String" }], + qr[\Q structure: "ts Nullable(DateTime64(6)), tsn Nullable(DateTime64(3)), tstz DateTime64(6, 'UTC'), tstzn DateTime64(4, 'UTC'), date Date32, time Time64(6), timen Time64(3), ttz String, ttzn String, ival String" }], ); $node->psql(postgres => q{ @@ -324,7 +324,7 @@ qq{COPY datetime_arrays FROM 'file://$dir/nonesuch.csv'}, qr/nonesuch.csv doesn't exist/, qr[\QSELECT * FROM file], - qr[\Q structure: "ts Array(Nullable(DateTime64(6, 'UTC'))), tsn Array(Nullable(DateTime64(6, 'UTC'))), tstz Array(Nullable(DateTime64(6, 'UTC'))), tstzn Array(Nullable(DateTime64(6, 'UTC'))), date Array(Nullable(Date32)), time Array(Nullable(Time64(6))), timen Array(Nullable(Time64(6))), ttz Array(Nullable(String)), ttzn Array(Nullable(String)), ival Array(Nullable(String))" }], + qr[\Q structure: "ts Array(Nullable(DateTime64(6))), tsn Array(Nullable(DateTime64(3))), tstz Array(Nullable(DateTime64(6, 'UTC'))), tstzn Array(Nullable(DateTime64(4, 'UTC'))), date Array(Nullable(Date32)), time Array(Nullable(Time64(6))), timen Array(Nullable(Time64(3))), ttz Array(Nullable(String)), ttzn Array(Nullable(String)), ival Array(Nullable(String))" }], ); } diff --git a/test/expected/timestamp.out b/test/expected/timestamp.out index fbf3d118..86e103fe 100644 --- a/test/expected/timestamp.out +++ b/test/expected/timestamp.out @@ -146,6 +146,72 @@ SELECT pg_read_file(:'tsv_path'); (1 row) +/************************ No offset ************************/ +-- Text without offset reads as wall clock in Postgres timezone for timestamp, +-- UTC for timestamptz. ts stays 12:00 under every timezone. +\set naive_path :temp_dir /naive.tsv +\set naive_url file:// :naive_path +CREATE TABLE naive (id INT, ts TEXT, tstz TEXT); +INSERT INTO naive VALUES (1, '2026-08-28 12:00:00', '2026-08-28 12:00:00'); +COPY naive TO :'naive_url'; +SET timezone TO 'America/Los_Angeles'; +TRUNCATE dt2; +COPY dt2 FROM :'naive_url'; +SELECT * FROM dt2; + id | ts | tstz +----+---------------------+------------------------ + 1 | 2026-08-28 12:00:00 | 2026-08-28 05:00:00-07 +(1 row) + +SET timezone TO 'Asia/Tokyo'; +TRUNCATE dt2; +COPY dt2 FROM :'naive_url'; +SELECT * FROM dt2; + id | ts | tstz +----+---------------------+------------------------ + 1 | 2026-08-28 12:00:00 | 2026-08-28 21:00:00+09 +(1 row) + +-- Fixed offset, which chDB spells Fixed/UTC-07:00:00 +SET TIME ZONE -7; +TRUNCATE dt2; +COPY dt2 FROM :'naive_url'; +SELECT * FROM dt2; + id | ts | tstz +----+---------------------+------------------------ + 1 | 2026-08-28 12:00:00 | 2026-08-28 05:00:00-07 +(1 row) + +SET timezone TO 'UTC'; +/************************ Declared precision ************************/ +-- Inferred structure takes fractional digits from column typmod. +CREATE TABLE precisions ( + ts TIMESTAMP(3) NOT NULL, + tstz TIMESTAMPTZ(1) NOT NULL, + t TIME(4) NOT NULL, + tn TIME(2) NULL +); +INSERT INTO precisions +VALUES ('2026-07-23 20:43:27.363', '2026-07-23 20:43:50.9Z', '13:45:24.2821', '13:45:24.28'); +COPY precisions TO :'tsv_url'; +SELECT pg_read_file(:'tsv_path'); + pg_read_file +------------------------------------------------------------------------------------- + 2026-07-23T20:43:27.363Z 2026-07-23T20:43:50.9Z 13:45:24.2821 13:45:24.28+ + +(1 row) + +-- Parquet replaces parameterized Time64 with String. +\set parquet_url file:// :temp_dir /precisions.parquet +COPY precisions TO :'parquet_url' (format 'Parquet'); +TRUNCATE precisions; +COPY precisions FROM :'parquet_url' (format 'Parquet'); +SELECT * FROM precisions; + ts | tstz | t | tn +-------------------------+--------------------------+---------------+------------- + 2026-07-23 20:43:27.363 | 2026-07-23 20:43:50.9+00 | 13:45:24.2821 | 13:45:24.28 +(1 row) + /************************ Default scales ************************/ -- A DateTime64 or Time64 named without a scale takes chDB's default of -- milliseconds, truncating the microseconds Postgres holds. diff --git a/test/sql/timestamp.sql b/test/sql/timestamp.sql index 423c5993..865f2196 100644 --- a/test/sql/timestamp.sql +++ b/test/sql/timestamp.sql @@ -76,6 +76,54 @@ SELECT pg_read_file(:'tsv_path'); COPY date_times TO :'tsv_url' (structure 'id Int8, ts DateTime64(3, ''Japan''), tstz DateTime64(3, ''America/New_York'')'); SELECT pg_read_file(:'tsv_path'); +/************************ No offset ************************/ +-- Text without offset reads as wall clock in Postgres timezone for timestamp, +-- UTC for timestamptz. ts stays 12:00 under every timezone. +\set naive_path :temp_dir /naive.tsv +\set naive_url file:// :naive_path +CREATE TABLE naive (id INT, ts TEXT, tstz TEXT); +INSERT INTO naive VALUES (1, '2026-08-28 12:00:00', '2026-08-28 12:00:00'); +COPY naive TO :'naive_url'; + +SET timezone TO 'America/Los_Angeles'; +TRUNCATE dt2; +COPY dt2 FROM :'naive_url'; +SELECT * FROM dt2; + +SET timezone TO 'Asia/Tokyo'; +TRUNCATE dt2; +COPY dt2 FROM :'naive_url'; +SELECT * FROM dt2; + +-- Fixed offset, which chDB spells Fixed/UTC-07:00:00 +SET TIME ZONE -7; +TRUNCATE dt2; +COPY dt2 FROM :'naive_url'; +SELECT * FROM dt2; +SET timezone TO 'UTC'; + +/************************ Declared precision ************************/ +-- Inferred structure takes fractional digits from column typmod. +CREATE TABLE precisions ( + ts TIMESTAMP(3) NOT NULL, + tstz TIMESTAMPTZ(1) NOT NULL, + t TIME(4) NOT NULL, + tn TIME(2) NULL +); + +INSERT INTO precisions +VALUES ('2026-07-23 20:43:27.363', '2026-07-23 20:43:50.9Z', '13:45:24.2821', '13:45:24.28'); + +COPY precisions TO :'tsv_url'; +SELECT pg_read_file(:'tsv_path'); + +-- Parquet replaces parameterized Time64 with String. +\set parquet_url file:// :temp_dir /precisions.parquet +COPY precisions TO :'parquet_url' (format 'Parquet'); +TRUNCATE precisions; +COPY precisions FROM :'parquet_url' (format 'Parquet'); +SELECT * FROM precisions; + /************************ Default scales ************************/ -- A DateTime64 or Time64 named without a scale takes chDB's default of -- milliseconds, truncating the microseconds Postgres holds. diff --git a/vendor/pg-clickhouse-c b/vendor/pg-clickhouse-c index f45f0563..9aa10b06 160000 --- a/vendor/pg-clickhouse-c +++ b/vendor/pg-clickhouse-c @@ -1 +1 @@ -Subproject commit f45f05634627c2c0f148700fbe39f948e91efdea +Subproject commit 9aa10b06f061f41c92cbb720239a0bf44ef2a70e