From e397de581904a3f5d56ad05cb0678709cf5c6e8a Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:38:44 +0000 Subject: [PATCH] Replace PG to CH type mapping with pg-clickhouse-c generated table --- Makefile | 4 ++- dev/README.md | 4 +-- dev/type_table.awk | 70 ++++++++++++++++++++++++++++++--------- doc/chdb_hook.md | 82 +++++++++++++++++++++------------------------- 4 files changed, 96 insertions(+), 64 deletions(-) diff --git a/Makefile b/Makefile index 4c45f9e..938be01 100644 --- a/Makefile +++ b/Makefile @@ -156,9 +156,11 @@ uninstall-libchdb: format: $(wildcard src/*.c src/*.h src/helper/*.c) @$(CLANG_FORMAT) --style=file:.clang-format -i $^ -.PHONY: type-table # Regenerate the chDB to Postgres table of doc/chdb_hook.md. +.PHONY: type-table # Regenerate the data type tables of doc/chdb_hook.md. type-table: @(cd $(PGCH_DIR) && ./gen_type_table.awk) | dev/type_table.awk doc/chdb_hook.md + @(cd $(PGCH_DIR) && ./gen_type_table.awk -v section=ENCODE) | \ + dev/type_table.awk -v section=ENCODE doc/chdb_hook.md .PHONY: clang-tidy # Run clang-tidy static analysis (requires compile_commands.json) clang-tidy: compile_commands.json diff --git a/dev/README.md b/dev/README.md index 92f3892..e85afe6 100644 --- a/dev/README.md +++ b/dev/README.md @@ -11,8 +11,8 @@ development of the chdb extensions. ## Data Type Table -`type_table.awk` rewrites the inferred data type table in -[doc/chdb_hook.md](../doc/chdb_hook.md) from the table that pg-clickhouse-c +`type_table.awk` rewrites both data type tables in +[doc/chdb_hook.md](../doc/chdb_hook.md) from the tables that pg-clickhouse-c generates out of its own regression test: ```sh diff --git a/dev/type_table.awk b/dev/type_table.awk index c925855..181310a 100755 --- a/dev/type_table.awk +++ b/dev/type_table.awk @@ -1,16 +1,22 @@ #!/usr/bin/awk -f -# Filter the pg-clickhouse-c type table into the chDB to Postgres table of -# doc/chdb_hook.md. pgch_pg_type_for reports pseudo types no column holds, -# where the command declares text and reads the fields as its items +# Filter pg-clickhouse-c type tables into doc/chdb_hook.md. +# pgch_pg_type_for reports pseudo types no column holds; CREATE TABLE uses text. +# COPY uses default type options but not upstream's encoder # -# usage: (cd vendor/pg-clickhouse-c && ./gen_type_table.awk) | -# dev/type_table.awk [markdown-file] +# usage: (cd vendor/pg-clickhouse-c && ./gen_type_table.awk [-v section=ENCODE]) | +# dev/type_table.awk [-v section=ENCODE] [markdown-file] function die(msg) { print "type_table: " msg > "/dev/stderr" exit (failed = 1) } +function joined(c, out) { + out = $2 + for (c = 2; c <= COLS; c++) out = out "|" $(c + 1) + return out +} + function read_row(got) { got = (getline < "/dev/stdin") if (got < 0) die("cannot read standard input") @@ -52,8 +58,34 @@ function rule(c, dashes, out) { BEGIN { FS = " *\\| *" - COLS = 4 - header = "ClickHouse|Default PostgreSQL|Additional read targets|Notes" + if (section == "") section = "TYPE" + begin = section "-TABLE-BEGIN" + end = section "-TABLE-END" + + if (section == "TYPE") { + COLS = 4 + header = "ClickHouse|Default PostgreSQL|Additional read targets|Notes" + } else if (section == "ENCODE") { + COLS = 3 + header = "PostgreSQL|Default ClickHouse|Notes" + } else { + die("unknown section " section) + } + + note["numeric"] = "Also when precision exceeds 76 digits." + note["numeric(12,6)"] = "Precision and scale carry over." + note["inet"] = "Override with `IPv4` or `IPv6` if data contains only one or the other." + note["interval"] = "Override with an `Interval` unit such as `IntervalDay`." + note["json"] = "Override with `JSON` if data contains only objects." + note["jsonb"] = "Override with `JSON` if data contains only objects." + note["time"] = "Override with `String` for formats that don't support times." + note["timestamp"] = "Converted from session time zone." + note["point"] = "Same two coordinates as Postgres." + note["lseg"] = "A line of exactly two points." + note["path"] = "A closed path repeats its first point." + note["polygon"] = "A ring closes implicitly, as a polygon does." + note["box"] = "The two corners, sorted as Postgres sorts." + note["line"] = "The equation `Ax + By + C = 0`." upstream["Map(K,V)"] = "record[]|One record per pair" ours["Map(K,V)"] = "text[][]|One row of text items per pair" @@ -63,16 +95,18 @@ BEGIN { ours["Tuple(...)"] = "text[]|Fields become text items" if (!read_row()) die("no table on standard input") - if ($2 "|" $3 "|" $4 "|" $5 != header) { - die("header <" $2 "|" $3 "|" $4 "|" $5 "> is not <" header ">") - } + if (joined() != header) die("header <" joined() "> is not <" header ">") + sub(/(Default )?ClickHouse/, "chDB") keep_row(0) if (!read_row() || $2 !~ /^-+$/) die("no rule under the header") while (read_row()) { type = $2 - if (type in ours) { + seen[type] = 1 + if (section == "ENCODE") { + $4 = note[type] + } else if (type in ours) { if ($3 "|" $5 != upstream[type]) { die("swap for " type " expects <" upstream[type] ">, " \ "got <" $3 "|" $5 ">") @@ -80,13 +114,17 @@ BEGIN { split(ours[type], swap, "|") $3 = swap[1] $5 = swap[2] - swapped[type] = 1 } - if ($3 ~ /^record(\[\])*$/) die("no swap for pseudo type row " type) + if (section == "TYPE" && $3 ~ /^record(\[\])*$/) { + die("no swap for pseudo type row " type) + } keep_row(++rows) } for (type in ours) { - if (!(type in swapped)) die("no row for " type) + if (section == "TYPE" && !(type in seen)) die("no row for " type) + } + for (type in note) { + if (section == "ENCODE" && !(type in seen)) die("no row for " type) } table = row(0) "\n" rule() @@ -99,8 +137,8 @@ BEGIN { } } -/TYPE-TABLE-BEGIN/ { doc = doc $0 "\n" table "\n"; spliced = 1; skip = 1; next } -/TYPE-TABLE-END/ { skip = 0 } +index($0, begin) { doc = doc $0 "\n" table "\n"; spliced = 1; skip = 1; next } +index($0, end) { skip = 0 } !skip { doc = doc $0 "\n" } END { diff --git a/doc/chdb_hook.md b/doc/chdb_hook.md index 2cacd7d..2390422 100644 --- a/doc/chdb_hook.md +++ b/doc/chdb_hook.md @@ -462,50 +462,42 @@ Postgres types to reasonable chDB equivalents. When they don't match your use case, specify the [structure](#structure) to override the generated types with those you need. -| Postgres | chDB | Notes | -| ----------- | ---------------------------------------- | ---------------------------------------------------------------------- | -| boolean | Bool | | -| name | String | | -| text | String | | -| inet | String | Override with `IPv4` or `IPv6` if data contains only one or the other. | -| cidr | String | | -| macaddr | String | | -| macaddr8 | String | | -| interval | String | Override with an `Interval` unit such as `IntervalDay`. | -| tsvector | String | | -| tsquery | String | | -| jsonpath | String | | -| money | String | | -| enum | String | | -| varchar | String | | -| varbit | String | | -| char | FixedString | | -| bit | FixedString | | -| bpchar | String | | -| int2 | Int16 | | -| int4 | Int32 | | -| int8 | Int64 | | -| oid | UInt32 | | -| oid8 | UInt64 | | -| xid8 | UInt64 | | -| json | String | Override with `JSON` if data contains only objects. | -| jsonb | String | Override with `JSON` if data contains only objects. | -| float4 | Float32 | | -| float8 | Float64 | | -| date | Date32 | | -| time | Time64(6) | Override with `String` for formats that don't support times. | -| timetz | String | | -| timestamp | DateTime64(6) | Declared with the `UTC` time zone, converted from session time zone. | -| timestamptz | DateTime64(6) | Declared with the `UTC` time zone. | -| numeric | Decimal | | -| uuid | UUID | | -| point | `Point` | Same two coordinates as Postgres. | -| lseg | `LineString` | A line of exactly two points. | -| path | `LineString` | A closed path repeats its first point. | -| polygon | `Ring` | A ring closes implicitly, as a polygon does. | -| box | `Tuple(high Point, low Point)` | The two corners, sorted as Postgres sorts. | -| circle | `Tuple(center Point, radius Float64)` | | -| line | `Tuple(a Float64, b Float64, c Float64)` | The equation `Ax + By + C = 0`. | + +| PostgreSQL | chDB | Notes | +|------------------|----------------------------------------|------------------------------------------------------------------------| +| boolean | Bool | | +| smallint | Int16 | | +| integer | Int32 | | +| bigint | Int64 | | +| oid | UInt32 | | +| xid8 | UInt64 | | +| oid8 | UInt64 | | +| real | Float32 | | +| double precision | Float64 | | +| numeric | Decimal256(38) | Also when precision exceeds 76 digits. | +| numeric(12,6) | Decimal(12,6) | Precision and scale carry over. | +| text | String | | +| bytea | String | | +| date | Date32 | | +| time | Time64(6) | Override with `String` for formats that don't support times. | +| timestamp | DateTime64(6, 'UTC') | Converted from session time zone. | +| timestamptz | DateTime64(6, 'UTC') | | +| interval | String | Override with an `Interval` unit such as `IntervalDay`. | +| uuid | UUID | | +| json | String | Override with `JSON` if data contains only objects. | +| jsonb | String | Override with `JSON` if data contains only objects. | +| inet | String | Override with `IPv4` or `IPv6` if data contains only one or the other. | +| point | Point | Same two coordinates as Postgres. | +| lseg | LineString | A line of exactly two points. | +| path | LineString | A closed path repeats its first point. | +| polygon | Ring | A ring closes implicitly, as a polygon does. | +| box | Tuple(high Point, low Point) | The two corners, sorted as Postgres sorts. | +| circle | Tuple(center Point, radius Float64) | | +| line | Tuple(a Float64, b Float64, c Float64) | The equation `Ax + By + C = 0`. | + + +Types absent from this table, such as `name`, `varchar`, `char`, `bit`, +`timetz`, `money`, and enums, map to `String`. Array types map to `Array`s of the mapped element type. ClickHouse constrains nullability per column while Postgres constrains it per array, so elements are @@ -572,7 +564,7 @@ casts. Empty cells still allow those casts. These targets describe reads; writes follow separate conversion rules. -| ClickHouse | Default PostgreSQL | Additional read targets | Notes | +| chDB | Default PostgreSQL | Additional read targets | Notes | |------------------------------|-----------------------------|-------------------------------------------|------------------------------------------------------| | Array(T) | T[] | | One PG array type per depth | | BFloat16 | real | | |