From 1f1e315b3e6df342233c31495bd6de20c9201efa Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Wed, 4 Nov 2026 14:12:27 -0800 Subject: [PATCH 1/3] feat: session ID columns on hyperdx_sessions, severity text from severity number --- .../session-id-columns-severity-text.md | 14 +++ docker/otel-collector/config.yaml | 15 +++ .../schema/seed/00004_hyperdx_sessions.sql | 19 ++- .../infer-text-from-number/assert_query.sql | 1 + .../infer-text-from-number/expected.snap | 11 ++ .../infer-text-from-number/input.json | 108 ++++++++++++++++++ .../skip-infer/expected.snap | 2 +- .../otel-collector/severity-inference.bats | 6 + 8 files changed, 165 insertions(+), 11 deletions(-) create mode 100644 .changeset/session-id-columns-severity-text.md create mode 100644 smoke-tests/otel-collector/data/severity-inference/infer-text-from-number/assert_query.sql create mode 100644 smoke-tests/otel-collector/data/severity-inference/infer-text-from-number/expected.snap create mode 100644 smoke-tests/otel-collector/data/severity-inference/infer-text-from-number/input.json diff --git a/.changeset/session-id-columns-severity-text.md b/.changeset/session-id-columns-severity-text.md new file mode 100644 index 0000000000..27cce1c831 --- /dev/null +++ b/.changeset/session-id-columns-severity-text.md @@ -0,0 +1,14 @@ +--- +'@hyperdx/otel-collector': patch +--- + +feat: session ID columns on `hyperdx_sessions`, severity text from severity number + +The `hyperdx_sessions` seed adds materialized `rum.sessionId` and `session.id` +columns with skip indexes, so the sessions list and replay lookups no longer +read the table's attribute maps. Only newly created tables get them; existing +tables need an `ALTER TABLE ... ADD COLUMN` to benefit. + +Logs that carry a severity number but no severity text, such as those from the +new browser SDK, now get the matching level text (`info`, `error`, ...) instead +of an empty level. Severity text set by the producer is never overwritten. diff --git a/docker/otel-collector/config.yaml b/docker/otel-collector/config.yaml index fce203c602..fd4bca9878 100644 --- a/docker/otel-collector/config.yaml +++ b/docker/otel-collector/config.yaml @@ -117,6 +117,21 @@ processors: - set(log.severity_text, "info") where log.severity_number == 0 - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_number == 0 + - context: log + error_mode: ignore + conditions: + - severity_text == "" and severity_number > 0 + statements: + - set(log.severity_text, "trace") where log.severity_number < 5 + - set(log.severity_text, "debug") where log.severity_number >= 5 and + log.severity_number < 9 + - set(log.severity_text, "info") where log.severity_number >= 9 and + log.severity_number < 13 + - set(log.severity_text, "warn") where log.severity_number >= 13 and + log.severity_number < 17 + - set(log.severity_text, "error") where log.severity_number >= 17 and + log.severity_number < 21 + - set(log.severity_text, "fatal") where log.severity_number >= 21 - context: log error_mode: ignore statements: diff --git a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql index 076375c597..7c4d1433ab 100644 --- a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql +++ b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql @@ -1,6 +1,5 @@ -- +goose Up -CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions -( +CREATE TABLE IF NOT EXISTS $ { DATABASE }.hyperdx_sessions ( `Timestamp` DateTime64(9) CODEC(Delta, ZSTD(3)), `TimestampTime` DateTime DEFAULT toDateTime(Timestamp), `TraceId` String CODEC(ZSTD(3)), @@ -17,7 +16,11 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions `ScopeVersion` LowCardinality(String) CODEC(ZSTD(3)), `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `__hdx_materialized_session.id` String MATERIALIZED LogAttributes ['session.id'] CODEC(ZSTD(3)), + `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes ['rum.sessionId'] CODEC(ZSTD(3)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_session_id __hdx_materialized_session.id TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, @@ -25,11 +28,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 -) -ENGINE = MergeTree -PARTITION BY toDate(TimestampTime) -PRIMARY KEY (ServiceName, TimestampTime) -ORDER BY (ServiceName, TimestampTime, Timestamp) -TTL TimestampTime + ${SESSIONS_TTL} -SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; - +) ENGINE = MergeTree PARTITION BY toDate(TimestampTime) PRIMARY KEY (ServiceName, TimestampTime) +ORDER BY + (ServiceName, TimestampTime, Timestamp) TTL TimestampTime + $ { SESSIONS_TTL } SETTINGS index_granularity = 8192, + ttl_only_drop_parts = 1; \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-text-from-number/assert_query.sql b/smoke-tests/otel-collector/data/severity-inference/infer-text-from-number/assert_query.sql new file mode 100644 index 0000000000..712c70d0e2 --- /dev/null +++ b/smoke-tests/otel-collector/data/severity-inference/infer-text-from-number/assert_query.sql @@ -0,0 +1 @@ +SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-text-from-number' ORDER BY (toStartOfFiveMinutes(Timestamp), Timestamp) FORMAT CSV diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-text-from-number/expected.snap b/smoke-tests/otel-collector/data/severity-inference/infer-text-from-number/expected.snap new file mode 100644 index 0000000000..dad30759a6 --- /dev/null +++ b/smoke-tests/otel-collector/data/severity-inference/infer-text-from-number/expected.snap @@ -0,0 +1,11 @@ +"trace",1,"number 1 only" +"trace",4,"number 4 only" +"debug",5,"number 5 only" +"info",9,"number 9 only" +"warn",13,"number 13 only" +"error",17,"number 17 only" +"error",20,"number 20 only" +"fatal",21,"number 21 only" +"fatal",24,"number 24 only" +"error",17,"producer text wins" +"notice",9,"producer text wins even when it disagrees with the number" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/severity-inference/infer-text-from-number/input.json b/smoke-tests/otel-collector/data/severity-inference/infer-text-from-number/input.json new file mode 100644 index 0000000000..ee1a69f231 --- /dev/null +++ b/smoke-tests/otel-collector/data/severity-inference/infer-text-from-number/input.json @@ -0,0 +1,108 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "severity-inference" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "infer-text-from-number" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1901999590000000000", + "severity_number": 1, + "body": { + "stringValue": "number 1 only" + } + }, + { + "timeUnixNano": "1901999590000000001", + "severity_number": 4, + "body": { + "stringValue": "number 4 only" + } + }, + { + "timeUnixNano": "1901999590000000002", + "severity_number": 5, + "body": { + "stringValue": "number 5 only" + } + }, + { + "timeUnixNano": "1901999590000000003", + "severity_number": 9, + "body": { + "stringValue": "number 9 only" + } + }, + { + "timeUnixNano": "1901999590000000004", + "severity_number": 13, + "body": { + "stringValue": "number 13 only" + } + }, + { + "timeUnixNano": "1901999590000000005", + "severity_number": 17, + "body": { + "stringValue": "number 17 only" + } + }, + { + "timeUnixNano": "1901999590000000006", + "severity_number": 20, + "body": { + "stringValue": "number 20 only" + } + }, + { + "timeUnixNano": "1901999590000000007", + "severity_number": 21, + "body": { + "stringValue": "number 21 only" + } + }, + { + "timeUnixNano": "1901999590000000008", + "severity_number": 24, + "body": { + "stringValue": "number 24 only" + } + }, + { + "timeUnixNano": "1901999590000000009", + "severity_number": 17, + "severity_text": "ERROR", + "body": { + "stringValue": "producer text wins" + } + }, + { + "timeUnixNano": "1901999590000000010", + "severity_number": 9, + "severity_text": "notice", + "body": { + "stringValue": "producer text wins even when it disagrees with the number" + } + } + ] + } + ] + } + ] +} diff --git a/smoke-tests/otel-collector/data/severity-inference/skip-infer/expected.snap b/smoke-tests/otel-collector/data/severity-inference/skip-infer/expected.snap index b5b2d7b7e1..fbab437976 100644 --- a/smoke-tests/otel-collector/data/severity-inference/skip-infer/expected.snap +++ b/smoke-tests/otel-collector/data/severity-inference/skip-infer/expected.snap @@ -1,2 +1,2 @@ "debug",0,"2025-05-03T03:23:14Z [INFO] this should skip because the severity_text is already set" -"",9,"[warn] this should also be skipped because a severity_number was set even if it's missing text" \ No newline at end of file +"info",9,"[warn] this should also be skipped because a severity_number was set even if it's missing text" \ No newline at end of file diff --git a/smoke-tests/otel-collector/severity-inference.bats b/smoke-tests/otel-collector/severity-inference.bats index b5683efe1d..032aa3112f 100644 --- a/smoke-tests/otel-collector/severity-inference.bats +++ b/smoke-tests/otel-collector/severity-inference.bats @@ -45,6 +45,12 @@ load 'test_helpers/assertions.bash' assert_test_data "data/severity-inference/skip-infer" } +@test "should derive severity text from severity number when only the number is set" { + emit_otel_data "http://localhost:4318" "data/severity-inference/infer-text-from-number" + sleep 1 + assert_test_data "data/severity-inference/infer-text-from-number" +} + @test "should not infer severity from keywords embedded mid-word" { emit_otel_data "http://localhost:4318" "data/severity-inference/no-infer-substring" sleep 1 From a923178ff55a2cd906e34f29388ee30005114868 Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Wed, 4 Nov 2026 15:18:01 -0800 Subject: [PATCH 2/3] feat: copy rum.sessionId to session.id and add session.id column to otel_logs --- .../session-id-columns-severity-text.md | 15 ++- docker/otel-collector/config.yaml | 32 ++++- docker/otel-collector/custom.config.yaml | 12 +- .../schema/seed/00002_otel_logs.sql | 2 + .../schema/seed/00002_otel_logs_compat.sql | 2 + .../schema/seed/00004_hyperdx_sessions.sql | 19 +-- packages/otel-collector/README.md | 10 +- .../custom-pipeline-config.yaml | 14 ++- .../session-id-copy/logs/assert_query.sql | 10 ++ .../data/session-id-copy/logs/expected.snap | 5 + .../data/session-id-copy/logs/input.json | 118 ++++++++++++++++++ .../session-id-copy/replay/assert_query.sql | 10 ++ .../data/session-id-copy/replay/expected.snap | 2 + .../data/session-id-copy/replay/input.json | 75 +++++++++++ .../session-id-copy/spans/assert_query.sql | 10 ++ .../data/session-id-copy/spans/expected.snap | 3 + .../data/session-id-copy/spans/input.json | 107 ++++++++++++++++ .../otel-collector/session-id-copy.bats | 26 ++++ 18 files changed, 444 insertions(+), 28 deletions(-) create mode 100644 smoke-tests/otel-collector/data/session-id-copy/logs/assert_query.sql create mode 100644 smoke-tests/otel-collector/data/session-id-copy/logs/expected.snap create mode 100644 smoke-tests/otel-collector/data/session-id-copy/logs/input.json create mode 100644 smoke-tests/otel-collector/data/session-id-copy/replay/assert_query.sql create mode 100644 smoke-tests/otel-collector/data/session-id-copy/replay/expected.snap create mode 100644 smoke-tests/otel-collector/data/session-id-copy/replay/input.json create mode 100644 smoke-tests/otel-collector/data/session-id-copy/spans/assert_query.sql create mode 100644 smoke-tests/otel-collector/data/session-id-copy/spans/expected.snap create mode 100644 smoke-tests/otel-collector/data/session-id-copy/spans/input.json create mode 100644 smoke-tests/otel-collector/session-id-copy.bats diff --git a/.changeset/session-id-columns-severity-text.md b/.changeset/session-id-columns-severity-text.md index 27cce1c831..b92efb96b4 100644 --- a/.changeset/session-id-columns-severity-text.md +++ b/.changeset/session-id-columns-severity-text.md @@ -2,12 +2,19 @@ '@hyperdx/otel-collector': patch --- -feat: session ID columns on `hyperdx_sessions`, severity text from severity number +feat: session ID columns, `session.id` from `rum.sessionId`, severity text The `hyperdx_sessions` seed adds materialized `rum.sessionId` and `session.id` -columns with skip indexes, so the sessions list and replay lookups no longer -read the table's attribute maps. Only newly created tables get them; existing -tables need an `ALTER TABLE ... ADD COLUMN` to benefit. +columns with skip indexes, and the `otel_logs` seed adds a materialized +`session.id` column with a skip index, so session queries no longer read the +attribute maps. Only newly created tables get them; existing tables need an +`ALTER TABLE ... ADD COLUMN` to benefit. + +Spans, logs and session replay events from older `@hyperdx/browser` SDKs, which +carry the session id only in the `rum.sessionId` resource attribute, now also +get the OpenTelemetry `session.id` attribute. An existing `session.id` is never +overwritten and `rum.sessionId` is kept. If you override pipeline `processors:` +lists in `CUSTOM_OTELCOL_CONFIG_FILE`, add `transform/session_id` to keep this. Logs that carry a severity number but no severity text, such as those from the new browser SDK, now get the matching level text (`info`, `error`, ...) instead diff --git a/docker/otel-collector/config.yaml b/docker/otel-collector/config.yaml index fd4bca9878..48ac20fd8a 100644 --- a/docker/otel-collector/config.yaml +++ b/docker/otel-collector/config.yaml @@ -137,6 +137,30 @@ processors: statements: # Normalize the severity_text case - set(log.severity_text, ConvertCase(log.severity_text, "lower")) + # Older @hyperdx/browser SDKs put the session id only in the + # `rum.sessionId` resource attribute. Copy it to the OTel `session.id` + # attribute so newer and older SDK data can be queried by one key. An + # existing `session.id` (e.g. from a JSON body merged by `transform`) is + # never overwritten, and `rum.sessionId` is kept. + transform/session_id: + trace_statements: + - context: span + error_mode: ignore + statements: + - set(span.attributes["session.id"], + resource.attributes["rum.sessionId"]) where + span.attributes["session.id"] == nil and + IsString(resource.attributes["rum.sessionId"]) and + Len(resource.attributes["rum.sessionId"]) > 0 + log_statements: + - context: log + error_mode: ignore + statements: + - set(log.attributes["session.id"], + resource.attributes["rum.sessionId"]) where + log.attributes["session.id"] == nil and + IsString(resource.attributes["rum.sessionId"]) and + Len(resource.attributes["rum.sessionId"]) > 0 resourcedetection: detectors: - env @@ -198,10 +222,12 @@ service: # merge wins. See https://github.com/hyperdxio/hyperdx/pull/2351. pipelines: traces: - processors: [memory_limiter, batch] + processors: [memory_limiter, transform/session_id, batch] metrics: processors: [memory_limiter, batch] + # transform/session_id runs after transform so a `session.id` parsed from + # a JSON body wins. Replay rows skip transform but still need the copy. logs/out-default: - processors: [memory_limiter, transform, batch] + processors: [memory_limiter, transform, transform/session_id, batch] logs/out-rrweb: - processors: [memory_limiter, batch] + processors: [memory_limiter, transform/session_id, batch] diff --git a/docker/otel-collector/custom.config.yaml b/docker/otel-collector/custom.config.yaml index 94c6219135..7a71b3c344 100644 --- a/docker/otel-collector/custom.config.yaml +++ b/docker/otel-collector/custom.config.yaml @@ -24,13 +24,13 @@ receivers: # service: # pipelines: # traces: -# processors: [memory_limiter/custom, batch] +# processors: [memory_limiter/custom, transform/session_id, batch] # metrics: # processors: [memory_limiter/custom, batch] # logs/out-default: -# processors: [memory_limiter/custom, transform, batch] +# processors: [memory_limiter/custom, transform, transform/session_id, batch] # logs/out-rrweb: -# processors: [memory_limiter/custom, batch] +# processors: [memory_limiter/custom, transform/session_id, batch] # # Example: tune `batch` for lower-latency exports. # (See packages/otel-collector/README.md for env-var alternatives.) @@ -44,15 +44,15 @@ receivers: # service: # pipelines: # traces: -# processors: [memory_limiter, batch/lowlatency] +# processors: [memory_limiter, transform/session_id, batch/lowlatency] # logs/out-default: -# processors: [memory_limiter, transform, batch/lowlatency] +# processors: [memory_limiter, transform, transform/session_id, batch/lowlatency] exporters: # fork data to otel-collector-json service (DEV ONLY) otlphttp/json: endpoint: http://otel-collector-json:4318 headers: - authorization: "super-secure-ingestion-api-key" + authorization: 'super-secure-ingestion-api-key' compression: gzip service: pipelines: diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql index db8b68092a..12ec3fbe37 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql @@ -25,10 +25,12 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(3)), `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(3)), `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(3)), + `__hdx_materialized_session.id` String MATERIALIZED LogAttributes['session.id'] CODEC(ZSTD(3)), `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))), `ScopeAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ScopeAttributes::Array(Tuple(String, String))), `LogAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), LogAttributes::Array(Tuple(String, String))), INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), + INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'), diff --git a/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql index 20a6d05bf5..a8660f9f7b 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql @@ -26,7 +26,9 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(3)), `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(3)), `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(3)), + `__hdx_materialized_session.id` String MATERIALIZED LogAttributes['session.id'] CODEC(ZSTD(3)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_session_id __hdx_materialized_session.id TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, diff --git a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql index 7c4d1433ab..c523cbdade 100644 --- a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql +++ b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql @@ -1,5 +1,6 @@ -- +goose Up -CREATE TABLE IF NOT EXISTS $ { DATABASE }.hyperdx_sessions ( +CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions +( `Timestamp` DateTime64(9) CODEC(Delta, ZSTD(3)), `TimestampTime` DateTime DEFAULT toDateTime(Timestamp), `TraceId` String CODEC(ZSTD(3)), @@ -16,8 +17,8 @@ CREATE TABLE IF NOT EXISTS $ { DATABASE }.hyperdx_sessions ( `ScopeVersion` LowCardinality(String) CODEC(ZSTD(3)), `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), - `__hdx_materialized_session.id` String MATERIALIZED LogAttributes ['session.id'] CODEC(ZSTD(3)), - `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes ['rum.sessionId'] CODEC(ZSTD(3)), + `__hdx_materialized_session.id` String MATERIALIZED LogAttributes['session.id'] CODEC(ZSTD(3)), + `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(3)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_session_id __hdx_materialized_session.id TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1, @@ -28,7 +29,11 @@ CREATE TABLE IF NOT EXISTS $ { DATABASE }.hyperdx_sessions ( INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 -) ENGINE = MergeTree PARTITION BY toDate(TimestampTime) PRIMARY KEY (ServiceName, TimestampTime) -ORDER BY - (ServiceName, TimestampTime, Timestamp) TTL TimestampTime + $ { SESSIONS_TTL } SETTINGS index_granularity = 8192, - ttl_only_drop_parts = 1; \ No newline at end of file +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +PRIMARY KEY (ServiceName, TimestampTime) +ORDER BY (ServiceName, TimestampTime, Timestamp) +TTL TimestampTime + ${SESSIONS_TTL} +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + diff --git a/packages/otel-collector/README.md b/packages/otel-collector/README.md index d200712ccf..5c8f05dcfb 100644 --- a/packages/otel-collector/README.md +++ b/packages/otel-collector/README.md @@ -433,13 +433,13 @@ processors: service: pipelines: traces: - processors: [memory_limiter/custom, batch] + processors: [memory_limiter/custom, transform/session_id, batch] metrics: processors: [memory_limiter/custom, batch] logs/out-default: - processors: [memory_limiter/custom, transform, batch] + processors: [memory_limiter/custom, transform, transform/session_id, batch] logs/out-rrweb: - processors: [memory_limiter/custom, batch] + processors: [memory_limiter/custom, transform/session_id, batch] ``` After restart, the collector instantiates `memory_limiter/custom` (and not @@ -466,9 +466,9 @@ processors: service: pipelines: traces: - processors: [memory_limiter, batch/lowlatency] + processors: [memory_limiter, transform/session_id, batch/lowlatency] logs/out-default: - processors: [memory_limiter, transform, batch/lowlatency] + processors: [memory_limiter, transform, transform/session_id, batch/lowlatency] ``` Notes: diff --git a/smoke-tests/otel-collector/custom-pipeline-config.yaml b/smoke-tests/otel-collector/custom-pipeline-config.yaml index 85e8474452..d42f695dc4 100644 --- a/smoke-tests/otel-collector/custom-pipeline-config.yaml +++ b/smoke-tests/otel-collector/custom-pipeline-config.yaml @@ -18,10 +18,18 @@ processors: service: pipelines: traces: - processors: [memory_limiter/custom, batch/lowlatency] + processors: + [memory_limiter/custom, transform/session_id, batch/lowlatency] metrics: processors: [memory_limiter/custom, batch/lowlatency] logs/out-default: - processors: [memory_limiter/custom, transform, batch/lowlatency] + processors: + [ + memory_limiter/custom, + transform, + transform/session_id, + batch/lowlatency, + ] logs/out-rrweb: - processors: [memory_limiter/custom, batch/lowlatency] + processors: + [memory_limiter/custom, transform/session_id, batch/lowlatency] diff --git a/smoke-tests/otel-collector/data/session-id-copy/logs/assert_query.sql b/smoke-tests/otel-collector/data/session-id-copy/logs/assert_query.sql new file mode 100644 index 0000000000..4c329e7dba --- /dev/null +++ b/smoke-tests/otel-collector/data/session-id-copy/logs/assert_query.sql @@ -0,0 +1,10 @@ +SELECT + Body, + mapContains(LogAttributes, 'session.id'), + LogAttributes['session.id'], + `__hdx_materialized_session.id` +FROM otel_logs +WHERE ResourceAttributes['suite-id'] = 'session-id-copy' + AND ResourceAttributes['test-id'] = 'logs' +ORDER BY Timestamp +FORMAT CSV diff --git a/smoke-tests/otel-collector/data/session-id-copy/logs/expected.snap b/smoke-tests/otel-collector/data/session-id-copy/logs/expected.snap new file mode 100644 index 0000000000..e338798798 --- /dev/null +++ b/smoke-tests/otel-collector/data/session-id-copy/logs/expected.snap @@ -0,0 +1,5 @@ +"copied from rum.sessionId",1,"abc","abc" +"own session.id wins",1,"app1","app1" +"{""session.id"":""body1"",""message"":""json body session.id wins""}",1,"body1","body1" +"new sdk session.id unchanged",1,"xyz","xyz" +"no session",0,"","" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/session-id-copy/logs/input.json b/smoke-tests/otel-collector/data/session-id-copy/logs/input.json new file mode 100644 index 0000000000..2495b40a5d --- /dev/null +++ b/smoke-tests/otel-collector/data/session-id-copy/logs/input.json @@ -0,0 +1,118 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "session-id-copy" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "logs" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "old-browser-sdk" + } + }, + { + "key": "rum.sessionId", + "value": { + "stringValue": "abc" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1901999590000000000", + "body": { + "stringValue": "copied from rum.sessionId" + } + }, + { + "timeUnixNano": "1901999590000000001", + "body": { + "stringValue": "own session.id wins" + }, + "attributes": [ + { + "key": "session.id", + "value": { + "stringValue": "app1" + } + } + ] + }, + { + "timeUnixNano": "1901999590000000002", + "body": { + "stringValue": "{\"session.id\":\"body1\",\"message\":\"json body session.id wins\"}" + } + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "session-id-copy" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "logs" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "new-browser-sdk" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1901999590000000003", + "body": { + "stringValue": "new sdk session.id unchanged" + }, + "attributes": [ + { + "key": "session.id", + "value": { + "stringValue": "xyz" + } + } + ] + }, + { + "timeUnixNano": "1901999590000000004", + "body": { + "stringValue": "no session" + } + } + ] + } + ] + } + ] +} diff --git a/smoke-tests/otel-collector/data/session-id-copy/replay/assert_query.sql b/smoke-tests/otel-collector/data/session-id-copy/replay/assert_query.sql new file mode 100644 index 0000000000..56b2b342ab --- /dev/null +++ b/smoke-tests/otel-collector/data/session-id-copy/replay/assert_query.sql @@ -0,0 +1,10 @@ +SELECT + Body, + LogAttributes['session.id'], + `__hdx_materialized_session.id`, + `__hdx_materialized_rum.sessionId` +FROM hyperdx_sessions +WHERE ResourceAttributes['suite-id'] = 'session-id-copy' + AND ResourceAttributes['test-id'] = 'replay' +ORDER BY Timestamp +FORMAT CSV diff --git a/smoke-tests/otel-collector/data/session-id-copy/replay/expected.snap b/smoke-tests/otel-collector/data/session-id-copy/replay/expected.snap new file mode 100644 index 0000000000..2411f76aac --- /dev/null +++ b/smoke-tests/otel-collector/data/session-id-copy/replay/expected.snap @@ -0,0 +1,2 @@ +"replay event copied from rum.sessionId","abc","abc","abc" +"replay event own session.id wins","app1","app1","abc" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/session-id-copy/replay/input.json b/smoke-tests/otel-collector/data/session-id-copy/replay/input.json new file mode 100644 index 0000000000..68715f3f58 --- /dev/null +++ b/smoke-tests/otel-collector/data/session-id-copy/replay/input.json @@ -0,0 +1,75 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "session-id-copy" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "replay" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "old-browser-sdk" + } + }, + { + "key": "rum.sessionId", + "value": { + "stringValue": "abc" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1901999590000000000", + "body": { + "stringValue": "replay event copied from rum.sessionId" + }, + "attributes": [ + { + "key": "rr-web.event", + "value": { + "stringValue": "2" + } + } + ] + }, + { + "timeUnixNano": "1901999590000000001", + "body": { + "stringValue": "replay event own session.id wins" + }, + "attributes": [ + { + "key": "rr-web.event", + "value": { + "stringValue": "3" + } + }, + { + "key": "session.id", + "value": { + "stringValue": "app1" + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/smoke-tests/otel-collector/data/session-id-copy/spans/assert_query.sql b/smoke-tests/otel-collector/data/session-id-copy/spans/assert_query.sql new file mode 100644 index 0000000000..d409609c0b --- /dev/null +++ b/smoke-tests/otel-collector/data/session-id-copy/spans/assert_query.sql @@ -0,0 +1,10 @@ +SELECT + SpanName, + mapContains(SpanAttributes, 'session.id'), + SpanAttributes['session.id'], + ResourceAttributes['rum.sessionId'] +FROM otel_traces +WHERE ResourceAttributes['suite-id'] = 'session-id-copy' + AND ResourceAttributes['test-id'] = 'spans' +ORDER BY Timestamp +FORMAT CSV diff --git a/smoke-tests/otel-collector/data/session-id-copy/spans/expected.snap b/smoke-tests/otel-collector/data/session-id-copy/spans/expected.snap new file mode 100644 index 0000000000..f23472608d --- /dev/null +++ b/smoke-tests/otel-collector/data/session-id-copy/spans/expected.snap @@ -0,0 +1,3 @@ +"copied from rum.sessionId",1,"abc","abc" +"own session.id wins",1,"app1","abc" +"no session",0,"","" \ No newline at end of file diff --git a/smoke-tests/otel-collector/data/session-id-copy/spans/input.json b/smoke-tests/otel-collector/data/session-id-copy/spans/input.json new file mode 100644 index 0000000000..f200299ddc --- /dev/null +++ b/smoke-tests/otel-collector/data/session-id-copy/spans/input.json @@ -0,0 +1,107 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "session-id-copy" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "spans" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "old-browser-sdk" + } + }, + { + "key": "rum.sessionId", + "value": { + "stringValue": "abc" + } + } + ] + }, + "scopeSpans": [ + { + "scope": {}, + "spans": [ + { + "traceId": "6c9f0f0a1b2c3d4e5f60718293a4b5c6", + "spanId": "a1a2a3a4a5a6a7a8", + "parentSpanId": "", + "name": "copied from rum.sessionId", + "kind": 1, + "startTimeUnixNano": "1901999590000000000", + "endTimeUnixNano": "1901999590000100000" + }, + { + "traceId": "6c9f0f0a1b2c3d4e5f60718293a4b5c6", + "spanId": "b1b2b3b4b5b6b7b8", + "parentSpanId": "a1a2a3a4a5a6a7a8", + "name": "own session.id wins", + "kind": 1, + "startTimeUnixNano": "1901999590000000001", + "endTimeUnixNano": "1901999590000100000", + "attributes": [ + { + "key": "session.id", + "value": { + "stringValue": "app1" + } + } + ] + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "session-id-copy" + } + }, + { + "key": "test-id", + "value": { + "stringValue": "spans" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "backend" + } + } + ] + }, + "scopeSpans": [ + { + "scope": {}, + "spans": [ + { + "traceId": "7d0a1b2c3d4e5f60718293a4b5c6d7e8", + "spanId": "c1c2c3c4c5c6c7c8", + "parentSpanId": "", + "name": "no session", + "kind": 2, + "startTimeUnixNano": "1901999590000000002", + "endTimeUnixNano": "1901999590000100000" + } + ] + } + ] + } + ] +} diff --git a/smoke-tests/otel-collector/session-id-copy.bats b/smoke-tests/otel-collector/session-id-copy.bats new file mode 100644 index 0000000000..0e407cd3ac --- /dev/null +++ b/smoke-tests/otel-collector/session-id-copy.bats @@ -0,0 +1,26 @@ +#!/usr/bin/env bats + +# transform/session_id copies the old @hyperdx/browser `rum.sessionId` +# resource attribute into the OTel `session.id` attribute on spans, logs and +# replay rows. An existing `session.id` is never overwritten. + +load 'test_helpers/utilities.bash' +load 'test_helpers/assertions.bash' + +@test "session id copy: spans get session.id from rum.sessionId" { + emit_otel_data "http://localhost:4318" "data/session-id-copy/spans" "traces" + wait_for_rows 9000 "SELECT count() FROM otel_traces WHERE ResourceAttributes['suite-id'] = 'session-id-copy' AND ResourceAttributes['test-id'] = 'spans'" 3 + assert_test_data "data/session-id-copy/spans" +} + +@test "session id copy: logs get session.id from rum.sessionId, own and JSON body values win" { + emit_otel_data "http://localhost:4318" "data/session-id-copy/logs" + wait_for_rows 9000 "SELECT count() FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'session-id-copy' AND ResourceAttributes['test-id'] = 'logs'" 5 + assert_test_data "data/session-id-copy/logs" +} + +@test "session id copy: replay rows in hyperdx_sessions get session.id from rum.sessionId" { + emit_otel_data "http://localhost:4318" "data/session-id-copy/replay" + wait_for_rows 9000 "SELECT count() FROM hyperdx_sessions WHERE ResourceAttributes['suite-id'] = 'session-id-copy' AND ResourceAttributes['test-id'] = 'replay'" 2 + assert_test_data "data/session-id-copy/replay" +} From fc1ad18fbe4550f59147bf3604a991f5eee9dad5 Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Wed, 4 Nov 2026 16:53:38 -0800 Subject: [PATCH 3/3] feat: text indexes for hyperdx_sessions session ids, with a compat seed --- .../schema/seed/00004_hyperdx_sessions.sql | 4 +- .../seed/00004_hyperdx_sessions_compat.sql | 40 ++++++++++ packages/otel-collector/cmd/migrate/main.go | 41 +++++++++- .../otel-collector/cmd/migrate/main_test.go | 75 +++++++++++++++++++ 4 files changed, 156 insertions(+), 4 deletions(-) create mode 100644 docker/otel-collector/schema/seed/00004_hyperdx_sessions_compat.sql diff --git a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql index c523cbdade..e14258ce0d 100644 --- a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql +++ b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql @@ -20,8 +20,8 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions `__hdx_materialized_session.id` String MATERIALIZED LogAttributes['session.id'] CODEC(ZSTD(3)), `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(3)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_session_id __hdx_materialized_session.id TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_session_id __hdx_materialized_session.id TYPE text(tokenizer = 'array'), + INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, diff --git a/docker/otel-collector/schema/seed/00004_hyperdx_sessions_compat.sql b/docker/otel-collector/schema/seed/00004_hyperdx_sessions_compat.sql new file mode 100644 index 0000000000..ddc75a1c02 --- /dev/null +++ b/docker/otel-collector/schema/seed/00004_hyperdx_sessions_compat.sql @@ -0,0 +1,40 @@ +-- +goose Up +-- Compatibility schema for ClickHouse < 26.2 (no full text search indexes) +CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions +( + `Timestamp` DateTime64(9) CODEC(Delta, ZSTD(3)), + `TimestampTime` DateTime DEFAULT toDateTime(Timestamp), + `TraceId` String CODEC(ZSTD(3)), + `SpanId` String CODEC(ZSTD(3)), + `TraceFlags` UInt8, + `SeverityText` LowCardinality(String) CODEC(ZSTD(3)), + `SeverityNumber` UInt8, + `ServiceName` LowCardinality(String) CODEC(ZSTD(3)), + `Body` String CODEC(ZSTD(3)), + `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(3)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(3)), + `ScopeName` String CODEC(ZSTD(3)), + `ScopeVersion` LowCardinality(String) CODEC(ZSTD(3)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(3)), + `__hdx_materialized_session.id` String MATERIALIZED LogAttributes['session.id'] CODEC(ZSTD(3)), + `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(3)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_session_id __hdx_materialized_session.id TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +PRIMARY KEY (ServiceName, TimestampTime) +ORDER BY (ServiceName, TimestampTime, Timestamp) +TTL TimestampTime + ${SESSIONS_TTL} +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index fc4e514ed0..b67e373099 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -718,6 +718,37 @@ func removeCompatTracesSchema(tempDir string) error { return nil } +// swapSessionsSchemaForCompat replaces the full-text-search sessions schema +// with the compatibility variant (bloom_filter session id indexes) in the +// processed temp directory. It removes 00004_hyperdx_sessions.sql and renames +// 00004_hyperdx_sessions_compat.sql to take its place, so goose runs the +// compat schema instead. +func swapSessionsSchemaForCompat(tempDir string) error { + fullTextPath := filepath.Join(tempDir, "00004_hyperdx_sessions.sql") + compatPath := filepath.Join(tempDir, "00004_hyperdx_sessions_compat.sql") + + if err := os.Remove(fullTextPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove full text sessions schema: %w", err) + } + + if err := os.Rename(compatPath, fullTextPath); err != nil { + return fmt.Errorf("failed to rename compat sessions schema: %w", err) + } + + return nil +} + +// removeCompatSessionsSchema removes the compat sessions schema file from the +// temp directory when full text search is supported, so goose doesn't run both +// schemas. +func removeCompatSessionsSchema(tempDir string) error { + compatPath := filepath.Join(tempDir, "00004_hyperdx_sessions_compat.sql") + if err := os.Remove(compatPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove compat sessions schema: %w", err) + } + return nil +} + // listSQLFiles lists all SQL files in a directory for logging purposes func listSQLFiles(dir string) ([]string, error) { entries, err := os.ReadDir(dir) @@ -796,7 +827,7 @@ func main() { } defer os.RemoveAll(tempDir) - // Select the appropriate logs and traces schemas based on ClickHouse version + // Select the appropriate logs, traces and sessions schemas based on ClickHouse version if supportsFullTextSearch(chMajor, chMinor) { if err := removeCompatLogsSchema(tempDir); err != nil { log.Fatalf("Failed to remove compat logs schema: %v", err) @@ -804,14 +835,20 @@ func main() { if err := removeCompatTracesSchema(tempDir); err != nil { log.Fatalf("Failed to remove compat traces schema: %v", err) } + if err := removeCompatSessionsSchema(tempDir); err != nil { + log.Fatalf("Failed to remove compat sessions schema: %v", err) + } } else { - log.Printf("ClickHouse %d.%d < 26.2, falling back to compatibility logs and traces schemas", chMajor, chMinor) + log.Printf("ClickHouse %d.%d < 26.2, falling back to compatibility logs, traces and sessions schemas", chMajor, chMinor) if err := swapLogsSchemaForCompat(tempDir); err != nil { log.Fatalf("Failed to swap logs schema: %v", err) } if err := swapTracesSchemaForCompat(tempDir); err != nil { log.Fatalf("Failed to swap traces schema: %v", err) } + if err := swapSessionsSchemaForCompat(tempDir); err != nil { + log.Fatalf("Failed to swap sessions schema: %v", err) + } } if os.Getenv("ENABLE_PROMQL") != "true" { diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go index ba066ef6b8..84e42687d5 100644 --- a/packages/otel-collector/cmd/migrate/main_test.go +++ b/packages/otel-collector/cmd/migrate/main_test.go @@ -1133,6 +1133,81 @@ func TestRemoveCompatLogsSchema(t *testing.T) { }) } +func TestSwapSessionsSchemaForCompat(t *testing.T) { + t.Run("swaps compat over full text", func(t *testing.T) { + dir := t.TempDir() + fullTextPath := filepath.Join(dir, "00004_hyperdx_sessions.sql") + compatPath := filepath.Join(dir, "00004_hyperdx_sessions_compat.sql") + + if err := os.WriteFile(fullTextPath, []byte("FULL TEXT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(compatPath, []byte("COMPAT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + + if err := swapSessionsSchemaForCompat(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got, err := os.ReadFile(fullTextPath) + if err != nil { + t.Fatalf("failed to read swapped file: %v", err) + } + if string(got) != "COMPAT SCHEMA" { + t.Errorf("expected compat content, got %q", string(got)) + } + if _, err := os.Stat(compatPath); !os.IsNotExist(err) { + t.Error("compat file should have been renamed away") + } + }) + + t.Run("errors when compat file is missing", func(t *testing.T) { + dir := t.TempDir() + fullTextPath := filepath.Join(dir, "00004_hyperdx_sessions.sql") + + if err := os.WriteFile(fullTextPath, []byte("FULL TEXT SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + + if err := swapSessionsSchemaForCompat(dir); err == nil { + t.Fatal("expected error when compat file is missing") + } + }) +} + +func TestRemoveCompatSessionsSchema(t *testing.T) { + t.Run("removes compat file and preserves others", func(t *testing.T) { + dir := t.TempDir() + fullTextPath := filepath.Join(dir, "00004_hyperdx_sessions.sql") + compatPath := filepath.Join(dir, "00004_hyperdx_sessions_compat.sql") + + if err := os.WriteFile(fullTextPath, []byte("FULL TEXT"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(compatPath, []byte("COMPAT"), 0644); err != nil { + t.Fatal(err) + } + + if err := removeCompatSessionsSchema(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if _, err := os.Stat(compatPath); !os.IsNotExist(err) { + t.Error("compat file should have been removed") + } + if _, err := os.Stat(fullTextPath); err != nil { + t.Error("full text file should still exist") + } + }) + + t.Run("no error when compat file does not exist", func(t *testing.T) { + if err := removeCompatSessionsSchema(t.TempDir()); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) +} + // --------------------------------------------------------------------------- // processSchemaDir // ---------------------------------------------------------------------------