Replace SQLite/Postgres with a log-structured metadata store

Cargo.toml

@@ -14,21 +14,56 @@ members = [
 [workspace.package]
 license = "Apache-2.0"
 
+# Enforced across every crate rather than left to whoever runs clippy locally.
+# The set is deliberately one that passes clean today: a lint config carrying
+# standing warnings just teaches people to scroll past it.
+[workspace.lints.rust]
+# There is no unsafe in this workspace and no reason for any: forbid rather
+# than deny, so it cannot be re-enabled with a local #[allow].
+unsafe_code = "forbid"
+unused_qualifications = "warn"
+unreachable_pub = "warn"
+
+[workspace.lints.clippy]
+all = { level = "warn", priority = -1 }
+# Silent truncation is a bug class this codebase has already been bitten by
+# (a page limit wrapping through usize), so casts are checked rather than
+# trusted.
+cast_possible_truncation = "warn"
+cast_sign_loss = "warn"
+cast_possible_wrap = "warn"
+
+unwrap_used = "warn"
+expect_used = "warn"
+panic = "warn"
+indexing_slicing = "warn"
+
 [workspace.dependencies]
+# OpenTelemetry. Traces are exported over OTLP to a collector; the whole stack
+# is inert unless OTEL_EXPORTER_OTLP_ENDPOINT is set, so a deployment that wants
+# no telemetry pays nothing for it.
+opentelemetry = "0.32"
+opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] }
+opentelemetry-otlp = { version = "0.32", default-features = false, features = ["trace", "metrics", "logs", "grpc-tonic"] }
+tracing-opentelemetry = "0.33"
+opentelemetry-appender-tracing = "0.32"
 axum = { version = "0.7", features = ["macros"] }
 tower = { version = "0.4", features = ["util"] }
 tower-http = { version = "0.5", features = ["trace", "cors"] }
 # "full" was previously requested but nothing here uses tokio::process/
 # signal (checked via grep) -- uc-server does use tokio::fs (local_path
 # read/write in main.rs), so that stays. hyper/reqwest/axum will still
 # pull whatever tokio features *they* need transitively regardless.
-tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time", "fs"] }
+# "signal" for graceful shutdown: without a SIGTERM handler the default
+# disposition kills the process instantly and in-flight requests are severed.
+tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time", "fs", "signal"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "uuid", "chrono"] }
 uuid = { version = "1", features = ["v4", "v7", "serde"] }
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] }
 chrono = { version = "0.4", features = ["serde"] }
+async-trait = "0.1"
 jsonwebtoken = "9"
 casbin = { version = "2", features = ["runtime-tokio"] }
 tracing = "0.1"
@@ -45,7 +80,12 @@ rsa = { version = "0.9", features = ["sha2", "pkcs5"] }
 # Cloud SDK deps (feature-gated in uc-credentials)
 aws-config = { version = "1", default-features = false, features = ["rustls", "rt-tokio"] }
 aws-sdk-sts = { version = "1", default-features = false, features = ["rustls", "rt-tokio"] }
-aws-sdk-s3 = { version = "1", default-features = false, features = ["rustls", "rt-tokio"] }
+# Floored rather than left at "1": the log store's put_if_absent needs
+# PutObject's if_none_match, which older 1.x releases do not expose. Cargo.lock
+# is gitignored here, so a fresh clone resolves whatever is current and nothing
+# else records the requirement. 1.144 is the version this was built and tested
+# against; the true minimum is lower but has not been verified.
+aws-sdk-s3 = { version = "1.144", default-features = false, features = ["rustls", "rt-tokio"] }
 
 [profile.release]
 lto = true

README.md

@@ -11,14 +11,11 @@ unitycatalog-rs/
 │   ├── uc-errors/       ErrorCode enum, UcError, UC/Delta wire error shapes
 │   ├── uc-types/        Privilege, UriScheme, TokenType, SecurableType
 │   ├── uc-openapi/      Serde types from all.yaml + control.yaml + delta.yaml
-│   ├── uc-db/           sqlx row structs + repositories (SQLite / Postgres)
+│   ├── uc-db/           row structs, repositories, log-structured store
 │   ├── uc-auth/         JWT (RS512) + Casbin RBAC
 │   ├── uc-credentials/  AWS/Azure/GCP credential vending
 │   ├── uc-api/          Axum routers — catalog, control, delta APIs
 │   └── uc-server/       Binary: startup wiring, CLI args, serve
-├── migrations/
-│   ├── sqlite/          DDL for SQLite (default)
-│   └── postgres/        DDL for PostgreSQL
 ├── tests/python/        Pytest integration tests
 ├── scripts/
 │   └── seed.py          Seeds sample data (unity catalog + default schema)
@@ -29,7 +26,7 @@ unitycatalog-rs/
 | Concern | Crate |
 |---|---|
 | HTTP server | `axum 0.7` + `tower-http` |
-| Database | `sqlx 0.7` (SQLite default, Postgres via feature flag) |
+| Storage | log-structured object store (S3 / MinIO), materialised in memory |
 | Auth | `jsonwebtoken 9` (RS512 JWT) + `casbin` (RBAC) |
 | Serialization | `serde` + `serde_json` |
 | Cloud credentials | `aws-sdk-sts` (always compiled in; vending toggled at runtime via `--enable-aws-credentials`, default on) |
@@ -42,23 +39,32 @@ unitycatalog-rs/
 cargo build
 ```
 
-For Postgres instead of SQLite:
+There is one backend and no feature flags to choose it.
+
+### 2. Run the server
+
+Key material comes from a secret store, so generate some once:
 
 ```bash
-cargo build --no-default-features --features postgres
+./target/debug/uc-server --generate-key-file ./etc/conf/keys.json
 ```
 
-### 2. Run the server
+Then point it at an object store (`AWS_ENDPOINT_URL` redirects to MinIO):
 
 ```bash
+AWS_ENDPOINT_URL=http://localhost:9000 \
 ./target/debug/uc-server \
   --port 8080 \
-  --config-dir ./etc/conf \
-  --database-url "sqlite:./etc/db/uc.db?mode=rwc" \
+  --storage-root s3://my-bucket/my-org \
+  --key-file ./etc/conf/keys.json \
   --no-auth
 ```
 
-RSA keys are generated automatically on first start under `--config-dir`.
+Key material is never written to the object store, and there is no option to do
+so. Credentials vended by this server are bucket-scoped, so a private key in
+that bucket would be readable by anything holding one. A missing key file is a
+startup error, never a cue to generate: silently minting a new keypair
+invalidates every token already issued.
 
 ### 3. Seed sample data
 
@@ -121,31 +127,89 @@ Iceberg REST catalog (`/api/2.1/unity-catalog/iceberg/*`).
 
 RBAC uses [Casbin](https://casbin.org/) with a hierarchical model: Metastore → Catalog → Schema → Table/Volume/Function/Model.
 
-## Database
+## Storage
+
+Metadata is an append-only JSONL commit log with periodic checkpoints, laid out
+like Delta's `_delta_log`, materialised in memory at startup. There is no
+database, no driver, no migrations and no local state.
+
+Concurrency rests on conditional writes (`If-None-Match: *`), so it needs S3
+(August 2024 or later) or MinIO. Delta commits are partitioned into a log per
+table, matching Delta's own layout.
+
+Multiple replicas may share one log. Writes are safe — a stale replica loses the
+conditional write, replays and retries — but reads are only eventually
+consistent, bounded by `--refresh-interval-secs`.
+
+See [docs/log-structured-metadata.md](docs/log-structured-metadata.md) for the
+design and its limits.
+
+## Observability
 
-SQLite is the default (zero setup). Switch to Postgres at compile time:
+Traces are exported over OTLP when `OTEL_EXPORTER_OTLP_ENDPOINT` is set, and the
+whole stack is inert otherwise — no exporter, no batch processor, no span layer.
+Configured through the standard `OTEL_*` variables rather than bespoke flags.
 
 ```bash
-cargo build --no-default-features --features postgres
+OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 \
+OTEL_SERVICE_NAME=uc-server \
+./target/release/uc-server --storage-root s3://bucket/org --oidc-issuer https://...
 ```
 
-Migrations run automatically on startup from `migrations/sqlite/` or `migrations/postgres/`.
+All three signals go over the same OTLP pipeline.
+
+**Traces.** `http.request` wraps each request, with `store.commit` (carrying
+`uc.operation`, the version that landed, and how many attempts contention cost),
+`store.catch_up`, and the individual `s3.*` object operations nested beneath it.
+Enough to see whether a slow request was spent in the object store or waiting on
+a conditional-write retry.
+
+**Logs.** Exported over OTLP rather than scraped from stdout, so each record
+carries the trace and span id of the request that produced it — a log you cannot
+pivot to its trace is not much better than a log on its own. stdout keeps its
+human-readable output for when the collector itself is the problem.
+
+**Metrics.** Deliberately only resident state:
+
+| Metric | Why |
+|---|---|
+| `uc.store.entities` (by `uc.kind`) | The whole catalog is in memory, so entity count decides whether the process fits its limit |
+| `uc.store.version` | Replay position of the in-memory snapshot |
+
+Everything event-shaped — request rate, latency, commit contention — is already
+on the spans, and the collector's spanmetrics connector derives RED metrics from
+those. Emitting both would be double instrumentation that can disagree with
+itself.
+
+`SIGTERM` drains in-flight requests, then flushes all three signals before exit.
 
 ## CLI Options
 
 ```
---port          Port to listen on (default: 8080)
---config-dir    Path to config directory — RSA keys, JWKS, token (default: ./etc/conf)
---database-url  SQLite or Postgres connection string
---no-auth       Disable JWT/RBAC enforcement
+--port                    Port to listen on (default: 8080)
+--storage-root            Object-store root, as s3://bucket/prefix
+--key-file PATH           JWT signing key material, the path a mounted secret
+                          arrives at. Required. A missing file is an error,
+                          never a cue to generate.
+--generate-key-file PATH  Write fresh key material to PATH and exit, for loading
+                          into a secret store. Refuses to overwrite an existing
+                          file.
+--config-dir              Config directory — holds the dev admin token
+                          (default: ./etc/conf)
+--refresh-interval-secs   Refresh the in-memory snapshot from the log every N
+                          seconds (default 0, off). Only useful with more than
+                          one replica on the same log.
+--no-auth                 Disable JWT/RBAC enforcement
 ```
 
 ## Development
 
 ```bash
-cargo check          # fast type check
-cargo test --lib     # unit tests across the workspace (JWT, serde, error mapping, ...)
-cargo build          # full build
+cargo check                                  # fast type check
+cargo test --lib                             # unit tests across the workspace
+cargo build                                  # full build
+
+cargo test -p uc-db --test test_repos          # repo layer, end to end
 ```
 
 ## License

crates/uc-api/Cargo.toml

@@ -4,11 +4,6 @@ version = "0.1.0"
 edition = "2021"
 license.workspace = true
 
-[features]
-default = ["sqlite"]
-sqlite = ["uc-db/sqlite", "uc-auth/sqlite"]
-postgres = ["uc-db/postgres", "uc-auth/postgres"]
-
 [dependencies]
 uc-errors = { path = "../uc-errors" }
 uc-types = { path = "../uc-types" }
@@ -18,19 +13,18 @@ uc-auth = { path = "../uc-auth" }
 uc-credentials = { path = "../uc-credentials" }
 axum = { workspace = true }
 tower = { workspace = true }
-tower-http = { workspace = true }
 tokio = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 uuid = { workspace = true }
 chrono = { workspace = true }
-tracing = { workspace = true }
-thiserror = { workspace = true }
-sqlx = { workspace = true }
 
 [dev-dependencies]
 tokio = { workspace = true }
 uuid = { workspace = true }
 base64 = { workspace = true }
 jsonwebtoken = { workspace = true }
 chrono = { workspace = true }
+
+[lints]
+workspace = true

crates/uc-api/src/catalog_api/catalogs.rs

@@ -42,7 +42,7 @@ pub async fn create(
     }
 
     validate_sql_name(&req.name)?;
-    let id = Uuid::new_v4();
+    let id = Uuid::now_v7();
     let now = chrono::Utc::now().timestamp_millis();
     let creator = if state.auth_enabled {
         Some(claims.sub.as_str())
@@ -105,7 +105,13 @@ pub async fn list(
     State(state): State<AppState>,
     Query(params): Query<ListParams>,
 ) -> Result<Json<ListCatalogsResponse>, UcError> {
-    let max = params.max_results.unwrap_or(50).min(1000);
+    // A non-positive max_results means "unspecified", not "an empty page". It
+    // used to reach the repo layer and underflow there.
+    let max = params
+        .max_results
+        .filter(|n| *n > 0)
+        .unwrap_or(50)
+        .min(1000);
     let (rows, next_token) = catalog::list(&state.pool, params.page_token.as_deref(), max).await?;
 
     let catalogs = rows
@@ -288,17 +294,14 @@ pub async fn delete(
 }
 
 /// Delete all children of a schema (tables, volumes, functions, models) without deleting the schema itself.
-async fn delete_schema_children(
-    pool: &uc_db::AnyPool,
-    schema_id: uuid::Uuid,
-) -> Result<(), UcError> {
+async fn delete_schema_children(pool: &uc_db::AnyPool, schema_id: Uuid) -> Result<(), UcError> {
     use uc_db::repos::{function, model, table, volume};
 
     // Delete tables (with columns and properties)
     let (tables, _) = table::list(pool, schema_id, None, 10000).await?;
     for t in tables {
         table::delete_columns(pool, t.id).await?;
-        uc_db::repos::property::delete_for_entity(pool, t.id, "table").await?;
+        property::delete_for_entity(pool, t.id, "table").await?;
         table::delete(pool, t.id).await?;
     }
     // Delete volumes

crates/uc-api/src/catalog_api/credentials.rs

@@ -36,7 +36,7 @@ pub async fn create(
         )
         .await?;
     }
-    let id = Uuid::new_v4();
+    let id = Uuid::now_v7();
     let now = now_ms();
     let credential_json = serde_json::to_string(&req.aws_iam_role).unwrap_or_default();
     let row = CredentialRow {
@@ -79,7 +79,13 @@ pub async fn list(
         )
         .await?;
     }
-    let max = params.max_results.unwrap_or(50).min(1000);
+    // A non-positive max_results means "unspecified", not "an empty page". It
+    // used to reach the repo layer and underflow there.
+    let max = params
+        .max_results
+        .filter(|n| *n > 0)
+        .unwrap_or(50)
+        .min(1000);
     let (rows, next_token) =
         credential::list(&state.pool, params.page_token.as_deref(), max).await?;
     let credentials = rows.into_iter().map(to_cred_info).collect();
@@ -117,25 +123,25 @@ pub async fn update(
     let existing = credential::get_by_name(&state.pool, &name).await?;
     if state.auth_enabled {
         let user = get_user(&state, &claims.sub).await?;
-        require(&state, user.id, existing.id, uc_types::Privilege::Owner).await?;
+        require(&state, user.id, existing.id, Privilege::Owner).await?;
     }
     let effective_name = req.new_name.as_deref().unwrap_or(&name);
     let now = now_ms();
     let new_credential_json = req
         .aws_iam_role
         .as_ref()
         .map(|r| serde_json::to_string(r).unwrap_or_default());
-    sqlx::query(
-        "UPDATE uc_credentials SET name=COALESCE($1,name), comment=COALESCE($2,comment), owner=COALESCE($3,owner), credential=COALESCE($4,credential), updated_at=$5, updated_by=$6 WHERE id=$7"
+    credential::update(
+        &state.pool,
+        existing.id,
+        req.new_name.as_deref(),
+        req.comment.as_deref(),
+        req.owner.as_deref(),
+        new_credential_json.as_deref(),
+        now,
+        auth_sub(&state, &claims),
     )
-    .bind(req.new_name.as_deref())
-    .bind(req.comment.as_deref())
-    .bind(req.owner.as_deref())
-    .bind(new_credential_json.as_deref())
-    .bind(now)
-    .bind(auth_sub(&state, &claims))
-    .bind(existing.id)
-    .execute(state.pool.as_ref()).await.map_err(crate::db_err)?;
+    .await?;
     let updated = credential::get_by_name(&state.pool, effective_name).await?;
     Ok(Json(to_cred_info(updated)))
 }