diff --git a/ops/selfhost-supabase/nichedb-db b/ops/selfhost-supabase/nichedb-db index 997c847..0e1d659 100755 --- a/ops/selfhost-supabase/nichedb-db +++ b/ops/selfhost-supabase/nichedb-db @@ -488,7 +488,8 @@ cmd_indexes() { # source (the app builds some after boot), and those must exist here too. src_exec pg_dump -U postgres -d "$SOURCE_DB" --schema-only --no-owner --no-privileges --section=post-data > "$d/post.sql" python3 "$HERE/lib/split_post_data.py" "$d/post.sql" "$d/pk.now.sql" "$d/rest.sql" >&2 - if ! diff -q "$d/pk.sql" "$d/pk.now.sql" >/dev/null; then + # pg_dump stamps each dump with a fresh \restrict token: compare without it. + if ! diff -q <(grep -v '^\\' "$d/pk.sql") <(grep -v '^\\' "$d/pk.now.sql") >/dev/null; then warn "primary keys changed on the source since 'schema'; a new table needs the subscription refreshed" fi log "Building indexes, unique and foreign keys on the target (detached; replication keeps streaming)" @@ -560,10 +561,13 @@ cmd_cutover() { log "Preflight" all_ready || die "not every table is in state r: $(sync_states | tr '\n' ' ')" [ "$(tgt_q "select subenabled from pg_subscription where subname = '$SUB'")" = t ] || die "subscription is disabled: see 'nichedb-db status'" - local si ti - si=$(src_q "select count(*) from pg_indexes where schemaname = 'public'") - ti=$(tgt_q "select count(*) from pg_indexes where schemaname = 'public'") - [ "$si" = "$ti" ] || die "indexes: source $si, target $ti. Run/finish 'nichedb-db indexes' first" + # Valid indexes only: a failed CREATE INDEX CONCURRENTLY leaves an invalid, + # empty index on the source that pg_dump omits and nothing should copy. + local si ti q + q="select count(*) from pg_index i join pg_class c on c.oid = i.indrelid join pg_namespace n on n.oid = c.relnamespace where n.nspname = 'public' and i.indisvalid" + si=$(src_q "$q") + ti=$(tgt_q "$q") + [ "$si" = "$ti" ] || die "valid indexes: source $si, target $ti. Run/finish 'nichedb-db indexes' first" cmd_verify >/dev/null 2>&1 || die "schemas differ: run 'nichedb-db verify' and reconcile first" local lag lag=$(slot_lag_bytes) @@ -595,18 +599,29 @@ cmd_cutover() { cmd_verify || warn "verify reported differences; check before trusting the target" log "4/6 Dropping the subscription (also drops the slot on the source)" - tgt_q "drop subscription $SUB" + if ! tgt_q "drop subscription $SUB"; then + warn "drop subscription failed; detaching the slot and dropping locally" + tgt_q "alter subscription $SUB disable; alter subscription $SUB set (slot_name = none); drop subscription $SUB" + fi + # An orphaned slot would hold WAL on the source (capped at SLOT_KEEP_GB, but + # still): make sure nothing is left behind. + local left + left=$(src_q "select string_agg(slot_name, ',') from pg_replication_slots where not active") + if [ -n "$left" ]; then + src_q "select pg_drop_replication_slot(slot_name) from pg_replication_slots where not active" >/dev/null + note "dropped leftover slot(s) on the source: $left" + fi + note "slots left on the source: $(src_q "select coalesce(string_agg(slot_name||' active='||active, ', '), 'none') from pg_replication_slots")" if [ "$SOURCE_MODE" = railway ]; then - log "5/6 Railway app -> the new database, web only (the worker moves here)" - local old - old=$(rw_vars "$RW_APP_SERVICE" true | jq -r '.DATABASE_URL') - [ "${OLD_DATABASE_URL:-}" ] || save_state OLD_DATABASE_URL "$old" - rw_set_var "$RW_APP_SERVICE" ROLES web - rw_set_var "$RW_APP_SERVICE" DATABASE_URL "$(target_url)" - note "DATABASE_URL updated (was $old), ROLES=web; Railway is deploying" - wait_for_deploy "$RW_APP_SERVICE" - curl -fsS --max-time 20 "$APP_HEALTH_URL" >/dev/null && note "Railway app healthy on the new database" || warn "$APP_HEALTH_URL did not answer ok" + log "5/6 Railway app stays on its read-only copy for the DNS TTL, web only" + # Not repointed at the new database: dev2's 5432 is allowlisted and + # Railway's egress addresses are not fixed. Stale resolvers get reads + # from Railway for up to ten minutes; writes there fail until they move. + if [ "$(rw_vars "$RW_APP_SERVICE" | jq -r '.ROLES')" != web ]; then + rw_set_var "$RW_APP_SERVICE" ROLES web + wait_for_deploy "$RW_APP_SERVICE" + fi log "6/6 The app on this box, then DNS" cmd_app_deploy --start @@ -614,7 +629,7 @@ cmd_cutover() { via=$(curl -sk --max-time 20 --resolve "$APP_DOMAIN:443:$TARGET_IP" "https://$APP_DOMAIN/healthz" || true) [ "$via" = ok ] || die "https://$APP_DOMAIN via $TARGET_IP answered '$via', not ok; DNS NOT moved (Railway still serves it from the new DB)" cmd_app_dns "$TARGET_IP" - note "DNS moved; Railway keeps answering old resolvers for the TTL, from the same database" + note "DNS moved; Railway answers stale resolvers for the TTL from its read-only copy" note "app connections on the target: $(tgt_q "select count(*) from pg_stat_activity where usename = 'postgres' and client_addr is not null and client_addr <> '127.0.0.1'")" else note "5/6 and 6/6 skipped (rehearsal: no app to repoint)" @@ -642,9 +657,8 @@ cmd_rollback() { warn "Rollback points the app back at Railway. Writes made on the new server since the cutover are NOT copied back." confirm "Roll back to the Railway database?" if [ "$SOURCE_MODE" = railway ]; then - rw_set_var "$RW_APP_SERVICE" DATABASE_URL "${OLD_DATABASE_URL:-\${{Postgres.DATABASE_URL\}}}" rw_set_var "$RW_APP_SERVICE" ROLES web,worker - note "DATABASE_URL restored, ROLES=web,worker" + note "ROLES=web,worker on Railway again" tgt_exec "cd $APP_DIR 2>/dev/null && docker compose -f docker-compose.app.yml --env-file deploy.env stop app" 2>/dev/null || true note "app on the box stopped" if [ -n "${APP_DNS_FLIPPED:-}" ]; then @@ -694,8 +708,10 @@ porkbun_keys() { # exports PORKBUN_API_KEY / PORKBUN_SECRET_API_KEY from cli-too app_env_from_railway() { # the app's real environment is Railway's, rendered rw_vars "$RW_APP_SERVICE" | jq -r 'to_entries[] | select(.key | test("^RAILWAY_") | not) | "\(.key)=\(.value)"' | - grep -Ev '^(DATABASE_URL|REDIS_URL|PORT|NODE_ENV)=' + grep -Ev '^(DATABASE_URL|REDIS_URL|PORT|NODE_ENV|ROLES)=' printf 'DATABASE_URL=%s\n' "$(target_url | sed "s|@${DB_DOMAIN}:|@${APP_DB_HOST:-$DB_DOMAIN}:|")" + # Railway may be running web-only while the copy runs; the box takes both. + printf 'ROLES=web,worker\n' } cmd_app_deploy() { # app-deploy [--start] clone/pull + build; starts only with --start @@ -792,8 +808,8 @@ In order: app-deploy clone + build the app under ~/www/nichedb.dev (not started) app-cert cert for nichedb.dev + www over Porkbun DNS-01 (before DNS moves) app-vhost nginx vhost -> 127.0.0.1:3010 - cutover source read-only, drain, sequences; then Railway app -> - new DB (web only), start the app here, move DNS at Porkbun + cutover source read-only, drain, sequences; start the app on the + box, move DNS at Porkbun (Railway serves reads for the TTL) rollback point the app back at Railway and make it writable cleanup drop publication/slot and the TCP proxy (after you're sure)