ops: cutover fixes from the live run

ops/selfhost-supabase/nichedb-db

@@ -488,7 +488,8 @@ cmd_indexes() {
   # source (the app builds some after boot), and those must exist here too.
   src_exec pg_dump -U postgres -d "$SOURCE_DB" --schema-only --no-owner --no-privileges --section=post-data > "$d/post.sql"
   python3 "$HERE/lib/split_post_data.py" "$d/post.sql" "$d/pk.now.sql" "$d/rest.sql" >&2
-  if ! diff -q "$d/pk.sql" "$d/pk.now.sql" >/dev/null; then
+  # pg_dump stamps each dump with a fresh \restrict token: compare without it.
+  if ! diff -q <(grep -v '^\\' "$d/pk.sql") <(grep -v '^\\' "$d/pk.now.sql") >/dev/null; then
     warn "primary keys changed on the source since 'schema'; a new table needs the subscription refreshed"
   fi
   log "Building indexes, unique and foreign keys on the target (detached; replication keeps streaming)"
@@ -560,10 +561,13 @@ cmd_cutover() {
   log "Preflight"
   all_ready || die "not every table is in state r: $(sync_states | tr '\n' ' ')"
   [ "$(tgt_q "select subenabled from pg_subscription where subname = '$SUB'")" = t ] || die "subscription is disabled: see 'nichedb-db status'"
-  local si ti
-  si=$(src_q "select count(*) from pg_indexes where schemaname = 'public'")
-  ti=$(tgt_q "select count(*) from pg_indexes where schemaname = 'public'")
-  [ "$si" = "$ti" ] || die "indexes: source $si, target $ti. Run/finish 'nichedb-db indexes' first"
+  # Valid indexes only: a failed CREATE INDEX CONCURRENTLY leaves an invalid,
+  # empty index on the source that pg_dump omits and nothing should copy.
+  local si ti q
+  q="select count(*) from pg_index i join pg_class c on c.oid = i.indrelid join pg_namespace n on n.oid = c.relnamespace where n.nspname = 'public' and i.indisvalid"
+  si=$(src_q "$q")
+  ti=$(tgt_q "$q")
+  [ "$si" = "$ti" ] || die "valid indexes: source $si, target $ti. Run/finish 'nichedb-db indexes' first"
   cmd_verify >/dev/null 2>&1 || die "schemas differ: run 'nichedb-db verify' and reconcile first"
   local lag
   lag=$(slot_lag_bytes)
@@ -595,26 +599,37 @@ cmd_cutover() {
   cmd_verify || warn "verify reported differences; check before trusting the target"
 
   log "4/6 Dropping the subscription (also drops the slot on the source)"
-  tgt_q "drop subscription $SUB"
+  if ! tgt_q "drop subscription $SUB"; then
+    warn "drop subscription failed; detaching the slot and dropping locally"
+    tgt_q "alter subscription $SUB disable; alter subscription $SUB set (slot_name = none); drop subscription $SUB"
+  fi
+  # An orphaned slot would hold WAL on the source (capped at SLOT_KEEP_GB, but
+  # still): make sure nothing is left behind.
+  local left
+  left=$(src_q "select string_agg(slot_name, ',') from pg_replication_slots where not active")
+  if [ -n "$left" ]; then
+    src_q "select pg_drop_replication_slot(slot_name) from pg_replication_slots where not active" >/dev/null
+    note "dropped leftover slot(s) on the source: $left"
+  fi
+  note "slots left on the source: $(src_q "select coalesce(string_agg(slot_name||' active='||active, ', '), 'none') from pg_replication_slots")"
 
   if [ "$SOURCE_MODE" = railway ]; then
-    log "5/6 Railway app -> the new database, web only (the worker moves here)"
-    local old
-    old=$(rw_vars "$RW_APP_SERVICE" true | jq -r '.DATABASE_URL')
-    [ "${OLD_DATABASE_URL:-}" ] || save_state OLD_DATABASE_URL "$old"
-    rw_set_var "$RW_APP_SERVICE" ROLES web
-    rw_set_var "$RW_APP_SERVICE" DATABASE_URL "$(target_url)"
-    note "DATABASE_URL updated (was $old), ROLES=web; Railway is deploying"
-    wait_for_deploy "$RW_APP_SERVICE"
-    curl -fsS --max-time 20 "$APP_HEALTH_URL" >/dev/null && note "Railway app healthy on the new database" || warn "$APP_HEALTH_URL did not answer ok"
+    log "5/6 Railway app stays on its read-only copy for the DNS TTL, web only"
+    # Not repointed at the new database: dev2's 5432 is allowlisted and
+    # Railway's egress addresses are not fixed. Stale resolvers get reads
+    # from Railway for up to ten minutes; writes there fail until they move.
+    if [ "$(rw_vars "$RW_APP_SERVICE" | jq -r '.ROLES')" != web ]; then
+      rw_set_var "$RW_APP_SERVICE" ROLES web
+      wait_for_deploy "$RW_APP_SERVICE"
+    fi
 
     log "6/6 The app on this box, then DNS"
     cmd_app_deploy --start
     local via
     via=$(curl -sk --max-time 20 --resolve "$APP_DOMAIN:443:$TARGET_IP" "https://$APP_DOMAIN/healthz" || true)
     [ "$via" = ok ] || die "https://$APP_DOMAIN via $TARGET_IP answered '$via', not ok; DNS NOT moved (Railway still serves it from the new DB)"
     cmd_app_dns "$TARGET_IP"
-    note "DNS moved; Railway keeps answering old resolvers for the TTL, from the same database"
+    note "DNS moved; Railway answers stale resolvers for the TTL from its read-only copy"
     note "app connections on the target: $(tgt_q "select count(*) from pg_stat_activity where usename = 'postgres' and client_addr is not null and client_addr <> '127.0.0.1'")"
   else
     note "5/6 and 6/6 skipped (rehearsal: no app to repoint)"
@@ -642,9 +657,8 @@ cmd_rollback() {
   warn "Rollback points the app back at Railway. Writes made on the new server since the cutover are NOT copied back."
   confirm "Roll back to the Railway database?"
   if [ "$SOURCE_MODE" = railway ]; then
-    rw_set_var "$RW_APP_SERVICE" DATABASE_URL "${OLD_DATABASE_URL:-\${{Postgres.DATABASE_URL\}}}"
     rw_set_var "$RW_APP_SERVICE" ROLES web,worker
-    note "DATABASE_URL restored, ROLES=web,worker"
+    note "ROLES=web,worker on Railway again"
     tgt_exec "cd $APP_DIR 2>/dev/null && docker compose -f docker-compose.app.yml --env-file deploy.env stop app" 2>/dev/null || true
     note "app on the box stopped"
     if [ -n "${APP_DNS_FLIPPED:-}" ]; then
@@ -694,8 +708,10 @@ porkbun_keys() { # exports PORKBUN_API_KEY / PORKBUN_SECRET_API_KEY from cli-too
 
 app_env_from_railway() { # the app's real environment is Railway's, rendered
   rw_vars "$RW_APP_SERVICE" | jq -r 'to_entries[] | select(.key | test("^RAILWAY_") | not) | "\(.key)=\(.value)"' |
-    grep -Ev '^(DATABASE_URL|REDIS_URL|PORT|NODE_ENV)='
+    grep -Ev '^(DATABASE_URL|REDIS_URL|PORT|NODE_ENV|ROLES)='
   printf 'DATABASE_URL=%s\n' "$(target_url | sed "s|@${DB_DOMAIN}:|@${APP_DB_HOST:-$DB_DOMAIN}:|")"
+  # Railway may be running web-only while the copy runs; the box takes both.
+  printf 'ROLES=web,worker\n'
 }
 
 cmd_app_deploy() { # app-deploy [--start]   clone/pull + build; starts only with --start
@@ -792,8 +808,8 @@ In order:
   app-deploy            clone + build the app under ~/www/nichedb.dev (not started)
   app-cert              cert for nichedb.dev + www over Porkbun DNS-01 (before DNS moves)
   app-vhost             nginx vhost -> 127.0.0.1:3010
-  cutover               source read-only, drain, sequences; then Railway app ->
-                        new DB (web only), start the app here, move DNS at Porkbun
+  cutover               source read-only, drain, sequences; start the app on the
+                        box, move DNS at Porkbun (Railway serves reads for the TTL)
   rollback              point the app back at Railway and make it writable
   cleanup               drop publication/slot and the TCP proxy (after you're sure)