diff --git a/backend/src/__tests__/infrastructure/sse/eventBus-resilience.test.ts b/backend/src/__tests__/infrastructure/sse/eventBus-resilience.test.ts index 43379266..0d100497 100644 --- a/backend/src/__tests__/infrastructure/sse/eventBus-resilience.test.ts +++ b/backend/src/__tests__/infrastructure/sse/eventBus-resilience.test.ts @@ -95,6 +95,28 @@ describe('EventBus Resilience & Degraded Mode (No Redis)', () => { return { client, enqueuedChunks }; } + it('notifies each local SSE client before closing its stream', () => { + const bus = new EventBus(); + const clients = ['first', 'second'].map(id => createMockClient(id, id)); + for (const { client } of clients) bus.register(client); + + bus.broadcastShutdown(); + + for (const { client, enqueuedChunks } of clients) { + expect(enqueuedChunks).toHaveLength(1); + const frame = enqueuedChunks[0]; + const delay = Number(frame.match(/^retry: (\d+)$/m)?.[1]); + expect(delay).toBeGreaterThanOrEqual(2000); + expect(delay).toBeLessThanOrEqual(5000); + expect(frame).toContain('event: server_shutdown'); + expect(frame).toContain(JSON.stringify({ reconnectDelayMs: delay })); + expect(client.controller.enqueue).toHaveBeenCalledBefore(vi.mocked(client.controller.close)); + } + expect(bus.getMetrics().activeConnections).toBe(0); + expect(() => bus.register(createMockClient('late', 'late').client)).toThrow('shutting down'); + bus.shutdown(); + }); + it('skips Redis PubSub and does not instantiate ioredis when REDIS_URL is absent', () => { delete process.env.REDIS_URL; diff --git a/backend/src/__tests__/integration/aiAuditPersistence.test.ts b/backend/src/__tests__/integration/aiAuditPersistence.test.ts index d7ade5c4..739b2b41 100644 --- a/backend/src/__tests__/integration/aiAuditPersistence.test.ts +++ b/backend/src/__tests__/integration/aiAuditPersistence.test.ts @@ -26,6 +26,20 @@ const auditEntry = { }; describe('AI audit persistence', () => { + it('waits for an audit write that the caller did not await', async () => { + const audit = new AIResponseAuditLogClass(); + let completed = false; + const write = audit.log(auditEntry).then(entry => { + completed = true; + return entry; + }); + + await audit.flushPendingWrites(); + + expect(completed).toBe(true); + expect(await write).toMatchObject({ userId: auditEntry.userId }); + }); + it('persists metadata for a fresh service instance and excludes raw content', async () => { const writer = new AIResponseAuditLogClass(); const entry = await writer.log(auditEntry); diff --git a/backend/src/__tests__/integration/setup.ts b/backend/src/__tests__/integration/setup.ts index 3506bcbf..84daac7e 100644 --- a/backend/src/__tests__/integration/setup.ts +++ b/backend/src/__tests__/integration/setup.ts @@ -28,7 +28,7 @@ afterAll(async () => { } }); -const [{ getRedisClient }, { logger }, { User, UserXp }] = await Promise.all([ +const [{ getRedisClient, disconnectRedis }, { logger }, { User, UserXp }] = await Promise.all([ import('../../shared/redis/index.js'), import('../../shared/logger.js'), import('../../db/models/index.js'), @@ -53,9 +53,7 @@ beforeAll(async () => { }); afterAll(async () => { - // Disconnect Redis - const redis = getRedisClient(); - await redis.quit(); + await disconnectRedis(); }); beforeEach(async () => { diff --git a/backend/src/__tests__/integration/sseHandshake.test.ts b/backend/src/__tests__/integration/sseHandshake.test.ts index 1ec3f2b5..49a5dd2d 100644 --- a/backend/src/__tests__/integration/sseHandshake.test.ts +++ b/backend/src/__tests__/integration/sseHandshake.test.ts @@ -68,9 +68,7 @@ describe('Security: SSE Handshake Ticket Protocol', () => { }); afterAll(async () => { - server.closeAllConnections?.(); - await new Promise((resolve) => server.close(() => resolve())); - await orchestrator.shutdown(); + await orchestrator.shutdown(server); }); it('should have successfully obtained a test access token', () => { @@ -141,4 +139,18 @@ describe('Security: SSE Handshake Ticket Protocol', () => { const status = await testSseConnection(fakeTicket); expect(status).toBe(401); }); + + it('delivers the shutdown event before ending the HTTP stream', async () => { + const ticketResponse = await fetch(`${API_BASE}/api/auth/sse-handshake`, { + method: 'POST', + headers: { Authorization: `Bearer ${accessToken}` }, + }); + const { data } = await ticketResponse.json() as { data: { ticket: string } }; + const response = await fetch(`${API_BASE}/api/events?ticket=${data.ticket}`); + expect(response.status).toBe(200); + + const [body] = await Promise.all([response.text(), orchestrator.shutdown(server)]); + expect(body).toMatch(/event: server_shutdown\nretry: \d+\ndata: {"reconnectDelayMs":\d+}/); + expect(server.listening).toBe(false); + }); }); diff --git a/backend/src/__tests__/worker-lifecycle/api-startup.test.ts b/backend/src/__tests__/worker-lifecycle/api-startup.test.ts index bce0fa9f..b242b48c 100644 --- a/backend/src/__tests__/worker-lifecycle/api-startup.test.ts +++ b/backend/src/__tests__/worker-lifecycle/api-startup.test.ts @@ -1,14 +1,16 @@ import { afterEach, beforeEach, expect, it, vi } from 'vitest'; const state = vi.hoisted(() => ({ connect: vi.fn(), start: vi.fn(), stop: vi.fn(), queuesClose: vi.fn(), redisClose: vi.fn(), flush: vi.fn(), + auditFlush: vi.fn(), sseDrain: vi.fn(), sseClose: vi.fn(), mongoClose: vi.fn(), setSync: vi.fn(), setXp: vi.fn(), ready: true, })); vi.mock('../../config/env.js', () => ({ env: {} })); -vi.mock('../../db/connection.js', () => ({ connectDatabase: state.connect })); +vi.mock('../../db/connection.js', () => ({ connectDatabase: state.connect, disconnectDatabase: state.mongoClose })); vi.mock('../../modules/runtime-state/unifiedRuntimeState.service.js', () => ({ unifiedRuntimeStateService: { flushPendingEvents: state.flush } })); +vi.mock('../../modules/ai/audit/AIResponseAuditLog.js', () => ({ AIResponseAuditLog: { flushPendingWrites: state.auditFlush } })); vi.mock('../../db/migrations/index-setup.js', () => ({ setupProductionIndexes: vi.fn().mockResolvedValue(undefined) })); vi.mock('../../shared/logger.js', () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } })); -vi.mock('../../shared/sse/index.js', () => ({ eventBus: { shutdown: vi.fn() } })); +vi.mock('../../shared/sse/index.js', () => ({ eventBus: { broadcastShutdown: state.sseDrain, shutdown: state.sseClose } })); vi.mock('../../shared/redis/index.js', () => ({ getRedisClient: () => ({ status: 'ready' }), disconnectRedis: state.redisClose, getRedisHealth: () => ({ status: state.ready ? 'connected' : 'disconnected' }), @@ -25,16 +27,22 @@ vi.mock('../../shared/runtime/infrastructureRegistry.js', () => ({ setPlatformSyncWorkerStatus: state.setSync, setXpWorkerStatus: state.setXp, setSchedulerStatus: vi.fn(), setSseStatus: vi.fn(), })); import express from 'express'; +import { createServer } from 'node:http'; let startup: typeof import('../../shared/runtime/orchestrator.js')['startup']; +let registerShutdownSignals: typeof import('../../shared/runtime/orchestrator.js')['registerShutdownSignals']; let shutdown: typeof import('../../shared/runtime/orchestrator.js')['shutdown']; beforeEach(async () => { vi.resetModules(); state.ready = true; for (const item of Object.values(state)) if (typeof item === 'function') item.mockReset().mockResolvedValue(undefined); vi.stubEnv('ENABLE_WORKERS', 'true'); - ({ startup, shutdown } = await import('../../shared/runtime/orchestrator.js')); + ({ startup, shutdown, registerShutdownSignals } = await import('../../shared/runtime/orchestrator.js')); +}); +afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + vi.useRealTimers(); }); -afterEach(() => vi.unstubAllEnvs()); it('starts the shared registry and drains it before closing queue and Redis handles', async () => { const result = await startup(express()); expect(state.start).toHaveBeenCalledOnce(); @@ -44,7 +52,8 @@ it('starts the shared registry and drains it before closing queue and Redis hand expect(state.queuesClose.mock.invocationCallOrder[0]).toBeGreaterThan(state.stop.mock.invocationCallOrder[0]); expect(state.flush).toHaveBeenCalledOnce(); expect(state.flush.mock.invocationCallOrder[0]).toBeGreaterThan(state.queuesClose.mock.invocationCallOrder[0]); - expect(state.redisClose.mock.invocationCallOrder[0]).toBeGreaterThan(state.flush.mock.invocationCallOrder[0]); + expect(state.redisClose).toHaveBeenCalledAfter(state.flush); + expect(state.redisClose).toHaveBeenCalledAfter(state.auditFlush); }); it.each(['disabled', 'offline'])('does not start consumers when %s', async reason => { if (reason === 'disabled') vi.stubEnv('ENABLE_WORKERS', 'false'); else state.ready = false; @@ -83,3 +92,58 @@ it('prevents late startup from creating workers after shutdown and drains once', expect(state.redisClose).toHaveBeenCalledOnce(); expect(state.flush).toHaveBeenCalledOnce(); }); + +it.each(['SIGTERM', 'SIGINT'] as const)('drains resources in order when it receives %s', async signal => { + const server = createServer(); + const httpClose = vi.spyOn(server, 'close').mockImplementation(callback => { + callback?.(); + return server; + }); + const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never); + let finishMongoClose!: () => void; + state.mongoClose.mockReturnValueOnce(new Promise(resolve => { finishMongoClose = resolve; })); + registerShutdownSignals(server); + const sigtermHandler = process.listeners('SIGTERM').at(-1)!; + const sigintHandler = process.listeners('SIGINT').at(-1)!; + + try { + const handler = signal === 'SIGTERM' ? sigtermHandler : sigintHandler; + handler(signal); + handler(signal); + const draining = shutdown(); + await vi.waitFor(() => expect(state.mongoClose).toHaveBeenCalledOnce()); + expect(exit).not.toHaveBeenCalled(); + finishMongoClose(); + await draining; + + expect(httpClose).toHaveBeenCalledOnce(); + expect(state.stop).toHaveBeenCalledExactlyOnceWith(5000); + const calls = [ + httpClose, state.sseDrain, state.stop, state.queuesClose, state.flush, + state.sseClose, state.redisClose, state.mongoClose, exit, + ].map(mock => mock.mock.invocationCallOrder[0]); + expect(calls).toEqual([...calls].sort((a, b) => a - b)); + expect(exit).toHaveBeenCalledExactlyOnceWith(0); + } finally { + process.off('SIGTERM', sigtermHandler); + process.off('SIGINT', sigintHandler); + } +}); + +it('keeps closing resources when individual phases time out or reject late', async () => { + vi.useFakeTimers(); + let rejectWorkerDrain!: (error: Error) => void; + state.stop.mockReturnValueOnce(new Promise((_, reject) => { rejectWorkerDrain = reject; })); + state.flush.mockReturnValueOnce(new Promise(() => {})); + state.redisClose.mockRejectedValueOnce(new Error('Redis failed')); + + const draining = expect(shutdown()).rejects.toThrow('BullMQ worker drain timed out'); + await vi.advanceTimersByTimeAsync(5000); + expect(state.queuesClose).toHaveBeenCalledOnce(); + expect(state.redisClose).not.toHaveBeenCalled(); + rejectWorkerDrain(new Error('late worker failure')); + await vi.advanceTimersByTimeAsync(3000); + await draining; + expect(state.mongoClose).toHaveBeenCalledOnce(); + expect(vi.getTimerCount()).toBe(0); +}); diff --git a/backend/src/db/connection.ts b/backend/src/db/connection.ts index ebf94a41..05fbcc04 100644 --- a/backend/src/db/connection.ts +++ b/backend/src/db/connection.ts @@ -21,9 +21,10 @@ export const MONGO_OPTIONS: mongoose.ConnectOptions = { }; let reconnectTimer: NodeJS.Timeout | null = null; +let disconnecting = false; export function scheduleMongoReconnect(delayMs = 5000): void { - if (reconnectTimer) return; + if (reconnectTimer || disconnecting) return; reconnectTimer = setTimeout(async () => { reconnectTimer = null; if (mongoose.connection.readyState === 0) { @@ -41,6 +42,7 @@ export function scheduleMongoReconnect(delayMs = 5000): void { } export async function connectDatabase(): Promise { + disconnecting = false; try { logger.info('Attempting MongoDB connection...', { readyState: mongoose.connection.readyState, @@ -61,6 +63,7 @@ export async function connectDatabase(): Promise { } export async function disconnectDatabase(): Promise { + disconnecting = true; if (reconnectTimer) { clearTimeout(reconnectTimer); reconnectTimer = null; @@ -99,8 +102,3 @@ mongoose.connection.on('disconnected', () => { logger.warn('MongoDB disconnected event received', { readyState: mongoose.connection.readyState }); scheduleMongoReconnect(); }); - -process.on('SIGINT', async () => { - await disconnectDatabase(); - process.exit(0); -}); diff --git a/backend/src/index.ts b/backend/src/index.ts index 23d84259..1a057196 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -19,7 +19,7 @@ import { getRedisHealth } from './shared/redis/index.js'; import { validateStartup } from './shared/startup-validation.js'; import { getOrCreateQueue, QueueNames } from './shared/jobs/index.js'; import { getWorkerStatus, getXpWorkerStatus } from './shared/jobs/index.js'; -import { orchestrator } from './shared/runtime/index.js'; +import { orchestrator, registerShutdownSignals } from './shared/runtime/index.js'; import { getInfrastructureState, setApiStatus, type InfrastructureState } from './shared/runtime/infrastructureRegistry.js'; import routes from './routes/index.js'; import { getRegisteredWorkerStatuses } from './shared/jobs/workers.js'; @@ -134,41 +134,13 @@ async function bootstrap() { process.exit(1); }); + registerShutdownSignals(server); + // Phase 6.5: Start orchestrator in background (non-blocking) orchestrator.startup(app).catch((err) => { logger.error('Orchestrator startup failed (server still running in degraded mode)', err); setApiStatus('failed', err instanceof Error ? err.message : String(err)); }); - - // Phase 7: Graceful shutdown — drain infrastructure before exiting - let shuttingDown = false; - const shutdown = async () => { - if (shuttingDown) return; - shuttingDown = true; - logger.info('Shutting down... stopping infrastructure'); - - // Stop consumers immediately rather than waiting for lingering HTTP clients. - const timeout = setTimeout(() => { - logger.error('Forced shutdown after timeout'); - process.exit(1); - }, 10_000); - try { - await Promise.all([ - new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())), - orchestrator.shutdown(), - ]); - clearTimeout(timeout); - logger.info('Shutdown complete'); - process.exit(0); - } catch (error) { - clearTimeout(timeout); - logger.error('Shutdown failed', error); - process.exit(1); - } - }; - - process.on('SIGTERM', shutdown); - process.on('SIGINT', shutdown); } // --------------------------------------------------------------------------- diff --git a/backend/src/modules/ai/audit/AIResponseAuditLog.ts b/backend/src/modules/ai/audit/AIResponseAuditLog.ts index 40f808ed..fc391eb9 100644 --- a/backend/src/modules/ai/audit/AIResponseAuditLog.ts +++ b/backend/src/modules/ai/audit/AIResponseAuditLog.ts @@ -101,7 +101,25 @@ type AuditLogInput = Omit>; export class AIResponseAuditLogClass { + private readonly pendingWrites = new Set>(); + async log(entry: AuditLogInput): Promise { + const write = this.persist(entry); + this.pendingWrites.add(write); + try { + return await write; + } finally { + this.pendingWrites.delete(write); + } + } + + async flushPendingWrites(): Promise { + while (this.pendingWrites.size) { + await Promise.all(this.pendingWrites); + } + } + + private async persist(entry: AuditLogInput): Promise { try { // Persist metadata only. Prompts, generated text and provider errors can contain user data. const record = await AuditLogModel.create({ diff --git a/backend/src/shared/jobs/queueFactory.ts b/backend/src/shared/jobs/queueFactory.ts index a3d555b0..4942b0e5 100644 --- a/backend/src/shared/jobs/queueFactory.ts +++ b/backend/src/shared/jobs/queueFactory.ts @@ -15,9 +15,10 @@ export function getAllQueues(): Queue[] { } export async function closeAllQueues(): Promise { - const { dlqService } = await import('./dlq.service.js'); - await dlqService.close(); - await QueueRegistry.closeAllQueues(); + const dlqClose = import('./dlq.service.js').then(({ dlqService }) => dlqService.close()); + const results = await Promise.allSettled([dlqClose, QueueRegistry.closeAllQueues()]); + const errors = results.flatMap(result => result.status === 'rejected' ? [result.reason] : []); + if (errors.length) throw new AggregateError(errors, 'Queue cleanup failed'); } // Pre-registered queue getters diff --git a/backend/src/shared/jobs/workers.ts b/backend/src/shared/jobs/workers.ts index ac90e6ca..410c0006 100644 --- a/backend/src/shared/jobs/workers.ts +++ b/backend/src/shared/jobs/workers.ts @@ -14,10 +14,10 @@ export async function startAllWorkers(): Promise { catch (error) { await stopAllWorkers().catch(() => undefined); throw error; } } -export async function stopAllWorkers(): Promise { +export async function stopAllWorkers(timeoutMs?: number): Promise { const current = runtime; if (!current) return; - try { await current.stop(); } + try { await current.stop(timeoutMs); } finally { if (runtime === current) runtime = undefined; } } diff --git a/backend/src/shared/redis/client.ts b/backend/src/shared/redis/client.ts index 1df61f77..f01cb8c2 100644 --- a/backend/src/shared/redis/client.ts +++ b/backend/src/shared/redis/client.ts @@ -176,9 +176,17 @@ export function getRedisClient(): Redis { } export async function disconnectRedis(): Promise { - if (!_client) return; - await _client.quit(); - _client = null; + const client = _client; + if (!client) return; + try { + if (client.status === 'ready') await client.quit(); + } finally { + client.disconnect(); + if (_client === client) { + _client = null; + _connecting = false; + } + } logger.info('[redis] Disconnected', { event: 'redis_disconnected' }); } diff --git a/backend/src/shared/runtime/index.ts b/backend/src/shared/runtime/index.ts index 49a14483..f946e6b4 100644 --- a/backend/src/shared/runtime/index.ts +++ b/backend/src/shared/runtime/index.ts @@ -1,5 +1,5 @@ // src/shared/runtime/index.ts — Runtime infrastructure barrel -export { orchestrator } from './orchestrator.js'; +export { orchestrator, registerShutdownSignals } from './orchestrator.js'; export { getInfrastructureState, isOverallDegraded, diff --git a/backend/src/shared/runtime/orchestrator.ts b/backend/src/shared/runtime/orchestrator.ts index bdcd4762..3bbe892b 100644 --- a/backend/src/shared/runtime/orchestrator.ts +++ b/backend/src/shared/runtime/orchestrator.ts @@ -1,11 +1,11 @@ // src/shared/runtime/orchestrator.ts — Boot orchestrator (entry point for index.ts) // Wires multi-phase startup + graceful shutdown into a single callable. +import type { Server } from 'node:http'; import type { Express } from 'express'; import mongoose from 'mongoose'; -import { connectDatabase, sanitizeMongoError, scheduleMongoReconnect } from '../../db/connection.js'; +import { connectDatabase, disconnectDatabase, sanitizeMongoError, scheduleMongoReconnect } from '../../db/connection.js'; import { setupProductionIndexes } from '../../db/migrations/index-setup.js'; -import { env } from '../../config/env.js'; import { logger } from '../logger.js'; import { eventBus } from '../sse/index.js'; import { getRedisClient, disconnectRedis, getRedisHealth } from '../redis/index.js'; @@ -58,6 +58,7 @@ export interface StartupResult { async function bootMongo(retries = 3, retryDelayMs = 2000): Promise { setMongoStatus('initializing'); for (let attempt = 1; attempt <= retries; attempt++) { + assertStarting(); try { logger.info(`[startup] MongoDB connection attempt ${attempt}/${retries}...`); await connectDatabase(); @@ -79,6 +80,7 @@ async function bootMongo(retries = 3, retryDelayMs = 2000): Promise { logger.info('[startup] MongoDB connected and ready', { event: 'mongodb_connected', readyState: mongoose.connection.readyState }); return true; } catch (err) { + assertStarting(); const msg = err instanceof Error ? err.message : String(err); const safeMsg = typeof sanitizeMongoError === 'function' ? sanitizeMongoError(msg) : msg; logger.error(`[startup] MongoDB connection attempt ${attempt}/${retries} failed: ${safeMsg}`); @@ -233,40 +235,121 @@ export async function startup(app: Express): Promise { // Shutdown // --------------------------------------------------------------------------- -export function shutdown(): Promise { +export function shutdown(server?: Server): Promise { shutdownRequested = true; - shutdownPromise ??= performShutdown(); + shutdownPromise ??= performShutdown(server); return shutdownPromise; } -async function performShutdown(): Promise { +export function registerShutdownSignals(server: Server): void { + const handleSignal = () => { + if (shutdownRequested) return; + + // The individual phase budgets total 14 seconds. + const watchdog = setTimeout(() => { + logger.error('[shutdown] Forced shutdown after timeout'); + process.exit(1); + }, 15_000); + + void shutdown(server).then( + () => { + clearTimeout(watchdog); + process.exit(0); + }, + (error: unknown) => { + clearTimeout(watchdog); + logger.error('[shutdown] Shutdown failed', error); + process.exit(1); + }, + ); + }; + + process.on('SIGTERM', handleSignal); + process.on('SIGINT', handleSignal); +} + +async function performShutdown(server?: Server): Promise { + const startedAt = Date.now(); + const failures: string[] = []; + let firstError: unknown; logger.info('[shutdown] Initiating graceful shutdown'); - stopSyncScheduler(); - stopMissionScheduler(); // Stop mission scheduler - setSchedulerStatus('stopped'); + async function runPhase(name: string, timeoutMs: number, task: () => void | Promise): Promise { + let timer: ReturnType | undefined; + try { + await Promise.race([ + Promise.resolve().then(task), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${name} timed out after ${timeoutMs}ms`)), timeoutMs); + }), + ]); + return true; + } catch (error) { + if (failures.length === 0) firstError = error; + failures.push(name); + logger.error(`[shutdown] ${name} failed`, error); + return false; + } finally { + clearTimeout(timer); + } + } - let workerError: unknown; - try { await stopAllWorkers(); } catch (error) { workerError = error; } - setPlatformSyncWorkerStatus('stopped'); - setXpWorkerStatus('stopped'); + setApiStatus('stopped'); + const trafficDrained = await runPhase('HTTP and SSE drain', 2_000, async () => { + const httpClosed = server && new Promise((resolve, reject) => { + server.close(error => error ? reject(error) : resolve()); + }); + try { + stopSyncScheduler(); + stopMissionScheduler(); + setSchedulerStatus('stopped'); + eventBus.broadcastShutdown(); + } finally { + await httpClosed; + } + setSseStatus('stopped'); + }); + if (!trafficDrained) server?.closeAllConnections(); - await closeAllQueues(); - setQueuesStatus('stopped'); + // Worker.close() stops local consumption and drains active jobs. A global + // Queue.pause() would also stop healthy workers in other replicas. + if (await runPhase('BullMQ worker drain', 5_000, () => stopAllWorkers(5_000))) { + setPlatformSyncWorkerStatus('stopped'); + setXpWorkerStatus('stopped'); + } + if (await runPhase('BullMQ queue close', 1_000, closeAllQueues)) { + setQueuesStatus('stopped'); + } - try { - const { unifiedRuntimeStateService } = await import('../../modules/runtime-state/unifiedRuntimeState.service.js'); - await unifiedRuntimeStateService.flushPendingEvents(); - } catch (error) { workerError ??= error; } - await disconnectRedis(); - setRedisStatus('stopped'); + await runPhase('Runtime state and audit flush', 3_000, async () => { + const results = await Promise.allSettled([ + import('../../modules/runtime-state/unifiedRuntimeState.service.js') + .then(({ unifiedRuntimeStateService }) => unifiedRuntimeStateService.flushPendingEvents()), + import('../../modules/ai/audit/AIResponseAuditLog.js') + .then(({ AIResponseAuditLog }) => AIResponseAuditLog.flushPendingWrites()), + ]); + const errors = results.flatMap(result => result.status === 'rejected' ? [result.reason] : []); + if (errors.length === 1) throw errors[0]; + if (errors.length) throw new AggregateError(errors, 'Pending writes failed to flush'); + }); - eventBus.shutdown?.(); - setSseStatus('stopped'); + await runPhase('Redis teardown', 1_000, async () => { + try { + eventBus.shutdown(); + } finally { + await disconnectRedis(); + } + setRedisStatus('stopped'); + }); + await runPhase('MongoDB teardown', 2_000, disconnectDatabase); setApiStatus('stopped'); - if (workerError) throw workerError; - logger.info('[shutdown] Graceful shutdown complete', { event: 'shutdown_complete' }); + const metrics = { durationMs: Date.now() - startedAt, failedPhases: failures }; + if (failures.length) { + logger.warn('[shutdown] Graceful shutdown finished with errors', metrics); + throw firstError; + } + logger.info('[shutdown] Graceful shutdown complete', { event: 'shutdown_complete', ...metrics }); } // --------------------------------------------------------------------------- diff --git a/backend/src/shared/sse/eventBus.ts b/backend/src/shared/sse/eventBus.ts index a1040310..bb7ee513 100644 --- a/backend/src/shared/sse/eventBus.ts +++ b/backend/src/shared/sse/eventBus.ts @@ -278,6 +278,7 @@ const encoder = new TextEncoder(); // --------------------------------------------------------------------------- class EventBus extends EventEmitter { + private draining = false; private clients = new Map(); private heartbeatInterval: NodeJS.Timeout | null = null; private readonly HEARTBEAT_INTERVAL_MS = 25_000; @@ -326,6 +327,7 @@ class EventBus extends EventEmitter { // Persistent subscriber const subscriber = createRedisConnection(connectionOptions) as any; + this.redisSubscriber = subscriber; let isSubscribed = false; let subscriptionPromise: Promise | null = null; @@ -393,10 +395,11 @@ class EventBus extends EventEmitter { // Warning logged with context inside ensureSubscribed }); } - this.redisSubscriber = subscriber as any; + if (this.draining) return; // Shared singleton publisher const publisher = createRedisConnection(connectionOptions) as any; + this.redisPublisher = publisher; (publisher as any).on('error', (err: Error) => { logger.error('[sse-redis] Publisher error', err); }); @@ -406,7 +409,7 @@ class EventBus extends EventEmitter { await (publisher as any).connect().catch((err: unknown) => { logger.warn('[sse-redis] Publisher connect failed', { error: err instanceof Error ? err.message : String(err) }); }); - this.redisPublisher = publisher as any; + if (this.draining) return; logger.info('[sse] Redis PubSub & Publisher singletons initialized', { channel: SSE_CHANNEL }); } catch (err) { @@ -470,6 +473,7 @@ class EventBus extends EventEmitter { // --------------------------------------------------------------------------- register(client: SseClient): void { + if (this.draining) throw new Error('Server is shutting down'); if (this.clients.size >= this.MAX_CLIENTS) { logger.warn('[sse] Max clients reached, rejecting connection', { currentClients: this.clients.size, @@ -1013,6 +1017,21 @@ class EventBus extends EventEmitter { void this.emitRuntimeStatePatch(userId, state); } + broadcastShutdown(): void { + this.draining = true; + for (const client of this.clients.values()) { + const reconnectDelayMs = 2000 + Math.floor(Math.random() * 3001); + const data = JSON.stringify({ reconnectDelayMs }); + try { + client.controller.enqueue(encoder.encode(`event: server_shutdown\nretry: ${reconnectDelayMs}\ndata: ${data}\n\n`)); + client.controller.close(); + } catch { + // The response may already have disconnected. + } + this.unregister(client.id); + } + } + shutdown(): void { this.stopHeartbeat(); for (const { timer } of this.pendingPatches.values()) { diff --git a/backend/src/shared/sse/sseHandler.ts b/backend/src/shared/sse/sseHandler.ts index a44443c2..dde2cd63 100644 --- a/backend/src/shared/sse/sseHandler.ts +++ b/backend/src/shared/sse/sseHandler.ts @@ -239,9 +239,9 @@ export async function handleSseRequest(req: Request, res: Response): Promise ({ // --------------------------------------------------------------------------- // EventSource test double (jsdom does not implement EventSource) // --------------------------------------------------------------------------- -class MockEventSource { +class MockEventSource extends EventTarget { static instances: MockEventSource[] = []; url: string; @@ -33,6 +33,7 @@ class MockEventSource { onerror: ((ev: Event) => void) | null = null; constructor(url: string) { + super(); this.url = url; MockEventSource.instances.push(this); } @@ -127,6 +128,20 @@ describe('computeBackoffDelay', () => { }); describe('reconnection backoff', () => { + it('uses the server shutdown delay before requesting a fresh ticket', async () => { + await connect(); + const source = latest(); + source.dispatchEvent(new MessageEvent('server_shutdown', { + data: JSON.stringify({ reconnectDelayMs: 3500 }), + })); + source.fail(); + expect(source.closed).toBe(true); + await advance(3499); + expect(getSSEHandshakeTicket).toHaveBeenCalledTimes(1); + await advance(1); + expect(getSSEHandshakeTicket).toHaveBeenCalledTimes(2); + }); + it('waits a jittered exponential delay between consecutive failures', async () => { await connect(); expect(MockEventSource.instances).toHaveLength(1); diff --git a/frontend/src/lib/realtime/sse-manager.ts b/frontend/src/lib/realtime/sse-manager.ts index 1b30d395..21bbf120 100644 --- a/frontend/src/lib/realtime/sse-manager.ts +++ b/frontend/src/lib/realtime/sse-manager.ts @@ -207,7 +207,7 @@ function enterOfflineState(): void { setConnectionState({ connected: false, reconnecting: false, offline: true, error: null }); } -function scheduleReconnect(): void { +function scheduleReconnect(serverDelayMs?: number): void { if (!shouldConnect) return; clearReconnectTimer(); @@ -221,7 +221,7 @@ function scheduleReconnect(): void { return; } - const delay = computeBackoffDelay(reconnectAttempts); + const delay = serverDelayMs ?? computeBackoffDelay(reconnectAttempts); reconnectAttempts++; setConnectionState({ connected: false, @@ -434,6 +434,22 @@ export const useSseStore = create((set, get) => ({ } }; + source.addEventListener('server_shutdown', (event) => { + if (source !== eventSource || !(event instanceof MessageEvent)) return; + let delay: unknown; + try { + delay = JSON.parse(event.data)?.reconnectDelayMs; + } catch { + // Use the normal reconnect backoff if the control frame is malformed. + } + closeEventSource(); + scheduleReconnect( + typeof delay === 'number' && Number.isInteger(delay) && delay >= 2000 && delay <= 5000 + ? delay + : undefined + ); + }); + source.onerror = () => { if (source !== eventSource) return; closeEventSource();