Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,28 @@ describe('EventBus Resilience & Degraded Mode (No Redis)', () => {
return { client, enqueuedChunks };
}

it('notifies each local SSE client before closing its stream', () => {
const bus = new EventBus();
const clients = ['first', 'second'].map(id => createMockClient(id, id));
for (const { client } of clients) bus.register(client);

bus.broadcastShutdown();

for (const { client, enqueuedChunks } of clients) {
expect(enqueuedChunks).toHaveLength(1);
const frame = enqueuedChunks[0];
const delay = Number(frame.match(/^retry: (\d+)$/m)?.[1]);
expect(delay).toBeGreaterThanOrEqual(2000);
expect(delay).toBeLessThanOrEqual(5000);
expect(frame).toContain('event: server_shutdown');
expect(frame).toContain(JSON.stringify({ reconnectDelayMs: delay }));
expect(client.controller.enqueue).toHaveBeenCalledBefore(vi.mocked(client.controller.close));
}
expect(bus.getMetrics().activeConnections).toBe(0);
expect(() => bus.register(createMockClient('late', 'late').client)).toThrow('shutting down');
bus.shutdown();
});

it('skips Redis PubSub and does not instantiate ioredis when REDIS_URL is absent', () => {
delete process.env.REDIS_URL;

Expand Down
14 changes: 14 additions & 0 deletions backend/src/__tests__/integration/aiAuditPersistence.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,20 @@ const auditEntry = {
};

describe('AI audit persistence', () => {
it('waits for an audit write that the caller did not await', async () => {
const audit = new AIResponseAuditLogClass();
let completed = false;
const write = audit.log(auditEntry).then(entry => {
completed = true;
return entry;
});

await audit.flushPendingWrites();

expect(completed).toBe(true);
expect(await write).toMatchObject({ userId: auditEntry.userId });
});

it('persists metadata for a fresh service instance and excludes raw content', async () => {
const writer = new AIResponseAuditLogClass();
const entry = await writer.log(auditEntry);
Expand Down
6 changes: 2 additions & 4 deletions backend/src/__tests__/integration/setup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ afterAll(async () => {
}
});

const [{ getRedisClient }, { logger }, { User, UserXp }] = await Promise.all([
const [{ getRedisClient, disconnectRedis }, { logger }, { User, UserXp }] = await Promise.all([
import('../../shared/redis/index.js'),
import('../../shared/logger.js'),
import('../../db/models/index.js'),
Expand All @@ -53,9 +53,7 @@ beforeAll(async () => {
});

afterAll(async () => {
// Disconnect Redis
const redis = getRedisClient();
await redis.quit();
await disconnectRedis();
});

beforeEach(async () => {
Expand Down
18 changes: 15 additions & 3 deletions backend/src/__tests__/integration/sseHandshake.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,7 @@ describe('Security: SSE Handshake Ticket Protocol', () => {
});

afterAll(async () => {
server.closeAllConnections?.();
await new Promise<void>((resolve) => server.close(() => resolve()));
await orchestrator.shutdown();
await orchestrator.shutdown(server);
});

it('should have successfully obtained a test access token', () => {
Expand Down Expand Up @@ -141,4 +139,18 @@ describe('Security: SSE Handshake Ticket Protocol', () => {
const status = await testSseConnection(fakeTicket);
expect(status).toBe(401);
});

it('delivers the shutdown event before ending the HTTP stream', async () => {
const ticketResponse = await fetch(`${API_BASE}/api/auth/sse-handshake`, {
method: 'POST',
headers: { Authorization: `Bearer ${accessToken}` },
});
const { data } = await ticketResponse.json() as { data: { ticket: string } };
const response = await fetch(`${API_BASE}/api/events?ticket=${data.ticket}`);
expect(response.status).toBe(200);

const [body] = await Promise.all([response.text(), orchestrator.shutdown(server)]);
expect(body).toMatch(/event: server_shutdown\nretry: \d+\ndata: {"reconnectDelayMs":\d+}/);
expect(server.listening).toBe(false);
});
});
74 changes: 69 additions & 5 deletions backend/src/__tests__/worker-lifecycle/api-startup.test.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
const state = vi.hoisted(() => ({
connect: vi.fn(), start: vi.fn(), stop: vi.fn(), queuesClose: vi.fn(), redisClose: vi.fn(), flush: vi.fn(),
auditFlush: vi.fn(), sseDrain: vi.fn(), sseClose: vi.fn(), mongoClose: vi.fn(),
setSync: vi.fn(), setXp: vi.fn(), ready: true,
}));
vi.mock('../../config/env.js', () => ({ env: {} }));
vi.mock('../../db/connection.js', () => ({ connectDatabase: state.connect }));
vi.mock('../../db/connection.js', () => ({ connectDatabase: state.connect, disconnectDatabase: state.mongoClose }));
vi.mock('../../modules/runtime-state/unifiedRuntimeState.service.js', () => ({ unifiedRuntimeStateService: { flushPendingEvents: state.flush } }));
vi.mock('../../modules/ai/audit/AIResponseAuditLog.js', () => ({ AIResponseAuditLog: { flushPendingWrites: state.auditFlush } }));
vi.mock('../../db/migrations/index-setup.js', () => ({ setupProductionIndexes: vi.fn().mockResolvedValue(undefined) }));
vi.mock('../../shared/logger.js', () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } }));
vi.mock('../../shared/sse/index.js', () => ({ eventBus: { shutdown: vi.fn() } }));
vi.mock('../../shared/sse/index.js', () => ({ eventBus: { broadcastShutdown: state.sseDrain, shutdown: state.sseClose } }));
vi.mock('../../shared/redis/index.js', () => ({
getRedisClient: () => ({ status: 'ready' }), disconnectRedis: state.redisClose,
getRedisHealth: () => ({ status: state.ready ? 'connected' : 'disconnected' }),
Expand All @@ -25,16 +27,22 @@ vi.mock('../../shared/runtime/infrastructureRegistry.js', () => ({
setPlatformSyncWorkerStatus: state.setSync, setXpWorkerStatus: state.setXp, setSchedulerStatus: vi.fn(), setSseStatus: vi.fn(),
}));
import express from 'express';
import { createServer } from 'node:http';
let startup: typeof import('../../shared/runtime/orchestrator.js')['startup'];
let registerShutdownSignals: typeof import('../../shared/runtime/orchestrator.js')['registerShutdownSignals'];
let shutdown: typeof import('../../shared/runtime/orchestrator.js')['shutdown'];
beforeEach(async () => {
vi.resetModules();
state.ready = true;
for (const item of Object.values(state)) if (typeof item === 'function') item.mockReset().mockResolvedValue(undefined);
vi.stubEnv('ENABLE_WORKERS', 'true');
({ startup, shutdown } = await import('../../shared/runtime/orchestrator.js'));
({ startup, shutdown, registerShutdownSignals } = await import('../../shared/runtime/orchestrator.js'));
});
afterEach(() => {
vi.unstubAllEnvs();
vi.restoreAllMocks();
vi.useRealTimers();
});
afterEach(() => vi.unstubAllEnvs());
it('starts the shared registry and drains it before closing queue and Redis handles', async () => {
const result = await startup(express());
expect(state.start).toHaveBeenCalledOnce();
Expand All @@ -44,7 +52,8 @@ it('starts the shared registry and drains it before closing queue and Redis hand
expect(state.queuesClose.mock.invocationCallOrder[0]).toBeGreaterThan(state.stop.mock.invocationCallOrder[0]);
expect(state.flush).toHaveBeenCalledOnce();
expect(state.flush.mock.invocationCallOrder[0]).toBeGreaterThan(state.queuesClose.mock.invocationCallOrder[0]);
expect(state.redisClose.mock.invocationCallOrder[0]).toBeGreaterThan(state.flush.mock.invocationCallOrder[0]);
expect(state.redisClose).toHaveBeenCalledAfter(state.flush);
expect(state.redisClose).toHaveBeenCalledAfter(state.auditFlush);
});
it.each(['disabled', 'offline'])('does not start consumers when %s', async reason => {
if (reason === 'disabled') vi.stubEnv('ENABLE_WORKERS', 'false'); else state.ready = false;
Expand Down Expand Up @@ -83,3 +92,58 @@ it('prevents late startup from creating workers after shutdown and drains once',
expect(state.redisClose).toHaveBeenCalledOnce();
expect(state.flush).toHaveBeenCalledOnce();
});

it.each(['SIGTERM', 'SIGINT'] as const)('drains resources in order when it receives %s', async signal => {
const server = createServer();
const httpClose = vi.spyOn(server, 'close').mockImplementation(callback => {
callback?.();
return server;
});
const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never);
let finishMongoClose!: () => void;
state.mongoClose.mockReturnValueOnce(new Promise<void>(resolve => { finishMongoClose = resolve; }));
registerShutdownSignals(server);
const sigtermHandler = process.listeners('SIGTERM').at(-1)!;
const sigintHandler = process.listeners('SIGINT').at(-1)!;

try {
const handler = signal === 'SIGTERM' ? sigtermHandler : sigintHandler;
handler(signal);
handler(signal);
const draining = shutdown();
await vi.waitFor(() => expect(state.mongoClose).toHaveBeenCalledOnce());
expect(exit).not.toHaveBeenCalled();
finishMongoClose();
await draining;

expect(httpClose).toHaveBeenCalledOnce();
expect(state.stop).toHaveBeenCalledExactlyOnceWith(5000);
const calls = [
httpClose, state.sseDrain, state.stop, state.queuesClose, state.flush,
state.sseClose, state.redisClose, state.mongoClose, exit,
].map(mock => mock.mock.invocationCallOrder[0]);
expect(calls).toEqual([...calls].sort((a, b) => a - b));
expect(exit).toHaveBeenCalledExactlyOnceWith(0);
} finally {
process.off('SIGTERM', sigtermHandler);
process.off('SIGINT', sigintHandler);
}
});

it('keeps closing resources when individual phases time out or reject late', async () => {
vi.useFakeTimers();
let rejectWorkerDrain!: (error: Error) => void;
state.stop.mockReturnValueOnce(new Promise((_, reject) => { rejectWorkerDrain = reject; }));
state.flush.mockReturnValueOnce(new Promise(() => {}));
state.redisClose.mockRejectedValueOnce(new Error('Redis failed'));

const draining = expect(shutdown()).rejects.toThrow('BullMQ worker drain timed out');
await vi.advanceTimersByTimeAsync(5000);
expect(state.queuesClose).toHaveBeenCalledOnce();
expect(state.redisClose).not.toHaveBeenCalled();
rejectWorkerDrain(new Error('late worker failure'));
await vi.advanceTimersByTimeAsync(3000);
await draining;
expect(state.mongoClose).toHaveBeenCalledOnce();
expect(vi.getTimerCount()).toBe(0);
});
10 changes: 4 additions & 6 deletions backend/src/db/connection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@ export const MONGO_OPTIONS: mongoose.ConnectOptions = {
};

let reconnectTimer: NodeJS.Timeout | null = null;
let disconnecting = false;

export function scheduleMongoReconnect(delayMs = 5000): void {
if (reconnectTimer) return;
if (reconnectTimer || disconnecting) return;
reconnectTimer = setTimeout(async () => {
reconnectTimer = null;
if (mongoose.connection.readyState === 0) {
Expand All @@ -41,6 +42,7 @@ export function scheduleMongoReconnect(delayMs = 5000): void {
}

export async function connectDatabase(): Promise<void> {
disconnecting = false;
try {
logger.info('Attempting MongoDB connection...', {
readyState: mongoose.connection.readyState,
Expand All @@ -61,6 +63,7 @@ export async function connectDatabase(): Promise<void> {
}

export async function disconnectDatabase(): Promise<void> {
disconnecting = true;
if (reconnectTimer) {
clearTimeout(reconnectTimer);
reconnectTimer = null;
Expand Down Expand Up @@ -99,8 +102,3 @@ mongoose.connection.on('disconnected', () => {
logger.warn('MongoDB disconnected event received', { readyState: mongoose.connection.readyState });
scheduleMongoReconnect();
});

process.on('SIGINT', async () => {
await disconnectDatabase();
process.exit(0);
});
34 changes: 3 additions & 31 deletions backend/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { getRedisHealth } from './shared/redis/index.js';
import { validateStartup } from './shared/startup-validation.js';
import { getOrCreateQueue, QueueNames } from './shared/jobs/index.js';
import { getWorkerStatus, getXpWorkerStatus } from './shared/jobs/index.js';
import { orchestrator } from './shared/runtime/index.js';
import { orchestrator, registerShutdownSignals } from './shared/runtime/index.js';
import { getInfrastructureState, setApiStatus, type InfrastructureState } from './shared/runtime/infrastructureRegistry.js';
import routes from './routes/index.js';
import { getRegisteredWorkerStatuses } from './shared/jobs/workers.js';
Expand Down Expand Up @@ -134,41 +134,13 @@ async function bootstrap() {
process.exit(1);
});

registerShutdownSignals(server);

// Phase 6.5: Start orchestrator in background (non-blocking)
orchestrator.startup(app).catch((err) => {
logger.error('Orchestrator startup failed (server still running in degraded mode)', err);
setApiStatus('failed', err instanceof Error ? err.message : String(err));
});

// Phase 7: Graceful shutdown — drain infrastructure before exiting
let shuttingDown = false;
const shutdown = async () => {
if (shuttingDown) return;
shuttingDown = true;
logger.info('Shutting down... stopping infrastructure');

// Stop consumers immediately rather than waiting for lingering HTTP clients.
const timeout = setTimeout(() => {
logger.error('Forced shutdown after timeout');
process.exit(1);
}, 10_000);
try {
await Promise.all([
new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve())),
orchestrator.shutdown(),
]);
clearTimeout(timeout);
logger.info('Shutdown complete');
process.exit(0);
} catch (error) {
clearTimeout(timeout);
logger.error('Shutdown failed', error);
process.exit(1);
}
};

process.on('SIGTERM', shutdown);
process.on('SIGINT', shutdown);
}

// ---------------------------------------------------------------------------
Expand Down
18 changes: 18 additions & 0 deletions backend/src/modules/ai/audit/AIResponseAuditLog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,25 @@ type AuditLogInput = Omit<AuditLogEntry, 'logId' | 'timestamp' | 'prompt' | 'sys
& Partial<Pick<AuditLogEntry, 'prompt' | 'systemPrompt' | 'response'>>;

export class AIResponseAuditLogClass {
private readonly pendingWrites = new Set<Promise<AuditLogEntry | undefined>>();

async log(entry: AuditLogInput): Promise<AuditLogEntry | undefined> {
const write = this.persist(entry);
this.pendingWrites.add(write);
try {
return await write;
} finally {
this.pendingWrites.delete(write);
}
}

async flushPendingWrites(): Promise<void> {
while (this.pendingWrites.size) {
await Promise.all(this.pendingWrites);
}
}

private async persist(entry: AuditLogInput): Promise<AuditLogEntry | undefined> {
try {
// Persist metadata only. Prompts, generated text and provider errors can contain user data.
const record = await AuditLogModel.create({
Expand Down
7 changes: 4 additions & 3 deletions backend/src/shared/jobs/queueFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@ export function getAllQueues(): Queue[] {
}

export async function closeAllQueues(): Promise<void> {
const { dlqService } = await import('./dlq.service.js');
await dlqService.close();
await QueueRegistry.closeAllQueues();
const dlqClose = import('./dlq.service.js').then(({ dlqService }) => dlqService.close());
const results = await Promise.allSettled([dlqClose, QueueRegistry.closeAllQueues()]);
const errors = results.flatMap(result => result.status === 'rejected' ? [result.reason] : []);
if (errors.length) throw new AggregateError(errors, 'Queue cleanup failed');
}

// Pre-registered queue getters
Expand Down
4 changes: 2 additions & 2 deletions backend/src/shared/jobs/workers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ export async function startAllWorkers(): Promise<void> {
catch (error) { await stopAllWorkers().catch(() => undefined); throw error; }
}

export async function stopAllWorkers(): Promise<void> {
export async function stopAllWorkers(timeoutMs?: number): Promise<void> {
const current = runtime;
if (!current) return;
try { await current.stop(); }
try { await current.stop(timeoutMs); }
finally { if (runtime === current) runtime = undefined; }
}

Expand Down
14 changes: 11 additions & 3 deletions backend/src/shared/redis/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,17 @@ export function getRedisClient(): Redis {
}

export async function disconnectRedis(): Promise<void> {
if (!_client) return;
await _client.quit();
_client = null;
const client = _client;
if (!client) return;
try {
if (client.status === 'ready') await client.quit();
} finally {
client.disconnect();
if (_client === client) {
_client = null;
_connecting = false;
}
}
logger.info('[redis] Disconnected', { event: 'redis_disconnected' });
}

Expand Down
2 changes: 1 addition & 1 deletion backend/src/shared/runtime/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// src/shared/runtime/index.ts — Runtime infrastructure barrel
export { orchestrator } from './orchestrator.js';
export { orchestrator, registerShutdownSignals } from './orchestrator.js';
export {
getInfrastructureState,
isOverallDegraded,
Expand Down
Loading
Loading